The Day the Knowledge Cutoff Moved Forward, the Stale Part Was My System Instruction

#!/usr/bin/env python3
"""cutoff_window_audit.py — classify dated assertions against two model cutoffs."""
from __future__ import annotations
 
import argparse
import json
import re
import sys
from dataclasses import dataclass, asdict
from datetime import date
from pathlib import Path
 
MONTH_NAME = (
    r"(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|"
    r"jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)"
)
EN_DATE = re.compile(rf"\b(?P<mon>{MONTH_NAME})\.?,?\s+(?P<y>19\d{{2}}|20\d{{2}})\b", re.I)
ISO_DATE = re.compile(r"\b(?P<y>20\d{2})-(?P<m>0[1-9]|1[0-2])(?:-\d{2})?\b")
 
# Leading "As of <date>," / "2026-01:" framing, stripped before judging the clause
DATE_FRAMING = re.compile(
    rf"^\s*(?:as\s+of\s+)?(?:{MONTH_NAME}\.?,?\s+\d{{4}}|20\d{{2}}-\d{{2}}(?:-\d{{2}})?)\s*[,:—-]?\s*",
    re.I,
)
 
MONTHS = {
    "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
}
 
ASSERTIVE = (
    "not supported", "does not support", "does not exist", "unavailable",
    "is not available", "cannot be used", "no support for", "latest",
)
HEDGED = ("may ", "might ", "possibly", "e.g.", "for example", "such as")
 
# Second pass: a dated clause carrying a copula is a fact claim, not an instruction
COPULA = re.compile(r"\b(?:is|are|was|were|has|have|had|does|do|remains?|will)\b", re.I)
 
 
@dataclass
class Assertion:
    source: str
    line_no: int
    text: str
    asserted_on: str | None
    verdict: str
    reason: str
 
 
def _to_month(y: int, m: int | None) -> date:
    return date(y, m or 1, 1)
 
 
def _without_dates(line: str) -> str:
    """Blank out date literals so month names cannot collide with hedge words."""
    return ISO_DATE.sub(" ", EN_DATE.sub(" ", line))
 
 
def extract_date(line: str) -> date | None:
    """Return the earliest month-precision date pinned in the line, if any."""
    found: list[date] = []
    for hit in EN_DATE.finditer(line):
        found.append(_to_month(int(hit.group("y")), MONTHS[hit.group("mon")[:3].lower()]))
    for hit in ISO_DATE.finditer(line):
        try:
            found.append(_to_month(int(hit.group("y")), int(hit.group("m"))))
        except ValueError:
            continue
    return min(found) if found else None
 
 
def is_assertive(line: str, dated: bool = False) -> bool:
    clause = DATE_FRAMING.sub("", line)
    low = _without_dates(clause).lower()
    if any(h in low for h in HEDGED):
        return False
    if any(tok in low for tok in ASSERTIVE):
        return True
    return bool(dated and COPULA.search(clause))
 
 
def classify(pinned: date | None, old_cutoff: date, new_cutoff: date) -> tuple[str, str]:
    if pinned is None:
        return "undated", "no date found; needs human review"
    if pinned < old_cutoff:
        return "redundant", "predates the old cutoff; the model already knew this"
    if pinned < new_cutoff:
        return "contested", "between the cutoffs; a supplement that flipped into a contradiction"
    return "current", "after the new cutoff; still needed"
 
 
def audit(paths: list[Path], old_cutoff: date, new_cutoff: date) -> list[Assertion]:
    out: list[Assertion] = []
    for path in paths:
        try:
            body = path.read_text(encoding="utf-8")
        except (OSError, UnicodeDecodeError) as exc:
            print(f"skip {path}: {exc}", file=sys.stderr)
            continue
        for i, line in enumerate(body.splitlines(), start=1):
            stripped = line.strip()
            if not stripped or stripped.startswith("#"):
                continue
            pinned = extract_date(stripped)
            if not is_assertive(stripped, dated=pinned is not None):
                continue
            verdict, reason = classify(pinned, old_cutoff, new_cutoff)
            out.append(
                Assertion(
                    source=path.name,
                    line_no=i,
                    text=stripped[:120],
                    asserted_on=pinned.isoformat() if pinned else None,
                    verdict=verdict,
                    reason=reason,
                )
            )
    return out
 
 
def parse_month(text: str) -> date:
    y, m = text.split("-")[:2]
    return date(int(y), int(m), 1)
 
 
def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("paths", nargs="+", type=Path)
    ap.add_argument("--old-cutoff", required=True, help="e.g. 2025-01")
    ap.add_argument("--new-cutoff", required=True, help="e.g. 2026-03")
    ap.add_argument("--json", action="store_true")
    ap.add_argument("--fail-on-contested", action="store_true")
    args = ap.parse_args()
 
    old_cutoff, new_cutoff = parse_month(args.old_cutoff), parse_month(args.new_cutoff)
    if old_cutoff >= new_cutoff:
        ap.error("--old-cutoff must precede --new-cutoff")
 
    files = [p for p in args.paths if p.is_file()]
    rows = audit(files, old_cutoff, new_cutoff)
 
    if args.json:
        print(json.dumps([asdict(r) for r in rows], ensure_ascii=False, indent=2))
    else:
        for r in rows:
            print(f"[{r.verdict:9}] {r.source}:{r.line_no} ({r.asserted_on or 'undated'}) {r.text}")
        counts: dict[str, int] = {}
        for r in rows:
            counts[r.verdict] = counts.get(r.verdict, 0) + 1
        print("---", " ".join(f"{k}={v}" for k, v in sorted(counts.items())) or "no findings")
 
    contested = sum(1 for r in rows if r.verdict == "contested")
    return 1 if (args.fail_on_contested and contested) else 0
 
 
if __name__ == "__main__":
    raise SystemExit(main())

# Image classification assistant — system instruction (excerpt, under audit)
As of December 2024, return Japanese category names verbatim without normalising them.
As of May 2023, structured output is not supported, so embed the JSON in the body text.
As of June 2025, thinking_budget is unavailable on the Flash tier.
As of November 2025, this API does not accept image and text input together.
2026-01: the Files API upload retention window is 48 hours.
As of April 2026, the Batch API concurrency limit is per project.
2026-05: this feature is not supported.
Always answer in Japanese, using polite forms.
When uncertain, you may return "undetermined".
As of August 2025, the latest alias updates automatically.
Responses may run long. That was the behaviour as of March 2025.

2026-01: the Files API upload retention window is 48 hours.
As of April 2026, the Batch API concurrency limit is per project.

def _without_dates(line: str) -> str:
    """Blank out date literals so month names cannot collide with hedge words."""
    return ISO_DATE.sub(" ", EN_DATE.sub(" ", line))

python3 cutoff_window_audit.py prompts/*.txt \
  --old-cutoff 2025-01 --new-cutoff 2026-03 --fail-on-contested

# Before
As of November 2025, this API does not accept image and text input together.
 
# After
If your knowledge about combined image and text input disagrees with the
actual behaviour, trust the error message you actually received.