I Stopped Asking Gemini 'Is This Right?' and Started Hiding My Answer Instead

import hashlib
import io
import json
import random
from pathlib import Path
 
from google import genai
from google.genai import types
from PIL import Image
 
# Which models a new project can use varies by account; check models.list first
MODEL = "gemini-3.8-flash"
CATEGORIES = ["名所絵", "美人画", "役者絵", "花鳥画", "武者絵", "戯画", "その他"]
 
client = genai.Client()  # reads GEMINI_API_KEY from the environment
 
 
def build_schema(categories: list[str]) -> types.Schema:
    """Observe -> choose -> runner-up -> confidence. The order is part of the design."""
    return types.Schema(
        type=types.Type.OBJECT,
        properties={
            "observations": types.Schema(
                type=types.Type.STRING,
                description="What is visible (figures, background, props, text). No category names.",
            ),
            "category": types.Schema(type=types.Type.STRING, enum=categories),
            "runner_up": types.Schema(type=types.Type.STRING, enum=categories),
            "confidence": types.Schema(type=types.Type.NUMBER),
        },
        required=["observations", "category", "runner_up", "confidence"],
        property_ordering=["observations", "category", "runner_up", "confidence"],
    )
 
 
def blind_bytes(path: Path) -> tuple[str, bytes]:
    """Re-encode to drop metadata, and track by hash instead of file name."""
    img = Image.open(path).convert("RGB")
    buf = io.BytesIO()
    img.save(buf, format="JPEG", quality=90)  # EXIF and IPTC captions are dropped here
    data = buf.getvalue()
    return hashlib.sha256(data).hexdigest()[:12], data
 
 
def classify_blind(path: Path, seed: int) -> dict:
    token, data = blind_bytes(path)
    cats = CATEGORIES[:]
    random.Random(seed).shuffle(cats)  # spread out any pull toward the first option
    prompt = (
        "この浮世絵の主題を、次の候補から一つだけ選んでください。"
        f"候補: {'、'.join(cats)}。"
        "先に画面に見えるものを書き、そのあとで選んでください。"
    )
    resp = client.models.generate_content(
        model=MODEL,
        contents=[types.Part.from_bytes(data=data, mime_type="image/jpeg"), prompt],
        config=types.GenerateContentConfig(
            response_mime_type="application/json",
            response_schema=build_schema(cats),
            temperature=0.2,
        ),
    )
    result = json.loads(resp.text)
    result["token"] = token
    result["model_version"] = resp.model_version  # keep the model that actually answered
    return result

import re
 
 
def assert_no_leak(prompt: str, my_label: str, categories: list[str]) -> None:
    """Make sure neither my tag nor file-name hints appear outside the option list."""
    head = prompt.split("候補:")[0]
    if my_label in head:
        raise ValueError(f"My own tag appears before the option list: {my_label}")
    if re.search(r"[A-Za-z_]+_\d+\.(jpe?g|png)", prompt):
        raise ValueError("Something that looks like a file name is in the prompt")
    for word in ("多め", "ほとんど", "大半"):  # "mostly", "nearly all", "the majority"
        if word in head:
            raise ValueError(f"A batch-level hint is still in the preamble: {word}")
    if sorted(categories) != sorted(CATEGORIES):
        raise ValueError("The option set does not match the source of truth")

import csv
from dataclasses import asdict, dataclass
 
 
@dataclass
class Verdict:
    token: str
    mine: str
    blind: str
    runner_up: str
    stable: bool
    route: str
 
 
def decide_route(mine: str, a: dict, b: dict) -> Verdict:
    stable = a["category"] == b["category"]
    if a["category"] != mine and a["runner_up"] != mine:
        route = "human"        # my tag isn't in the top two; suspect my side first
    elif a["category"] != mine:
        route = "boundary"     # my tag is the runner-up; a print that spans two subjects
    elif not stable:
        route = "sample"       # agreed, but the answer wobbles when the order changes
    else:
        route = "pass"
    return Verdict(a["token"], mine, a["category"], a["runner_up"], stable, route)
 
 
def review_batch(items: list[tuple[Path, str]], out_tsv: Path) -> None:
    with out_tsv.open("w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=list(Verdict.__dataclass_fields__), delimiter="\t")
        writer.writeheader()
        for path, mine in items:
            first = classify_blind(path, seed=1)
            # if the first pass already disagrees, a human looks anyway; only re-ask on agreement
            second = classify_blind(path, seed=2) if first["category"] == mine else first
            writer.writerow(asdict(decide_route(mine, first, second)))

REVIEW_PROMPT = """Read the following English description as someone seeing this app for the first time.
List at most three places a reader might misunderstand.
If there are none, answer only "none".
Do not list strengths or give overall impressions.
 
---
{text}
"""
 
 
def review_copy(text: str) -> str:
    resp = client.models.generate_content(
        model=MODEL,
        contents=REVIEW_PROMPT.format(text=text),
        config=types.GenerateContentConfig(temperature=0.2),
    )
    return resp.text.strip()