モデル入れ替えの合否を一致率で決めると、カテゴリ分布のずれは通過します

import os, json, math
from collections import Counter
from google import genai
from google.genai import types
 
CATEGORIES = ["landscape", "night-sky", "flower", "animal", "abstract"]  # 実際は30件
OLD_MODEL = "gemini-3.5-flash"
NEW_MODEL = "gemini-3.7-flash"
 
client = genai.Client(api_key=os.environ.get("GEMINI_API_KEY", "YOUR_API_KEY"))
 
SCHEMA = types.Schema(
    type=types.Type.OBJECT,
    required=["category"],
    properties={"category": types.Schema(type=types.Type.STRING, enum=CATEGORIES)},
)
 
def classify(model: str, image_bytes: bytes) -> str | None:
    """1枚を1カテゴリへ。失敗は None を返し、その画像は対を作らず捨てる。"""
    try:
        res = client.models.generate_content(
            model=model,
            contents=[
                types.Part.from_bytes(data=image_bytes, mime_type="image/jpeg"),
                "この壁紙に最も合うカテゴリを1つだけ選んでください。",
            ],
            config=types.GenerateContentConfig(
                response_mime_type="application/json",
                response_schema=SCHEMA,
            ),
        )
        return json.loads(res.text)["category"]
    except Exception as exc:  # 片方だけ成功した対は使えないので握って捨てる
        print(f"[warn] {model}: {type(exc).__name__}: {exc}")
        return None

Z_975 = 1.959964  # 両側 5%
 
def paired_report(pairs: list[tuple[str, str]]) -> list[dict]:
    """pairs は (旧ラベル, 新ラベル) の並び。対を作れなかった画像は呼び出し前に除く。"""
    n = len(pairs)
    if n < 200:
        raise ValueError(f"対の数が {n} 件では判定に足りません（最低200件）")
 
    old_share = Counter(o for o, _ in pairs)
    rows = []
    for cat in CATEGORIES:
        d = [(1 if nw == cat else 0) - (1 if od == cat else 0) for od, nw in pairs]
        mean = sum(d) / n
        var = sum((x - mean) ** 2 for x in d) / (n - 1)
        se = math.sqrt(var / n) if var > 0 else 0.0
        base = old_share[cat] / n
        rows.append({
            "category": cat,
            "old_share": base,
            "delta": mean,                       # 構成比の変化（絶対値）
            "relative": mean / base if base else 0.0,
            "significant": se > 0 and abs(mean) / se > Z_975,
            "ci_half_width": Z_975 * se,
        })
    return rows