Rebuilding the CTR Denominator After AI Search Fans Out Your Queries

# fanout_score.py - score a query for "AI fan-out likeness" using the string alone
# Deliberately ignores clicks and impressions to avoid circular reasoning
import csv
import re
import sys
from dataclasses import dataclass, field
 
# Function words that chain attributes together; rare in human queries
JOINERS = (" vs ", " versus ", " for ", " and ", " with ", " without ")
 
# Words people use when something is broken. Strong negative signal.
INTENT_MARKERS = (
    "error", "fix", "not working", "failed", "how to", "why", "cannot", "can't",
    "エラー", "できない", "直し方", "とは", "使い方",
)
 
# A trailing year usually means a human wanting current information
YEAR = re.compile(r"\b(19|20)\d{2}\b")
 
# Nouns that ask a system to enumerate; skewed toward generated sub-queries
ATTRIBUTE_NOUNS = re.compile(
    r"\b(features|capabilities|use cases|target audience|overview|benefits)\b"
)
 
 
@dataclass
class Verdict:
    query: str
    score: int
    band: str
    reasons: list = field(default_factory=list)
 
 
def fanout_score(query: str) -> Verdict:
    q = query.lower().strip()
    tokens = q.split()
    score, reasons = 0, []
 
    if len(tokens) >= 6:
        score += 1
        reasons.append(f"tokens>=6 ({len(tokens)})")
    if len(tokens) >= 8:
        score += 1
        reasons.append(f"tokens>=8 ({len(tokens)})")
 
    joiners = [j.strip() for j in JOINERS if j in f" {q} "]
    if joiners:
        score += 1
        reasons.append("joiner:" + "/".join(joiners))
 
    hits = [m for m in INTENT_MARKERS if m in q]
    if hits:
        score -= 2
        reasons.append("intent:" + "/".join(hits))
 
    if YEAR.search(q):
        score -= 1
        reasons.append("year-suffix")
 
    if ATTRIBUTE_NOUNS.search(q):
        score += 1
        reasons.append("attribute-noun")
 
    # 3 or more: fan-out. 1 or less: human. 2: undecided.
    band = "fanout" if score >= 3 else ("human" if score <= 1 else "ambiguous")
    return Verdict(query=query, score=score, band=band, reasons=reasons)
 
 
def main(path: str) -> None:
    with open(path, newline="", encoding="utf-8-sig") as f:
        for row in csv.DictReader(f):
            v = fanout_score(row["Top queries"])
            print(f"{v.band:9s} score={v.score:2d}  {v.query}")
            print(f"            {v.reasons}")
 
 
if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else "Queries.csv")

ambiguous score= 2  rork.com core business features target audience
            ['tokens>=6 (6)', 'attribute-noun']
fanout    score= 3  rork max swiftui features and ai capabilities
            ['tokens>=6 (7)', 'joiner:and', 'attribute-noun']
ambiguous score= 2  onspace vs rork for complex app logic
            ['tokens>=6 (7)', 'joiner:vs/for']
human     score= 1  expo audio continue playing screen locked react native 2025
            ['tokens>=6 (9)', 'tokens>=8 (9)', 'year-suffix']
fanout    score= 4  rork max swiftui features and native app generation
            ['tokens>=6 (8)', 'joiner:and', 'attribute-noun', 'tokens>=8 (8)']

# adjudicate.py - send only the ambiguous rows to the model
from google import genai
from pydantic import BaseModel
 
client = genai.Client(api_key="YOUR_API_KEY")
 
SCHEMA_PROMPT = """You are analysing search logs.
Decide how likely it is that a human typed the given query into a search box.
 
Guidance:
- Human queries are short and contain only the words the person needs
- Stacked attribute nouns (features / capabilities / target audience) and
  phrases with the comparison axis embedded are typical of AI-generated
  sub-queries
- A long query that looks like a pasted error message is human
 
You are given no click or impression data. Judge from the wording alone."""
 
 
class Judgement(BaseModel):
    likely_human: bool
    confidence: float  # 0.0-1.0
    reason: str
 
 
def adjudicate(query: str) -> Judgement:
    response = client.models.generate_content(
        model="gemini-3.7-flash",
        contents=f"{SCHEMA_PROMPT}\n\nQuery: {query}",
        config={
            "response_mime_type": "application/json",
            "response_schema": Judgement,
        },
    )
    return Judgement.model_validate_json(response.text)
 
 
AMBIGUOUS = [
    "rork.com core business features target audience",
    "onspace vs rork for complex app logic",
]
 
for q in AMBIGUOUS:
    j = adjudicate(q)
    # Low-confidence calls are not adopted; they carry over to next week
    verdict = "human" if j.likely_human else "fanout"
    status = verdict if j.confidence >= 0.7 else "hold"
    print(f"{status:7s} conf={j.confidence:.2f}  {q}\n        {j.reason}")