Stop Losing Silently-Failed Jobs in Your Unattended Gemini API Pipeline

RETRYABLE_STATUS = {429, 500, 502, 503, 504}
PERMANENT_STATUS = {400, 403, 404, 422}
 
 
def classify_failure(status_code: int, message: str) -> str:
    """Classify a failure as 'retryable' / 'permanent' / 'unknown'."""
    text = (message or "").upper()
 
    # Failures caused by the input itself return the same result every time.
    if status_code in PERMANENT_STATUS:
        return "permanent"
    if "INVALID_ARGUMENT" in text or "FAILED_PRECONDITION" in text:
        return "permanent"
    if "SAFETY" in text or "BLOCKED" in text:
        return "permanent"
 
    # Transient load or network issues may recover after a delay.
    if status_code in RETRYABLE_STATUS:
        return "retryable"
    if "DEADLINE_EXCEEDED" in text or "UNAVAILABLE" in text:
        return "retryable"
 
    return "unknown"

import random
 
 
class ApiError(Exception):
    def __init__(self, status_code: int, message: str):
        super().__init__(message)
        self.status_code = status_code
        self.message = message
 
 
def run_with_dead_letter(job_id: str, payload: dict, model: str,
                         call_model, max_attempts: int = 4):
    """Retry a model call; park permanent failures and exhausted retries.
 
    Assumes call_model(payload) -> dict, raising ApiError on failure.
    """
    attempt = 0
    while True:
        attempt += 1
        try:
            return call_model(payload)
        except ApiError as exc:
            kind = classify_failure(exc.status_code, exc.message)
 
            if kind == "permanent":
                park(job_id, payload, model,
                     error_class="permanent:%d" % exc.status_code,
                     error_message=exc.message, attempts=attempt)
                return None
 
            if attempt >= max_attempts:
                park(job_id, payload, model,
                     error_class="exhausted:%d" % exc.status_code,
                     error_message=exc.message, attempts=attempt)
                return None
 
            # retryable / unknown get a cautious backoff.
            backoff = min(2 ** attempt, 30) + random.uniform(0, 1)
            time.sleep(backoff)

def replay(call_model, limit: int = 20, dry_run: bool = True,
           path: str = DLQ_PATH) -> dict:
    """Replay parked jobs. Meant to be run by hand after a fix."""
    con = sqlite3.connect(path)
    con.row_factory = sqlite3.Row
    rows = con.execute(
        "SELECT * FROM dead_letter WHERE status = 'parked' "
        "ORDER BY last_seen ASC LIMIT ?",
        (limit,),
    ).fetchall()
 
    recovered, still_failing = 0, 0
    for row in rows:
        payload = json.loads(row["payload_json"])
        if dry_run:
            print("[dry-run] would replay %s (%s)" % (row["job_id"], row["error_class"]))
            continue
        try:
            call_model(payload)
            con.execute(
                "UPDATE dead_letter SET status = 'recovered', last_seen = ? "
                "WHERE job_id = ?",
                (time.time(), row["job_id"]),
            )
            recovered += 1
        except ApiError:
            still_failing += 1
 
    con.commit()
    con.close()
    return {"scanned": len(rows), "recovered": recovered,
            "still_failing": still_failing, "dry_run": dry_run}

def dlq_summary(path: str = DLQ_PATH) -> None:
    """Log the dead-letter breakdown once a day."""
    con = sqlite3.connect(path)
    rows = con.execute(
        "SELECT error_class, COUNT(*) AS n FROM dead_letter "
        "WHERE status = 'parked' GROUP BY error_class ORDER BY n DESC"
    ).fetchall()
    con.close()
 
    if not rows:
        print("dead-letter: clean (0 parked)")
        return
    total = sum(n for _, n in rows)
    print("dead-letter: %d parked" % total)
    for error_class, n in rows:
        print("  %4d  %s" % (n, error_class))