Guarding Gemini API Responses in CI: Snapshot and Semantic Regression Testing

# test_structure.py
import json
import pytest
from gemini_client import client
from google.genai import types
 
@pytest.mark.vcr  # record/replay the API response (below)
def test_sentiment_schema(snapshot):
    """Verify the JSON output schema is preserved."""
    resp = client.models.generate_content(
        model="gemini-2.5-flash",
        contents='Sentiment of "I really enjoyed this product" as JSON. '
                 'Fields: sentiment, confidence, explanation',
        config=types.GenerateContentConfig(
            temperature=0,
            response_mime_type="application/json",
        ),
    )
    parsed = json.loads(resp.text)
 
    # Snapshot the shape, not the content
    schema = {
        "has_sentiment": "sentiment" in parsed,
        "sentiment_valid": parsed.get("sentiment") in ("positive", "negative", "neutral"),
        "confidence_is_float": isinstance(parsed.get("confidence"), float),
        "has_explanation": "explanation" in parsed,
        "finish_reason": str(resp.candidates[0].finish_reason),
    }
    assert schema == snapshot

pytest test_structure.py --snapshot-update   # first: create snapshot
pytest test_structure.py                      # after: diff check

# semantic_snapshot.py
import numpy as np
from google import genai
from google.genai import types
 
client = genai.Client(api_key="YOUR_GEMINI_API_KEY")
 
def embed(text: str) -> np.ndarray:
    r = client.models.embed_content(
        model="gemini-embedding-2",
        contents=text,
        config=types.EmbedContentConfig(output_dimensionality=256),
    )
    v = np.array(r.embeddings[0].values, dtype=np.float32)
    return v / np.linalg.norm(v)  # normalize so cosine == dot product
 
def cosine(a: np.ndarray, b: np.ndarray) -> float:
    return float(np.dot(a, b))

# test_semantic.py
import numpy as np
import pytest
from pathlib import Path
from gemini_client import classify_sentiment
from semantic_snapshot import embed, cosine
 
BASELINE = Path("baseline/sentiment_review.npy")
THRESHOLD = 0.88  # determined empirically (below)
 
@pytest.mark.vcr
def test_sentiment_semantic_stability():
    current = classify_sentiment("Shipping was slow, but support was courteous.")
    cur_vec = embed(current)
 
    if not BASELINE.exists():
        BASELINE.parent.mkdir(exist_ok=True)
        np.save(BASELINE, cur_vec)
        pytest.skip("Created a new baseline; comparing from next run.")
 
    base_vec = np.load(BASELINE)
    sim = cosine(cur_vec, base_vec)
    assert sim >= THRESHOLD, f"Semantic drift: cosine={sim:.3f} < {THRESHOLD}"

# pytest.ini
[pytest]
addopts = --record-mode=none

# conftest.py — keep API keys out of cassettes
import pytest
 
@pytest.fixture(scope="module")
def vcr_config():
    return {
        "filter_headers": [("x-goog-api-key", "REDACTED"),
                           ("authorization", "REDACTED")],
        "filter_query_parameters": [("key", "REDACTED")],
        "record_mode": "none",
    }

# When adding a new test: hit the real API exactly once to record
pytest test_semantic.py --record-mode=once
git add tests/cassettes/   # commit the cassette
# In CI from then on: replay only, no network
pytest --record-mode=none

# calibrate_threshold.py — measure the floor of natural variation
import numpy as np
from itertools import combinations
from gemini_client import classify_sentiment
from semantic_snapshot import embed, cosine
 
samples = [embed(classify_sentiment("Shipping was slow, but support was courteous."))
           for _ in range(20)]
sims = [cosine(a, b) for a, b in combinations(samples, 2)]
print(f"min={min(sims):.3f}  mean={np.mean(sims):.3f}  p05={np.percentile(sims,5):.3f}")

def stable_assert(check, attempts=3, need_fail=2):
    fails = sum(0 if check() else 1 for _ in range(attempts))
    assert fails < need_fail, f"{fails}/{attempts} failed: likely a real regression"

# Before: pin content with exact equality → goes red on every model update
def test_summary():
    out = summarize("...long text...")
    assert out == "This article explains Gemini testing techniques."

# After: monitor structure (layer one) and meaning (layer two) separately
@pytest.mark.vcr
def test_summary_structure(snapshot):
    out = summarize_json("...long text...")      # {"summary": str, "topics": [...]}
    parsed = json.loads(out)
    assert {
        "has_summary": isinstance(parsed.get("summary"), str),
        "topics_is_list": isinstance(parsed.get("topics"), list),
        "topics_nonempty": len(parsed.get("topics", [])) > 0,
    } == snapshot
 
@pytest.mark.vcr
def test_summary_semantic():
    out = summarize_json("...long text...")
    sim = cosine(embed(out), np.load("baseline/summary.npy"))
    assert sim >= 0.86

pytest --snapshot-update
git add -p __snapshots__/ baseline/    # inspect each diff before staging
git commit -m "test: update snapshots for prompt schema change (intentional)"