Automating Accessibility Audits with the Gemini API — A Design That Survives False Positives

# pip install google-genai playwright pydantic
# playwright install chromium
 
from __future__ import annotations
 
import json
import hashlib
from typing import Literal
 
import google.genai as genai
from google.genai import types
from playwright.sync_api import sync_playwright
from pydantic import BaseModel, Field
 
client = genai.Client(api_key="YOUR_GEMINI_API_KEY")

CONTRAST_PROBE = """
() => {
  const srgb = (c) => {
    const v = c / 255;
    return v <= 0.03928 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4);
  };
  const luminance = ([r, g, b]) =>
    0.2126 * srgb(r) + 0.7152 * srgb(g) + 0.0722 * srgb(b);
 
  const parse = (s) => (s.match(/[\\d.]+/g) || []).map(Number);
 
  // Resolve transparent backgrounds by walking ancestors.
  // Skip this and every element looks like it sits on white.
  const effectiveBg = (el) => {
    let node = el;
    while (node && node !== document.documentElement) {
      const c = parse(getComputedStyle(node).backgroundColor);
      if (c.length >= 3 && (c[3] === undefined || c[3] > 0)) return c.slice(0, 3);
      node = node.parentElement;
    }
    return [255, 255, 255];
  };
 
  const ratio = (a, b) => {
    const [hi, lo] = [luminance(a), luminance(b)].sort((x, y) => y - x);
    return (hi + 0.05) / (lo + 0.05);
  };
 
  const out = [];
  for (const el of document.querySelectorAll('p, a, span, li, button, h1, h2, h3, h4, label')) {
    const text = (el.innerText || '').trim();
    if (!text) continue;
    const cs = getComputedStyle(el);
    if (cs.visibility === 'hidden' || cs.display === 'none') continue;
 
    const size = parseFloat(cs.fontSize);
    const bold = parseInt(cs.fontWeight, 10) >= 700;
    // "Large text" is >= 24px, or >= 18.66px when bold
    const isLarge = size >= 24 || (size >= 18.66 && bold);
    const required = isLarge ? 3.0 : 4.5;
 
    const r = ratio(parse(cs.color).slice(0, 3), effectiveBg(el));
    if (r < required) {
      out.push({
        selector: el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
        text: text.slice(0, 60),
        ratio: Math.round(r * 100) / 100,
        required,
        font_px: size,
      });
    }
  }
  return out;
}
"""
 
 
def measure_contrast(url: str) -> list[dict]:
    """Measure contrast ratios. No AI involved, at any point."""
    with sync_playwright() as p:
        browser = p.chromium.launch()
        page = browser.new_page(viewport={"width": 1280, "height": 720})
        page.goto(url, wait_until="networkidle")
        violations = page.evaluate(CONTRAST_PROBE)
        browser.close()
    return violations

TARGET_SIZE_PROBE = """
() => {
  const MIN = 24;
  const out = [];
  for (const el of document.querySelectorAll('a, button, input, select, [role="button"]')) {
    const r = el.getBoundingClientRect();
    if (r.width === 0 || r.height === 0) continue;      // skip hidden elements
    if (r.width < MIN || r.height < MIN) {
      out.push({
        selector: el.tagName.toLowerCase(),
        label: (el.innerText || el.getAttribute('aria-label') || '').slice(0, 40),
        width: Math.round(r.width),
        height: Math.round(r.height),
      });
    }
  }
  return out;
}
"""

class Violation(BaseModel):
    criterion: str = Field(description="WCAG success criterion number, e.g. 1.1.1")
    severity: Literal["critical", "major", "minor"]
    selector: str = Field(description="A CSS selector that locates the problem")
    excerpt: str = Field(description="Markup excerpt, 120 characters or fewer")
    reason: str = Field(description="Why this fails the criterion")
    fix: str = Field(description="A concrete fix, as a code fragment where possible")
 
 
class SemanticAudit(BaseModel):
    violations: list[Violation]
    uncertain: list[str] = Field(description="Places a human should verify")
 
 
SEMANTIC_PROMPT = """You are a WCAG 2.2 auditor.
Inspect the HTML below for **only those criteria that require reading context**.
 
Check:
- 2.4.4 Link Purpose: link text such as "here" or "details" that hides its destination
- 1.1.1 Alt text quality: non-empty alt that fails to describe the image
  (for example alt="image1", alt="photo")
- 1.3.1 Info and Relationships: elements that look like headings but are not h1-h6,
  or look like tables but are not table
- 3.3.2 Labels: form labels that do not explain what the control does
- 1.4.1 Use of Color: meaning conveyed by color alone, e.g. "red fields are required"
 
Do NOT check (already measured elsewhere):
- contrast ratio, element dimensions, presence of alt, presence of lang
 
If you are not confident, put the item in `uncertain` rather than `violations`.
Never invent a violation to fill the list.
 
HTML:
{html}
"""
 
 
def audit_semantics(clean_html: str) -> SemanticAudit:
    response = client.models.generate_content(
        model="gemini-2.5-flash",
        contents=SEMANTIC_PROMPT.format(html=clean_html),
        config=types.GenerateContentConfig(
            temperature=0.0,          # auditing needs no creativity
            seed=42,                  # further suppress run-to-run drift
            response_mime_type="application/json",
            response_schema=SemanticAudit,
        ),
    )
    return SemanticAudit.model_validate_json(response.text)

LANDMARKS = ["header", "nav", "main", "aside", "footer", "form"]
MAX_CHARS = 12000
 
 
def split_by_landmark(page) -> list[tuple[str, str]]:
    """Chunk HTML per landmark; split an oversized main further by section."""
    chunks: list[tuple[str, str]] = []
    for tag in LANDMARKS:
        for i, handle in enumerate(page.query_selector_all(tag)):
            html = handle.evaluate("el => el.outerHTML")
            if len(html) <= MAX_CHARS:
                chunks.append((f"{tag}[{i}]", html))
                continue
            for j, sub in enumerate(handle.query_selector_all("section, article")):
                sub_html = sub.evaluate("el => el.outerHTML")
                chunks.append((f"{tag}[{i}]>section[{j}]", sub_html[:MAX_CHARS]))
    return chunks

def _focus_clip(page, pad: int = 24) -> dict:
    """Return a rectangle around the currently focused element."""
    box = page.evaluate(
        "() => { const r = document.activeElement.getBoundingClientRect();"
        "  return {x: r.x, y: r.y, width: r.width, height: r.height}; }"
    )
    return {
        "x": max(box["x"] - pad, 0),
        "y": max(box["y"] - pad, 0),
        "width": box["width"] + pad * 2,
        "height": box["height"] + pad * 2,
    }
 
 
def capture_focus_states(url: str, steps: int = 8) -> list[bytes]:
    """Tab through the page, photographing each focus state."""
    shots: list[bytes] = []
    with sync_playwright() as p:
        browser = p.chromium.launch()
        page = browser.new_page(viewport={"width": 1280, "height": 720})
        page.goto(url, wait_until="networkidle")
        for _ in range(steps):
            page.keyboard.press("Tab")
            page.wait_for_timeout(120)   # let transition animations settle
            focused = page.evaluate("() => document.activeElement?.tagName")
            if not focused or focused == "BODY":
                break
            shots.append(page.screenshot(clip=_focus_clip(page)))
        browser.close()
    return shots
 
 
def review_focus_visibility(shots: list[bytes]) -> str:
    parts = [types.Part.from_bytes(data=s, mime_type="image/png") for s in shots]
    parts.append(
        "Each image is a close-up of a focus position reached by pressing Tab. "
        "For each image, answer whether the focused element's outline is visually "
        "distinguishable from the background: 'clear', 'faint', or 'invisible'. "
        "Add one sentence of reasoning. Do not estimate contrast ratios."
    )
    response = client.models.generate_content(
        model="gemini-2.5-pro",
        contents=parts,
        config=types.GenerateContentConfig(temperature=0.0),
    )
    return response.text

def fingerprint(v: dict) -> str:
    """A fingerprint immune to line-number and wording jitter."""
    key = f"{v['criterion']}|{v['selector']}|{v.get('reason', '')[:40]}"
    return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
 
 
def diff_against_baseline(current: list[dict], baseline_path: str) -> dict:
    try:
        with open(baseline_path, encoding="utf-8") as f:
            known = set(json.load(f)["fingerprints"])
    except FileNotFoundError:
        known = set()
 
    seen = {fingerprint(v): v for v in current}
    new_ids = set(seen) - known
    fixed_ids = known - set(seen)
 
    return {
        "new": [seen[i] for i in new_ids],
        "fixed_count": len(fixed_ids),
        "total": len(seen),
    }

# a11y-suppress.yml
- criterion: "1.1.1"
  selector: "img.decorative"
  reason: "Decorative image; role=presentation already applied"
  expires: "2026-12-31"   # waivers expire, so they never become permanent excuses

# .github/workflows/a11y.yml
name: accessibility
 
on:
  pull_request:
    paths: ["app/**", "components/**", "styles/**"]
 
jobs:
  audit:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
 
      - run: pip install google-genai playwright pydantic && playwright install --with-deps chromium
 
      - name: Run audit
        env:
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
        run: python scripts/a11y_audit.py --url "${{ vars.PREVIEW_URL }}" --baseline .a11y/baseline.json
 
      # exit code 2 = new critical from the deterministic layer only
      # exit code 1 = new findings from the judgment layer (report only)
      - name: Comment findings
        if: always()
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('.a11y/report.md', 'utf8');
            await github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body,
            });

REPORT_PROMPT = """Below are accessibility audit results for a single page.
Write a Markdown report.
 
Structure:
1. Summary (three lines max: count of new violations, heaviest item only)
2. New criticals (with measured values, as a table)
3. New major / minor findings
4. Items requiring human review (uncertain plus visual flags)
5. Remaining known violations and the change since last run
 
Rules:
- Copy measured values (contrast ratio, target size) verbatim. Never round them.
- Never supply a number by inference.
- Express fixes as CSS or HTML fragments for the affected element.
 
Target: {url}
Measurements: {measured}
Semantic audit: {semantic}
Delta: {diff}
"""
 
 
def build_report(url: str, measured: dict, semantic: dict, diff: dict) -> str:
    response = client.models.generate_content(
        model="gemini-2.5-flash",
        contents=REPORT_PROMPT.format(
            url=url,
            measured=json.dumps(measured, ensure_ascii=False)[:6000],
            semantic=json.dumps(semantic, ensure_ascii=False)[:6000],
            diff=json.dumps(diff, ensure_ascii=False)[:3000],
        ),
        config=types.GenerateContentConfig(temperature=0.2),
    )
    return response.text