Gemini API の SLO と Error Budget を個人開発で運用する設計 — 4 サイトの可用性を Burn Rate 監視で守る

// src/lib/slo/recorder.ts — Gemini API 呼び出し結果を 1 分粒度で記録する
import type { KVNamespace } from "@cloudflare/workers-types";
 
export type SiteId = "claudelab" | "gemilab" | "rorklab" | "antigravitylab";
 
interface MinuteBucket {
  total: number;
  errors: number;
}
 
const BUCKET_TTL_SECONDS = 60 * 60 * 24 * 35; // 35 日保持（30 日 SLO + バッファ）
 
function bucketKey(site: SiteId, minute: number): string {
  return `slo:${site}:m:${minute}`;
}
 
function currentMinute(): number {
  return Math.floor(Date.now() / 60_000);
}
 
export async function recordCall(
  kv: KVNamespace,
  site: SiteId,
  success: boolean,
): Promise<void> {
  const minute = currentMinute();
  const key = bucketKey(site, minute);
 
  const existing = await kv.get<MinuteBucket>(key, "json");
  const next: MinuteBucket = existing
    ? { total: existing.total + 1, errors: existing.errors + (success ? 0 : 1) }
    : { total: 1, errors: success ? 0 : 1 };
 
  await kv.put(key, JSON.stringify(next), {
    expirationTtl: BUCKET_TTL_SECONDS,
  });
}

// src/lib/gemini/call.ts — 既存呼び出しに SLO 記録を埋め込む
export async function callGemini(
  env: Env,
  site: SiteId,
  prompt: string,
): Promise<string> {
  try {
    const response = await retryWithBackoff(() => rawCall(env, prompt));
    await recordCall(env.SLO_KV, site, true);
    return response;
  } catch (err) {
    await recordCall(env.SLO_KV, site, false);
    throw err;
  }
}

// src/lib/slo/burn-rate.ts
export interface SloConfig {
  site: SiteId;
  targetSuccessRate: number; // 例: 0.99
}
 
export interface WindowResult {
  windowMinutes: number;
  total: number;
  errors: number;
  errorRate: number;
  burnRate: number; // errorRate / (1 - targetSuccessRate)
}
 
async function readWindow(
  kv: KVNamespace,
  site: SiteId,
  windowMinutes: number,
): Promise<{ total: number; errors: number }> {
  const now = Math.floor(Date.now() / 60_000);
  const keys = Array.from({ length: windowMinutes }, (_, i) =>
    bucketKey(site, now - i),
  );
 
  const values = await Promise.all(
    keys.map((k) => kv.get<MinuteBucket>(k, "json")),
  );
 
  return values.reduce(
    (acc, v) => ({
      total: acc.total + (v?.total ?? 0),
      errors: acc.errors + (v?.errors ?? 0),
    }),
    { total: 0, errors: 0 },
  );
}
 
export async function computeBurnRate(
  kv: KVNamespace,
  cfg: SloConfig,
  windowMinutes: number,
): Promise<WindowResult> {
  const { total, errors } = await readWindow(kv, cfg.site, windowMinutes);
  const errorRate = total === 0 ? 0 : errors / total;
  const budget = 1 - cfg.targetSuccessRate;
  return {
    windowMinutes,
    total,
    errors,
    errorRate,
    burnRate: budget === 0 ? 0 : errorRate / budget,
  };
}

// src/lib/slo/alert.ts — Multi-Window 評価を行う
export type AlertLevel = "critical" | "high" | "medium" | "low" | "ok";
 
export async function evaluateAlerts(
  kv: KVNamespace,
  cfg: SloConfig,
): Promise<AlertLevel> {
  const w5 = await computeBurnRate(kv, cfg, 5);
  const w60 = await computeBurnRate(kv, cfg, 60);
  const w360 = await computeBurnRate(kv, cfg, 360);
  const w1440 = await computeBurnRate(kv, cfg, 1440);
 
  // 短時間と長時間の両方で同時にしきい値を超えたときだけ発火
  if (w5.burnRate >= 14.4 && w60.burnRate >= 6.0) return "critical";
  if (w60.burnRate >= 6.0 && w360.burnRate >= 1.0) return "high";
  if (w360.burnRate >= 1.0 && w1440.burnRate >= 0.5) return "medium";
  if (w1440.burnRate >= 0.5) return "low";
  return "ok";
}

// src/lib/gemini/select-model.ts
export async function selectModel(
  kv: KVNamespace,
  cfg: SloConfig,
): Promise<"gemini-2.5-pro" | "gemini-2.5-flash"> {
  const alert = await evaluateAlerts(kv, cfg);
  if (alert === "critical" || alert === "high") {
    return "gemini-2.5-flash";
  }
  return "gemini-2.5-pro";
}