Managed Agents が静かに止まった朝、ログに手がかりが何も残っていなかった — 走行を外側から追える観測層の設計

// run-ledger.ts — 1実行=1レコードの走行台帳
export type StepKind = "plan" | "tool_call" | "tool_result" | "code_exec" | "message" | "artifact";
 
export interface RunStep {
  seq: number;            // 0始まりの工程番号
  kind: StepKind;
  at: number;             // 観測時刻(epoch ms)
  label: string;          // 人が読める一行(例: "tool_call: fetch_reviews")
  meta?: Record<string, unknown>;
}
 
export interface RunRecord {
  runId: string;
  agentId: string;
  startedAt: number;
  lastProgressAt: number; // 最後にイベントが来た時刻 — 停止検知の軸
  status: "running" | "succeeded" | "failed" | "stalled";
  steps: RunStep[];
  failure?: { type: string; detail: string; at: number };
  endedAt?: number;
}
 
const KEY = (runId: string) => `run:${runId}`;
 
export async function initRun(kv: KVNamespace, runId: string, agentId: string): Promise<RunRecord> {
  const now = Date.now();
  const rec: RunRecord = {
    runId, agentId, startedAt: now, lastProgressAt: now,
    status: "running", steps: [],
  };
  // 90日保持。終端後は事後ログ生成が読みにいく
  await kv.put(KEY(runId), JSON.stringify(rec), { expirationTtl: 60 * 60 * 24 * 90 });
  return rec;
}
 
export async function loadRun(kv: KVNamespace, runId: string): Promise<RunRecord | null> {
  const raw = await kv.get(KEY(runId));
  return raw ? (JSON.parse(raw) as RunRecord) : null;
}
 
export async function saveRun(kv: KVNamespace, rec: RunRecord): Promise<void> {
  await kv.put(KEY(rec.runId), JSON.stringify(rec), { expirationTtl: 60 * 60 * 24 * 90 });
}

// ❌ Before — 終わるまで何も残らない。止まると手がかりゼロ
const op = await agents.run({ agentId, input });
const final = await op.result();        // ← ここで無音停止すると、以降のログは一生来ない
await recordEverything(final);

// ✅ After — イベントが来るたびに台帳へ追記。止まっても直前までは残る
import { initRun, loadRun, saveRun, type RunStep } from "./run-ledger";
 
// SDKごとに異なるイベント形を、台帳の語彙へ寄せる正規化アダプタ
function normalizeUpdate(raw: any): Omit<RunStep, "seq" | "at"> | null {
  if (!raw) return null;
  if (raw.type === "tool_call")   return { kind: "tool_call",   label: `tool_call: ${raw.name}`, meta: { args: raw.args } };
  if (raw.type === "tool_result") return { kind: "tool_result", label: `tool_result: ${raw.name}`, meta: { ok: raw.ok } };
  if (raw.type === "code")        return { kind: "code_exec",   label: `code_exec(${(raw.code ?? "").length}b)` };
  if (raw.type === "message")     return { kind: "message",     label: `message(${(raw.text ?? "").length}b)` };
  if (raw.type === "artifact")    return { kind: "artifact",    label: `artifact: ${raw.path}` };
  if (raw.type === "plan")        return { kind: "plan",        label: "plan updated" };
  return null;
}
 
export async function tapRun(kv: KVNamespace, runId: string, stream: AsyncIterable<any>) {
  for await (const raw of stream) {
    const step = normalizeUpdate(raw);
    if (!step) continue;
    const rec = await loadRun(kv, runId);
    if (!rec || rec.status !== "running") break;
    const now = Date.now();
    rec.steps.push({ seq: rec.steps.length, at: now, ...step });
    rec.lastProgressAt = now;             // ← 進捗の事実をここへ必ず反映
    await saveRun(kv, rec);
  }
}

// stall-detector.ts — 進捗が途絶えた走行を拾う低頻度ポーラー
const IDLE_LIMIT_MS = 90_000;     // 90秒進捗が来なければ「無音停止の疑い」
const HARD_WALL_MS  = 15 * 60_000; // 15分を超えたら工程の進捗に関わらず打ち切り候補
 
export async function sweepStalls(kv: KVNamespace, onStall: (rec: RunRecord) => Promise<void>) {
  const list = await kv.list({ prefix: "run:" });
  const now = Date.now();
  for (const key of list.keys) {
    const rec = await loadRun(kv, key.name.slice(4));
    if (!rec || rec.status !== "running") continue;
 
    const idle = now - rec.lastProgressAt;
    const wall = now - rec.startedAt;
    if (idle < IDLE_LIMIT_MS && wall < HARD_WALL_MS) continue;
 
    rec.status = "stalled";
    rec.endedAt = now;
    rec.failure = {
      type: wall >= HARD_WALL_MS ? "wall_clock_exceeded" : "silent_idle",
      detail: `idle=${Math.round(idle / 1000)}s wall=${Math.round(wall / 1000)}s lastStep=${rec.steps.at(-1)?.label ?? "-"}`,
      at: now,
    };
    await saveRun(kv, rec);
    await onStall(rec);   // 通知＋（必要なら）オペレーションのキャンセルへ
  }
}

// postmortem.ts — 終端の台帳を、読める事後ログへ畳む
const TERMINAL = new Set(["succeeded", "failed", "stalled"]);
 
export function classify(rec: RunRecord): string {
  if (rec.status === "succeeded") return "ok";
  if (rec.failure?.type === "silent_idle") return "silent_idle";
  if (rec.failure?.type === "wall_clock_exceeded") return "wall_clock_exceeded";
  const lastTool = [...rec.steps].reverse().find((s) => s.kind === "tool_result");
  if (lastTool && (lastTool.meta as any)?.ok === false) return "tool_error";
  const detail = (rec.failure?.detail ?? "").toLowerCase();
  if (detail.includes("quota") || detail.includes("429")) return "quota_block";
  if (detail.includes("safety") || detail.includes("blocked")) return "safety_block";
  if (detail.includes("give up") || detail.includes("cannot")) return "agent_gaveup";
  return "unknown";
}
 
export function buildPostmortem(rec: RunRecord) {
  const klass = classify(rec);
  const durationS = Math.round(((rec.endedAt ?? Date.now()) - rec.startedAt) / 1000);
  // 「想定内」は件数集計へ、「要調査」は工程列ごと残す
  const watched = klass === "silent_idle" || klass === "wall_clock_exceeded" || klass === "unknown";
  return {
    runId: rec.runId,
    agentId: rec.agentId,
    class: klass,
    durationS,
    stepCount: rec.steps.length,
    lastStep: rec.steps.at(-1)?.label ?? "-",
    // 要調査だけ工程列を丸ごと、それ以外は要約だけ残してログ量を抑える
    trace: watched ? rec.steps.map((s) => `${s.seq}:${s.kind}:${s.label}`) : undefined,
    detail: rec.failure?.detail,
  };
}