The Morning a Managed Agent Stalled and Left No Trace — Building a Run-Observability Layer Outside the Sandbox

// run-ledger.ts — one execution = one record
export type StepKind = "plan" | "tool_call" | "tool_result" | "code_exec" | "message" | "artifact";
 
export interface RunStep {
  seq: number;            // zero-based step number
  kind: StepKind;
  at: number;             // observed time (epoch ms)
  label: string;          // human-readable one-liner (e.g. "tool_call: fetch_reviews")
  meta?: Record<string, unknown>;
}
 
export interface RunRecord {
  runId: string;
  agentId: string;
  startedAt: number;
  lastProgressAt: number; // when the last event arrived — the axis for stall detection
  status: "running" | "succeeded" | "failed" | "stalled";
  steps: RunStep[];
  failure?: { type: string; detail: string; at: number };
  endedAt?: number;
}
 
const KEY = (runId: string) => `run:${runId}`;
 
export async function initRun(kv: KVNamespace, runId: string, agentId: string): Promise<RunRecord> {
  const now = Date.now();
  const rec: RunRecord = {
    runId, agentId, startedAt: now, lastProgressAt: now,
    status: "running", steps: [],
  };
  // keep 90 days; the postmortem builder reads it after a terminal state
  await kv.put(KEY(runId), JSON.stringify(rec), { expirationTtl: 60 * 60 * 24 * 90 });
  return rec;
}
 
export async function loadRun(kv: KVNamespace, runId: string): Promise<RunRecord | null> {
  const raw = await kv.get(KEY(runId));
  return raw ? (JSON.parse(raw) as RunRecord) : null;
}
 
export async function saveRun(kv: KVNamespace, rec: RunRecord): Promise<void> {
  await kv.put(KEY(rec.runId), JSON.stringify(rec), { expirationTtl: 60 * 60 * 24 * 90 });
}

// ❌ Before — nothing is left until it finishes. If it stalls, you have zero clues.
const op = await agents.run({ agentId, input });
const final = await op.result();        // ← if it goes silent here, no further log ever arrives
await recordEverything(final);

// ✅ After — append to the ledger on every event. Even on a stall, you keep up to the last step.
import { initRun, loadRun, saveRun, type RunStep } from "./run-ledger";
 
// a normalizing adapter that pulls each SDK's event shape toward the ledger's vocabulary
function normalizeUpdate(raw: any): Omit<RunStep, "seq" | "at"> | null {
  if (!raw) return null;
  if (raw.type === "tool_call")   return { kind: "tool_call",   label: `tool_call: ${raw.name}`, meta: { args: raw.args } };
  if (raw.type === "tool_result") return { kind: "tool_result", label: `tool_result: ${raw.name}`, meta: { ok: raw.ok } };
  if (raw.type === "code")        return { kind: "code_exec",   label: `code_exec(${(raw.code ?? "").length}b)` };
  if (raw.type === "message")     return { kind: "message",     label: `message(${(raw.text ?? "").length}b)` };
  if (raw.type === "artifact")    return { kind: "artifact",    label: `artifact: ${raw.path}` };
  if (raw.type === "plan")        return { kind: "plan",        label: "plan updated" };
  return null;
}
 
export async function tapRun(kv: KVNamespace, runId: string, stream: AsyncIterable<any>) {
  for await (const raw of stream) {
    const step = normalizeUpdate(raw);
    if (!step) continue;
    const rec = await loadRun(kv, runId);
    if (!rec || rec.status !== "running") break;
    const now = Date.now();
    rec.steps.push({ seq: rec.steps.length, at: now, ...step });
    rec.lastProgressAt = now;             // ← always reflect the fact of progress here
    await saveRun(kv, rec);
  }
}

// stall-detector.ts — a low-frequency poller for runs whose progress has ceased
const IDLE_LIMIT_MS = 90_000;     // no progress for 90s ⇒ "suspected silent stall"
const HARD_WALL_MS  = 15 * 60_000; // past 15 min, a cutoff candidate regardless of step progress
 
export async function sweepStalls(kv: KVNamespace, onStall: (rec: RunRecord) => Promise<void>) {
  const list = await kv.list({ prefix: "run:" });
  const now = Date.now();
  for (const key of list.keys) {
    const rec = await loadRun(kv, key.name.slice(4));
    if (!rec || rec.status !== "running") continue;
 
    const idle = now - rec.lastProgressAt;
    const wall = now - rec.startedAt;
    if (idle < IDLE_LIMIT_MS && wall < HARD_WALL_MS) continue;
 
    rec.status = "stalled";
    rec.endedAt = now;
    rec.failure = {
      type: wall >= HARD_WALL_MS ? "wall_clock_exceeded" : "silent_idle",
      detail: `idle=${Math.round(idle / 1000)}s wall=${Math.round(wall / 1000)}s lastStep=${rec.steps.at(-1)?.label ?? "-"}`,
      at: now,
    };
    await saveRun(kv, rec);
    await onStall(rec);   // notify, and (if needed) cancel the operation
  }
}

// postmortem.ts — fold a terminal ledger into a readable postmortem
const TERMINAL = new Set(["succeeded", "failed", "stalled"]);
 
export function classify(rec: RunRecord): string {
  if (rec.status === "succeeded") return "ok";
  if (rec.failure?.type === "silent_idle") return "silent_idle";
  if (rec.failure?.type === "wall_clock_exceeded") return "wall_clock_exceeded";
  const lastTool = [...rec.steps].reverse().find((s) => s.kind === "tool_result");
  if (lastTool && (lastTool.meta as any)?.ok === false) return "tool_error";
  const detail = (rec.failure?.detail ?? "").toLowerCase();
  if (detail.includes("quota") || detail.includes("429")) return "quota_block";
  if (detail.includes("safety") || detail.includes("blocked")) return "safety_block";
  if (detail.includes("give up") || detail.includes("cannot")) return "agent_gaveup";
  return "unknown";
}
 
export function buildPostmortem(rec: RunRecord) {
  const klass = classify(rec);
  const durationS = Math.round(((rec.endedAt ?? Date.now()) - rec.startedAt) / 1000);
  // count the "expected" ones; keep the full step run for "investigate"
  const watched = klass === "silent_idle" || klass === "wall_clock_exceeded" || klass === "unknown";
  return {
    runId: rec.runId,
    agentId: rec.agentId,
    class: klass,
    durationS,
    stepCount: rec.steps.length,
    lastStep: rec.steps.at(-1)?.label ?? "-",
    // keep the whole step run only for the watched ones; summarize the rest to hold down log volume
    trace: watched ? rec.steps.map((s) => `${s.seq}:${s.kind}:${s.label}`) : undefined,
    detail: rec.failure?.detail,
  };
}