fix: recover terminal runner outbox

2026-06-20 16:12:25 +08:00
parent 11ac140e90
commit 909e8b31f8
2 changed files with 421 additions and 38 deletions
@@ -1,7 +1,7 @@
 import { spawn } from "node:child_process";
 import { AgentRunError } from "../common/errors.js";
 import { redactJson, redactText } from "../common/redaction.js";
-import type { CommandRecord, FailureKind, JsonRecord, JsonValue, RunnerJobRecord, TerminalStatus } from "../common/types.js";
+import type { BackendEvent, CommandRecord, EventType, FailureKind, JsonRecord, JsonValue, RunnerJobRecord, TerminalStatus } from "../common/types.js";
 import { nowIso, stableHash } from "../common/validation.js";
 import type { AgentRunStore } from "./store.js";
 import { isTerminalCommandState, isTerminalRunStatus } from "./store.js";
@@ -37,6 +37,35 @@ interface K8sList {
  items?: K8sObject[];
 }

+interface CommandTerminalReport {
+  terminalStatus: TerminalStatus;
+  failureKind: FailureKind | null;
+  failureMessage: string | null;
+  threadId?: string;
+  turnId?: string;
+}
+
+interface TerminalOutboxRecord {
+  outboxKey: string;
+  phase: string | null;
+  runId: string;
+  commandId: string;
+  runnerId: string;
+  runnerJobId: string | null;
+  attemptId: string;
+  report: CommandTerminalReport;
+  terminalRun: boolean;
+  events: BackendEvent[];
+}
+
+interface TerminalOutboxSearch {
+  outbox: TerminalOutboxRecord | null;
+  lineCount: number;
+  candidateCount: number;
+  parseFailedCount: number;
+  stdoutHash: string;
+}
+
 export async function reconcileRunnerJobsOnce(input: RunnerReconcilerOptions): Promise<JsonRecord> {
  const limit = clampLimit(input.limit ?? 20);
  const jobs = await input.store.listRunnerJobsForReconciliation(limit);
@@ -45,7 +74,9 @@ export async function reconcileRunnerJobsOnce(input: RunnerReconcilerOptions): P
  let runClosureCount = 0;

  for (const job of jobs) {
-    const observation = await observeRunnerJob(job, { ...(input.namespace ? { namespace: input.namespace } : {}), ...(input.kubectlCommand ? { kubectlCommand: input.kubectlCommand } : {}) });
+    let observation = await observeRunnerJob(job, { ...(input.namespace ? { namespace: input.namespace } : {}), ...(input.kubectlCommand ? { kubectlCommand: input.kubectlCommand } : {}) });
+    const terminalOutbox = await recoverTerminalOutboxIfNeeded(input.store, job, observation, { ...(input.namespace ? { namespace: input.namespace } : {}), ...(input.kubectlCommand ? { kubectlCommand: input.kubectlCommand } : {}) });
+    if (terminalOutbox) observation = mergeTerminalOutboxObservation(observation, terminalOutbox);
    await input.store.updateRunnerJobResult(job.id, { observation });
    if (stringValue(observation.category) === "runner-job-observe-failed") observeFailedCount++;
    const runClosure = await closeOpenRunWhenCommandTerminal(input.store, job);
@@ -59,6 +90,7 @@ export async function reconcileRunnerJobsOnce(input: RunnerReconcilerOptions): P
      observedRunnerPhase: stringValue(observation.observedRunnerPhase) ?? "unknown",
      terminalReportState: stringValue(observation.terminalReportState) ?? "unknown",
      runReportState: stringValue(observation.runReportState) ?? "unknown",
+      terminalOutboxState: terminalOutboxState(observation),
      runClosure,
      valuesPrinted: false,
    });
@@ -77,6 +109,114 @@ export async function reconcileRunnerJobsOnce(input: RunnerReconcilerOptions): P
  };
 }

+async function recoverTerminalOutboxIfNeeded(store: AgentRunStore, job: RunnerJobRecord, observation: JsonRecord, input: { namespace?: string; kubectlCommand?: string }): Promise<JsonRecord | null> {
+  const phase = stringValue(observation.observedRunnerPhase);
+  if (phase !== "k8s:succeeded" && phase !== "k8s:failed") return null;
+  try {
+    const command = await store.getCommand(job.commandId);
+    if (isTerminalCommandState(command.state)) {
+      return { state: "command-terminal", terminalReportState: "terminal-command-already-recorded", commandState: command.state, valuesPrinted: false };
+    }
+    const namespace = input.namespace ?? job.namespace;
+    const kubectlCommand = input.kubectlCommand ?? "kubectl";
+    const logs = await getRunnerJobLogs(kubectlCommand, namespace, job.jobName);
+    if (logs.code !== 0) {
+      return {
+        state: "logs-unavailable",
+        terminalReportState: "terminal-outbox-unavailable",
+        runReportState: "not-updated",
+        failureKind: "infra-failed",
+        failureMessage: redactText(logs.stderr || `kubectl logs failed with code ${logs.code}`).slice(0, 300),
+        valuesPrinted: false,
+      };
+    }
+    const search = findLatestTerminalOutbox(logs.stdout, job);
+    if (!search.outbox) {
+      return {
+        state: "not-found",
+        terminalReportState: "terminal-outbox-missing",
+        runReportState: "not-updated",
+        lineCount: search.lineCount,
+        candidateCount: search.candidateCount,
+        parseFailedCount: search.parseFailedCount,
+        stdoutHash: search.stdoutHash,
+        valuesPrinted: false,
+      };
+    }
+    const eventReplay = await replayTerminalOutboxEvents(store, search.outbox);
+    const nextCommand = await store.finishCommand(job.commandId, search.outbox.report);
+    const runRecovery = await recoverTerminalOutboxRun(store, search.outbox);
+    return {
+      state: "recovered",
+      terminalReportState: "terminal-outbox-replayed",
+      runReportState: stringValue(runRecovery.state) ?? "not-updated",
+      outboxKey: search.outbox.outboxKey,
+      phase: search.outbox.phase,
+      commandState: nextCommand.state,
+      terminalStatus: search.outbox.report.terminalStatus,
+      failureKind: search.outbox.report.failureKind,
+      failureMessageHash: search.outbox.report.failureMessage ? stableHash({ message: search.outbox.report.failureMessage }) : null,
+      terminalRun: search.outbox.terminalRun,
+      lineCount: search.lineCount,
+      candidateCount: search.candidateCount,
+      parseFailedCount: search.parseFailedCount,
+      eventReplay,
+      runRecovery,
+      valuesPrinted: false,
+    };
+  } catch (error) {
+    return {
+      state: "recovery-failed",
+      terminalReportState: "terminal-outbox-recovery-failed",
+      runReportState: "not-updated",
+      failureKind: "infra-failed",
+      failureMessage: error instanceof Error ? redactText(error.message).slice(0, 300) : "unknown terminal outbox recovery failure",
+      valuesPrinted: false,
+    };
+  }
+}
+
+async function replayTerminalOutboxEvents(store: AgentRunStore, outbox: TerminalOutboxRecord): Promise<JsonRecord> {
+  const existing = await store.listEvents(outbox.runId, 0, 1_000);
+  const existingKeys = new Set(existing.map((event) => eventContentKey({ type: event.type, payload: event.payload })));
+  let appendedCount = 0;
+  let skippedCount = 0;
+  for (const event of outbox.events) {
+    const key = eventContentKey(event);
+    if (existingKeys.has(key)) {
+      skippedCount++;
+      continue;
+    }
+    await store.appendEvent(outbox.runId, event.type, event.payload);
+    existingKeys.add(key);
+    appendedCount++;
+  }
+  return { eventCount: outbox.events.length, appendedCount, skippedCount, valuesPrinted: false };
+}
+
+async function recoverTerminalOutboxRun(store: AgentRunStore, outbox: TerminalOutboxRecord): Promise<JsonRecord> {
+  if (outbox.terminalRun !== true) return { state: "not-requested", valuesPrinted: false };
+  const run = await store.getRun(outbox.runId);
+  if (isTerminalRunStatus(run.status)) return { state: "already-terminal", runStatus: run.status, valuesPrinted: false };
+  const next = await store.finishRun(outbox.runId, outbox.report);
+  return { state: "terminal-outbox-replayed", runStatus: next.status, terminalStatus: outbox.report.terminalStatus, valuesPrinted: false };
+}
+
+function mergeTerminalOutboxObservation(observation: JsonRecord, terminalOutbox: JsonRecord): JsonRecord {
+  return {
+    ...observation,
+    terminalOutbox,
+    terminalReportState: stringValue(terminalOutbox.terminalReportState) ?? stringValue(observation.terminalReportState) ?? "unknown",
+    runReportState: stringValue(terminalOutbox.runReportState) ?? stringValue(observation.runReportState) ?? "unknown",
+    evidenceLevel: terminalOutbox.state === "recovered" ? "high" : observation.evidenceLevel ?? "medium",
+    valuesPrinted: false,
+  };
+}
+
+function terminalOutboxState(observation: JsonRecord): string {
+  return stringValue(recordValue(observation.terminalOutbox)?.state) ?? "not-checked";
+}
+
 export function startRunnerJobReconciler(input: RunnerReconcilerLoopOptions): () => void {
  const intervalMs = Math.max(1_000, input.intervalMs);
  let stopped = false;
@@ -256,6 +396,158 @@ async function getK8sList(kubectlCommand: string, resource: string, namespace: s
  return Array.isArray(parsed.items) ? parsed.items.filter((item) => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
 }

+async function getRunnerJobLogs(kubectlCommand: string, namespace: string, jobName: string): Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }> {
+  return await kubectlRun(kubectlCommand, ["logs", `job/${jobName}`, "-n", namespace, "--tail=2000"]);
+}
+
+function findLatestTerminalOutbox(stdout: string, job: RunnerJobRecord): TerminalOutboxSearch {
+  let outbox: TerminalOutboxRecord | null = null;
+  let lineCount = 0;
+  let candidateCount = 0;
+  let parseFailedCount = 0;
+  for (const line of stdout.split(/\r?\n/u)) {
+    if (line.trim().length === 0) continue;
+    lineCount++;
+    if (!line.includes("terminal.outbox")) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      parseFailedCount++;
+      continue;
+    }
+    const candidate = terminalOutboxFromRecord(parsed);
+    if (!candidate) continue;
+    candidateCount++;
+    if (matchesTerminalOutbox(candidate, job)) outbox = candidate;
+  }
+  return { outbox, lineCount, candidateCount, parseFailedCount, stdoutHash: stableHash({ stdoutTail: stdout.slice(-20_000), stdoutLength: stdout.length }) };
+}
+
+function terminalOutboxFromRecord(value: unknown): TerminalOutboxRecord | null {
+  const record = recordValue(value);
+  if (!record || record.label !== "terminal.outbox") return null;
+  const outboxKey = stringValue(record.outboxKey);
+  const runId = stringValue(record.runId);
+  const commandId = stringValue(record.commandId);
+  const runnerId = stringValue(record.runnerId);
+  const attemptId = stringValue(record.attemptId);
+  const reportRecord = recordValue(record.report);
+  if (!outboxKey || !runId || !commandId || !runnerId || !attemptId || !reportRecord) return null;
+  const report = terminalReportFromRecord(reportRecord);
+  if (!report) return null;
+  const events = Array.isArray(record.events) ? record.events.map(eventFromRecord).filter((event): event is BackendEvent => event !== null) : [];
+  return {
+    outboxKey,
+    phase: stringValue(record.phase),
+    runId,
+    commandId,
+    runnerId,
+    runnerJobId: stringValue(record.runnerJobId),
+    attemptId,
+    report,
+    terminalRun: record.terminalRun === true,
+    events,
+  };
+}
+
+function terminalReportFromRecord(record: JsonRecord): CommandTerminalReport | null {
+  const terminalStatus = terminalStatusValue(record.terminalStatus);
+  const failureKind = failureKindValue(record.failureKind);
+  if (!terminalStatus || failureKind === undefined) return null;
+  const threadId = stringValue(record.threadId);
+  const turnId = stringValue(record.turnId);
+  return {
+    terminalStatus,
+    failureKind,
+    failureMessage: nullableString(record.failureMessage),
+    ...(threadId ? { threadId } : {}),
+    ...(turnId ? { turnId } : {}),
+  };
+}
+
+function eventFromRecord(value: unknown): BackendEvent | null {
+  const record = recordValue(value);
+  if (!record) return null;
+  const type = eventTypeValue(record.type);
+  const payload = recordValue(record.payload);
+  if (!type || !payload) return null;
+  return { type, payload };
+}
+
+function matchesTerminalOutbox(outbox: TerminalOutboxRecord, job: RunnerJobRecord): boolean {
+  if (outbox.runId !== job.runId || outbox.commandId !== job.commandId) return false;
+  if (outbox.runnerJobId && outbox.runnerJobId !== job.id) return false;
+  return true;
+}
+
+function eventContentKey(event: BackendEvent): string {
+  return stableHash({ type: event.type, payload: stripTerminalOutboxEventPayload(event.payload) });
+}
+
+function stripTerminalOutboxEventPayload(payload: JsonRecord): JsonRecord {
+  const next: JsonRecord = {};
+  for (const [key, value] of Object.entries(payload)) {
+    if (key === "terminalOutboxKey" || key === "terminalOutboxIndex") continue;
+    next[key] = value;
+  }
+  return next;
+}
+
+function recordValue(value: unknown): JsonRecord | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonRecord : null;
+}
+
+const eventTypes = new Set<string>(["backend_status", "assistant_message", "tool_call", "command_output", "diff", "error", "terminal_status"]);
+const terminalStatuses = new Set<string>(["completed", "failed", "blocked", "cancelled"]);
+const failureKinds = new Set<string>([
+  "auth-missing",
+  "auth-failed",
+  "schema-invalid",
+  "tenant-policy-denied",
+  "secret-unavailable",
+  "prompt-unavailable",
+  "prompt-too-large",
+  "required-skill-unavailable",
+  "skill-unavailable",
+  "runner-lease-conflict",
+  "backend-failed",
+  "backend-protocol-error",
+  "backend-spawn-failed",
+  "backend-json-parse-error",
+  "backend-response-invalid",
+  "thread-resume-failed",
+  "backend-timeout",
+  "provider-auth-failed",
+  "provider-rate-limited",
+  "provider-stream-disconnected",
+  "provider-http-error",
+  "provider-invalid-tool-call",
+  "provider-compact-unsupported",
+  "provider-unavailable",
+  "infra-failed",
+  "session-store-evicted",
+  "cancelled",
+]);
+
+function eventTypeValue(value: JsonValue | undefined): EventType | null {
+  return typeof value === "string" && eventTypes.has(value) ? value as EventType : null;
+}
+
+function terminalStatusValue(value: JsonValue | undefined): TerminalStatus | null {
+  return typeof value === "string" && terminalStatuses.has(value) ? value as TerminalStatus : null;
+}
+
+function failureKindValue(value: JsonValue | undefined): FailureKind | null | undefined {
+  if (value === null) return null;
+  if (typeof value === "string" && failureKinds.has(value)) return value as FailureKind;
+  return undefined;
+}
+
+function nullableString(value: JsonValue | undefined): string | null {
+  return typeof value === "string" ? value : null;
+}
+
 async function kubectlRun(kubectlCommand: string, args: string[]): Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }> {
  const child = spawn(kubectlCommand, args, { stdio: ["ignore", "pipe", "pipe"] });
  let stdout = "";