fix: 增强终态失败分类可见性

2026-06-11 02:50:42 +08:00
parent 6843e7517b
commit ae8c89e099
4 changed files with 288 additions and 15 deletions
@@ -25,6 +25,17 @@ interface AssistantReplySummary {
  outputTruncated: boolean;
 }

+interface TerminalClassificationInput {
+  terminal: TerminalStatus | null;
+  terminalSource: string;
+  failureKind: FailureKind | null;
+  failureMessage: string | null;
+  timeoutBudget: JsonRecord;
+  transportDisconnect: RunEvent | null;
+  lastActivity: JsonRecord | null;
+  command: CommandRecord | null;
+}
+
 export async function buildRunResult(store: AgentRunStore, runId: string, commandId?: string): Promise<JsonRecord> {
  const run = await store.getRun(runId);
  const command = await selectCommand(store, runId, commandId);
@@ -44,6 +55,7 @@ export async function buildRunResult(store: AgentRunStore, runId: string, comman
  const reply = assistantReply(scopedEvents);
  const blocker = terminal === "blocked" || terminal === "failed" ? { failureKind, message: failureMessage, details: failureDetails } : null;
  const liveness = livenessSnapshot(run, command, events, scopedEvents, terminal, failureKind, failureMessage);
+  const terminalClassification = terminalClassificationSummary({ terminal, terminalSource, failureKind, failureMessage, liveness });
  const steerDelivery = command?.type === "steer" ? steerDeliverySummary(events, command.id) : null;
  return {
    runId: run.id,
@@ -75,6 +87,7 @@ export async function buildRunResult(store: AgentRunStore, runId: string, comman
    failureKind,
    failureMessage,
    failureDetails,
+    terminalClassification,
    blocker,
    liveness,
    ...(steerDelivery ? { steerDelivery } : {}),
@@ -103,6 +116,7 @@ function livenessSnapshot(run: RunRecord, command: CommandRecord | null, events:
  const transportDisconnect = latestTransportDisconnect(scopedEvents);
  const lastActivity = livenessActivitySummary(lastCommandActivity, nowMs);
  const timeoutBudget = timeoutBudgetSummary(run, command, terminal, failureKind, nowMs);
+  const terminalClassification = terminalClassificationFromEvidence({ terminal, terminalSource: "liveness", failureKind, failureMessage, timeoutBudget, transportDisconnect, lastActivity, command });
  const phase = livenessPhase({ active, command, lastVisibleActivity, leaseExpired: lease.leaseExpired, transportDisconnect, timeoutBudget, lastActivity });
  const afterSeq = lastEvent?.seq ?? 0;
  return {
@@ -119,6 +133,7 @@ function livenessSnapshot(run: RunRecord, command: CommandRecord | null, events:
    lastActivity,
    lastCommandActivity: lastActivity,
    timeoutBudget,
+    terminalClassification,
    lease,
    transportDisconnect: transportDisconnect ? livenessActivitySummary(transportDisconnect, nowMs) : null,
    recoveryActions: recoveryActions({ run, command, afterSeq, active, terminal, failureKind, failureMessage }),
@@ -146,6 +161,103 @@ function livenessPhase(input: { active: boolean; command: CommandRecord | null;
  return "waiting-model";
 }

+function terminalClassificationSummary(input: { terminal: TerminalStatus | null; terminalSource: string; failureKind: FailureKind | null; failureMessage: string | null; liveness: JsonRecord }): JsonRecord {
+  const livenessClassification = jsonRecordValue(input.liveness.terminalClassification);
+  return {
+    ...(livenessClassification ?? {}),
+    terminalStatus: input.terminal,
+    terminalSource: input.terminalSource,
+    failureKind: input.failureKind,
+    failureMessage: input.failureMessage ? boundedTextSummary(input.failureMessage, { limitChars: 240 }).text as string : null,
+    valuesPrinted: false,
+  };
+}
+
+function terminalClassificationFromEvidence(input: TerminalClassificationInput): JsonRecord {
+  const timeoutState = stringJsonValue(input.timeoutBudget.state);
+  const hardTimeout = input.failureKind === "backend-timeout" || timeoutState === "timed-out";
+  const providerKind = providerFailureCategory(input.failureKind);
+  const cancelled = input.terminal === "cancelled" || input.failureKind === "cancelled";
+  const taskFailure = input.terminal === "failed" && input.failureKind !== null && !hardTimeout && !providerKind && !infrastructureFailureKind(input.failureKind);
+  let category = "unknown";
+  let confidence = "low";
+  let providerEvidence = "not-applicable";
+  let reason = "terminal state is not yet available";
+
+  if (input.terminal === "completed") {
+    category = "completed";
+    confidence = "high";
+    reason = "command completed successfully";
+  } else if (cancelled) {
+    category = "cancelled";
+    confidence = "high";
+    reason = "terminal status or failureKind is cancelled";
+  } else if (providerKind) {
+    category = providerKind;
+    confidence = "high";
+    providerEvidence = "failure-kind";
+    reason = `failureKind ${input.failureKind} is provider-specific`;
+  } else if (hardTimeout && input.transportDisconnect) {
+    category = "execution-hard-timeout";
+    confidence = "medium";
+    providerEvidence = "observed-transport-disconnect";
+    reason = "hard timeout is terminal and a backend transport/app-server close event was observed, but existing events do not prove the model provider caused it";
+  } else if (hardTimeout) {
+    category = "execution-hard-timeout";
+    confidence = "high";
+    providerEvidence = "insufficient";
+    reason = "hard timeout is terminal; no provider-specific failure event was recorded";
+  } else if (input.terminal === "blocked") {
+    category = "blocked";
+    confidence = "high";
+    reason = `terminal status is blocked${input.failureKind ? ` with failureKind ${input.failureKind}` : ""}`;
+  } else if (taskFailure) {
+    category = "task-failed";
+    confidence = "medium";
+    reason = `terminal failure is not timeout, cancellation, provider-specific, or infrastructure-classified${input.failureKind ? `; failureKind=${input.failureKind}` : ""}`;
+  } else if (input.terminal === "failed" && infrastructureFailureKind(input.failureKind)) {
+    category = "infrastructure-failed";
+    confidence = "medium";
+    reason = `failureKind ${input.failureKind} is infrastructure/backend classified`;
+  }
+
+  return {
+    category,
+    confidence,
+    providerEvidence,
+    providerInterruption: providerEvidence === "failure-kind" || providerEvidence === "observed-transport-disconnect" ? providerEvidence : "not-established",
+    providerInterruptionKnown: providerEvidence === "failure-kind",
+    providerInterruptionReason: providerEvidence === "failure-kind"
+      ? "provider-specific failureKind is authoritative"
+      : providerEvidence === "observed-transport-disconnect"
+        ? "transport disconnect was observed, but current events cannot distinguish provider outage from runner/backend shutdown during timeout"
+        : providerEvidence === "insufficient"
+          ? "no provider-specific error or disconnect evidence was recorded"
+          : null,
+    hardTimeout,
+    timeoutState,
+    transportDisconnectObserved: Boolean(input.transportDisconnect),
+    transportDisconnectSeq: input.transportDisconnect?.seq ?? null,
+    lastActivityKind: stringJsonValue(input.lastActivity?.activityKind),
+    lastActivitySeq: numberJsonValue(input.lastActivity?.sourceSeq),
+    commandId: input.command?.id ?? null,
+    reason,
+    valuesPrinted: false,
+  };
+}
+
+function providerFailureCategory(failureKind: FailureKind | null): string | null {
+  if (!failureKind) return null;
+  if (failureKind === "provider-stream-disconnected") return "provider-interrupted";
+  if (failureKind.startsWith("provider-")) return "provider-failed";
+  return null;
+}
+
+function infrastructureFailureKind(failureKind: FailureKind | null): boolean {
+  if (!failureKind) return false;
+  return failureKind.startsWith("backend-") || failureKind === "runner-lease-conflict" || failureKind === "infra-failed" || failureKind === "thread-resume-failed";
+}
+
 function timeoutBudgetSummary(run: RunRecord, command: CommandRecord | null, terminal: TerminalStatus | null, failureKind: FailureKind | null, nowMs: number): JsonRecord {
  const timeoutMs = typeof run.executionPolicy.timeoutMs === "number" && Number.isFinite(run.executionPolicy.timeoutMs) && run.executionPolicy.timeoutMs > 0 ? Math.trunc(run.executionPolicy.timeoutMs) : null;
  const startedAt = command?.acknowledgedAt ?? command?.createdAt ?? run.updatedAt ?? run.createdAt;
@@ -295,6 +407,14 @@ function numberJsonValue(value: JsonValue | undefined): number | null {
  return typeof value === "number" && Number.isFinite(value) ? value : null;
 }

+function stringJsonValue(value: JsonValue | undefined): string | null {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function jsonRecordValue(value: unknown): JsonRecord | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonRecord : null;
+}
+
 function steerDeliverySummary(events: RunEvent[], commandId: string): JsonRecord {
  const related = events.filter((event) => event.payload.commandId === commandId);
  const completed = latestPhaseEvent(related, "turn/steer:completed");
@@ -144,12 +144,14 @@ async function queueTaskSupervisor(store: AgentRunStore, task: JsonRecord): Prom
    const liveness = asJsonRecord(result.liveness);
    const lastActivity = asJsonRecord(liveness?.lastActivity ?? liveness?.lastCommandActivity);
    const timeoutBudget = asJsonRecord(liveness?.timeoutBudget);
+    const terminalClassification = asJsonRecord(result.terminalClassification ?? liveness?.terminalClassification);
    return {
      runId: stringJsonValue(result.runId),
      commandId: stringJsonValue(result.commandId),
      status: stringJsonValue(result.status),
      terminalStatus: stringJsonValue(result.terminalStatus),
      failureKind: stringJsonValue(result.failureKind),
+      terminalClassification: terminalClassification ? compactTerminalClassification(terminalClassification) : null,
      phase: stringJsonValue(liveness?.phase),
      active: liveness?.active === true,
      lastSeq: numberJsonValue(liveness?.lastSeq ?? result.lastSeq),
@@ -190,6 +192,22 @@ function compactTimeoutBudget(budget: JsonRecord): JsonRecord {
  };
 }

+function compactTerminalClassification(record: JsonRecord): JsonRecord {
+  return {
+    category: stringJsonValue(record.category),
+    confidence: stringJsonValue(record.confidence),
+    providerEvidence: stringJsonValue(record.providerEvidence),
+    providerInterruption: stringJsonValue(record.providerInterruption),
+    providerInterruptionKnown: record.providerInterruptionKnown === true,
+    providerInterruptionReason: boundedJsonString(record.providerInterruptionReason, 240),
+    hardTimeout: record.hardTimeout === true,
+    transportDisconnectObserved: record.transportDisconnectObserved === true,
+    transportDisconnectSeq: numberJsonValue(record.transportDisconnectSeq),
+    reason: boundedJsonString(record.reason, 240),
+    valuesPrinted: false,
+  };
+}
+
 function compactRecoveryActions(value: JsonValue | undefined): JsonValue[] {
  if (!Array.isArray(value)) return [];
  return value.slice(0, 5).map((item) => {