import type { CommandRecord, JsonRecord, JsonValue, RunEvent, RunRecord, RunnerJobRecord } from "../common/types.js"; import { boundedTextSummary } from "../common/output.js"; export interface RunDiagnosisInput { run: RunRecord; command: CommandRecord | null; latestJob: RunnerJobRecord | null; events: RunEvent[]; terminalClassification: JsonRecord | null; liveness: JsonRecord | null; terminalStatus: string | null; failureKind: string | null; failureMessage: string | null; } export function runDiagnosis(input: RunDiagnosisInput): JsonRecord { const nowMs = Date.now(); const lease = recordAt(input.liveness, "lease") ?? leaseFromRun(input.run, nowMs); const timeoutBudget = recordAt(input.liveness, "timeoutBudget"); const staleClaimed = input.run.status === "claimed" && booleanValue(lease.leaseExpired) === true; const terminalCommandOpenRun = input.run.status === "claimed" && input.terminalStatus !== null; const runnerJob = input.latestJob ? runnerJobReference(input.latestJob, input.events) : null; const runnerLost = staleClaimed && (runnerJob === null || runnerJobPhaseIndicatesLost(runnerJob.phase)); const session = sessionReference(input.run); const providerEvidence = stringValue(input.terminalClassification?.providerEvidence) ?? "not-applicable"; const providerInterruption = stringValue(input.terminalClassification?.providerInterruption) ?? "not-established"; const category = diagnosisCategory({ staleClaimed, runnerLost, terminalCommandOpenRun, providerEvidence, terminalCategory: stringValue(input.terminalClassification?.category) }); const recoveryActions = recoveryActionsForDiagnosis({ run: input.run, command: input.command, latestJob: input.latestJob, session, runnerLost, staleClaimed, terminalCommandOpenRun, failureKind: input.failureKind, lastSeq: numberValue(input.liveness?.lastSeq) ?? 0 }); return { category, staleClaimed, runnerLost, terminalCommandOpenRun, evidenceLevel: evidenceLevel(category, providerEvidence, runnerLost, staleClaimed, terminalCommandOpenRun), providerEvidence, providerInterruption, providerInterruptionKnown: input.terminalClassification?.providerInterruptionKnown === true, terminalCategory: stringValue(input.terminalClassification?.category), terminalStatus: input.terminalStatus, failureKind: input.failureKind, failureMessage: input.failureMessage ? boundedTextSummary(input.failureMessage, { limitChars: 240 }).text as string : null, run: { runId: input.run.id, status: input.run.status, claimedBy: input.run.claimedBy, leaseExpiresAt: input.run.leaseExpiresAt, leaseExpired: booleanOrNull(lease.leaseExpired), leaseRemainingMs: numberValue(lease.leaseRemainingMs), valuesPrinted: false, }, command: input.command ? { commandId: input.command.id, state: input.command.state, terminalStatus: input.terminalStatus, acknowledgedAt: input.command.acknowledgedAt ?? null, updatedAt: input.command.updatedAt, valuesPrinted: false, } : null, runnerJob, session, timeoutBudget: timeoutBudget ? compactRecord(timeoutBudget, ["state", "timeoutKind", "timeoutMs", "elapsedMs", "idleElapsedMs", "remainingMs", "startedAt", "idleStartedAt", "lastActivityAt", "lastActivitySeq", "commandElapsedMs", "runElapsedMs", "source"]) : null, recoveryActions, valuesPrinted: false, }; } export function runnerJobDiagnosis(job: RunnerJobRecord, events: RunEvent[] = []): JsonRecord { const observation = runnerJobObservation(job, events); const phase = stringValue(observation.phase) ?? "unknown"; const notStarted = phase === "created" || phase === "recorded"; const runnerLostSuspected = notStarted || runnerJobPhaseIndicatesLost(phase); return { category: stringValue(observation.category) ?? (notStarted ? "runner-job-created" : phase.startsWith("terminal:") ? "runner-job-terminal" : "runner-job-observed"), runnerLostSuspected, phase, evidenceLevel: stringValue(observation.evidenceLevel) ?? (notStarted ? "medium" : "high"), lastObservedSeq: numberValue(observation.lastObservedSeq), lastObservedAt: stringValue(observation.lastObservedAt), lastObservedKind: stringValue(observation.lastObservedKind), terminalReportState: stringValue(observation.terminalReportState), runReportState: stringValue(observation.runReportState), runId: job.runId, commandId: job.commandId, runnerJobId: job.id, attemptId: job.attemptId, runnerId: job.runnerId, jobName: job.jobName, namespace: job.namespace, logPath: stringValue(recordAt(job.result, "runner")?.logPath), nextActions: [ recoveryDescriptor({ action: "inspect-run", operation: "describe", resourceKind: "run", resourceName: job.runId, runId: job.runId }), recoveryDescriptor({ action: "inspect-command", operation: "describe", resourceKind: "command", resourceName: job.commandId, runId: job.runId, commandId: job.commandId }), recoveryDescriptor({ action: "poll-events", operation: "events", resourceKind: "run", resourceName: job.runId, runId: job.runId, commandId: job.commandId, afterSeq: 0, limit: 100 }), ], valuesPrinted: false, }; } export function runnerJobObservation(job: RunnerJobRecord, events: RunEvent[] = []): JsonRecord { const terminalEvent = latestTerminalEvent(events, job.commandId); const terminalStatus = stringValue(terminalEvent?.payload.terminalStatus) ?? stringValue(latestTerminalStatusFromResult(job.result)); if (terminalStatus) { return { phase: `terminal:${terminalStatus}`, category: "runner-job-terminal", terminalStatus, failureKind: stringValue(terminalEvent?.payload.failureKind), startedAt: stringValue(firstObservedRunnerEvent(job, events)?.createdAt), finishedAt: stringValue(terminalEvent?.createdAt), lastObservedSeq: numberValue(terminalEvent?.seq), lastObservedAt: stringValue(terminalEvent?.createdAt), lastObservedKind: eventKind(terminalEvent), evidenceLevel: "high", valuesPrinted: false, }; } const observed = relevantRunnerJobEvents(job, events); const lastObserved = observed.at(-1) ?? null; if (lastObserved) { return { phase: "running", category: "runner-job-running", terminalStatus: null, failureKind: null, startedAt: stringValue(observed[0]?.createdAt), finishedAt: null, lastObservedSeq: numberValue(lastObserved.seq), lastObservedAt: stringValue(lastObserved.createdAt), lastObservedKind: eventKind(lastObserved), evidenceLevel: "high", valuesPrinted: false, }; } const reconcilerObservation = recordAt(job.result, "observation"); const reconcilerPhase = stringValue(reconcilerObservation?.observedRunnerPhase) ?? stringValue(reconcilerObservation?.phase); if (reconcilerObservation && reconcilerPhase) { return { phase: reconcilerPhase, category: stringValue(reconcilerObservation.category) ?? "runner-job-observed", terminalStatus: stringValue(reconcilerObservation.terminalStatus), failureKind: stringValue(reconcilerObservation.failureKind), startedAt: stringValue(recordAt(reconcilerObservation, "k8s")?.startTime), finishedAt: stringValue(recordAt(reconcilerObservation, "k8s")?.completionTime), lastObservedSeq: null, lastObservedAt: stringValue(reconcilerObservation.lastK8sObservedAt) ?? stringValue(reconcilerObservation.lastObservedAt), lastObservedKind: stringValue(reconcilerObservation.lastObservedKind) ?? `manager-reconciler:${reconcilerPhase}`, terminalReportState: stringValue(reconcilerObservation.terminalReportState), runReportState: stringValue(reconcilerObservation.runReportState), evidenceLevel: stringValue(reconcilerObservation.evidenceLevel) ?? "high", valuesPrinted: false, }; } const created = recordAt(job.result, "kubernetes")?.created === true; return { phase: created ? "created" : "recorded", category: created ? "runner-job-created" : "runner-job-recorded", terminalStatus: null, failureKind: null, startedAt: null, finishedAt: null, lastObservedSeq: null, lastObservedAt: null, lastObservedKind: null, evidenceLevel: created ? "medium" : "low", valuesPrinted: false, }; } function diagnosisCategory(input: { staleClaimed: boolean; runnerLost: boolean; terminalCommandOpenRun: boolean; providerEvidence: string; terminalCategory: string | null }): string { if (input.runnerLost) return "runner-lost"; if (input.staleClaimed) return "stale-claimed"; if (input.terminalCommandOpenRun) return "terminal-command-open-run"; if (input.providerEvidence === "failure-kind") return "provider-interruption-known"; if (input.providerEvidence === "observed-transport-disconnect") return "provider-interruption-unknown"; if (input.terminalCategory) return input.terminalCategory; return "unknown"; } function evidenceLevel(category: string, providerEvidence: string, runnerLost: boolean, staleClaimed: boolean, terminalCommandOpenRun: boolean): string { if (runnerLost || staleClaimed || terminalCommandOpenRun) return "high"; if (providerEvidence === "failure-kind") return "high"; if (providerEvidence === "observed-transport-disconnect") return "medium"; if (category === "completed" || category === "cancelled") return "high"; return "low"; } function recoveryActionsForDiagnosis(input: { run: RunRecord; command: CommandRecord | null; latestJob: RunnerJobRecord | null; session: JsonRecord; runnerLost: boolean; staleClaimed: boolean; terminalCommandOpenRun: boolean; failureKind: string | null; lastSeq: number }): JsonRecord[] { const actions: JsonRecord[] = []; if (input.latestJob) actions.push(recoveryDescriptor({ action: "inspect-runner-job", operation: "describe", resourceKind: "runnerjob", resourceName: input.latestJob.id, runId: input.run.id, commandId: input.command?.id ?? input.latestJob.commandId, runnerJobId: input.latestJob.id })); if (input.command) actions.push(recoveryDescriptor({ action: "inspect-command", operation: "result", resourceKind: "command", resourceName: input.command.id, runId: input.run.id, commandId: input.command.id })); actions.push(recoveryDescriptor({ action: "poll-events", operation: "events", resourceKind: "run", resourceName: input.run.id, runId: input.run.id, commandId: input.command?.id ?? null, afterSeq: input.lastSeq, limit: 100 })); const sessionId = stringValue(input.session.sessionId); if (sessionId) actions.push(recoveryDescriptor({ action: "continue-session", operation: "send", resourceKind: "session", resourceName: sessionId, runId: input.run.id, commandId: input.command?.id ?? null, sessionId, inputKind: "prompt" })); else actions.push({ action: "session-unavailable", operation: "operator-decision", resourceKind: "run", resourceName: input.run.id, runId: input.run.id, commandId: input.command?.id ?? null, reason: "sessionRef=null", reasonHint: "当前 run 没有 sessionRef,管理者只能从 run/events/command/runner-job 读取 trace 后重新提交;这表示该任务不可同 session 续跑。", valuesPrinted: false }); if (input.runnerLost || input.staleClaimed || input.terminalCommandOpenRun) actions.push(recoveryDescriptor({ action: "refresh-queue-or-resubmit", operation: "operator-decision", resourceKind: sessionId ? "session" : "run", resourceName: sessionId ?? input.run.id, runId: input.run.id, commandId: input.command?.id ?? null, sessionId, reason: input.failureKind ?? "stale-runner-state", reasonHint: "先用 queue refresh/show 对齐 attempt;有 sessionId 时继续同一 session,没有 sessionId 才重新派发。" })); return actions.slice(0, 6); } function recoveryDescriptor(input: { action: string; operation: string; resourceKind: string; resourceName: string; runId?: string | null; commandId?: string | null; runnerJobId?: string | null; sessionId?: string | null; afterSeq?: number | null; limit?: number | null; reason?: string | null; reasonHint?: string | null; inputKind?: string | null }): JsonRecord { return { action: input.action, operation: input.operation, resourceKind: input.resourceKind, resourceName: input.resourceName, runId: input.runId ?? null, commandId: input.commandId ?? null, runnerJobId: input.runnerJobId ?? null, sessionId: input.sessionId ?? null, ...(input.afterSeq !== undefined ? { afterSeq: input.afterSeq } : {}), ...(input.limit !== undefined ? { limit: input.limit } : {}), ...(input.reason ? { reason: input.reason } : {}), ...(input.reasonHint ? { reasonHint: input.reasonHint } : {}), ...(input.inputKind ? { inputKind: input.inputKind } : {}), valuesPrinted: false, }; } function runnerJobReference(job: RunnerJobRecord, events: RunEvent[]): JsonRecord { const observation = runnerJobObservation(job, events); const terminalStatus = stringValue(observation.terminalStatus); return { runnerJobId: job.id, attemptId: job.attemptId, runnerId: job.runnerId, namespace: job.namespace, jobName: job.jobName, phase: stringValue(observation.phase) ?? "unknown", terminalStatus, startedAt: stringValue(observation.startedAt), finishedAt: stringValue(observation.finishedAt), lastObservedSeq: numberValue(observation.lastObservedSeq), lastObservedAt: stringValue(observation.lastObservedAt), lastObservedKind: stringValue(observation.lastObservedKind), terminalReportState: stringValue(observation.terminalReportState), runReportState: stringValue(observation.runReportState), logPath: stringValue(recordAt(job.result, "runner")?.logPath), valuesPrinted: false, }; } function runnerJobPhaseIndicatesLost(value: JsonValue | undefined): boolean { const phase = stringValue(value); return phase === "created" || phase === "recorded" || phase === "k8s:failed" || phase === "k8s:missing" || phase === "k8s:succeeded" || phase === "k8s:observe-failed"; } function sessionReference(run: RunRecord): JsonRecord { if (!run.sessionRef) return { sessionId: null, sessionRefNull: true, sessionPath: null, valuesPrinted: false }; return { sessionId: run.sessionRef.sessionId, sessionRefNull: false, sessionPath: `/api/v1/sessions/${run.sessionRef.sessionId}`, valuesPrinted: false }; } function leaseFromRun(run: RunRecord, nowMs: number): JsonRecord { const leaseExpiresMs = run.leaseExpiresAt ? Date.parse(run.leaseExpiresAt) : NaN; const hasLease = Boolean(run.claimedBy && run.leaseExpiresAt && Number.isFinite(leaseExpiresMs)); return { leaseExpired: run.claimedBy ? (hasLease ? leaseExpiresMs <= nowMs : true) : null, leaseRemainingMs: hasLease ? Math.max(0, leaseExpiresMs - nowMs) : null, valuesPrinted: false, }; } function latestTerminalStatusFromResult(result: JsonRecord): JsonValue | null { const value = result.terminalStatus; return typeof value === "string" ? value : null; } function latestTerminalEvent(events: RunEvent[], commandId: string): RunEvent | null { for (const event of [...events].reverse()) { const payload = event.payload; if (payload?.commandId && payload.commandId !== commandId) continue; if (event.type === "terminal_status") return event; if (event.type === "backend_status" && payload?.phase === "command-terminal" && payload.commandId === commandId) return event; } return null; } function relevantRunnerJobEvents(job: RunnerJobRecord, events: RunEvent[]): RunEvent[] { return events.filter((event) => isRunnerJobActivity(job, event)); } function firstObservedRunnerEvent(job: RunnerJobRecord, events: RunEvent[]): RunEvent | null { return relevantRunnerJobEvents(job, events)[0] ?? null; } function isRunnerJobActivity(job: RunnerJobRecord, event: RunEvent): boolean { const payload = event.payload; if (payload?.phase === "runner-job-created") return false; if (payload?.runnerId === job.runnerId || payload?.attemptId === job.attemptId) return true; if (payload?.commandId === job.commandId) { if (event.type === "tool_call" || event.type === "assistant_message" || event.type === "command_output" || event.type === "error") return true; if (event.type === "backend_status" && typeof payload.phase === "string" && payload.phase !== "command-created") return true; } return false; } function eventKind(event: RunEvent | null): string | null { if (!event) return null; const phase = stringValue(event.payload?.phase); if (phase) return `${event.type}:${phase}`; const toolName = stringValue(event.payload?.toolName); if (toolName) return `${event.type}:${toolName}`; return event.type; } function compactRecord(record: JsonRecord, keys: string[]): JsonRecord { const result: JsonRecord = {}; for (const key of keys) { const value = record[key]; if (value === undefined) continue; if (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || value === null) result[key] = value; } result.valuesPrinted = false; return result; } function recordAt(record: JsonRecord | null | undefined, key: string): JsonRecord | null { const value = record?.[key]; return typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonRecord : null; } function stringValue(value: JsonValue | undefined): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function numberValue(value: JsonValue | undefined): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } function booleanValue(value: JsonValue | undefined): boolean | null { return typeof value === "boolean" ? value : null; } function booleanOrNull(value: JsonValue | undefined): boolean | null { return typeof value === "boolean" ? value : null; }