fix: expose command result failure in otel (#222)

src/mgr/server.ts

@@ -428,6 +428,29 @@ function compactRecoveryActions(value: JsonValue | undefined): JsonValue[] {
   });
 }
 
+function commandResultOtelAttributes(result: JsonValue): JsonRecord {
+  const record = asJsonRecord(result);
+  const terminalClassification = asJsonRecord(record.terminalClassification);
+  const diagnosis = asJsonRecord(record.diagnosis);
+  const runnerJob = asJsonRecord(diagnosis?.runnerJob);
+  const failureMessage = stringJsonValue(record.failureMessage);
+  return {
+    terminalStatus: stringJsonValue(record.terminalStatus),
+    terminalSource: stringJsonValue(record.terminalSource),
+    failureKind: stringJsonValue(record.failureKind),
+    failureMessage: failureMessage === null ? null : failureMessage.slice(0, 240),
+    terminalCategory: stringJsonValue(terminalClassification?.category),
+    commandState: stringJsonValue(record.commandState),
+    runStatus: stringJsonValue(record.runStatus),
+    runnerId: stringJsonValue(record.runnerId),
+    runnerJobId: stringJsonValue(runnerJob?.runnerJobId),
+    jobName: stringJsonValue(record.jobName),
+    eventCount: numberJsonValue(record.eventCount),
+    scopedEventCount: numberJsonValue(record.scopedEventCount),
+    runnerJobCount: numberJsonValue(record.runnerJobCount),
+  };
+}
+
 async function route({ method, url, body, store, sourceCommit, authSummary, runnerJobDefaults, sessionPvcDefaults, providerProfileDefaults, toolCredentialDefaults, runnerReconcilerOptions, aipodSpecDir }: { method: string; url: URL; body: unknown; store: AgentRunStore; sourceCommit: string; authSummary?: JsonRecord; runnerJobDefaults?: NonNullable<ManagerServerOptions["runnerJobDefaults"]>; sessionPvcDefaults?: NonNullable<ManagerServerOptions["sessionPvcOptions"]>; providerProfileDefaults?: NonNullable<ManagerServerOptions["providerProfileOptions"]>; toolCredentialDefaults?: NonNullable<ManagerServerOptions["toolCredentialOptions"]>; runnerReconcilerOptions?: RunnerReconcilerRuntimeOptions; aipodSpecDir?: string }): Promise<JsonValue> {
   const path = url.pathname;
   if (method === "GET" && (path === "/health" || path === "/health/live" || path === "/health/readiness")) {
@@ -703,7 +726,7 @@ async function route({ method, url, body, store, sourceCommit, authSummary, runn
     const commandId = url.searchParams.get("commandId") ?? undefined;
     const command = commandId ? await store.getCommand(commandId) : null;
     const result = await buildRunResult(store, runId, commandId) as JsonValue;
-    void emitAgentRunOtelSpan("command_result", run, process.env, { command, startTimeMs: startedAt, kind: 2, attributes: { "http.method": "GET", "http.route": "/api/v1/runs/:runId/result", "http.status_code": 200, terminalStatus: typeof result === "object" && result !== null && !Array.isArray(result) ? (result as JsonRecord).terminalStatus ?? null : null } });
+    void emitAgentRunOtelSpan("command_result", run, process.env, { command, startTimeMs: startedAt, kind: 2, attributes: { "http.method": "GET", "http.route": "/api/v1/runs/:runId/result", "http.status_code": 200, ...commandResultOtelAttributes(result) } });
     return result;
   }
   const runCancelMatch = path.match(/^\/api\/v1\/runs\/([^/]+)\/cancel$/u);
@@ -768,7 +791,7 @@ async function route({ method, url, body, store, sourceCommit, authSummary, runn
     const commandId = commandResultMatch[2] ?? "";
     const [run, command] = await Promise.all([store.getRun(runId), store.getCommand(commandId)]);
     const result = await buildRunResult(store, runId, commandId) as JsonValue;
-    void emitAgentRunOtelSpan("command_result", run, process.env, { command, startTimeMs: startedAt, kind: 2, attributes: { "http.method": "GET", "http.route": "/api/v1/runs/:runId/commands/:commandId/result", "http.status_code": 200, terminalStatus: typeof result === "object" && result !== null && !Array.isArray(result) ? (result as JsonRecord).terminalStatus ?? null : null } });
+    void emitAgentRunOtelSpan("command_result", run, process.env, { command, startTimeMs: startedAt, kind: 2, attributes: { "http.method": "GET", "http.route": "/api/v1/runs/:runId/commands/:commandId/result", "http.status_code": 200, ...commandResultOtelAttributes(result) } });
     return result;
   }
   if (method === "POST" && path === "/api/v1/runners/register") return await store.registerRunner(asRecord(body ?? {}, "runner")) as unknown as JsonValue;