// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/platform-infra-observability.ts. // Moved mechanically from scripts/src/platform-infra-observability.ts:624-982 for #903. // SPEC: PJ2026-01060501 OTel追踪 draft-2026-06-19-p0. // Responsibility: YAML-first platform-infra OpenTelemetry tracing control commands. import { Buffer } from "node:buffer"; import { readFileSync } from "node:fs"; import type { UniDeskConfig } from "../config"; import { rootPath } from "../config"; import type { RenderedCliResult } from "../output"; import { compactCapture, createYamlFieldReader, numberField, parseJsonOutput, redactSensitiveUnknown, shQuote, capture, } from "../platform-infra-ops-library"; import type { DiagnoseCodeAgentOptions, ObservabilityTarget, SearchOptions, TraceOptions } from "./types"; import { resolveTarget } from "./actions"; import { asPlainRecord, compactDiagnoseCodeAgentResult } from "./apply-status-scripts"; import { readObservabilityConfig } from "./config"; import { diagnoseCodeAgentScript, searchScript, traceScript } from "./diagnose-code-agent-script"; import { buildDiagnoseCommand, buildSearchCommand, buildTraceCommand, compactSearchResult, formatTable, httpTableRows, isoFromUnixNano, joinValues, numberFromUnknown, numberValue, searchTraceStatus, shortenEnd, shortenMiddle, spanColumnAttr, spanDetail, textValue, traceSpanDurationMs, traceSpanImportanceScore, uniqueSpanRecords } from "./manifest"; import { targetSummary } from "./summary"; import { asArray } from "./trace-script"; export async function trace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { if (options.traceId === null) throw new Error("observability trace requires --trace-id "); const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const tracePath = observability.probes.traceQueryPathTemplate.replaceAll("{{traceId}}", encodeURIComponent(options.traceId)); const result = await capture(config, target.route, ["sh"], traceScript(observability, target, tracePath, options)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && parsed?.ok === true; if (!options.raw) { return renderTraceTable({ ok, target, options, tracePath, query: { backend: observability.traceBackend.type, service: observability.traceBackend.serviceName, path: tracePath, }, result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : asPlainRecord(redactSensitiveUnknown(parsed)) ?? {}, }); } return { ok, action: "platform-infra-observability-trace", mutation: false, target: targetSummary(target), traceId: options.traceId, query: { backend: observability.traceBackend.type, service: observability.traceBackend.serviceName, path: tracePath, }, result: redactSensitiveUnknown(parsed), }; } export async function search(config: UniDeskConfig, options: SearchOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null; const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080); const effectiveTempoQuery = options.query ?? (businessTraceId === null ? inferSearchTempoQuery(options) : `{ .traceId = "${businessTraceId}" }`); const effectiveOptions: SearchOptions = { ...options, query: effectiveTempoQuery, lookbackMinutes: effectiveLookbackMinutes, }; const endMillis = options.endAt === null ? Date.now() : Date.parse(options.endAt); if (!Number.isFinite(endMillis)) throw new Error("--end-at must be an ISO timestamp parseable by Date.parse"); const endSeconds = Math.floor(endMillis / 1000); const startSeconds = endSeconds - (effectiveLookbackMinutes * 60); const params = new URLSearchParams({ start: String(startSeconds), end: String(endSeconds), limit: String(options.candidateLimit), }); if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery); const searchPath = `/api/search?${params.toString()}`; const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && parsed?.ok === true; const query = { backend: observability.traceBackend.type, service: observability.traceBackend.serviceName, path: searchPath, grep: options.grep, tempoQuery: effectiveTempoQuery, explicitTempoQuery: options.query, pathFilter: options.path, statusFilter: options.status, lookbackMinutes: effectiveLookbackMinutes, requestedLookbackMinutes: options.lookbackMinutes, endAt: new Date(endSeconds * 1000).toISOString(), businessTraceId, mode: businessTraceId === null ? "candidate-grep" : "business-trace-exact", grepQueryInference: options.query === null && businessTraceId === null && effectiveTempoQuery !== null ? "inferred-from-grep-or-filters" : null, grepCoverage: options.grep === null ? null : "candidate traces are fetched by tempoQuery, then each scanned trace is matched against raw trace body, span name, status message, route and full span attributes", candidateLimit: options.candidateLimit, limit: options.limit, }; if (!options.full && !options.raw) { return renderSearchTable({ ok, target, options, query, result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : compactSearchResult(parsed), }); } const exposedResult = parsed === null ? compactCapture(result, { full: true }) : options.raw ? redactSensitiveUnknown(parsed) : { ...compactSearchFullResult(parsed), outputMode: "bounded-compact-json", rawDisclosure: "Use --raw only for Tempo/API envelope or parser debugging.", }; return { ok, action: "platform-infra-observability-search", mutation: false, target: targetSummary(target), query, result: exposedResult, }; } function compactSearchFullResult(value: unknown): Record { const source = asPlainRecord(value) ?? {}; return { ok: source.ok === true, grep: source.grep ?? null, tempoQuery: source.tempoQuery ?? null, pathFilter: source.pathFilter ?? null, statusFilter: source.statusFilter ?? null, matchingActive: source.matchingActive ?? null, grepCoverage: source.grepCoverage ?? null, grepQueryInference: source.grepQueryInference ?? null, limit: source.limit ?? null, candidateLimit: source.candidateLimit ?? null, searchParseOk: source.searchParseOk ?? null, candidateTraceCount: source.candidateTraceCount ?? null, scannedTraceCount: source.scannedTraceCount ?? null, matchedTraceCount: source.matchedTraceCount ?? null, scanStopped: source.scanStopped ?? null, traces: asArray(source.traces).slice(0, 5).map(compactSearchFullTrace), omittedTraceCount: Math.max(0, asArray(source.traces).length - 5), truncated: source.truncated ?? null, next: source.next ?? null, searchStderrTail: source.searchStderrTail ?? null, disclosure: { defaultView: "bounded search full summary; span arrays and raw Tempo bodies are omitted", expand: "query one trace with platform-infra observability trace --trace-id --grep --full; use --raw for parser/debug only", }, }; } function compactSearchFullTrace(value: unknown): Record { const trace = asPlainRecord(value) ?? {}; const meta = asPlainRecord(trace.meta); return { traceId: trace.traceId ?? null, startAt: trace.startAt ?? isoFromUnixNano(meta?.startTimeUnixNano), durationMs: trace.durationMs ?? meta?.durationMs ?? null, queryOk: trace.queryOk ?? null, parseOk: trace.parseOk ?? null, rawMatched: trace.rawMatched ?? null, bodyBytes: trace.bodyBytes ?? null, spanCount: trace.spanCount ?? null, serviceCount: trace.serviceCount ?? null, services: asArray(trace.services).slice(0, 4), businessTraceIds: asArray(trace.businessTraceIds).slice(0, 4), errorSpanCount: trace.errorSpanCount ?? null, matchedSpanCount: trace.matchedSpanCount ?? null, spanNameCounts: asArray(trace.spanNameCounts).slice(0, 4).map(compactNameCount), matchedSpanNames: [...new Set(asArray(trace.matchedSpans).map((span) => textValue(asPlainRecord(span)?.name)).filter((name) => name !== "-"))].slice(0, 4), }; } function compactNameCount(value: unknown): Record { const item = asPlainRecord(value) ?? {}; return { name: item.name ?? null, count: item.count ?? null, }; } export function inferSearchTempoQuery(options: SearchOptions): string | null { const filters: string[] = []; if (options.path !== null) filters.push(`.http.route = ${JSON.stringify(options.path)}`); if (options.status !== null) filters.push(`.http.response.status_code = ${options.status}`); if (filters.length > 0) return `{ ${filters.join(" && ")} }`; const grep = options.grep?.trim() ?? ""; if (!grep) return null; const keyValue = grep.match(/^([A-Za-z0-9_.-]+)=(.+)$/u); if (keyValue !== null) { const key = keyValue[1] ?? ""; const value = (keyValue[2] ?? "").trim(); const traceQlKey = key === "name" || key.startsWith("resource.") ? key : `.${key.replace(/^span[.]/u, "")}`; if (/^(?:name|resource[.][A-Za-z0-9_.-]+|[.][A-Za-z0-9_.-]+)$/u.test(traceQlKey) && value.length > 0 && value.length <= 200) return `{ ${traceQlKey} = ${traceQlLiteral(value)} }`; } if (grep.startsWith("/") && !grep.includes("\n") && grep.length <= 300) return `{ .http.route = ${JSON.stringify(grep)} }`; if (/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,180}$/u.test(grep)) { if (isKnownTraceAttributeKey(grep)) return `{ .${grep} != nil }`; if (grep.includes(".") || grep.includes(":")) return `{ name = ${JSON.stringify(grep)} }`; if (/^(?:hwlab|agentrun|opencode|platform|code-queue|unidesk)[A-Za-z0-9-]*$/u.test(grep) || /-(?:api|web|proxy|runner|manager|service|mgr|collector|tempo)$/u.test(grep)) return `{ resource.service.name = ${JSON.stringify(grep)} }`; } return null; } const knownTraceAttributeKeys = new Set([ "opencode.proxy.phase", "opencode.proxy.streaming", "opencode.proxy.ticket_accepted", "opencode.proxy.sse.directory_rewrite_enabled", "opencode.proxy.sse.directory_rewrite_from", "opencode.proxy.sse.directory_rewrite_to", "opencode.provider.sse.content_chunks", "opencode.provider.sse.content_chars", "opencode.provider.sse.output_data_lines", "opencode.provider.sse.done_lines", "opencode.provider.sse.json_errors", "opencode.provider.sse.reasoning_only_choices_dropped", "workbench.sync.contract_version", "workbench.sync.since_outbox_seq", "workbench.sync.cursor_outbox_seq", "workbench.sync.event_count", "workbench.sync.entity_family_count", "workbench.sync.entity_families", "workbench.sync.max_entity_version", "workbench.realtime.authority", "workbench.projection.revision", "workbench.terminal.seal", "workbench.detail.projection", ]); function isKnownTraceAttributeKey(value: string): boolean { return knownTraceAttributeKeys.has(value) || value.startsWith("opencode.proxy.sse.") || value.startsWith("opencode.provider.sse."); } function traceQlLiteral(value: string): string { const unquoted = stripOuterQuotes(value.trim()); if (/^(?:true|false)$/iu.test(unquoted)) return unquoted.toLowerCase(); if (/^-?(?:0|[1-9][0-9]*)(?:[.][0-9]+)?$/u.test(unquoted)) return unquoted; if (unquoted === "nil") return "nil"; return JSON.stringify(unquoted); } function stripOuterQuotes(value: string): string { if (value.length >= 2 && ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'")))) { return value.slice(1, -1); } return value; } export function businessTraceIdFromSearchText(value: string | null): string | null { const match = value?.match(/\btrc_[A-Za-z0-9_-]+\b/u); return match?.[0] ?? null; } function diagnoseTraceSearchFilters(options: DiagnoseCodeAgentOptions): string[] { const filters: string[] = []; if (options.businessTraceId !== null) filters.push(`.traceId = ${JSON.stringify(options.businessTraceId)}`); if (options.runId !== null) filters.push(`.runId = ${JSON.stringify(options.runId)}`); if (options.commandId !== null) filters.push(`.commandId = ${JSON.stringify(options.commandId)}`); if (options.sessionId !== null) filters.push(`.sessionId = ${JSON.stringify(options.sessionId)}`); if (options.runnerJobId !== null) filters.push(`.runnerJobId = ${JSON.stringify(options.runnerJobId)}`); return filters; } function diagnoseTraceSearchMode(options: DiagnoseCodeAgentOptions): string { if (options.businessTraceId !== null && options.runId === null && options.commandId === null && options.sessionId === null && options.runnerJobId === null) return "business-trace-id"; return "trace-attribute-query"; } export async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const endSeconds = Math.floor(Date.now() / 1000); const startSeconds = endSeconds - (options.lookbackMinutes * 60); let searchPath: string | null = null; const searchFilters = diagnoseTraceSearchFilters(options); const tempoQuery = searchFilters.length > 0 ? `{ ${searchFilters.join(" && ")} }` : null; if (options.traceId === null && tempoQuery !== null) { const params = new URLSearchParams({ start: String(startSeconds), end: String(endSeconds), limit: String(options.candidateLimit), q: tempoQuery, }); searchPath = `/api/search?${params.toString()}`; } const result = await capture(config, target.route, ["sh"], diagnoseCodeAgentScript(observability, target, searchPath, options)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && parsed?.ok === true; const query = { backend: observability.traceBackend.type, service: observability.traceBackend.serviceName, businessTraceId: options.businessTraceId, traceId: options.traceId, runId: options.runId, commandId: options.commandId, sessionId: options.sessionId, runnerJobId: options.runnerJobId, mode: diagnoseTraceSearchMode(options), tempoQuery, queryClauses: searchFilters, path: searchPath, lookbackMinutes: options.lookbackMinutes, candidateLimit: options.candidateLimit, limit: options.limit, }; if (!options.full && !options.raw) { return renderDiagnoseCodeAgentTable({ ok, target, options, query, result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : compactDiagnoseCodeAgentResult(parsed), }); } const exposedResult = parsed === null ? compactCapture(result, { full: true }) : options.raw ? redactSensitiveUnknown(parsed) : { ...compactDiagnoseCodeAgentResult(parsed), outputMode: "bounded-compact-json", rawDisclosure: "Use --raw only for Tempo/API envelope or CLI parser debugging.", }; return { ok, action: "platform-infra-observability-diagnose-code-agent", mutation: false, target: targetSummary(target), query, result: exposedResult, }; } export function renderTraceTable(input: { ok: boolean; target: ObservabilityTarget; options: TraceOptions; tracePath: string; query: Record; result: Record; }): RenderedCliResult { const spanSource = input.result.grep === null || input.result.grep === undefined ? [...asArray(input.result.errorSpans), ...asArray(input.result.spans)] : asArray(input.result.spans); const rowLimit = input.options.full ? Math.max(10, Math.min(input.options.limit, 24)) : 10; const allSpans = [...asArray(input.result.errorSpans), ...asArray(input.result.spans)] .map((span) => asPlainRecord(span) ?? {}); const spanRows = uniqueSpanRecords(spanSource) .sort((left, right) => traceSpanImportanceScore(right) - traceSpanImportanceScore(left)) .slice(0, rowLimit) .map((span) => { const attrs = asPlainRecord(span.attributes); const durationMs = traceSpanDurationMs(span, attrs); return [ shortenEnd(textValue(span.name), 40), shortenEnd(textValue(span.service), 22), shortenEnd(spanColumnAttr(attrs, ["http.route", "workbench.ui.route"]), 34), spanColumnAttr(attrs, ["http.response.status_code", "http.status_code", "http.response.status_class"]), numberValue(durationMs), spanColumnAttr(attrs, ["workbench.ui.resource_duration_ms"]), spanColumnAttr(attrs, ["workbench.ui.resource_fetch_to_request_ms"]), spanColumnAttr(attrs, ["workbench.ui.resource_request_wait_ms"]), spanColumnAttr(attrs, ["workbench.ui.resource_response_transfer_ms"]), shortenEnd(spanColumnAttr(attrs, ["workbench.ui.resource_next_hop_protocol"]), 14), spanDetail(span, input.options.full ? 150 : 220), ]; }); const runnerRows = traceRunnerRows(allSpans); const readRows = traceReadWindowRows(allSpans); const sessionListRows = traceSessionListRows(allSpans); const sessionMessageRows = traceSessionMessageRows(allSpans); const turnRows = traceTurnStatusRows(allSpans); const projectionRows = traceProjectionRows(allSpans); const projectionSyncRows = traceProjectionSyncRows(allSpans); const providerRows = traceProviderDecisionRows(allSpans); const countRows = asArray(input.result.spanNameCounts).slice(0, 8).map((item) => { const row = asPlainRecord(item) ?? {}; return [shortenEnd(textValue(row.name), 64), textValue(row.count)]; }); const identity = asPlainRecord(input.result.identity) ?? {}; const identityRows = ["runId", "commandId", "runnerJobId", "runnerId", "backendProfile", "sourceCommit", "jobName", "podName"] .map((key) => [key, joinValues(identity[key], 88)]) .filter((row) => row[1] !== "-"); const next = asPlainRecord(input.result.next); const lines = [ `platform-infra observability trace (${input.ok ? "ok" : "not-ok"})`, "", formatTable(["TRACE", "SPANS", "ERR", "MATCH", "SERVICES", "BUSINESS_TRACE"], [[ shortenMiddle(input.options.traceId ?? "-", 20), textValue(input.result.spanCount), textValue(input.result.errorSpanCount), textValue(input.result.matchedSpanCount), joinValues(input.result.services, 44), joinValues(input.result.businessTraceIds, 34), ]]), "", "Identity:", formatTable(["FIELD", "VALUE"], identityRows.length > 0 ? identityRows : [["-", "-"]]), "", "Provider decision:", formatTable(["SERVICE", "PROFILE", "SOURCE", "DEFAULT", "MODEL", "ADAPTER", "RUNNER_NS", "SESSION"], providerRows.length > 0 ? providerRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Key spans:", formatTable(["NAME", "SERVICE", "ROUTE", "STATUS", "DUR_MS", "RES_MS", "FETCH_REQ", "REQ_WAIT", "RESP_XFER", "PROTO", "DETAIL"], spanRows.length > 0 ? spanRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Runner terminal/errors:", formatTable(["NAME", "SERVICE", "TERMINAL", "FAILURE", "RETRY", "ROUTE", "HTTP", "MESSAGE"], runnerRows.length > 0 ? runnerRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Trace read windows:", formatTable(["TRACE", "SESSION", "SINCE", "LIMIT", "RETURNED", "RANGE", "TOTAL", "MORE", "FULL", "LAST"], readRows.length > 0 ? readRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Session list reads:", formatTable(["SOURCE", "COUNT", "FALLBACK", "RATIO", "EMPTY_PREVIEW", "INCLUDE", "SESSIONS", "TRACES"], sessionListRows.length > 0 ? sessionListRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Session message reads:", formatTable(["SESSION", "RETURNED", "TOTAL", "ROLES", "USER_PREFIX", "ADJ_SAME", "USER", "AGENT", "TRACES"], sessionMessageRows.length > 0 ? sessionMessageRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Turn status reads:", formatTable(["TRACE", "SESSION", "TURN", "STATUS", "PHASE", "ROUTE", "HTTP"], turnRows.length > 0 ? turnRows : [["-", "-", "-", "-", "-", "-", "-"]]), "", "Projection writes:", formatTable(["NAME", "TRACE", "SESSION", "TURN", "STATUS", "EVENT", "PROJECTED", "SOURCE", "MESSAGE"], projectionRows.length > 0 ? projectionRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Projection sync cursors:", formatTable(["SERVICE", "TRACE", "SESSION", "AFTER", "END", "EVENTS", "RAW", "MAX", "LAST", "TERM", "AUTH"], projectionSyncRows.length > 0 ? projectionSyncRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Span name counts:", formatTable(["NAME", "COUNT"], countRows.length > 0 ? countRows : [["-", "-"]]), "", "Summary:", ` target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)} path=${input.tracePath}`, ` grep=${textValue(input.result.grep)} limit=${textValue(input.result.limit)} traceFound=${textValue(input.result.traceFound)} parseOk=${textValue(input.result.parseOk)}`, "", "Next:", ]; const nextCommands = [ textValue(next?.fullSummary), textValue(next?.grepProviderStream), textValue(next?.raw), ].filter((item) => item !== "-"); if (nextCommands.length > 0) { for (const command of nextCommands.slice(0, 3)) lines.push(` ${command}`); } else { lines.push(` ${buildTraceCommand(input.target, input.options, true)}`); } lines.push("", "Disclosure:"); lines.push(" trace output is a bounded table; --full expands row counts and diagnostic tables, --raw returns backend JSON."); return { ok: input.ok, command: "platform-infra observability trace", contentType: "text/plain", renderedText: lines.join("\n"), }; } function traceRunnerRows(spans: Record[]): string[][] { const rows = spans.filter((span) => { const name = textValue(span.name); const attrs = asPlainRecord(span.attributes); return name.startsWith("runner_terminal") || name.startsWith("runner_error") || name === "command_result" || (name.startsWith("codex_stdio") && spanColumnAttr(attrs, ["failureKind", "terminalStatus"]) !== "-"); }).map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenEnd(textValue(span.name), 34), shortenEnd(textValue(span.service), 20), shortenEnd(spanColumnAttr(attrs, ["terminalStatus"]), 12), shortenEnd(spanColumnAttr(attrs, ["failureKind"]), 24), shortenEnd(spanColumnAttr(attrs, ["willRetry"]), 8), shortenEnd(spanColumnAttr(attrs, ["http.route"]), 34), shortenEnd(spanColumnAttr(attrs, ["http.status_code", "http.response.status_code"]), 8), shortenEnd(spanColumnAttr(attrs, ["errorSummary", "message", "error.message", "exception.message", "statusMessage"]), 80), ]; }); return dedupeRows(rows).slice(0, 12); } function traceProviderDecisionRows(spans: Record[]): string[][] { const rows = spans.filter((span) => { const attrs = asPlainRecord(span.attributes); return textValue(span.name) === "provider_decision" || spanColumnAttr(attrs, ["agent.chat.provider_profile", "providerProfile", "defaultProviderProfile"]) !== "-"; }).map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenEnd(textValue(span.service), 20), shortenEnd(spanColumnAttr(attrs, ["agent.chat.provider_profile", "providerProfile", "backendProfile", "defaultProviderProfile"]), 22), shortenEnd(spanColumnAttr(attrs, ["agent.chat.provider_profile_source", "providerProfileSource"]), 12), shortenEnd(spanColumnAttr(attrs, ["defaultProviderProfile"]), 22), shortenEnd(spanColumnAttr(attrs, ["agent.chat.model", "agent.chat.model_id", "agent.chat.provider_model", "model", "modelId", "providerModel"]), 28), shortenEnd(spanColumnAttr(attrs, ["adapter"]), 16), shortenEnd(spanColumnAttr(attrs, ["agentRunRunnerNamespace"]), 16), shortenMiddle(spanColumnAttr(attrs, ["agent.chat.session_id", "sessionId", "workbench.session_id"]), 24), ]; }); return dedupeRows(rows).slice(0, 8); } function traceReadWindowRows(spans: Record[]): string[][] { const rows = spans.filter((span) => textValue(span.name) === "trace_events_read").map((span) => { const attrs = asPlainRecord(span.attributes); const fromSeq = spanColumnAttr(attrs, ["fromSeq"]); const toSeq = spanColumnAttr(attrs, ["toSeq"]); return [ shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18), shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18), shortenEnd(spanColumnAttr(attrs, ["sinceSeq", "afterSeq"]), 8), shortenEnd(spanColumnAttr(attrs, ["limit"]), 8), shortenEnd(spanColumnAttr(attrs, ["returnedEvents", "rawEventCount"]), 8), fromSeq === "-" && toSeq === "-" ? "-" : `${fromSeq}..${toSeq}`, shortenEnd(spanColumnAttr(attrs, ["totalEvents"]), 8), shortenEnd(spanColumnAttr(attrs, ["hasMore"]), 6), shortenEnd(spanColumnAttr(attrs, ["fullTraceLoaded"]), 6), shortenEnd(spanColumnAttr(attrs, ["traceLastSeq", "maxSeq", "endSeq"]), 8), ]; }); return dedupeRows(rows).slice(0, 12); } function traceSessionListRows(spans: Record[]): string[][] { const rows = spans.filter((span) => textValue(span.name) === "session_list_read").map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenEnd(spanColumnAttr(attrs, ["source"]), 10), shortenEnd(spanColumnAttr(attrs, ["sessionCount"]), 8), shortenEnd(spanColumnAttr(attrs, ["fallbackTitleCount"]), 8), shortenEnd(spanColumnAttr(attrs, ["fallbackTitleRatio"]), 8), shortenEnd(spanColumnAttr(attrs, ["emptyPreviewCount"]), 8), shortenMiddle(spanColumnAttr(attrs, ["includeSessionId"]), 18), shortenMiddle(spanColumnAttr(attrs, ["sessionIds"]), 34), shortenMiddle(spanColumnAttr(attrs, ["traceIds"]), 34), ]; }); return dedupeRows(rows).slice(0, 12); } function traceSessionMessageRows(spans: Record[]): string[][] { const rows = spans.filter((span) => textValue(span.name) === "session_messages_read").map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18), shortenEnd(spanColumnAttr(attrs, ["returnedMessages"]), 8), shortenEnd(spanColumnAttr(attrs, ["totalMessages"]), 8), shortenEnd(spanColumnAttr(attrs, ["roleSequencePrefix"]), 32), shortenEnd(spanColumnAttr(attrs, ["consecutiveUserPrefix"]), 8), shortenEnd(spanColumnAttr(attrs, ["adjacentSameRoleCount"]), 8), shortenEnd(spanColumnAttr(attrs, ["userCount"]), 8), shortenEnd(spanColumnAttr(attrs, ["agentCount"]), 8), shortenMiddle(spanColumnAttr(attrs, ["traceIds"]), 34), ]; }); return dedupeRows(rows).slice(0, 12); } function traceTurnStatusRows(spans: Record[]): string[][] { const rows = spans.filter((span) => textValue(span.name) === "turn_status_read").map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18), shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18), shortenMiddle(spanColumnAttr(attrs, ["turnId"]), 18), shortenEnd(spanColumnAttr(attrs, ["status", "terminalStatus"]), 14), shortenEnd(spanColumnAttr(attrs, ["phase"]), 14), shortenEnd(spanColumnAttr(attrs, ["http.route"]), 34), shortenEnd(spanColumnAttr(attrs, ["http.status_code", "http.response.status_code"]), 8), ]; }); return dedupeRows(rows).slice(0, 12); } function traceProjectionRows(spans: Record[]): string[][] { const rows = spans.filter((span) => textValue(span.name).includes("projection")).map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenEnd(textValue(span.name), 34), shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18), shortenMiddle(spanColumnAttr(attrs, ["sessionId", "workbench.projection.session_id"]), 18), shortenMiddle(spanColumnAttr(attrs, ["turnId", "workbench.projection.turn_id"]), 18), shortenEnd(spanColumnAttr(attrs, ["status", "terminalStatus", "phase", "workbench.projection.phase"]), 14), shortenEnd(spanColumnAttr(attrs, ["eventType", "type", "workbench.projection.event_type"]), 18), shortenEnd(spanColumnAttr(attrs, ["projectedSeq", "workbench.projection.projected_seq"]), 10), shortenEnd(spanColumnAttr(attrs, ["sourceSeq", "workbench.projection.source_seq", "sourceEventId", "workbench.projection.source_event_id"]), 18), shortenMiddle(spanColumnAttr(attrs, ["messageId", "workbench.projection.message_id", "failureKind", "errorSummary"]), 24), ]; }); return dedupeRows(rows).slice(0, 12); } function traceProjectionSyncRows(spans: Record[]): string[][] { const rows = spans.filter((span) => { const name = textValue(span.name); const attrs = asPlainRecord(span.attributes); return name === "projection_sync" || name === "workbench.sync.replay" || spanColumnAttr(attrs, ["workbench.sync.contract_version", "workbench.sync.cursor_outbox_seq", "workbench.sync.event_count", "workbench.realtime.authority"]) !== "-"; }).map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenEnd(textValue(span.service), 18), shortenMiddle(spanColumnAttr(attrs, ["traceId", "workbench.trace_id"]), 18), shortenMiddle(spanColumnAttr(attrs, ["sessionId", "workbench.session_id"]), 18), shortenEnd(spanColumnAttr(attrs, ["afterSeq", "workbench.sync.since_outbox_seq"]), 8), shortenEnd(spanColumnAttr(attrs, ["endSeq", "workbench.sync.cursor_outbox_seq"]), 8), shortenEnd(spanColumnAttr(attrs, ["eventCount", "workbench.sync.event_count"]), 8), shortenEnd(spanColumnAttr(attrs, ["rawEventCount"]), 8), shortenEnd(spanColumnAttr(attrs, ["maxSeq", "workbench.sync.max_entity_version"]), 8), shortenEnd(spanColumnAttr(attrs, ["traceLastSeq"]), 8), shortenEnd(spanColumnAttr(attrs, ["terminalFromEvents", "terminalFromRawEvents", "terminalStatus", "workbench.terminal.seal"]), 12), shortenEnd(spanColumnAttr(attrs, ["workbench.realtime.authority", "workbench.sync.contract_version", "workbench.detail.projection"]), 18), ]; }); return dedupeRows(rows).slice(0, 16); } function dedupeRows(rows: string[][]): string[][] { const output: string[][] = []; const seen = new Set(); for (const row of rows) { const key = JSON.stringify(row); if (seen.has(key)) continue; seen.add(key); output.push(row); } return output; } export function renderSearchTable(input: { ok: boolean; target: ObservabilityTarget; options: SearchOptions; query: Record; result: Record; }): RenderedCliResult { const traces = asArray(input.result.traces).map((item) => asPlainRecord(item) ?? {}); const requestedLimit = numberFromUnknown(input.options.limit); const rowLimit = Math.max(1, Math.min(Number.isFinite(requestedLimit) ? Math.round(requestedLimit) : 12, 40)); const rows = traces.slice(0, rowLimit).map((trace) => { const meta = asPlainRecord(trace.meta) ?? {}; return [ shortenMiddle(textValue(trace.traceId), 32), shortenEnd(textValue(trace.startAt ?? isoFromUnixNano(meta.startTimeUnixNano)), 20), numberValue(trace.durationMs ?? meta.durationMs), searchTraceStatus(trace), numberValue(trace.spanCount), numberValue(trace.errorSpanCount), numberValue(trace.matchedSpanCount), joinValues(trace.services, 38), ]; }); const lines = [ `platform-infra observability search (${input.ok ? "ok" : "not-ok"})`, "", formatTable(["TRACE", "START", "DUR_MS", "STATUS", "SPANS", "ERR", "MATCH", "SERVICES"], rows.length > 0 ? rows : [["-", "-", "-", "no-match", "-", "-", "-", "-"]]), "", "Summary:", ` target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)}`, ` grep=${textValue(input.query.grep)} path=${textValue(input.query.pathFilter)} status=${textValue(input.query.statusFilter)} tempoQuery=${shortenMiddle(textValue(input.query.tempoQuery), 80)}`, ` mode=${textValue(input.query.mode)} businessTraceId=${textValue(input.query.businessTraceId)} lookbackMinutes=${textValue(input.query.lookbackMinutes)} requestedLookbackMinutes=${textValue(input.query.requestedLookbackMinutes)} endAt=${textValue(input.query.endAt)}`, ` candidateLimit=${textValue(input.query.candidateLimit)} limit=${textValue(input.query.limit)}`, ` candidates=${textValue(input.result.candidateTraceCount)} scanned=${textValue(input.result.scannedTraceCount)} matched=${textValue(input.result.matchedTraceCount)} stopped=${textValue(input.result.scanStopped)}`, ]; if (input.query.grep !== null) { lines.push(` grepCoverage=${textValue(input.query.grepCoverage)}`); if (input.query.grepQueryInference !== null) lines.push(` grepQueryInference=${textValue(input.query.grepQueryInference)}`); } const firstTraceId = traces.length > 0 ? textValue(traces[0].traceId) : ""; lines.push("", "Next:"); if (firstTraceId.length > 0 && firstTraceId !== "-") { lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${firstTraceId}`); } lines.push(` ${buildSearchCommand(input.target, input.options, true)}`); if (input.query.grep !== null && Number(input.result.matchedTraceCount ?? 0) === 0) { lines.push(` explicit TraceQL: bun scripts/cli.ts platform-infra observability search --target ${input.target.id} --query '{ resource.service.name = "" }' --lookback-minutes ${textValue(input.query.lookbackMinutes)} --candidate-limit ${textValue(input.query.candidateLimit)} --limit ${textValue(input.query.limit)}`); lines.push(` widen candidates: bun scripts/cli.ts platform-infra observability search --target ${input.target.id} --grep ${JSON.stringify(String(input.query.grep))} --lookback-minutes ${textValue(input.query.lookbackMinutes)} --candidate-limit 300 --limit ${textValue(input.query.limit)}`); } lines.push("", "Disclosure:"); lines.push(" default view is a bounded table; use --full for structured diagnosis JSON or trace --trace-id for one trace."); return { ok: input.ok, command: "platform-infra observability search", contentType: "text/plain", renderedText: lines.join("\n"), }; } export function renderDiagnoseCodeAgentTable(input: { ok: boolean; target: ObservabilityTarget; options: DiagnoseCodeAgentOptions; query: Record; result: Record; }): RenderedCliResult { const mapping = asPlainRecord(input.result.mapping); const diagnosticTrace = asPlainRecord(input.result.diagnosticTrace); const identity = asPlainRecord(input.result.identity); const agentrun = asPlainRecord(input.result.agentrun); const agentrunAuthority = asPlainRecord(agentrun?.authority); const projectionLag = asPlainRecord(input.result.projectionLag); const summary = asPlainRecord(input.result.summary); const evidence = asPlainRecord(input.result.evidence); const observabilityGap = asPlainRecord(input.result.observabilityGap); const servicePath = asPlainRecord(input.result.servicePath); const providerDecision = asPlainRecord(input.result.providerDecision); const rootCauses = asArray(input.result.rootCauseCandidates).map((item) => asPlainRecord(item) ?? {}); const http = asPlainRecord(input.result.http); const services = joinValues(input.result.services, 54); const traceId = textValue(mapping?.otelTraceId ?? input.query.traceId); const rootCause = textValue(summary?.rootCause ?? rootCauses[0]?.code); const rows = [[ shortenMiddle(traceId, 20), shortenEnd(rootCause, 34), textValue(agentrun?.terminalStatus), textValue(agentrun?.runnerProviderClassification), textValue(projectionLag?.status), textValue(evidence?.errorSpanCount), services, ]]; const rootRows = rootCauses.slice(0, 5).map((candidate) => [ shortenEnd(textValue(candidate.code), 40), textValue(candidate.confidence), shortenEnd(textValue(candidate.summary ?? candidate.label), 80), ]); const httpRows = httpTableRows(http); const expectedServices = asArray(observabilityGap?.expectedServices).map((item) => textValue(item)).filter((item) => item !== "-"); const seenServices = new Set(asArray(observabilityGap?.seenServices).map((item) => textValue(item)).filter((item) => item !== "-")); const missingServices = new Set(asArray(observabilityGap?.missingServices).map((item) => textValue(item)).filter((item) => item !== "-")); const serviceRows = expectedServices.map((service) => [ service, textValue(servicePath?.[service] ?? (missingServices.has(service) ? "missing" : seenServices.has(service) ? "reached" : "-")), missingServices.has(service) ? "missing" : seenServices.has(service) ? "seen" : "-", ]); const queryClauses = asArray(input.query.queryClauses).map((item) => textValue(item)).filter((item) => item !== "-"); const requestedRunId = textValue(input.query.runId); const requestedCommandId = textValue(input.query.commandId); const requestedSessionId = textValue(input.query.sessionId); const requestedRunnerJobId = textValue(input.query.runnerJobId); const observedRunId = textValue(identity?.runId); const observedCommandId = textValue(identity?.commandId); const observedSessionId = textValue(identity?.sessionId); const observedRunnerJobId = textValue(identity?.runnerJobId); const outcome = asPlainRecord(input.result.outcome); const outcomeAuthority = asPlainRecord(outcome?.authority); const outcomeProjection = asPlainRecord(outcome?.projection); const lines = [ `platform-infra observability diagnose-code-agent (${input.ok ? "ok" : "not-ok"})`, "", formatTable(["TRACE", "ROOT_CAUSE", "TERMINAL", "RUNNER", "PROJECTION", "ERR", "SERVICES"], rows), "", "Identity:", ` businessTraceId=${textValue(mapping?.businessTraceId ?? input.query.businessTraceId)} otelTraceId=${traceId}`, ` queryMode=${textValue(mapping?.mode ?? input.query.mode)} tempoQuery=${shortenEnd(textValue(mapping?.searchQuery ?? input.query.tempoQuery), 180)}`, ` queryClauses=${queryClauses.length > 0 ? queryClauses.join(" ; ") : "-"}`, ` requested runId=${requestedRunId} commandId=${requestedCommandId} sessionId=${requestedSessionId} runnerJobId=${requestedRunnerJobId}`, ` observed runId=${observedRunId} commandId=${observedCommandId} sessionId=${observedSessionId} runnerJobId=${observedRunnerJobId} runnerId=${textValue(identity?.runnerId)}`, ` backendProfile=${textValue(identity?.backendProfile)} sourceCommit=${shortenMiddle(textValue(identity?.sourceCommit), 20)}`, ` providerProfile=${textValue(providerDecision?.providerProfile)} model=${textValue(providerDecision?.model)} source=${textValue(providerDecision?.providerProfileSource)} adapter=${textValue(providerDecision?.adapter)} runnerNamespace=${textValue(providerDecision?.runnerNamespace)}`, "", "Diagnostic trace:", ` traceId=${textValue(diagnosticTrace?.traceId)} parentSpanId=${textValue(diagnosticTrace?.parentSpanId)} operation=${textValue(diagnosticTrace?.operation)} contextReady=${textValue(diagnosticTrace?.contextReady)}`, ` target businessTraceId=${textValue(diagnosticTrace?.targetBusinessTraceId)} commandId=${textValue(diagnosticTrace?.targetCommandId)} expectedLinkTraceId=${textValue(diagnosticTrace?.expectedTargetOtelTraceId)}`, ` diagnosticReads=${joinValues(diagnosticTrace?.headerReadPaths, 60)} ordinaryReads=${joinValues(diagnosticTrace?.ordinaryReadPaths, 40)}`, ` link=${shortenEnd(textValue(diagnosticTrace?.linkQueryClue), 180)}`, "", "Root causes:", formatTable(["CODE", "CONF", "SUMMARY"], rootRows.length > 0 ? rootRows : [["-", "-", "-"]]), "", "Service trace coverage:", formatTable(["SERVICE", "PATH", "SPAN"], serviceRows.length > 0 ? serviceRows : [["-", "-", "-"]]), ` observabilityGap=${textValue(observabilityGap?.status)} missing=${joinValues(observabilityGap?.missingServices, 60)} seen=${joinValues(observabilityGap?.seenServices, 80)}`, "", "HTTP:", formatTable(["METHOD", "ROUTE", "STATUS", "COUNT"], httpRows.length > 0 ? httpRows : [["-", "-", "-", "-"]]), "", "Summary:", ` outcome=${textValue(outcome?.typed)} effectiveStatus=${textValue(outcome?.effectiveStatus)} mutation=${textValue(outcome?.mutation)}`, ` durableAuthority kind=${textValue(outcomeAuthority?.kind)} status=${textValue(outcomeAuthority?.status)} source=${textValue(outcomeAuthority?.source)}`, ` projection kind=${textValue(outcomeProjection?.kind)} status=${textValue(outcomeProjection?.status)} lag=${textValue(outcomeProjection?.lag)} warning=${textValue(outcomeProjection?.warning)} blocking=${textValue(outcomeProjection?.blocking)}`, ` target=${input.target.id} spanCount=${textValue(input.result.spanCount)} servicePath=${joinValues(input.result.servicePath, 60)}`, ` agentrun=${textValue(agentrun?.terminalStatus)} failureKind=${textValue(agentrun?.failureKind)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`, ` authorityQuery namespace=${textValue(agentrunAuthority?.namespace)} source=${textValue(agentrunAuthority?.namespaceSource)} ok=${textValue(agentrunAuthority?.ok)} commandState=${textValue(agentrunAuthority?.commandState)} terminal=${textValue(agentrunAuthority?.terminalStatus)}`, ` readModel=${shortenEnd(JSON.stringify(input.result.hwlabReadModel ?? null), 140)}`, ]; const next = asPlainRecord(input.result.next); lines.push("", "Next:"); const nextCommands = [ textValue(diagnosticTrace?.traceCommand), textValue(diagnosticTrace?.targetTraceCommand), textValue(next?.diagnoseFull), textValue(next?.traceSummary), textValue(next?.traceReads), textValue(next?.sessionReads), textValue(next?.turnStatus), textValue(next?.projection), ].filter((item) => item !== "-"); if (nextCommands.length > 0) { for (const command of Array.from(new Set(nextCommands)).slice(0, 6)) lines.push(` ${command}`); } else { lines.push(` ${buildDiagnoseCommand(input.target, input.options, true)}`); if (traceId.length > 0 && traceId !== "-") lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${traceId}`); } lines.push("", "Disclosure:"); lines.push(" default view is a bounded table; use --full for structured diagnosis JSON or trace --grep for span-level drill-down."); return { ok: input.ok, command: "platform-infra observability diagnose-code-agent", contentType: "text/plain", renderedText: lines.join("\n"), }; }