Files
pikasTech-unidesk/scripts/src/platform-infra-observability/render.ts
T
2026-07-12 17:08:00 +02:00

833 lines
43 KiB
TypeScript

// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/platform-infra-observability.ts.
// Moved mechanically from scripts/src/platform-infra-observability.ts:624-982 for #903.
// SPEC: PJ2026-01060501 OTel追踪 draft-2026-06-19-p0.
// Responsibility: YAML-first platform-infra OpenTelemetry tracing control commands.
import { Buffer } from "node:buffer";
import { readFileSync } from "node:fs";
import type { UniDeskConfig } from "../config";
import { rootPath } from "../config";
import type { RenderedCliResult } from "../output";
import {
compactCapture,
createYamlFieldReader,
numberField,
parseJsonOutput,
redactSensitiveUnknown,
shQuote,
capture,
} from "../platform-infra-ops-library";
import type { DiagnoseCodeAgentOptions, ObservabilityTarget, SearchOptions, TraceOptions } from "./types";
import { resolveTarget } from "./actions";
import { asPlainRecord, compactDiagnoseCodeAgentResult } from "./apply-status-scripts";
import { readObservabilityConfig } from "./config";
import { diagnoseCodeAgentScript, searchScript, traceScript } from "./diagnose-code-agent-script";
import { buildDiagnoseCommand, buildSearchCommand, buildTraceCommand, compactSearchResult, formatTable, httpTableRows, isoFromUnixNano, joinValues, numberFromUnknown, numberValue, searchTraceStatus, shortenEnd, shortenMiddle, spanColumnAttr, spanDetail, textValue, traceSpanDurationMs, traceSpanImportanceScore, uniqueSpanRecords } from "./manifest";
import { targetSummary } from "./summary";
import { asArray } from "./trace-script";
export async function trace(config: UniDeskConfig, options: TraceOptions): Promise<Record<string, unknown> | RenderedCliResult> {
if (options.traceId === null) throw new Error("observability trace requires --trace-id <traceId>");
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
const tracePath = observability.probes.traceQueryPathTemplate.replaceAll("{{traceId}}", encodeURIComponent(options.traceId));
const result = await capture(config, target.route, ["sh"], traceScript(observability, target, tracePath, options));
const parsed = parseJsonOutput(result.stdout);
const ok = result.exitCode === 0 && parsed?.ok === true;
if (!options.raw) {
return renderTraceTable({
ok,
target,
options,
tracePath,
query: {
backend: observability.traceBackend.type,
service: observability.traceBackend.serviceName,
path: tracePath,
},
result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : asPlainRecord(redactSensitiveUnknown(parsed)) ?? {},
});
}
return {
ok,
action: "platform-infra-observability-trace",
mutation: false,
target: targetSummary(target),
traceId: options.traceId,
query: {
backend: observability.traceBackend.type,
service: observability.traceBackend.serviceName,
path: tracePath,
},
result: redactSensitiveUnknown(parsed),
};
}
export async function search(config: UniDeskConfig, options: SearchOptions): Promise<Record<string, unknown> | RenderedCliResult> {
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null;
const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080);
const effectiveTempoQuery = options.query ?? (businessTraceId === null ? inferSearchTempoQuery(options) : `{ .traceId = "${businessTraceId}" }`);
const effectiveOptions: SearchOptions = {
...options,
query: effectiveTempoQuery,
lookbackMinutes: effectiveLookbackMinutes,
};
const endMillis = options.endAt === null ? Date.now() : Date.parse(options.endAt);
if (!Number.isFinite(endMillis)) throw new Error("--end-at must be an ISO timestamp parseable by Date.parse");
const endSeconds = Math.floor(endMillis / 1000);
const startSeconds = endSeconds - (effectiveLookbackMinutes * 60);
const params = new URLSearchParams({
start: String(startSeconds),
end: String(endSeconds),
limit: String(options.candidateLimit),
});
if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery);
const searchPath = `/api/search?${params.toString()}`;
const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions));
const parsed = parseJsonOutput(result.stdout);
const ok = result.exitCode === 0 && parsed?.ok === true;
const query = {
backend: observability.traceBackend.type,
service: observability.traceBackend.serviceName,
path: searchPath,
grep: options.grep,
tempoQuery: effectiveTempoQuery,
explicitTempoQuery: options.query,
pathFilter: options.path,
statusFilter: options.status,
lookbackMinutes: effectiveLookbackMinutes,
requestedLookbackMinutes: options.lookbackMinutes,
endAt: new Date(endSeconds * 1000).toISOString(),
businessTraceId,
mode: businessTraceId === null ? "candidate-grep" : "business-trace-exact",
grepQueryInference: options.query === null && businessTraceId === null && effectiveTempoQuery !== null ? "inferred-from-grep-or-filters" : null,
grepCoverage: options.grep === null ? null : "candidate traces are fetched by tempoQuery, then each scanned trace is matched against raw trace body, span name, status message, route and full span attributes",
candidateLimit: options.candidateLimit,
limit: options.limit,
};
if (!options.full && !options.raw) {
return renderSearchTable({
ok,
target,
options,
query,
result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : compactSearchResult(parsed),
});
}
const exposedResult = parsed === null
? compactCapture(result, { full: true })
: options.raw
? redactSensitiveUnknown(parsed)
: {
...compactSearchFullResult(parsed),
outputMode: "bounded-compact-json",
rawDisclosure: "Use --raw only for Tempo/API envelope or parser debugging.",
};
return {
ok,
action: "platform-infra-observability-search",
mutation: false,
target: targetSummary(target),
query,
result: exposedResult,
};
}
function compactSearchFullResult(value: unknown): Record<string, unknown> {
const source = asPlainRecord(value) ?? {};
return {
ok: source.ok === true,
grep: source.grep ?? null,
tempoQuery: source.tempoQuery ?? null,
pathFilter: source.pathFilter ?? null,
statusFilter: source.statusFilter ?? null,
matchingActive: source.matchingActive ?? null,
grepCoverage: source.grepCoverage ?? null,
grepQueryInference: source.grepQueryInference ?? null,
limit: source.limit ?? null,
candidateLimit: source.candidateLimit ?? null,
searchParseOk: source.searchParseOk ?? null,
candidateTraceCount: source.candidateTraceCount ?? null,
scannedTraceCount: source.scannedTraceCount ?? null,
matchedTraceCount: source.matchedTraceCount ?? null,
scanStopped: source.scanStopped ?? null,
traces: asArray(source.traces).slice(0, 5).map(compactSearchFullTrace),
omittedTraceCount: Math.max(0, asArray(source.traces).length - 5),
truncated: source.truncated ?? null,
next: source.next ?? null,
searchStderrTail: source.searchStderrTail ?? null,
disclosure: {
defaultView: "bounded search full summary; span arrays and raw Tempo bodies are omitted",
expand: "query one trace with platform-infra observability trace --trace-id <traceId> --grep <text> --full; use --raw for parser/debug only",
},
};
}
function compactSearchFullTrace(value: unknown): Record<string, unknown> {
const trace = asPlainRecord(value) ?? {};
const meta = asPlainRecord(trace.meta);
return {
traceId: trace.traceId ?? null,
startAt: trace.startAt ?? isoFromUnixNano(meta?.startTimeUnixNano),
durationMs: trace.durationMs ?? meta?.durationMs ?? null,
queryOk: trace.queryOk ?? null,
parseOk: trace.parseOk ?? null,
rawMatched: trace.rawMatched ?? null,
bodyBytes: trace.bodyBytes ?? null,
spanCount: trace.spanCount ?? null,
serviceCount: trace.serviceCount ?? null,
services: asArray(trace.services).slice(0, 4),
businessTraceIds: asArray(trace.businessTraceIds).slice(0, 4),
errorSpanCount: trace.errorSpanCount ?? null,
matchedSpanCount: trace.matchedSpanCount ?? null,
spanNameCounts: asArray(trace.spanNameCounts).slice(0, 4).map(compactNameCount),
matchedSpanNames: [...new Set(asArray(trace.matchedSpans).map((span) => textValue(asPlainRecord(span)?.name)).filter((name) => name !== "-"))].slice(0, 4),
};
}
function compactNameCount(value: unknown): Record<string, unknown> {
const item = asPlainRecord(value) ?? {};
return {
name: item.name ?? null,
count: item.count ?? null,
};
}
export function inferSearchTempoQuery(options: SearchOptions): string | null {
const filters: string[] = [];
if (options.path !== null) filters.push(`.http.route = ${JSON.stringify(options.path)}`);
if (options.status !== null) filters.push(`.http.response.status_code = ${options.status}`);
if (filters.length > 0) return `{ ${filters.join(" && ")} }`;
const grep = options.grep?.trim() ?? "";
if (!grep) return null;
const keyValue = grep.match(/^([A-Za-z0-9_.-]+)=(.+)$/u);
if (keyValue !== null) {
const key = keyValue[1] ?? "";
const value = (keyValue[2] ?? "").trim();
const traceQlKey = key === "name" || key.startsWith("resource.") ? key : `.${key.replace(/^span[.]/u, "")}`;
if (/^(?:name|resource[.][A-Za-z0-9_.-]+|[.][A-Za-z0-9_.-]+)$/u.test(traceQlKey) && value.length > 0 && value.length <= 200) return `{ ${traceQlKey} = ${traceQlLiteral(value)} }`;
}
if (grep.startsWith("/") && !grep.includes("\n") && grep.length <= 300) return `{ .http.route = ${JSON.stringify(grep)} }`;
if (/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,180}$/u.test(grep)) {
if (isKnownTraceAttributeKey(grep)) return `{ .${grep} != nil }`;
if (grep.includes(".") || grep.includes(":")) return `{ name = ${JSON.stringify(grep)} }`;
if (/^(?:hwlab|agentrun|opencode|platform|code-queue|unidesk)[A-Za-z0-9-]*$/u.test(grep) || /-(?:api|web|proxy|runner|manager|service|mgr|collector|tempo)$/u.test(grep)) return `{ resource.service.name = ${JSON.stringify(grep)} }`;
}
return null;
}
const knownTraceAttributeKeys = new Set([
"opencode.proxy.phase",
"opencode.proxy.streaming",
"opencode.proxy.ticket_accepted",
"opencode.proxy.sse.directory_rewrite_enabled",
"opencode.proxy.sse.directory_rewrite_from",
"opencode.proxy.sse.directory_rewrite_to",
"opencode.provider.sse.content_chunks",
"opencode.provider.sse.content_chars",
"opencode.provider.sse.output_data_lines",
"opencode.provider.sse.done_lines",
"opencode.provider.sse.json_errors",
"opencode.provider.sse.reasoning_only_choices_dropped",
"workbench.sync.contract_version",
"workbench.sync.since_outbox_seq",
"workbench.sync.cursor_outbox_seq",
"workbench.sync.event_count",
"workbench.sync.entity_family_count",
"workbench.sync.entity_families",
"workbench.sync.max_entity_version",
"workbench.realtime.authority",
"workbench.projection.revision",
"workbench.terminal.seal",
"workbench.detail.projection",
]);
function isKnownTraceAttributeKey(value: string): boolean {
return knownTraceAttributeKeys.has(value) || value.startsWith("opencode.proxy.sse.") || value.startsWith("opencode.provider.sse.");
}
function traceQlLiteral(value: string): string {
const unquoted = stripOuterQuotes(value.trim());
if (/^(?:true|false)$/iu.test(unquoted)) return unquoted.toLowerCase();
if (/^-?(?:0|[1-9][0-9]*)(?:[.][0-9]+)?$/u.test(unquoted)) return unquoted;
if (unquoted === "nil") return "nil";
return JSON.stringify(unquoted);
}
function stripOuterQuotes(value: string): string {
if (value.length >= 2 && ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'")))) {
return value.slice(1, -1);
}
return value;
}
export function businessTraceIdFromSearchText(value: string | null): string | null {
const match = value?.match(/\btrc_[A-Za-z0-9_-]+\b/u);
return match?.[0] ?? null;
}
function diagnoseTraceSearchFilters(options: DiagnoseCodeAgentOptions): string[] {
const filters: string[] = [];
if (options.businessTraceId !== null) filters.push(`.traceId = ${JSON.stringify(options.businessTraceId)}`);
if (options.runId !== null) filters.push(`.runId = ${JSON.stringify(options.runId)}`);
if (options.commandId !== null) filters.push(`.commandId = ${JSON.stringify(options.commandId)}`);
if (options.sessionId !== null) filters.push(`.sessionId = ${JSON.stringify(options.sessionId)}`);
if (options.runnerJobId !== null) filters.push(`.runnerJobId = ${JSON.stringify(options.runnerJobId)}`);
return filters;
}
function diagnoseTraceSearchMode(options: DiagnoseCodeAgentOptions): string {
if (options.businessTraceId !== null && options.runId === null && options.commandId === null && options.sessionId === null && options.runnerJobId === null) return "business-trace-id";
return "trace-attribute-query";
}
export async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise<Record<string, unknown> | RenderedCliResult> {
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
const endSeconds = Math.floor(Date.now() / 1000);
const startSeconds = endSeconds - (options.lookbackMinutes * 60);
let searchPath: string | null = null;
const searchFilters = diagnoseTraceSearchFilters(options);
const tempoQuery = searchFilters.length > 0 ? `{ ${searchFilters.join(" && ")} }` : null;
if (options.traceId === null && tempoQuery !== null) {
const params = new URLSearchParams({
start: String(startSeconds),
end: String(endSeconds),
limit: String(options.candidateLimit),
q: tempoQuery,
});
searchPath = `/api/search?${params.toString()}`;
}
const result = await capture(config, target.route, ["sh"], diagnoseCodeAgentScript(observability, target, searchPath, options));
const parsed = parseJsonOutput(result.stdout);
const ok = result.exitCode === 0 && parsed?.ok === true;
const query = {
backend: observability.traceBackend.type,
service: observability.traceBackend.serviceName,
businessTraceId: options.businessTraceId,
traceId: options.traceId,
runId: options.runId,
commandId: options.commandId,
sessionId: options.sessionId,
runnerJobId: options.runnerJobId,
mode: diagnoseTraceSearchMode(options),
tempoQuery,
queryClauses: searchFilters,
path: searchPath,
lookbackMinutes: options.lookbackMinutes,
candidateLimit: options.candidateLimit,
limit: options.limit,
};
if (!options.full && !options.raw) {
return renderDiagnoseCodeAgentTable({
ok,
target,
options,
query,
result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : compactDiagnoseCodeAgentResult(parsed),
});
}
const exposedResult = parsed === null
? compactCapture(result, { full: true })
: options.raw
? redactSensitiveUnknown(parsed)
: {
...compactDiagnoseCodeAgentResult(parsed),
outputMode: "bounded-compact-json",
rawDisclosure: "Use --raw only for Tempo/API envelope or CLI parser debugging.",
};
return {
ok,
action: "platform-infra-observability-diagnose-code-agent",
mutation: false,
target: targetSummary(target),
query,
result: exposedResult,
};
}
export function renderTraceTable(input: {
ok: boolean;
target: ObservabilityTarget;
options: TraceOptions;
tracePath: string;
query: Record<string, unknown>;
result: Record<string, unknown>;
}): RenderedCliResult {
const spanSource = input.result.grep === null || input.result.grep === undefined
? [...asArray(input.result.errorSpans), ...asArray(input.result.spans)]
: asArray(input.result.spans);
const rowLimit = input.options.full
? Math.max(10, Math.min(input.options.limit, 24))
: 10;
const allSpans = [...asArray(input.result.errorSpans), ...asArray(input.result.spans)]
.map((span) => asPlainRecord(span) ?? {});
const spanRows = uniqueSpanRecords(spanSource)
.sort((left, right) => traceSpanImportanceScore(right) - traceSpanImportanceScore(left))
.slice(0, rowLimit)
.map((span) => {
const attrs = asPlainRecord(span.attributes);
const durationMs = traceSpanDurationMs(span, attrs);
return [
shortenEnd(textValue(span.name), 40),
shortenEnd(textValue(span.service), 22),
shortenEnd(spanColumnAttr(attrs, ["http.route", "workbench.ui.route"]), 34),
spanColumnAttr(attrs, ["http.response.status_code", "http.status_code", "http.response.status_class"]),
numberValue(durationMs),
spanColumnAttr(attrs, ["workbench.ui.resource_duration_ms"]),
spanColumnAttr(attrs, ["workbench.ui.resource_fetch_to_request_ms"]),
spanColumnAttr(attrs, ["workbench.ui.resource_request_wait_ms"]),
spanColumnAttr(attrs, ["workbench.ui.resource_response_transfer_ms"]),
shortenEnd(spanColumnAttr(attrs, ["workbench.ui.resource_next_hop_protocol"]), 14),
spanDetail(span, input.options.full ? 150 : 220),
];
});
const runnerRows = traceRunnerRows(allSpans);
const readRows = traceReadWindowRows(allSpans);
const sessionListRows = traceSessionListRows(allSpans);
const sessionMessageRows = traceSessionMessageRows(allSpans);
const turnRows = traceTurnStatusRows(allSpans);
const projectionRows = traceProjectionRows(allSpans);
const projectionSyncRows = traceProjectionSyncRows(allSpans);
const providerRows = traceProviderDecisionRows(allSpans);
const countRows = asArray(input.result.spanNameCounts).slice(0, 8).map((item) => {
const row = asPlainRecord(item) ?? {};
return [shortenEnd(textValue(row.name), 64), textValue(row.count)];
});
const identity = asPlainRecord(input.result.identity) ?? {};
const identityRows = ["runId", "commandId", "runnerJobId", "runnerId", "backendProfile", "sourceCommit", "jobName", "podName"]
.map((key) => [key, joinValues(identity[key], 88)])
.filter((row) => row[1] !== "-");
const next = asPlainRecord(input.result.next);
const lines = [
`platform-infra observability trace (${input.ok ? "ok" : "not-ok"})`,
"",
formatTable(["TRACE", "SPANS", "ERR", "MATCH", "SERVICES", "BUSINESS_TRACE"], [[
shortenMiddle(input.options.traceId ?? "-", 20),
textValue(input.result.spanCount),
textValue(input.result.errorSpanCount),
textValue(input.result.matchedSpanCount),
joinValues(input.result.services, 44),
joinValues(input.result.businessTraceIds, 34),
]]),
"",
"Identity:",
formatTable(["FIELD", "VALUE"], identityRows.length > 0 ? identityRows : [["-", "-"]]),
"",
"Provider decision:",
formatTable(["SERVICE", "PROFILE", "SOURCE", "DEFAULT", "MODEL", "ADAPTER", "RUNNER_NS", "SESSION"], providerRows.length > 0 ? providerRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Key spans:",
formatTable(["NAME", "SERVICE", "ROUTE", "STATUS", "DUR_MS", "RES_MS", "FETCH_REQ", "REQ_WAIT", "RESP_XFER", "PROTO", "DETAIL"], spanRows.length > 0 ? spanRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Runner terminal/errors:",
formatTable(["NAME", "SERVICE", "TERMINAL", "FAILURE", "RETRY", "ROUTE", "HTTP", "MESSAGE"], runnerRows.length > 0 ? runnerRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Trace read windows:",
formatTable(["TRACE", "SESSION", "SINCE", "LIMIT", "RETURNED", "RANGE", "TOTAL", "MORE", "FULL", "LAST"], readRows.length > 0 ? readRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Session list reads:",
formatTable(["SOURCE", "COUNT", "FALLBACK", "RATIO", "EMPTY_PREVIEW", "INCLUDE", "SESSIONS", "TRACES"], sessionListRows.length > 0 ? sessionListRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Session message reads:",
formatTable(["SESSION", "RETURNED", "TOTAL", "ROLES", "USER_PREFIX", "ADJ_SAME", "USER", "AGENT", "TRACES"], sessionMessageRows.length > 0 ? sessionMessageRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Turn status reads:",
formatTable(["TRACE", "SESSION", "TURN", "STATUS", "PHASE", "ROUTE", "HTTP"], turnRows.length > 0 ? turnRows : [["-", "-", "-", "-", "-", "-", "-"]]),
"",
"Projection writes:",
formatTable(["NAME", "TRACE", "SESSION", "TURN", "STATUS", "EVENT", "PROJECTED", "SOURCE", "MESSAGE"], projectionRows.length > 0 ? projectionRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Projection sync cursors:",
formatTable(["SERVICE", "TRACE", "SESSION", "AFTER", "END", "EVENTS", "RAW", "MAX", "LAST", "TERM", "AUTH"], projectionSyncRows.length > 0 ? projectionSyncRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
"",
"Span name counts:",
formatTable(["NAME", "COUNT"], countRows.length > 0 ? countRows : [["-", "-"]]),
"",
"Summary:",
` target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)} path=${input.tracePath}`,
` grep=${textValue(input.result.grep)} limit=${textValue(input.result.limit)} traceFound=${textValue(input.result.traceFound)} parseOk=${textValue(input.result.parseOk)}`,
"",
"Next:",
];
const nextCommands = [
textValue(next?.fullSummary),
textValue(next?.grepProviderStream),
textValue(next?.raw),
].filter((item) => item !== "-");
if (nextCommands.length > 0) {
for (const command of nextCommands.slice(0, 3)) lines.push(` ${command}`);
} else {
lines.push(` ${buildTraceCommand(input.target, input.options, true)}`);
}
lines.push("", "Disclosure:");
lines.push(" trace output is a bounded table; --full expands row counts and diagnostic tables, --raw returns backend JSON.");
return {
ok: input.ok,
command: "platform-infra observability trace",
contentType: "text/plain",
renderedText: lines.join("\n"),
};
}
function traceRunnerRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => {
const name = textValue(span.name);
const attrs = asPlainRecord(span.attributes);
return name.startsWith("runner_terminal")
|| name.startsWith("runner_error")
|| name === "command_result"
|| (name.startsWith("codex_stdio") && spanColumnAttr(attrs, ["failureKind", "terminalStatus"]) !== "-");
}).map((span) => {
const attrs = asPlainRecord(span.attributes);
return [
shortenEnd(textValue(span.name), 34),
shortenEnd(textValue(span.service), 20),
shortenEnd(spanColumnAttr(attrs, ["terminalStatus"]), 12),
shortenEnd(spanColumnAttr(attrs, ["failureKind"]), 24),
shortenEnd(spanColumnAttr(attrs, ["willRetry"]), 8),
shortenEnd(spanColumnAttr(attrs, ["http.route"]), 34),
shortenEnd(spanColumnAttr(attrs, ["http.status_code", "http.response.status_code"]), 8),
shortenEnd(spanColumnAttr(attrs, ["errorSummary", "message", "error.message", "exception.message", "statusMessage"]), 80),
];
});
return dedupeRows(rows).slice(0, 12);
}
function traceProviderDecisionRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => {
const attrs = asPlainRecord(span.attributes);
return textValue(span.name) === "provider_decision"
|| spanColumnAttr(attrs, ["agent.chat.provider_profile", "providerProfile", "defaultProviderProfile"]) !== "-";
}).map((span) => {
const attrs = asPlainRecord(span.attributes);
return [
shortenEnd(textValue(span.service), 20),
shortenEnd(spanColumnAttr(attrs, ["agent.chat.provider_profile", "providerProfile", "backendProfile", "defaultProviderProfile"]), 22),
shortenEnd(spanColumnAttr(attrs, ["agent.chat.provider_profile_source", "providerProfileSource"]), 12),
shortenEnd(spanColumnAttr(attrs, ["defaultProviderProfile"]), 22),
shortenEnd(spanColumnAttr(attrs, ["agent.chat.model", "agent.chat.model_id", "agent.chat.provider_model", "model", "modelId", "providerModel"]), 28),
shortenEnd(spanColumnAttr(attrs, ["adapter"]), 16),
shortenEnd(spanColumnAttr(attrs, ["agentRunRunnerNamespace"]), 16),
shortenMiddle(spanColumnAttr(attrs, ["agent.chat.session_id", "sessionId", "workbench.session_id"]), 24),
];
});
return dedupeRows(rows).slice(0, 8);
}
function traceReadWindowRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => textValue(span.name) === "trace_events_read").map((span) => {
const attrs = asPlainRecord(span.attributes);
const fromSeq = spanColumnAttr(attrs, ["fromSeq"]);
const toSeq = spanColumnAttr(attrs, ["toSeq"]);
return [
shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18),
shortenEnd(spanColumnAttr(attrs, ["sinceSeq", "afterSeq"]), 8),
shortenEnd(spanColumnAttr(attrs, ["limit"]), 8),
shortenEnd(spanColumnAttr(attrs, ["returnedEvents", "rawEventCount"]), 8),
fromSeq === "-" && toSeq === "-" ? "-" : `${fromSeq}..${toSeq}`,
shortenEnd(spanColumnAttr(attrs, ["totalEvents"]), 8),
shortenEnd(spanColumnAttr(attrs, ["hasMore"]), 6),
shortenEnd(spanColumnAttr(attrs, ["fullTraceLoaded"]), 6),
shortenEnd(spanColumnAttr(attrs, ["traceLastSeq", "maxSeq", "endSeq"]), 8),
];
});
return dedupeRows(rows).slice(0, 12);
}
function traceSessionListRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => textValue(span.name) === "session_list_read").map((span) => {
const attrs = asPlainRecord(span.attributes);
return [
shortenEnd(spanColumnAttr(attrs, ["source"]), 10),
shortenEnd(spanColumnAttr(attrs, ["sessionCount"]), 8),
shortenEnd(spanColumnAttr(attrs, ["fallbackTitleCount"]), 8),
shortenEnd(spanColumnAttr(attrs, ["fallbackTitleRatio"]), 8),
shortenEnd(spanColumnAttr(attrs, ["emptyPreviewCount"]), 8),
shortenMiddle(spanColumnAttr(attrs, ["includeSessionId"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["sessionIds"]), 34),
shortenMiddle(spanColumnAttr(attrs, ["traceIds"]), 34),
];
});
return dedupeRows(rows).slice(0, 12);
}
function traceSessionMessageRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => textValue(span.name) === "session_messages_read").map((span) => {
const attrs = asPlainRecord(span.attributes);
return [
shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18),
shortenEnd(spanColumnAttr(attrs, ["returnedMessages"]), 8),
shortenEnd(spanColumnAttr(attrs, ["totalMessages"]), 8),
shortenEnd(spanColumnAttr(attrs, ["roleSequencePrefix"]), 32),
shortenEnd(spanColumnAttr(attrs, ["consecutiveUserPrefix"]), 8),
shortenEnd(spanColumnAttr(attrs, ["adjacentSameRoleCount"]), 8),
shortenEnd(spanColumnAttr(attrs, ["userCount"]), 8),
shortenEnd(spanColumnAttr(attrs, ["agentCount"]), 8),
shortenMiddle(spanColumnAttr(attrs, ["traceIds"]), 34),
];
});
return dedupeRows(rows).slice(0, 12);
}
function traceTurnStatusRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => textValue(span.name) === "turn_status_read").map((span) => {
const attrs = asPlainRecord(span.attributes);
return [
shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["turnId"]), 18),
shortenEnd(spanColumnAttr(attrs, ["status", "terminalStatus"]), 14),
shortenEnd(spanColumnAttr(attrs, ["phase"]), 14),
shortenEnd(spanColumnAttr(attrs, ["http.route"]), 34),
shortenEnd(spanColumnAttr(attrs, ["http.status_code", "http.response.status_code"]), 8),
];
});
return dedupeRows(rows).slice(0, 12);
}
function traceProjectionRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => textValue(span.name).includes("projection")).map((span) => {
const attrs = asPlainRecord(span.attributes);
return [
shortenEnd(textValue(span.name), 34),
shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["sessionId", "workbench.projection.session_id"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["turnId", "workbench.projection.turn_id"]), 18),
shortenEnd(spanColumnAttr(attrs, ["status", "terminalStatus", "phase", "workbench.projection.phase"]), 14),
shortenEnd(spanColumnAttr(attrs, ["eventType", "type", "workbench.projection.event_type"]), 18),
shortenEnd(spanColumnAttr(attrs, ["projectedSeq", "workbench.projection.projected_seq"]), 10),
shortenEnd(spanColumnAttr(attrs, ["sourceSeq", "workbench.projection.source_seq", "sourceEventId", "workbench.projection.source_event_id"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["messageId", "workbench.projection.message_id", "failureKind", "errorSummary"]), 24),
];
});
return dedupeRows(rows).slice(0, 12);
}
function traceProjectionSyncRows(spans: Record<string, unknown>[]): string[][] {
const rows = spans.filter((span) => {
const name = textValue(span.name);
const attrs = asPlainRecord(span.attributes);
return name === "projection_sync"
|| name === "workbench.sync.replay"
|| spanColumnAttr(attrs, ["workbench.sync.contract_version", "workbench.sync.cursor_outbox_seq", "workbench.sync.event_count", "workbench.realtime.authority"]) !== "-";
}).map((span) => {
const attrs = asPlainRecord(span.attributes);
return [
shortenEnd(textValue(span.service), 18),
shortenMiddle(spanColumnAttr(attrs, ["traceId", "workbench.trace_id"]), 18),
shortenMiddle(spanColumnAttr(attrs, ["sessionId", "workbench.session_id"]), 18),
shortenEnd(spanColumnAttr(attrs, ["afterSeq", "workbench.sync.since_outbox_seq"]), 8),
shortenEnd(spanColumnAttr(attrs, ["endSeq", "workbench.sync.cursor_outbox_seq"]), 8),
shortenEnd(spanColumnAttr(attrs, ["eventCount", "workbench.sync.event_count"]), 8),
shortenEnd(spanColumnAttr(attrs, ["rawEventCount"]), 8),
shortenEnd(spanColumnAttr(attrs, ["maxSeq", "workbench.sync.max_entity_version"]), 8),
shortenEnd(spanColumnAttr(attrs, ["traceLastSeq"]), 8),
shortenEnd(spanColumnAttr(attrs, ["terminalFromEvents", "terminalFromRawEvents", "terminalStatus", "workbench.terminal.seal"]), 12),
shortenEnd(spanColumnAttr(attrs, ["workbench.realtime.authority", "workbench.sync.contract_version", "workbench.detail.projection"]), 18),
];
});
return dedupeRows(rows).slice(0, 16);
}
function dedupeRows(rows: string[][]): string[][] {
const output: string[][] = [];
const seen = new Set<string>();
for (const row of rows) {
const key = JSON.stringify(row);
if (seen.has(key)) continue;
seen.add(key);
output.push(row);
}
return output;
}
export function renderSearchTable(input: {
ok: boolean;
target: ObservabilityTarget;
options: SearchOptions;
query: Record<string, unknown>;
result: Record<string, unknown>;
}): RenderedCliResult {
const traces = asArray(input.result.traces).map((item) => asPlainRecord(item) ?? {});
const requestedLimit = numberFromUnknown(input.options.limit);
const rowLimit = Math.max(1, Math.min(Number.isFinite(requestedLimit) ? Math.round(requestedLimit) : 12, 40));
const rows = traces.slice(0, rowLimit).map((trace) => {
const meta = asPlainRecord(trace.meta) ?? {};
return [
shortenMiddle(textValue(trace.traceId), 32),
shortenEnd(textValue(trace.startAt ?? isoFromUnixNano(meta.startTimeUnixNano)), 20),
numberValue(trace.durationMs ?? meta.durationMs),
searchTraceStatus(trace),
numberValue(trace.spanCount),
numberValue(trace.errorSpanCount),
numberValue(trace.matchedSpanCount),
joinValues(trace.services, 38),
];
});
const lines = [
`platform-infra observability search (${input.ok ? "ok" : "not-ok"})`,
"",
formatTable(["TRACE", "START", "DUR_MS", "STATUS", "SPANS", "ERR", "MATCH", "SERVICES"], rows.length > 0 ? rows : [["-", "-", "-", "no-match", "-", "-", "-", "-"]]),
"",
"Summary:",
` target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)}`,
` grep=${textValue(input.query.grep)} path=${textValue(input.query.pathFilter)} status=${textValue(input.query.statusFilter)} tempoQuery=${shortenMiddle(textValue(input.query.tempoQuery), 80)}`,
` mode=${textValue(input.query.mode)} businessTraceId=${textValue(input.query.businessTraceId)} lookbackMinutes=${textValue(input.query.lookbackMinutes)} requestedLookbackMinutes=${textValue(input.query.requestedLookbackMinutes)} endAt=${textValue(input.query.endAt)}`,
` candidateLimit=${textValue(input.query.candidateLimit)} limit=${textValue(input.query.limit)}`,
` candidates=${textValue(input.result.candidateTraceCount)} scanned=${textValue(input.result.scannedTraceCount)} matched=${textValue(input.result.matchedTraceCount)} stopped=${textValue(input.result.scanStopped)}`,
];
if (input.query.grep !== null) {
lines.push(` grepCoverage=${textValue(input.query.grepCoverage)}`);
if (input.query.grepQueryInference !== null) lines.push(` grepQueryInference=${textValue(input.query.grepQueryInference)}`);
}
const firstTraceId = traces.length > 0 ? textValue(traces[0].traceId) : "";
lines.push("", "Next:");
if (firstTraceId.length > 0 && firstTraceId !== "-") {
lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${firstTraceId}`);
}
lines.push(` ${buildSearchCommand(input.target, input.options, true)}`);
if (input.query.grep !== null && Number(input.result.matchedTraceCount ?? 0) === 0) {
lines.push(` explicit TraceQL: bun scripts/cli.ts platform-infra observability search --target ${input.target.id} --query '{ resource.service.name = "<service>" }' --lookback-minutes ${textValue(input.query.lookbackMinutes)} --candidate-limit ${textValue(input.query.candidateLimit)} --limit ${textValue(input.query.limit)}`);
lines.push(` widen candidates: bun scripts/cli.ts platform-infra observability search --target ${input.target.id} --grep ${JSON.stringify(String(input.query.grep))} --lookback-minutes ${textValue(input.query.lookbackMinutes)} --candidate-limit 300 --limit ${textValue(input.query.limit)}`);
}
lines.push("", "Disclosure:");
lines.push(" default view is a bounded table; use --full for structured diagnosis JSON or trace --trace-id for one trace.");
return {
ok: input.ok,
command: "platform-infra observability search",
contentType: "text/plain",
renderedText: lines.join("\n"),
};
}
export function renderDiagnoseCodeAgentTable(input: {
ok: boolean;
target: ObservabilityTarget;
options: DiagnoseCodeAgentOptions;
query: Record<string, unknown>;
result: Record<string, unknown>;
}): RenderedCliResult {
const mapping = asPlainRecord(input.result.mapping);
const diagnosticTrace = asPlainRecord(input.result.diagnosticTrace);
const identity = asPlainRecord(input.result.identity);
const agentrun = asPlainRecord(input.result.agentrun);
const agentrunAuthority = asPlainRecord(agentrun?.authority);
const projectionLag = asPlainRecord(input.result.projectionLag);
const summary = asPlainRecord(input.result.summary);
const evidence = asPlainRecord(input.result.evidence);
const observabilityGap = asPlainRecord(input.result.observabilityGap);
const servicePath = asPlainRecord(input.result.servicePath);
const providerDecision = asPlainRecord(input.result.providerDecision);
const rootCauses = asArray(input.result.rootCauseCandidates).map((item) => asPlainRecord(item) ?? {});
const http = asPlainRecord(input.result.http);
const services = joinValues(input.result.services, 54);
const traceId = textValue(mapping?.otelTraceId ?? input.query.traceId);
const rootCause = textValue(summary?.rootCause ?? rootCauses[0]?.code);
const rows = [[
shortenMiddle(traceId, 20),
shortenEnd(rootCause, 34),
textValue(agentrun?.terminalStatus),
textValue(agentrun?.runnerProviderClassification),
textValue(projectionLag?.status),
textValue(evidence?.errorSpanCount),
services,
]];
const rootRows = rootCauses.slice(0, 5).map((candidate) => [
shortenEnd(textValue(candidate.code), 40),
textValue(candidate.confidence),
shortenEnd(textValue(candidate.summary ?? candidate.label), 80),
]);
const httpRows = httpTableRows(http);
const expectedServices = asArray(observabilityGap?.expectedServices).map((item) => textValue(item)).filter((item) => item !== "-");
const seenServices = new Set(asArray(observabilityGap?.seenServices).map((item) => textValue(item)).filter((item) => item !== "-"));
const missingServices = new Set(asArray(observabilityGap?.missingServices).map((item) => textValue(item)).filter((item) => item !== "-"));
const serviceRows = expectedServices.map((service) => [
service,
textValue(servicePath?.[service] ?? (missingServices.has(service) ? "missing" : seenServices.has(service) ? "reached" : "-")),
missingServices.has(service) ? "missing" : seenServices.has(service) ? "seen" : "-",
]);
const queryClauses = asArray(input.query.queryClauses).map((item) => textValue(item)).filter((item) => item !== "-");
const requestedRunId = textValue(input.query.runId);
const requestedCommandId = textValue(input.query.commandId);
const requestedSessionId = textValue(input.query.sessionId);
const requestedRunnerJobId = textValue(input.query.runnerJobId);
const observedRunId = textValue(identity?.runId);
const observedCommandId = textValue(identity?.commandId);
const observedSessionId = textValue(identity?.sessionId);
const observedRunnerJobId = textValue(identity?.runnerJobId);
const outcome = asPlainRecord(input.result.outcome);
const outcomeAuthority = asPlainRecord(outcome?.authority);
const outcomeProjection = asPlainRecord(outcome?.projection);
const lines = [
`platform-infra observability diagnose-code-agent (${input.ok ? "ok" : "not-ok"})`,
"",
formatTable(["TRACE", "ROOT_CAUSE", "TERMINAL", "RUNNER", "PROJECTION", "ERR", "SERVICES"], rows),
"",
"Identity:",
` businessTraceId=${textValue(mapping?.businessTraceId ?? input.query.businessTraceId)} otelTraceId=${traceId}`,
` queryMode=${textValue(mapping?.mode ?? input.query.mode)} tempoQuery=${shortenEnd(textValue(mapping?.searchQuery ?? input.query.tempoQuery), 180)}`,
` queryClauses=${queryClauses.length > 0 ? queryClauses.join(" ; ") : "-"}`,
` requested runId=${requestedRunId} commandId=${requestedCommandId} sessionId=${requestedSessionId} runnerJobId=${requestedRunnerJobId}`,
` observed runId=${observedRunId} commandId=${observedCommandId} sessionId=${observedSessionId} runnerJobId=${observedRunnerJobId} runnerId=${textValue(identity?.runnerId)}`,
` backendProfile=${textValue(identity?.backendProfile)} sourceCommit=${shortenMiddle(textValue(identity?.sourceCommit), 20)}`,
` providerProfile=${textValue(providerDecision?.providerProfile)} model=${textValue(providerDecision?.model)} source=${textValue(providerDecision?.providerProfileSource)} adapter=${textValue(providerDecision?.adapter)} runnerNamespace=${textValue(providerDecision?.runnerNamespace)}`,
"",
"Diagnostic trace:",
` traceId=${textValue(diagnosticTrace?.traceId)} parentSpanId=${textValue(diagnosticTrace?.parentSpanId)} operation=${textValue(diagnosticTrace?.operation)} contextReady=${textValue(diagnosticTrace?.contextReady)}`,
` target businessTraceId=${textValue(diagnosticTrace?.targetBusinessTraceId)} commandId=${textValue(diagnosticTrace?.targetCommandId)} expectedLinkTraceId=${textValue(diagnosticTrace?.expectedTargetOtelTraceId)}`,
` diagnosticReads=${joinValues(diagnosticTrace?.headerReadPaths, 60)} ordinaryReads=${joinValues(diagnosticTrace?.ordinaryReadPaths, 40)}`,
` link=${shortenEnd(textValue(diagnosticTrace?.linkQueryClue), 180)}`,
"",
"Root causes:",
formatTable(["CODE", "CONF", "SUMMARY"], rootRows.length > 0 ? rootRows : [["-", "-", "-"]]),
"",
"Service trace coverage:",
formatTable(["SERVICE", "PATH", "SPAN"], serviceRows.length > 0 ? serviceRows : [["-", "-", "-"]]),
` observabilityGap=${textValue(observabilityGap?.status)} missing=${joinValues(observabilityGap?.missingServices, 60)} seen=${joinValues(observabilityGap?.seenServices, 80)}`,
"",
"HTTP:",
formatTable(["METHOD", "ROUTE", "STATUS", "COUNT"], httpRows.length > 0 ? httpRows : [["-", "-", "-", "-"]]),
"",
"Summary:",
` outcome=${textValue(outcome?.typed)} effectiveStatus=${textValue(outcome?.effectiveStatus)} mutation=${textValue(outcome?.mutation)}`,
` durableAuthority kind=${textValue(outcomeAuthority?.kind)} status=${textValue(outcomeAuthority?.status)} source=${textValue(outcomeAuthority?.source)}`,
` projection kind=${textValue(outcomeProjection?.kind)} status=${textValue(outcomeProjection?.status)} lag=${textValue(outcomeProjection?.lag)} warning=${textValue(outcomeProjection?.warning)} blocking=${textValue(outcomeProjection?.blocking)}`,
` target=${input.target.id} spanCount=${textValue(input.result.spanCount)} servicePath=${joinValues(input.result.servicePath, 60)}`,
` agentrun=${textValue(agentrun?.terminalStatus)} failureKind=${textValue(agentrun?.failureKind)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`,
` authorityQuery namespace=${textValue(agentrunAuthority?.namespace)} source=${textValue(agentrunAuthority?.namespaceSource)} ok=${textValue(agentrunAuthority?.ok)} commandState=${textValue(agentrunAuthority?.commandState)} terminal=${textValue(agentrunAuthority?.terminalStatus)}`,
` readModel=${shortenEnd(JSON.stringify(input.result.hwlabReadModel ?? null), 140)}`,
];
const next = asPlainRecord(input.result.next);
lines.push("", "Next:");
const nextCommands = [
textValue(diagnosticTrace?.traceCommand),
textValue(diagnosticTrace?.targetTraceCommand),
textValue(next?.diagnoseFull),
textValue(next?.traceSummary),
textValue(next?.traceReads),
textValue(next?.sessionReads),
textValue(next?.turnStatus),
textValue(next?.projection),
].filter((item) => item !== "-");
if (nextCommands.length > 0) {
for (const command of Array.from(new Set(nextCommands)).slice(0, 6)) lines.push(` ${command}`);
} else {
lines.push(` ${buildDiagnoseCommand(input.target, input.options, true)}`);
if (traceId.length > 0 && traceId !== "-") lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${traceId}`);
}
lines.push("", "Disclosure:");
lines.push(" default view is a bounded table; use --full for structured diagnosis JSON or trace --grep for span-level drill-down.");
return {
ok: input.ok,
command: "platform-infra observability diagnose-code-agent",
contentType: "text/plain",
renderedText: lines.join("\n"),
};
}