diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 45bf6681..eccc1ddd 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -151,6 +151,7 @@ controlPlane: serviceAccount: agentrun-v01-runner jobNamePrefix: agentrun-v01-runner idleTimeoutMs: 600000 + missingTerminalAfterToolTimeoutMs: 60000 apiKeySecretRef: name: agentrun-v01-api-key key: HWLAB_API_KEY @@ -324,6 +325,7 @@ controlPlane: serviceAccount: agentrun-v02-runner jobNamePrefix: agentrun-v02-runner idleTimeoutMs: 172800000 + missingTerminalAfterToolTimeoutMs: 30000 apiKeySecretRef: name: agentrun-v02-api-key key: HWLAB_API_KEY diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 17c2a250..9751c04b 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -104,6 +104,7 @@ export interface AgentRunLaneSpec { readonly serviceAccount: string; readonly jobNamePrefix: string; readonly idleTimeoutMs: number; + readonly missingTerminalAfterToolTimeoutMs: number; readonly apiKeySecretRef: { readonly name: string; readonly key: string }; readonly egressProxyUrl: string | null; readonly noProxyExtra: readonly string[]; @@ -541,6 +542,7 @@ function parseDeployment(input: Record, path: string): AgentRun serviceAccount: stringField(runner, "serviceAccount", `${path}.runner`), jobNamePrefix: stringField(runner, "jobNamePrefix", `${path}.runner`), idleTimeoutMs: positiveIntegerField(runner, "idleTimeoutMs", `${path}.runner`), + missingTerminalAfterToolTimeoutMs: positiveIntegerField(runner, "missingTerminalAfterToolTimeoutMs", `${path}.runner`), apiKeySecretRef: parseSecretRef(recordField(runner, "apiKeySecretRef", `${path}.runner`), `${path}.runner.apiKeySecretRef`), egressProxyUrl: optionalStringField(runner, "egressProxyUrl", `${path}.runner`) ?? null, noProxyExtra: optionalStringArrayField(runner, "noProxyExtra", `${path}.runner`), diff --git a/scripts/src/agentrun-manifests.ts b/scripts/src/agentrun-manifests.ts index 087d3ef5..e464a6ac 100644 --- a/scripts/src/agentrun-manifests.ts +++ b/scripts/src/agentrun-manifests.ts @@ -439,6 +439,7 @@ function managerEnv(spec: AgentRunLaneSpec, sourceCommit: string, imageRef: stri { name: "AGENTRUN_RUNNER_SERVICE_ACCOUNT", value: spec.deployment.runner.serviceAccount }, { name: "AGENTRUN_RUNNER_JOB_NAME_PREFIX", value: spec.deployment.runner.jobNamePrefix }, { name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(spec.deployment.runner.idleTimeoutMs) }, + { name: "AGENTRUN_RUNNER_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS", value: String(spec.deployment.runner.missingTerminalAfterToolTimeoutMs) }, { name: "AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", value: String(spec.deployment.runner.retention.maxRunners) }, { name: "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER", value: spec.deployment.runner.retention.cleanupOrder }, { name: "AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", value: String(spec.deployment.runner.retention.activeHeartbeatMaxAgeMs) }, diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 72b9d14e..87d70e65 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -462,6 +462,7 @@ function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record): Record< const pagePerformance = record(analysis.pagePerformance); const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item)); const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item)); + const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item)); + const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item)); + const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item)); const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item)); const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item)); const id = result.id ?? "-"; @@ -7121,6 +7132,13 @@ function renderWebObserveAnalyzeResult(result: Record): Record< [[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]], ), "", + jsonlReadIssues.length === 0 + ? "JSONL_READ_ISSUES\n-" + : webObserveTable( + ["FILE", "CODE", "MESSAGE"], + jsonlReadIssues.map((item) => [item.file, item.code, item.message]), + ), + "", webObserveTable( ["TURN_TIMING", "VALUE"], [ @@ -7163,11 +7181,47 @@ function renderWebObserveAnalyzeResult(result: Record): Record< ["httpError", runtimeAlerts.httpErrorCount], ["requestFailed", runtimeAlerts.requestFailedCount], ["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount], + ["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount], ["consoleAlerts", runtimeAlerts.consoleAlertCount], ["executionErrors", runtimeAlerts.executionErrorCount], ], ), "", + domDiagnosticGroups.length === 0 + ? "DOM_DIAGNOSTIC_GROUPS\n-" + : webObserveTable( + ["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"], + domDiagnosticGroups.map((item) => [ + item.diagnosticCode, + item.count, + item.firstSeq, + item.lastSeq, + item.firstAt, + item.lastAt, + webObserveArray(item.promptIndexes).join(",") || "-", + item.traceId, + String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140), + ]), + ), + "", + domDiagnostics.length === 0 + ? "DOM_DIAGNOSTICS\n-" + : webObserveTable( + ["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"], + domDiagnostics.map((item) => [ + item.seq, + item.ts, + item.promptIndex, + item.diagnosticCode, + item.traceId, + item.httpStatus, + item.idleSeconds, + item.waitingFor, + item.lastEventLabel, + String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180), + ]), + ), + "", webObserveTable( ["PERF", "VALUE"], [ @@ -7202,6 +7256,94 @@ function renderWebObserveAnalyzeResult(result: Record): Record< return withWebObserveRendered(result, renderedText); } +function renderWebProbeScriptResult(result: Record): Record { + const summary = record(result.summary); + const issueEvidence = record(result.issueEvidence); + const probe = record(result.probe); + const script = record(probe.script); + const scriptResult = record(script.result); + const reportLoad = record(result.reportLoad); + const commandResult = record(result.result); + const recoveredArtifacts = record(result.recoveredArtifacts); + const recoveredArtifactSummary = record(recoveredArtifacts.artifacts); + const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item)); + const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item)); + const resultRows = webProbeScriptRecordRows(scriptResult, 12); + const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12); + const summaryRows = [ + ["ok", result.ok], + ["status", result.status], + ["degradedReason", result.degradedReason ?? "-"], + ["failureKind", result.failureKind ?? "-"], + ["scriptSource", result.scriptSource ?? "-"], + ["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"], + ["reportPath", reportLoad.path ?? probe.reportPath ?? "-"], + ["reportSha256", probe.reportSha256 ?? "-"], + ["scriptSha256", probe.scriptSha256 ?? "-"], + ["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"], + ["exitCode", commandResult.exitCode ?? "-"], + ]; + const renderedText = [ + webObserveTable( + ["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"], + [[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]], + ), + "", + webObserveTable(["FIELD", "VALUE"], summaryRows), + "", + resultRows.length === 0 + ? "SCRIPT_RESULT\n-" + : webObserveTable(["KEY", "VALUE"], resultRows), + "", + evidenceRows.length === 0 + ? "ISSUE_EVIDENCE\n-" + : webObserveTable(["KEY", "VALUE"], evidenceRows), + "", + steps.length === 0 + ? "STEPS\n-" + : webObserveTable( + ["STEP", "OK", "DETAIL"], + steps.map((item) => [ + item.name ?? item.step ?? item.label ?? "-", + item.ok ?? item.status ?? "-", + webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item), + ]), + ), + "", + recoveredItems.length === 0 + ? "ARTIFACTS\n-" + : webObserveTable( + ["KIND", "BYTES", "PATH"], + recoveredItems.map((item) => [item.kind, item.byteCount, item.path]), + ), + "", + "NEXT", + ` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`, + ` rerun: ${result.command ?? `hwlab nodes web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function webProbeScriptRecordRows(value: Record, limit: number): unknown[][] { + return Object.entries(value) + .slice(0, limit) + .map(([key, nested]) => [key, webProbeScriptPreview(nested)]); +} + +function webProbeScriptPreview(value: unknown): string { + if (value === null || value === undefined) return "-"; + if (typeof value === "string") return value.replace(/\s+/gu, " ").trim(); + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (Array.isArray(value)) { + const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", "); + return `[${value.length}] ${preview}`; + } + if (typeof value === "object") { + return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240); + } + return String(value); +} + function withWebObserveShortcuts(value: Record | null, id: string | null): Record | null { if (value === null || id === null) return value; return { @@ -7374,7 +7516,7 @@ function runNodeWebProbeScript( compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]); compactResult.stderrTail = redactKnownSecrets(result.stderr.slice(-2000), [material.password ?? ""]); } - return { + return renderWebProbeScriptResult({ ok: passed, status: passed ? "pass" : "blocked", command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`, @@ -7403,7 +7545,7 @@ function runNodeWebProbeScript( }, result: compactResult, valuesRedacted: true, - }; + }); } function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string { diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 9fec0dd0..ac266e12 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -982,13 +982,16 @@ function sleep(ms) { export function nodeWebObserveAnalyzerSource(): string { return String.raw`#!/usr/bin/env node import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import path from "node:path"; +import { createInterface } from "node:readline"; const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || process.argv[2] || ".state/web-observe/manual"); const analysisDir = path.join(stateDir, "analysis"); const reportJsonPath = path.join(analysisDir, "report.json"); const reportMdPath = path.join(analysisDir, "report.md"); +const jsonlReadIssues = []; const samples = await readJsonl(path.join(stateDir, "samples.jsonl")); const control = await readJsonl(path.join(stateDir, "control.jsonl")); const network = await readJsonl(path.join(stateDir, "network.jsonl")); @@ -1015,6 +1018,7 @@ const report = { manifest: compactManifest(manifest), heartbeat: compactHeartbeat(heartbeat), counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length }, + jsonlReadIssues, commandTimeline, transitions, sampleMetrics, @@ -1029,19 +1033,36 @@ const report = { await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +const compactRuntimeAlerts = { + ...runtimeAlerts.summary, + domDiagnostics: runtimeAlerts.domDiagnostics.slice(-80), + domDiagnosticsByFingerprint: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 80), +}; +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, jsonlReadIssues: jsonlReadIssues.slice(0, 20), sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: compactRuntimeAlerts, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } } async function readJsonl(file) { + const rows = []; try { - const text = await readFile(file, "utf8"); - return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => { - try { return JSON.parse(line); } catch { return { parseError: true, rawHash: sha256(line) }; } - }); - } catch { return []; } + const input = createReadStream(file, { encoding: "utf8" }); + const lines = createInterface({ input, crlfDelay: Infinity }); + for await (const line of lines) { + const trimmed = String(line || "").trim(); + if (!trimmed) continue; + try { + rows.push(JSON.parse(trimmed)); + } catch { + rows.push({ parseError: true, rawHash: sha256(trimmed) }); + } + } + } catch (error) { + if (error?.code === "ENOENT") return rows; + jsonlReadIssues.push({ file: path.basename(file), code: error?.code ?? error?.name ?? "jsonl_read_failed", message: String(error?.message || "JSONL read failed").slice(0, 240), valuesPrinted: false }); + } + return rows; } function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance) { @@ -1414,6 +1435,10 @@ function parseDomDiagnosticSummary(text) { const lastEventLabelMatch = value.match(/\blastEventLabel=([^\s；;，,)]+)/iu); const diagnosticCode = httpStatusMatch ? "http-" + httpStatusMatch[1] + : /projection-resume:sync-failed/iu.test(value) + ? "projection-resume-sync-failed" + : /AgentRun\s+GET\b[\s\S]*\/result\b[\s\S]*timed out after\s+\d+ms/iu.test(value) + ? "agentrun-result-timeout" : /turn\s*超过|无新活动/iu.test(value) ? "turn-idle-no-activity" : /Failed to fetch/iu.test(value) @@ -1564,13 +1589,15 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { pageErrorCount: pageErrors.length, networkErrorGroupCount: groupNetworkAlerts(httpErrors).length, requestFailedGroupCount: groupNetworkAlerts(requestFailed).length, + domDiagnosticGroupCount: groupDomDiagnostics(domDiagnostics).length, executionErrorGroupCount: groupExecutionErrors(executionErrors).length, baselineExecutionErrorGroupCount: groupExecutionErrors(baselineExecutionErrors).length, consoleAlertGroupCount: groupConsoleAlerts(consoleAlerts).length }, networkHttpErrorsByPath: groupNetworkAlerts(httpErrors), networkRequestFailedByPath: groupNetworkAlerts(requestFailed), - domDiagnostics: domDiagnostics.slice(0, 80), + domDiagnostics: domDiagnostics.slice(-80), + domDiagnosticsByFingerprint: groupDomDiagnostics(domDiagnostics).slice(0, 80), runtimeExecutionErrors: executionErrors.slice(0, 120), runtimeExecutionErrorsByCode: groupExecutionErrors(executionErrors), baselineRuntimeExecutionErrors: baselineExecutionErrors.slice(0, 80), @@ -1676,6 +1703,42 @@ function groupExecutionErrors(events) { return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.code).localeCompare(String(b.code))); } +function groupDomDiagnostics(events) { + const groups = new Map(); + for (const event of events) { + const key = [event.diagnosticCode || "-", event.traceId || "-", event.textHash || "-"].join(" "); + const group = groups.get(key) || { + diagnosticCode: event.diagnosticCode ?? null, + traceId: event.traceId ?? null, + textHash: event.textHash ?? null, + count: 0, + firstSeq: event.seq ?? null, + lastSeq: event.seq ?? null, + firstAt: event.ts ?? null, + lastAt: event.ts ?? null, + promptIndexes: [], + sources: [], + httpStatus: event.httpStatus ?? null, + idleSeconds: event.idleSeconds ?? null, + waitingFor: event.waitingFor ?? null, + lastEventLabel: event.lastEventLabel ?? null, + preview: event.preview ?? null, + valuesRedacted: true + }; + group.count += 1; + group.lastSeq = event.seq ?? group.lastSeq; + group.lastAt = event.ts ?? group.lastAt; + if (event.promptIndex && !group.promptIndexes.includes(event.promptIndex)) group.promptIndexes.push(event.promptIndex); + if (event.source && !group.sources.includes(event.source)) group.sources.push(event.source); + if (event.httpStatus && !group.httpStatus) group.httpStatus = event.httpStatus; + if (event.idleSeconds && !group.idleSeconds) group.idleSeconds = event.idleSeconds; + if (event.waitingFor && !group.waitingFor) group.waitingFor = event.waitingFor; + if (event.lastEventLabel && !group.lastEventLabel) group.lastEventLabel = event.lastEventLabel; + groups.set(key, group); + } + return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(b.lastAt ?? "").localeCompare(String(a.lastAt ?? ""))); +} + function consoleAlertEvent(item, promptTimes) { const text = String(item?.text || ""); const statusMatch = text.match(/\bstatus\s+of\s+([1-5][0-9]{2})\b/iu) || text.match(/\bHTTP\s+([1-5][0-9]{2})\b/iu); diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index b8fbcb21..d3170f5a 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -648,16 +648,24 @@ async function trace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); + const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null; + const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080); + const effectiveTempoQuery = options.query ?? (businessTraceId === null ? null : `{ .traceId = "${businessTraceId}" }`); + const effectiveOptions: SearchOptions = { + ...options, + query: effectiveTempoQuery, + lookbackMinutes: effectiveLookbackMinutes, + }; const endSeconds = Math.floor(Date.now() / 1000); - const startSeconds = endSeconds - (options.lookbackMinutes * 60); + const startSeconds = endSeconds - (effectiveLookbackMinutes * 60); const params = new URLSearchParams({ start: String(startSeconds), end: String(endSeconds), limit: String(options.candidateLimit), }); - if (options.query !== null) params.set("q", options.query); + if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery); const searchPath = `/api/search?${params.toString()}`; - const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, options)); + const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions)); const parsed = parseJsonOutput(result.stdout); const response = { ok: result.exitCode === 0 && parsed?.ok === true, @@ -669,8 +677,11 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); @@ -811,6 +827,9 @@ function renderSearchResult(response: Record): RenderedCliResul const lines: string[] = []; lines.push("COMMAND TARGET OK GREP_OR_QUERY MATCHED SCANNED CANDIDATES"); lines.push(`${pad("platform-infra observability search", 38)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(truncate(textValue(query.grep) || textValue(query.tempoQuery) || "-", 35), 35)} ${pad(textValue(result.matchedTraceCount) || "-", 8)} ${pad(textValue(result.scannedTraceCount) || "-", 8)} ${textValue(result.candidateTraceCount) || "-"}`); + if (textValue(query.mode) === "business-trace-exact") { + lines.push(`info: businessTraceId=${textValue(query.businessTraceId) || "-"} uses exact TraceQL and lookbackMinutes=${textValue(query.lookbackMinutes) || "-"} (requested=${textValue(query.requestedLookbackMinutes) || "-"})`); + } if (textValue(result.scanStopped)) lines.push(`warning: scanStopped=${textValue(result.scanStopped)}`); const truncated = recordValue(result.truncated); if (truncated.candidateTraces === true || truncated.matchedTraces === true) { @@ -880,7 +899,7 @@ function renderTraceResult(response: Record): RenderedCliResult lines.push("SERVICE DURATION_MS NAME DETAIL"); for (const item of errorSpans.slice(0, 8)) { const row = recordValue(item); - lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 160)}`); } } if (matchedSpans.length > 0) { @@ -889,7 +908,7 @@ function renderTraceResult(response: Record): RenderedCliResult lines.push("SERVICE DURATION_MS NAME DETAIL"); for (const item of matchedSpans.slice(0, 8)) { const row = recordValue(item); - lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 160)}`); } } lines.push(""); @@ -914,6 +933,7 @@ function traceSpanDetail(span: Record): string { "toolName", "exitCode", "durationMs", + "command", "waitingFor", "lastEventLabel", "idleMs", @@ -1799,7 +1819,7 @@ function searchScript(observability: ObservabilityConfig, target: ObservabilityT return ` set -u python3 - <<'PY' -import collections, json, subprocess, time +import collections, json, re, subprocess, time SEARCH_PROXY_PATH = ${JSON.stringify(searchProxyPath)} TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)} @@ -1811,6 +1831,7 @@ QUERY = ${queryLiteral} LIMIT = ${options.limit} CANDIDATE_LIMIT = ${options.candidateLimit} DEADLINE = time.time() + 50 +BUSINESS_TRACE_GREP = bool(GREP is not None and re.search(r"\\btrc_[A-Za-z0-9_-]+\\b", GREP)) IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", @@ -2037,6 +2058,15 @@ def trace_summary(trace_id, meta, body, rc, stderr): "stderrTail": (stderr or "")[-1000:], } +def rich_trace_rank(summary): + services = summary.get("services", []) + if not isinstance(services, list): + services = [] + span_count = summary.get("spanCount") if isinstance(summary.get("spanCount"), int) else 0 + error_count = summary.get("errorSpanCount") if isinstance(summary.get("errorSpanCount"), int) else 0 + matched_count = summary.get("matchedSpanCount") if isinstance(summary.get("matchedSpanCount"), int) else 0 + return (len(services), span_count, error_count, matched_count) + search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15) search_parsed = parse_json(search_body) search_parse_ok = isinstance(search_parsed, dict) @@ -2065,8 +2095,10 @@ for trace_id, meta in candidate_trace_ids: scanned.append(summary) if GREP is None or summary.get("rawMatched") is True or (summary.get("matchedSpanCount") or 0) > 0: matched.append(summary) - if len(matched) >= LIMIT and GREP is not None: + if len(matched) >= LIMIT and GREP is not None and not BUSINESS_TRACE_GREP: break +if BUSINESS_TRACE_GREP: + matched.sort(key=rich_trace_rank, reverse=True) payload = { "ok": search_rc == 0 and search_parse_ok, @@ -2074,6 +2106,7 @@ payload = { "searchProxyPath": SEARCH_PROXY_PATH, "grep": GREP, "tempoQuery": QUERY, + "businessTraceSearch": BUSINESS_TRACE_GREP, "limit": LIMIT, "candidateLimit": CANDIDATE_LIMIT, "searchParseOk": search_parse_ok,