From 9e8499824f5f61b1399782734113a540ec69c617 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 21 Jun 2026 18:02:37 +0000 Subject: [PATCH] fix: add web probe otel agentrun timeout diagnostics --- config/agentrun.yaml | 2 + scripts/src/agentrun-lanes.ts | 2 + scripts/src/agentrun-manifests.ts | 1 + scripts/src/hwlab-node-impl.ts | 354 +++++++++++++++++- .../hwlab-node-web-observe-runner-source.ts | 11 +- scripts/src/platform-infra-observability.ts | 52 ++- 6 files changed, 409 insertions(+), 13 deletions(-) diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 45bf6681..eccc1ddd 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -151,6 +151,7 @@ controlPlane: serviceAccount: agentrun-v01-runner jobNamePrefix: agentrun-v01-runner idleTimeoutMs: 600000 + missingTerminalAfterToolTimeoutMs: 60000 apiKeySecretRef: name: agentrun-v01-api-key key: HWLAB_API_KEY @@ -324,6 +325,7 @@ controlPlane: serviceAccount: agentrun-v02-runner jobNamePrefix: agentrun-v02-runner idleTimeoutMs: 172800000 + missingTerminalAfterToolTimeoutMs: 30000 apiKeySecretRef: name: agentrun-v02-api-key key: HWLAB_API_KEY diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 17c2a250..9751c04b 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -104,6 +104,7 @@ export interface AgentRunLaneSpec { readonly serviceAccount: string; readonly jobNamePrefix: string; readonly idleTimeoutMs: number; + readonly missingTerminalAfterToolTimeoutMs: number; readonly apiKeySecretRef: { readonly name: string; readonly key: string }; readonly egressProxyUrl: string | null; readonly noProxyExtra: readonly string[]; @@ -541,6 +542,7 @@ function parseDeployment(input: Record, path: string): AgentRun serviceAccount: stringField(runner, "serviceAccount", `${path}.runner`), jobNamePrefix: stringField(runner, "jobNamePrefix", `${path}.runner`), idleTimeoutMs: positiveIntegerField(runner, "idleTimeoutMs", `${path}.runner`), + missingTerminalAfterToolTimeoutMs: positiveIntegerField(runner, "missingTerminalAfterToolTimeoutMs", `${path}.runner`), apiKeySecretRef: parseSecretRef(recordField(runner, "apiKeySecretRef", `${path}.runner`), `${path}.runner.apiKeySecretRef`), egressProxyUrl: optionalStringField(runner, "egressProxyUrl", `${path}.runner`) ?? null, noProxyExtra: optionalStringArrayField(runner, "noProxyExtra", `${path}.runner`), diff --git a/scripts/src/agentrun-manifests.ts b/scripts/src/agentrun-manifests.ts index 087d3ef5..e464a6ac 100644 --- a/scripts/src/agentrun-manifests.ts +++ b/scripts/src/agentrun-manifests.ts @@ -439,6 +439,7 @@ function managerEnv(spec: AgentRunLaneSpec, sourceCommit: string, imageRef: stri { name: "AGENTRUN_RUNNER_SERVICE_ACCOUNT", value: spec.deployment.runner.serviceAccount }, { name: "AGENTRUN_RUNNER_JOB_NAME_PREFIX", value: spec.deployment.runner.jobNamePrefix }, { name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(spec.deployment.runner.idleTimeoutMs) }, + { name: "AGENTRUN_RUNNER_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS", value: String(spec.deployment.runner.missingTerminalAfterToolTimeoutMs) }, { name: "AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", value: String(spec.deployment.runner.retention.maxRunners) }, { name: "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER", value: spec.deployment.runner.retention.cleanupOrder }, { name: "AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", value: String(spec.deployment.runner.retention.activeHeartbeatMaxAgeMs) }, diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 34a092b4..45311314 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -7666,6 +7666,356 @@ function webObserveNextCommands(id: string): Record { }; } +function webObserveCell(value: unknown, maxLength = 96): string { + if (value === null || value === undefined) return "-"; + const text = typeof value === "string" ? value : String(value); + const compact = text.replace(/\s+/gu, " ").trim(); + if (compact.length === 0) return "-"; + if (compact.length <= maxLength) return compact; + return `${compact.slice(0, Math.max(1, maxLength - 1))}...`; +} + +function webObserveTable(headers: string[], rows: unknown[][]): string { + const stringRows = rows.map((row) => row.map((value) => webObserveCell(value))); + const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0))); + const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd(); + return [renderRow(headers), ...stringRows.map(renderRow)].join("\n"); +} + +function webObserveArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function withWebObserveRendered(result: Record, renderedText: string): Record { + return { + ...result, + renderedText, + contentType: "text/plain", + }; +} + +function renderWebObserveStatusResult(result: Record): Record { + const observer = record(result.observer); + const commandResult = record(result.result); + const manifest = record(observer.manifest); + const heartbeat = record(observer.heartbeat); + const tails = record(observer.tails); + const samples = webObserveArray(tails.samples); + const network = webObserveArray(tails.network); + const control = webObserveArray(tails.control); + const artifacts = webObserveArray(tails.artifacts); + const lastSample = record(samples[samples.length - 1]); + const failedNetwork = network + .map((item) => record(item)) + .filter((item) => item.type === "requestfailed" || typeof item.status === "number" && item.status >= 500); + const id = result.id ?? observer.id ?? manifest.jobId ?? heartbeat.jobId ?? "-"; + const resultSection = result.ok === true ? [] : [ + "", + webObserveTable( + ["RESULT", "VALUE"], + [ + ["exitCode", commandResult.exitCode ?? "-"], + ["timedOut", commandResult.timedOut ?? "-"], + ["stdoutTail", commandResult.stdoutTail ?? commandResult.stdout ?? "-"], + ["stderrTail", commandResult.stderrTail ?? commandResult.stderr ?? "-"], + ], + ), + ]; + const renderedText = [ + webObserveTable( + ["OBSERVER", "NODE", "LANE", "STATUS", "ALIVE", "PID", "SAMPLE", "COMMAND", "UPDATED"], + [[id, result.node, result.lane, manifest.status ?? heartbeat.status ?? result.status, observer.processAlive, observer.pid, heartbeat.sampleSeq, heartbeat.commandSeq, heartbeat.updatedAt]], + ), + "", + webObserveTable( + ["URL", "ROUTE_SESSION", "ACTIVE_SESSION", "MESSAGES", "TRACE_ROWS"], + [[heartbeat.currentUrl, lastSample.routeSessionId, lastSample.activeSessionId, lastSample.messageCount, lastSample.traceRowCount]], + ), + "", + webObserveTable( + ["TAIL", "COUNT", "DETAIL"], + [ + ["control", control.length, record(control[control.length - 1]).type ?? "-"], + ["samples", samples.length, record(samples[samples.length - 1]).ts ?? "-"], + ["network", network.length, failedNetwork.length === 0 ? "no recent failed/5xx tail" : `${failedNetwork.length} recent failed/5xx`], + ["artifacts", artifacts.length, record(artifacts[artifacts.length - 1]).sha256 ?? "-"], + ], + ), + ...resultSection, + "", + "NEXT", + ` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`, + ` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`, + ` command: bun scripts/cli.ts hwlab nodes web-probe observe command ${id} --type mark --label checkpoint`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function renderWebObserveCommandResult(result: Record): Record { + const observerCommand = record(result.observerCommand); + const observer = record(result.observer); + const id = result.id ?? "-"; + const renderedText = [ + webObserveTable( + ["OBSERVER", "NODE", "LANE", "COMMAND_ID", "TYPE", "STATUS", "DETAIL"], + [[id, result.node, result.lane, result.commandId, observerCommand.type, result.status, observer.queued === true ? "queued" : observer.phase ?? observer.status ?? "-"]], + ), + "", + webObserveTable( + ["INPUT", "VALUE"], + [ + ["label", observerCommand.label ?? "-"], + ["path", observerCommand.path ?? "-"], + ["provider", observerCommand.provider ?? "-"], + ["sessionId", observerCommand.sessionId ?? "-"], + ["text", observerCommand.textHash === null || observerCommand.textHash === undefined ? "-" : `${observerCommand.textBytes ?? "-"}B ${observerCommand.textHash}`], + ], + ), + "", + "NEXT", + ` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`, + ` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function renderWebObserveAnalyzeResult(result: Record): Record { + const analysis = record(result.analysis); + const counts = record(analysis.counts); + const sampleMetrics = record(analysis.sampleMetrics); + const runtimeAlerts = record(analysis.runtimeAlerts); + const pagePerformance = record(analysis.pagePerformance); + const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item)); + const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item)); + const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item)); + const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item)); + const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item)); + const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item)); + const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item)); + const id = result.id ?? "-"; + const renderedText = [ + webObserveTable( + ["OBSERVER", "NODE", "LANE", "STATUS", "REPORT_JSON", "REPORT_MD"], + [[id, result.node, result.lane, result.status, analysis.reportJsonSha256, analysis.reportMdSha256]], + ), + "", + webObserveTable( + ["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ROUNDS", "ROWS"], + [[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]], + ), + "", + jsonlReadIssues.length === 0 + ? "JSONL_READ_ISSUES\n-" + : webObserveTable( + ["FILE", "CODE", "MESSAGE"], + jsonlReadIssues.map((item) => [item.file, item.code, item.message]), + ), + "", + webObserveTable( + ["TURN_TIMING", "VALUE"], + [ + ["nonMonotonic", sampleMetrics.turnTimingNonMonotonicCount], + ["elapsedDecrease", sampleMetrics.turnTimingTotalElapsedDecreaseCount], + ["recentJump", sampleMetrics.turnTimingRecentUpdateJumpCount], + ["maxRecentStepSec", sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds], + ], + ), + "", + rounds.length === 0 + ? "ROUNDS\n-" + : webObserveTable( + ["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "DIAG", "TERM", "NONMONO", "JUMP"], + rounds.map((item) => [ + item.promptIndex, + item.promptCommandId, + item.sampleCount, + item.maxTotalElapsedSeconds, + item.lastTotalElapsedSeconds, + item.maxRecentUpdateSeconds, + item.lastRecentUpdateSeconds, + item.diagnosticSamples, + item.terminalSamples, + item.turnTimingNonMonotonicCount, + item.turnTimingRecentUpdateJumpCount, + ]), + ), + "", + turnColumns.length === 0 + ? "TURN_COLUMNS\n-" + : webObserveTable( + ["TURN", "PROMPT", "TRACE", "MESSAGE", "FIRST_SEQ", "LAST_SEQ", "LAST_TS"], + turnColumns.map((item) => [item.label ?? item.id, item.lastPromptIndex ?? item.promptIndex, item.traceId, item.messageId, item.firstSeq, item.lastSeq, item.lastTs]), + ), + "", + webObserveTable( + ["ALERT", "COUNT"], + [ + ["httpError", runtimeAlerts.httpErrorCount], + ["requestFailed", runtimeAlerts.requestFailedCount], + ["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount], + ["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount], + ["consoleAlerts", runtimeAlerts.consoleAlertCount], + ["executionErrors", runtimeAlerts.executionErrorCount], + ], + ), + "", + domDiagnosticGroups.length === 0 + ? "DOM_DIAGNOSTIC_GROUPS\n-" + : webObserveTable( + ["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"], + domDiagnosticGroups.map((item) => [ + item.diagnosticCode, + item.count, + item.firstSeq, + item.lastSeq, + item.firstAt, + item.lastAt, + webObserveArray(item.promptIndexes).join(",") || "-", + item.traceId, + String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140), + ]), + ), + "", + domDiagnostics.length === 0 + ? "DOM_DIAGNOSTICS\n-" + : webObserveTable( + ["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"], + domDiagnostics.map((item) => [ + item.seq, + item.ts, + item.promptIndex, + item.diagnosticCode, + item.traceId, + item.httpStatus, + item.idleSeconds, + item.waitingFor, + item.lastEventLabel, + String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180), + ]), + ), + "", + webObserveTable( + ["PERF", "VALUE"], + [ + ["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount], + ["slowPathCount", pagePerformance.slowPathCount], + ["slowSampleCount", pagePerformance.slowSampleCount], + ["worstP95Ms", pagePerformance.worstP95Ms], + ], + ), + "", + slowApis.length === 0 + ? "SLOW_API\n-" + : webObserveTable( + ["SLOW_API", "P50", "P75", "P95", ">5S", "COUNT"], + slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overFiveSecondCount, item.sampleCount]), + ), + "", + findings.length === 0 + ? "FINDINGS\n-" + : webObserveTable( + ["FINDING", "SEVERITY", "COUNT", "SUMMARY"], + findings.map((item) => [item.id, item.severity, item.count, item.summary]), + ), + "", + "REPORTS", + ` json: ${analysis.reportJsonPath ?? "-"}`, + ` md: ${analysis.reportMdPath ?? "-"}`, + "", + "NEXT", + ` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function renderWebProbeScriptResult(result: Record): Record { + const summary = record(result.summary); + const issueEvidence = record(result.issueEvidence); + const probe = record(result.probe); + const script = record(probe.script); + const scriptResult = record(script.result); + const reportLoad = record(result.reportLoad); + const commandResult = record(result.result); + const recoveredArtifacts = record(result.recoveredArtifacts); + const recoveredArtifactSummary = record(recoveredArtifacts.artifacts); + const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item)); + const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item)); + const resultRows = webProbeScriptRecordRows(scriptResult, 12); + const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12); + const summaryRows = [ + ["ok", result.ok], + ["status", result.status], + ["degradedReason", result.degradedReason ?? "-"], + ["failureKind", result.failureKind ?? "-"], + ["scriptSource", result.scriptSource ?? "-"], + ["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"], + ["reportPath", reportLoad.path ?? probe.reportPath ?? "-"], + ["reportSha256", probe.reportSha256 ?? "-"], + ["scriptSha256", probe.scriptSha256 ?? "-"], + ["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"], + ["exitCode", commandResult.exitCode ?? "-"], + ]; + const renderedText = [ + webObserveTable( + ["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"], + [[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]], + ), + "", + webObserveTable(["FIELD", "VALUE"], summaryRows), + "", + resultRows.length === 0 + ? "SCRIPT_RESULT\n-" + : webObserveTable(["KEY", "VALUE"], resultRows), + "", + evidenceRows.length === 0 + ? "ISSUE_EVIDENCE\n-" + : webObserveTable(["KEY", "VALUE"], evidenceRows), + "", + steps.length === 0 + ? "STEPS\n-" + : webObserveTable( + ["STEP", "OK", "DETAIL"], + steps.map((item) => [ + item.name ?? item.step ?? item.label ?? "-", + item.ok ?? item.status ?? "-", + webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item), + ]), + ), + "", + recoveredItems.length === 0 + ? "ARTIFACTS\n-" + : webObserveTable( + ["KIND", "BYTES", "PATH"], + recoveredItems.map((item) => [item.kind, item.byteCount, item.path]), + ), + "", + "NEXT", + ` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`, + ` rerun: ${result.command ?? `hwlab nodes web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function webProbeScriptRecordRows(value: Record, limit: number): unknown[][] { + return Object.entries(value) + .slice(0, limit) + .map(([key, nested]) => [key, webProbeScriptPreview(nested)]); +} + +function webProbeScriptPreview(value: unknown): string { + if (value === null || value === undefined) return "-"; + if (typeof value === "string") return value.replace(/\s+/gu, " ").trim(); + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (Array.isArray(value)) { + const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", "); + return `[${value.length}] ${preview}`; + } + if (typeof value === "object") { + return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240); + } + return String(value); +} + function withWebObserveShortcuts(value: Record | null, id: string | null): Record | null { if (value === null || id === null) return value; return { @@ -7838,7 +8188,7 @@ function runNodeWebProbeScript( compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]); compactResult.stderrTail = redactKnownSecrets(result.stderr.slice(-2000), [material.password ?? ""]); } - return { + return renderWebProbeScriptResult({ ok: passed, status: passed ? "pass" : "blocked", command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`, @@ -7867,7 +8217,7 @@ function runNodeWebProbeScript( }, result: compactResult, valuesRedacted: true, - }; + }); } function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string { diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 1be28a6e..46ff2e74 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -1080,7 +1080,13 @@ const report = { await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, readIssues: jsonlReadIssues.slice(0, 20), domDiagnosticGroups: runtimeAlerts.domDiagnosticsByText.slice(0, 20), turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +const compactRuntimeAlerts = { + ...runtimeAlerts.summary, + domDiagnostics: runtimeAlerts.domDiagnostics.slice(-80), + domDiagnosticsByText: runtimeAlerts.domDiagnosticsByText.slice(0, 80), + domDiagnosticsByFingerprint: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 80), +}; +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, jsonlReadIssues: jsonlReadIssues.slice(0, 20), readIssues: jsonlReadIssues.slice(0, 20), sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: compactRuntimeAlerts, domDiagnosticGroups: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 20), turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } @@ -1725,8 +1731,9 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { }, networkHttpErrorsByPath: groupNetworkAlerts(httpErrors), networkRequestFailedByPath: groupNetworkAlerts(requestFailed), - domDiagnostics: domDiagnostics.slice(0, 80), + domDiagnostics: domDiagnostics.slice(-80), domDiagnosticsByText: groupDomDiagnostics(domDiagnostics), + domDiagnosticsByFingerprint: groupDomDiagnostics(domDiagnostics).slice(0, 80), runtimeExecutionErrors: executionErrors.slice(0, 120), runtimeExecutionErrorsByCode: groupExecutionErrors(executionErrors), baselineRuntimeExecutionErrors: baselineExecutionErrors.slice(0, 80), diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index 8f93c45c..aa9cf3f0 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -679,17 +679,24 @@ async function trace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); + const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null; + const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080); + const effectiveTempoQuery = options.query ?? (businessTraceId === null ? inferSearchTempoQuery(options) : `{ .traceId = "${businessTraceId}" }`); + const effectiveOptions: SearchOptions = { + ...options, + query: effectiveTempoQuery, + lookbackMinutes: effectiveLookbackMinutes, + }; const endSeconds = Math.floor(Date.now() / 1000); - const startSeconds = endSeconds - (options.lookbackMinutes * 60); - const tempoQuery = options.query ?? inferSearchTempoQuery(options); + const startSeconds = endSeconds - (effectiveLookbackMinutes * 60); const params = new URLSearchParams({ start: String(startSeconds), end: String(endSeconds), limit: String(options.candidateLimit), }); - if (tempoQuery !== null) params.set("q", tempoQuery); + if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery); const searchPath = `/api/search?${params.toString()}`; - const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, options)); + const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && parsed?.ok === true; const query = { @@ -697,11 +704,14 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); @@ -876,7 +891,8 @@ function renderSearchTable(input: { "Summary:", ` target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)}`, ` grep=${textValue(input.query.grep)} path=${textValue(input.query.pathFilter)} status=${textValue(input.query.statusFilter)} tempoQuery=${shortenMiddle(textValue(input.query.tempoQuery), 80)}`, - ` lookbackMinutes=${textValue(input.query.lookbackMinutes)} candidateLimit=${textValue(input.query.candidateLimit)} limit=${textValue(input.query.limit)}`, + ` mode=${textValue(input.query.mode)} businessTraceId=${textValue(input.query.businessTraceId)} lookbackMinutes=${textValue(input.query.lookbackMinutes)} requestedLookbackMinutes=${textValue(input.query.requestedLookbackMinutes)}`, + ` candidateLimit=${textValue(input.query.candidateLimit)} limit=${textValue(input.query.limit)}`, ` candidates=${textValue(input.result.candidateTraceCount)} scanned=${textValue(input.result.scannedTraceCount)} matched=${textValue(input.result.matchedTraceCount)} stopped=${textValue(input.result.scanStopped)}`, ]; const firstTraceId = traces.length > 0 ? textValue(traces[0].traceId) : ""; @@ -1009,6 +1025,7 @@ function spanDetail(span: Record, maxLength: number): string { attrPart(attrs, "outputBytes"), attrPart(attrs, "durationMs"), attrPart(attrs, "cwd"), + attrPart(attrs, "command"), attrPart(attrs, "waitingFor"), attrPart(attrs, "lastEventLabel"), attrPart(attrs, "idleMs"), @@ -2192,7 +2209,7 @@ function searchScript(observability: ObservabilityConfig, target: ObservabilityT return ` set -u python3 - <<'PY' -import collections, json, subprocess, time +import collections, json, re, subprocess, time SEARCH_PROXY_PATH = ${JSON.stringify(searchProxyPath)} TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)} @@ -2206,6 +2223,7 @@ STATUS_FILTER = ${statusLiteral} LIMIT = ${options.limit} CANDIDATE_LIMIT = ${options.candidateLimit} DEADLINE = time.time() + 50 +BUSINESS_TRACE_GREP = bool(GREP is not None and re.search(r"\\btrc_[A-Za-z0-9_-]+\\b", GREP)) IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", @@ -2498,6 +2516,19 @@ def trace_summary(trace_id, meta, body, rc, stderr): "stderrTail": (stderr or "")[-1000:], } +def rich_trace_rank(summary): + services = set(summary.get("services") or []) + service_score = 0 + for service in ("agentrun-runner", "agentrun-manager", "hwlab-cloud-api"): + if service in services: + service_score += 1 + return ( + service_score, + int(summary.get("spanCount") or 0), + int(summary.get("errorSpanCount") or 0), + int(summary.get("matchedSpanCount") or 0), + ) + search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15) search_parsed = parse_json(search_body) search_parse_ok = isinstance(search_parsed, dict) @@ -2526,8 +2557,10 @@ for trace_id, meta in candidate_trace_ids: scanned.append(summary) if summary.get("matchingActive") is not True or summary.get("rawMatched") is True or (summary.get("matchedSpanCount") or 0) > 0: matched.append(summary) - if len(matched) >= LIMIT and summary.get("matchingActive") is True: + if len(matched) >= LIMIT and GREP is not None and not BUSINESS_TRACE_GREP: break +if BUSINESS_TRACE_GREP: + matched.sort(key=rich_trace_rank, reverse=True) payload = { "ok": search_rc == 0 and search_parse_ok, @@ -2537,6 +2570,7 @@ payload = { "tempoQuery": QUERY, "pathFilter": PATH_FILTER, "statusFilter": STATUS_FILTER, + "businessTraceSearch": BUSINESS_TRACE_GREP, "limit": LIMIT, "candidateLimit": CANDIDATE_LIMIT, "searchParseOk": search_parse_ok,