mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: expose web probe and agentrun timeout diagnostics

Browse Source
This commit is contained in:
Codex
2026-06-21 17:56:40 +00:00
parent a08dfea9da
commit 1789700f23
6 changed files with 262 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/agentrun.yaml
+2
View File
@@ -151,6 +151,7 @@ controlPlane:
serviceAccount: agentrun-v01-runner
jobNamePrefix: agentrun-v01-runner
idleTimeoutMs: 600000
missingTerminalAfterToolTimeoutMs: 60000
apiKeySecretRef:
name: agentrun-v01-api-key
key: HWLAB_API_KEY
@@ -324,6 +325,7 @@ controlPlane:
serviceAccount: agentrun-v02-runner
jobNamePrefix: agentrun-v02-runner
idleTimeoutMs: 172800000
missingTerminalAfterToolTimeoutMs: 30000
apiKeySecretRef:
name: agentrun-v02-api-key
key: HWLAB_API_KEY
scripts/src/agentrun-lanes.ts
+2
View File
@@ -104,6 +104,7 @@ export interface AgentRunLaneSpec {
readonly serviceAccount: string;
readonly jobNamePrefix: string;
readonly idleTimeoutMs: number;
readonly missingTerminalAfterToolTimeoutMs: number;
readonly apiKeySecretRef: { readonly name: string; readonly key: string };
readonly egressProxyUrl: string | null;
readonly noProxyExtra: readonly string[];
@@ -541,6 +542,7 @@ function parseDeployment(input: Record<string, unknown>, path: string): AgentRun
serviceAccount: stringField(runner, "serviceAccount", `${path}.runner`),
jobNamePrefix: stringField(runner, "jobNamePrefix", `${path}.runner`),
idleTimeoutMs: positiveIntegerField(runner, "idleTimeoutMs", `${path}.runner`),
missingTerminalAfterToolTimeoutMs: positiveIntegerField(runner, "missingTerminalAfterToolTimeoutMs", `${path}.runner`),
apiKeySecretRef: parseSecretRef(recordField(runner, "apiKeySecretRef", `${path}.runner`), `${path}.runner.apiKeySecretRef`),
egressProxyUrl: optionalStringField(runner, "egressProxyUrl", `${path}.runner`) ?? null,
noProxyExtra: optionalStringArrayField(runner, "noProxyExtra", `${path}.runner`),
scripts/src/agentrun-manifests.ts
+1
View File
@@ -439,6 +439,7 @@ function managerEnv(spec: AgentRunLaneSpec, sourceCommit: string, imageRef: stri
{ name: "AGENTRUN_RUNNER_SERVICE_ACCOUNT", value: spec.deployment.runner.serviceAccount },
{ name: "AGENTRUN_RUNNER_JOB_NAME_PREFIX", value: spec.deployment.runner.jobNamePrefix },
{ name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(spec.deployment.runner.idleTimeoutMs) },
{ name: "AGENTRUN_RUNNER_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS", value: String(spec.deployment.runner.missingTerminalAfterToolTimeoutMs) },
{ name: "AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", value: String(spec.deployment.runner.retention.maxRunners) },
{ name: "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER", value: spec.deployment.runner.retention.cleanupOrder },
{ name: "AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", value: String(spec.deployment.runner.retention.activeHeartbeatMaxAgeMs) },
scripts/src/hwlab-node-impl.ts
+145 -3
View File
@@ -462,6 +462,7 @@ function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown
image: spec.baseImage,
sourceImage: spec.baseImageSource ?? null,
},
serviceIds: spec.serviceIds,
buildkit: spec.buildkit === undefined ? null : {
sidecarImage: spec.buildkit.sidecarImage,
},
@@ -1813,6 +1814,10 @@ function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: st
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
}
function nodeRuntimeRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
return `${nodeRuntimePipelineRunName(spec, sourceCommit)}-r${Date.now().toString(36)}`.slice(0, 63);
}
function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
const suffix = `/${spec.runtimeRenderDir}`;
if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length);
@@ -2632,7 +2637,8 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
headProbe: compactRuntimeCommand(head.result),
};
}
const pipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
const pipelineRun = scoped.rerun ? nodeRuntimeRerunPipelineRunName(spec, sourceCommit) : basePipelineRun;
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun });
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun });
const before = getNodeRuntimePipelineRun(spec, pipelineRun);
@@ -2647,6 +2653,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
mode: "dry-run",
sourceCommit,
pipelineRun,
rerunOf: scoped.rerun ? basePipelineRun : null,
rerun: scoped.rerun,
before,
gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
@@ -2762,6 +2769,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
mutation: createOk,
sourceCommit,
pipelineRun,
rerunOf: scoped.rerun ? basePipelineRun : null,
rerun: scoped.rerun,
before,
gitMirror,
@@ -7107,6 +7115,9 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
const pagePerformance = record(analysis.pagePerformance);
const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item));
const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item));
const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item));
const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item));
const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item));
const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item));
const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item));
const id = result.id ?? "-";
@@ -7121,6 +7132,13 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
[[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]],
),
"",
jsonlReadIssues.length === 0
? "JSONL_READ_ISSUES\n-"
: webObserveTable(
["FILE", "CODE", "MESSAGE"],
jsonlReadIssues.map((item) => [item.file, item.code, item.message]),
),
"",
webObserveTable(
["TURN_TIMING", "VALUE"],
[
@@ -7163,11 +7181,47 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
["httpError", runtimeAlerts.httpErrorCount],
["requestFailed", runtimeAlerts.requestFailedCount],
["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount],
["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount],
["consoleAlerts", runtimeAlerts.consoleAlertCount],
["executionErrors", runtimeAlerts.executionErrorCount],
],
),
"",
domDiagnosticGroups.length === 0
? "DOM_DIAGNOSTIC_GROUPS\n-"
: webObserveTable(
["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"],
domDiagnosticGroups.map((item) => [
item.diagnosticCode,
item.count,
item.firstSeq,
item.lastSeq,
item.firstAt,
item.lastAt,
webObserveArray(item.promptIndexes).join(",") || "-",
item.traceId,
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140),
]),
),
"",
domDiagnostics.length === 0
? "DOM_DIAGNOSTICS\n-"
: webObserveTable(
["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"],
domDiagnostics.map((item) => [
item.seq,
item.ts,
item.promptIndex,
item.diagnosticCode,
item.traceId,
item.httpStatus,
item.idleSeconds,
item.waitingFor,
item.lastEventLabel,
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180),
]),
),
"",
webObserveTable(
["PERF", "VALUE"],
[
@@ -7202,6 +7256,94 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
return withWebObserveRendered(result, renderedText);
}
function renderWebProbeScriptResult(result: Record<string, unknown>): Record<string, unknown> {
const summary = record(result.summary);
const issueEvidence = record(result.issueEvidence);
const probe = record(result.probe);
const script = record(probe.script);
const scriptResult = record(script.result);
const reportLoad = record(result.reportLoad);
const commandResult = record(result.result);
const recoveredArtifacts = record(result.recoveredArtifacts);
const recoveredArtifactSummary = record(recoveredArtifacts.artifacts);
const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item));
const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item));
const resultRows = webProbeScriptRecordRows(scriptResult, 12);
const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12);
const summaryRows = [
["ok", result.ok],
["status", result.status],
["degradedReason", result.degradedReason ?? "-"],
["failureKind", result.failureKind ?? "-"],
["scriptSource", result.scriptSource ?? "-"],
["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"],
["reportPath", reportLoad.path ?? probe.reportPath ?? "-"],
["reportSha256", probe.reportSha256 ?? "-"],
["scriptSha256", probe.scriptSha256 ?? "-"],
["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"],
["exitCode", commandResult.exitCode ?? "-"],
];
const renderedText = [
webObserveTable(
["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"],
[[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]],
),
"",
webObserveTable(["FIELD", "VALUE"], summaryRows),
"",
resultRows.length === 0
? "SCRIPT_RESULT\n-"
: webObserveTable(["KEY", "VALUE"], resultRows),
"",
evidenceRows.length === 0
? "ISSUE_EVIDENCE\n-"
: webObserveTable(["KEY", "VALUE"], evidenceRows),
"",
steps.length === 0
? "STEPS\n-"
: webObserveTable(
["STEP", "OK", "DETAIL"],
steps.map((item) => [
item.name ?? item.step ?? item.label ?? "-",
item.ok ?? item.status ?? "-",
webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item),
]),
),
"",
recoveredItems.length === 0
? "ARTIFACTS\n-"
: webObserveTable(
["KIND", "BYTES", "PATH"],
recoveredItems.map((item) => [item.kind, item.byteCount, item.path]),
),
"",
"NEXT",
` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`,
` rerun: ${result.command ?? `hwlab nodes web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`,
].join("\n");
return withWebObserveRendered(result, renderedText);
}
function webProbeScriptRecordRows(value: Record<string, unknown>, limit: number): unknown[][] {
return Object.entries(value)
.slice(0, limit)
.map(([key, nested]) => [key, webProbeScriptPreview(nested)]);
}
function webProbeScriptPreview(value: unknown): string {
if (value === null || value === undefined) return "-";
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim();
if (typeof value === "number" || typeof value === "boolean") return String(value);
if (Array.isArray(value)) {
const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", ");
return `[${value.length}] ${preview}`;
}
if (typeof value === "object") {
return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240);
}
return String(value);
}
function withWebObserveShortcuts(value: Record<string, unknown> | null, id: string | null): Record<string, unknown> | null {
if (value === null || id === null) return value;
return {
@@ -7374,7 +7516,7 @@ function runNodeWebProbeScript(
compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]);
compactResult.stderrTail = redactKnownSecrets(result.stderr.slice(-2000), [material.password ?? ""]);
}
return {
return renderWebProbeScriptResult({
ok: passed,
status: passed ? "pass" : "blocked",
command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`,
@@ -7403,7 +7545,7 @@ function runNodeWebProbeScript(
},
result: compactResult,
valuesRedacted: true,
};
});
}
function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string {
scripts/src/hwlab-node-web-observe-runner-source.ts
+70 -7
View File
@@ -982,13 +982,16 @@ function sleep(ms) {
export function nodeWebObserveAnalyzerSource(): string {
return String.raw`#!/usr/bin/env node
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { createInterface } from "node:readline";
const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || process.argv[2] || ".state/web-observe/manual");
const analysisDir = path.join(stateDir, "analysis");
const reportJsonPath = path.join(analysisDir, "report.json");
const reportMdPath = path.join(analysisDir, "report.md");
const jsonlReadIssues = [];
const samples = await readJsonl(path.join(stateDir, "samples.jsonl"));
const control = await readJsonl(path.join(stateDir, "control.jsonl"));
const network = await readJsonl(path.join(stateDir, "network.jsonl"));
@@ -1015,6 +1018,7 @@ const report = {
manifest: compactManifest(manifest),
heartbeat: compactHeartbeat(heartbeat),
counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length },
jsonlReadIssues,
commandTimeline,
transitions,
sampleMetrics,
@@ -1029,19 +1033,36 @@ const report = {
await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 });
await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 });
const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]);
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
const compactRuntimeAlerts = {
...runtimeAlerts.summary,
domDiagnostics: runtimeAlerts.domDiagnostics.slice(-80),
domDiagnosticsByFingerprint: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 80),
};
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, jsonlReadIssues: jsonlReadIssues.slice(0, 20), sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: compactRuntimeAlerts, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
async function readJson(file) {
try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; }
}
async function readJsonl(file) {
const rows = [];
try {
const text = await readFile(file, "utf8");
return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
try { return JSON.parse(line); } catch { return { parseError: true, rawHash: sha256(line) }; }
});
} catch { return []; }
const input = createReadStream(file, { encoding: "utf8" });
const lines = createInterface({ input, crlfDelay: Infinity });
for await (const line of lines) {
const trimmed = String(line || "").trim();
if (!trimmed) continue;
try {
rows.push(JSON.parse(trimmed));
} catch {
rows.push({ parseError: true, rawHash: sha256(trimmed) });
}
}
} catch (error) {
if (error?.code === "ENOENT") return rows;
jsonlReadIssues.push({ file: path.basename(file), code: error?.code ?? error?.name ?? "jsonl_read_failed", message: String(error?.message || "JSONL read failed").slice(0, 240), valuesPrinted: false });
}
return rows;
}
function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance) {
@@ -1414,6 +1435,10 @@ function parseDomDiagnosticSummary(text) {
const lastEventLabelMatch = value.match(/\blastEventLabel=([^\s；;，,)]+)/iu);
const diagnosticCode = httpStatusMatch
? "http-" + httpStatusMatch[1]
: /projection-resume:sync-failed/iu.test(value)
? "projection-resume-sync-failed"
: /AgentRun\s+GET\b[\s\S]*\/result\b[\s\S]*timed out after\s+\d+ms/iu.test(value)
? "agentrun-result-timeout"
: /turn\s*超过|无新活动/iu.test(value)
? "turn-idle-no-activity"
: /Failed to fetch/iu.test(value)
@@ -1564,13 +1589,15 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) {
pageErrorCount: pageErrors.length,
networkErrorGroupCount: groupNetworkAlerts(httpErrors).length,
requestFailedGroupCount: groupNetworkAlerts(requestFailed).length,
domDiagnosticGroupCount: groupDomDiagnostics(domDiagnostics).length,
executionErrorGroupCount: groupExecutionErrors(executionErrors).length,
baselineExecutionErrorGroupCount: groupExecutionErrors(baselineExecutionErrors).length,
consoleAlertGroupCount: groupConsoleAlerts(consoleAlerts).length
},
networkHttpErrorsByPath: groupNetworkAlerts(httpErrors),
networkRequestFailedByPath: groupNetworkAlerts(requestFailed),
domDiagnostics: domDiagnostics.slice(0, 80),
domDiagnostics: domDiagnostics.slice(-80),
domDiagnosticsByFingerprint: groupDomDiagnostics(domDiagnostics).slice(0, 80),
runtimeExecutionErrors: executionErrors.slice(0, 120),
runtimeExecutionErrorsByCode: groupExecutionErrors(executionErrors),
baselineRuntimeExecutionErrors: baselineExecutionErrors.slice(0, 80),
@@ -1676,6 +1703,42 @@ function groupExecutionErrors(events) {
return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.code).localeCompare(String(b.code)));
}
function groupDomDiagnostics(events) {
const groups = new Map();
for (const event of events) {
const key = [event.diagnosticCode || "-", event.traceId || "-", event.textHash || "-"].join(" ");
const group = groups.get(key) || {
diagnosticCode: event.diagnosticCode ?? null,
traceId: event.traceId ?? null,
textHash: event.textHash ?? null,
count: 0,
firstSeq: event.seq ?? null,
lastSeq: event.seq ?? null,
firstAt: event.ts ?? null,
lastAt: event.ts ?? null,
promptIndexes: [],
sources: [],
httpStatus: event.httpStatus ?? null,
idleSeconds: event.idleSeconds ?? null,
waitingFor: event.waitingFor ?? null,
lastEventLabel: event.lastEventLabel ?? null,
preview: event.preview ?? null,
valuesRedacted: true
};
group.count += 1;
group.lastSeq = event.seq ?? group.lastSeq;
group.lastAt = event.ts ?? group.lastAt;
if (event.promptIndex && !group.promptIndexes.includes(event.promptIndex)) group.promptIndexes.push(event.promptIndex);
if (event.source && !group.sources.includes(event.source)) group.sources.push(event.source);
if (event.httpStatus && !group.httpStatus) group.httpStatus = event.httpStatus;
if (event.idleSeconds && !group.idleSeconds) group.idleSeconds = event.idleSeconds;
if (event.waitingFor && !group.waitingFor) group.waitingFor = event.waitingFor;
if (event.lastEventLabel && !group.lastEventLabel) group.lastEventLabel = event.lastEventLabel;
groups.set(key, group);
}
return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(b.lastAt ?? "").localeCompare(String(a.lastAt ?? "")));
}
function consoleAlertEvent(item, promptTimes) {
const text = String(item?.text || "");
const statusMatch = text.match(/\bstatus\s+of\s+([1-5][0-9]{2})\b/iu) || text.match(/\bHTTP\s+([1-5][0-9]{2})\b/iu);
scripts/src/platform-infra-observability.ts
+42 -9
View File
@@ -648,16 +648,24 @@ async function trace(config: UniDeskConfig, options: TraceOptions): Promise<Reco
async function search(config: UniDeskConfig, options: SearchOptions): Promise<Record<string, unknown> | RenderedCliResult> {
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null;
const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080);
const effectiveTempoQuery = options.query ?? (businessTraceId === null ? null : `{ .traceId = "${businessTraceId}" }`);
const effectiveOptions: SearchOptions = {
...options,
query: effectiveTempoQuery,
lookbackMinutes: effectiveLookbackMinutes,
};
const endSeconds = Math.floor(Date.now() / 1000);
const startSeconds = endSeconds - (options.lookbackMinutes * 60);
const startSeconds = endSeconds - (effectiveLookbackMinutes * 60);
const params = new URLSearchParams({
start: String(startSeconds),
end: String(endSeconds),
limit: String(options.candidateLimit),
});
if (options.query !== null) params.set("q", options.query);
if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery);
const searchPath = `/api/search?${params.toString()}`;
const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, options));
const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions));
const parsed = parseJsonOutput(result.stdout);
const response = {
ok: result.exitCode === 0 && parsed?.ok === true,
@@ -669,8 +677,11 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise<Re
service: observability.traceBackend.serviceName,
path: searchPath,
grep: options.grep,
tempoQuery: options.query,
lookbackMinutes: options.lookbackMinutes,
tempoQuery: effectiveTempoQuery,
lookbackMinutes: effectiveLookbackMinutes,
requestedLookbackMinutes: options.lookbackMinutes,
businessTraceId,
mode: businessTraceId === null ? "candidate-grep" : "business-trace-exact",
candidateLimit: options.candidateLimit,
limit: options.limit,
},
@@ -680,6 +691,11 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise<Re
return renderSearchResult(response);
}
function businessTraceIdFromSearchText(value: string | null): string | null {
const match = value?.match(/\btrc_[A-Za-z0-9_-]+\b/u);
return match?.[0] ?? null;
}
async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise<Record<string, unknown> | RenderedCliResult> {
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
@@ -811,6 +827,9 @@ function renderSearchResult(response: Record<string, unknown>): RenderedCliResul
const lines: string[] = [];
lines.push("COMMAND TARGET OK GREP_OR_QUERY MATCHED SCANNED CANDIDATES");
lines.push(`${pad("platform-infra observability search", 38)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(truncate(textValue(query.grep) || textValue(query.tempoQuery) || "-", 35), 35)} ${pad(textValue(result.matchedTraceCount) || "-", 8)} ${pad(textValue(result.scannedTraceCount) || "-", 8)} ${textValue(result.candidateTraceCount) || "-"}`);
if (textValue(query.mode) === "business-trace-exact") {
lines.push(`info: businessTraceId=${textValue(query.businessTraceId) || "-"} uses exact TraceQL and lookbackMinutes=${textValue(query.lookbackMinutes) || "-"} (requested=${textValue(query.requestedLookbackMinutes) || "-"})`);
}
if (textValue(result.scanStopped)) lines.push(`warning: scanStopped=${textValue(result.scanStopped)}`);
const truncated = recordValue(result.truncated);
if (truncated.candidateTraces === true || truncated.matchedTraces === true) {
@@ -880,7 +899,7 @@ function renderTraceResult(response: Record<string, unknown>): RenderedCliResult
lines.push("SERVICE DURATION_MS NAME DETAIL");
for (const item of errorSpans.slice(0, 8)) {
const row = recordValue(item);
lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`);
lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 160)}`);
}
}
if (matchedSpans.length > 0) {
@@ -889,7 +908,7 @@ function renderTraceResult(response: Record<string, unknown>): RenderedCliResult
lines.push("SERVICE DURATION_MS NAME DETAIL");
for (const item of matchedSpans.slice(0, 8)) {
const row = recordValue(item);
lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`);
lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 160)}`);
}
}
lines.push("");
@@ -914,6 +933,7 @@ function traceSpanDetail(span: Record<string, unknown>): string {
"toolName",
"exitCode",
"durationMs",
"command",
"waitingFor",
"lastEventLabel",
"idleMs",
@@ -1799,7 +1819,7 @@ function searchScript(observability: ObservabilityConfig, target: ObservabilityT
return `
set -u
python3 - <<'PY'
import collections, json, subprocess, time
import collections, json, re, subprocess, time
SEARCH_PROXY_PATH = ${JSON.stringify(searchProxyPath)}
TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)}
@@ -1811,6 +1831,7 @@ QUERY = ${queryLiteral}
LIMIT = ${options.limit}
CANDIDATE_LIMIT = ${options.candidateLimit}
DEADLINE = time.time() + 50
BUSINESS_TRACE_GREP = bool(GREP is not None and re.search(r"\\btrc_[A-Za-z0-9_-]+\\b", GREP))
IMPORTANT_ATTRS = [
"traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
"sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
@@ -2037,6 +2058,15 @@ def trace_summary(trace_id, meta, body, rc, stderr):
"stderrTail": (stderr or "")[-1000:],
}
def rich_trace_rank(summary):
services = summary.get("services", [])
if not isinstance(services, list):
services = []
span_count = summary.get("spanCount") if isinstance(summary.get("spanCount"), int) else 0
error_count = summary.get("errorSpanCount") if isinstance(summary.get("errorSpanCount"), int) else 0
matched_count = summary.get("matchedSpanCount") if isinstance(summary.get("matchedSpanCount"), int) else 0
return (len(services), span_count, error_count, matched_count)
search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15)
search_parsed = parse_json(search_body)
search_parse_ok = isinstance(search_parsed, dict)
@@ -2065,8 +2095,10 @@ for trace_id, meta in candidate_trace_ids:
scanned.append(summary)
if GREP is None or summary.get("rawMatched") is True or (summary.get("matchedSpanCount") or 0) > 0:
matched.append(summary)
if len(matched) >= LIMIT and GREP is not None:
if len(matched) >= LIMIT and GREP is not None and not BUSINESS_TRACE_GREP:
break
if BUSINESS_TRACE_GREP:
matched.sort(key=rich_trace_rank, reverse=True)
payload = {
"ok": search_rc == 0 and search_parse_ok,
@@ -2074,6 +2106,7 @@ payload = {
"searchProxyPath": SEARCH_PROXY_PATH,
"grep": GREP,
"tempoQuery": QUERY,
"businessTraceSearch": BUSINESS_TRACE_GREP,
"limit": LIMIT,
"candidateLimit": CANDIDATE_LIMIT,
"searchParseOk": search_parse_ok,