mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #617 from pikasTech/fix/web-probe-otel-agentrun-timeout-20260621

Browse Source 
修复 web-probe/OTel 诊断输出并注入 AgentRun 缺 terminal 超时
This commit is contained in:
Lyon
2026-06-22 02:03:23 +08:00
committed by GitHub
parent 38594174da 9e8499824f
commit 2454aec9c3
6 changed files with 409 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/agentrun.yaml
+2
View File
@@ -151,6 +151,7 @@ controlPlane:
serviceAccount: agentrun-v01-runner
jobNamePrefix: agentrun-v01-runner
idleTimeoutMs: 600000
missingTerminalAfterToolTimeoutMs: 60000
apiKeySecretRef:
name: agentrun-v01-api-key
key: HWLAB_API_KEY
@@ -324,6 +325,7 @@ controlPlane:
serviceAccount: agentrun-v02-runner
jobNamePrefix: agentrun-v02-runner
idleTimeoutMs: 172800000
missingTerminalAfterToolTimeoutMs: 30000
apiKeySecretRef:
name: agentrun-v02-api-key
key: HWLAB_API_KEY
scripts/src/agentrun-lanes.ts
+2
View File
@@ -104,6 +104,7 @@ export interface AgentRunLaneSpec {
readonly serviceAccount: string;
readonly jobNamePrefix: string;
readonly idleTimeoutMs: number;
readonly missingTerminalAfterToolTimeoutMs: number;
readonly apiKeySecretRef: { readonly name: string; readonly key: string };
readonly egressProxyUrl: string | null;
readonly noProxyExtra: readonly string[];
@@ -541,6 +542,7 @@ function parseDeployment(input: Record<string, unknown>, path: string): AgentRun
serviceAccount: stringField(runner, "serviceAccount", `${path}.runner`),
jobNamePrefix: stringField(runner, "jobNamePrefix", `${path}.runner`),
idleTimeoutMs: positiveIntegerField(runner, "idleTimeoutMs", `${path}.runner`),
missingTerminalAfterToolTimeoutMs: positiveIntegerField(runner, "missingTerminalAfterToolTimeoutMs", `${path}.runner`),
apiKeySecretRef: parseSecretRef(recordField(runner, "apiKeySecretRef", `${path}.runner`), `${path}.runner.apiKeySecretRef`),
egressProxyUrl: optionalStringField(runner, "egressProxyUrl", `${path}.runner`) ?? null,
noProxyExtra: optionalStringArrayField(runner, "noProxyExtra", `${path}.runner`),
scripts/src/agentrun-manifests.ts
+1
View File
@@ -439,6 +439,7 @@ function managerEnv(spec: AgentRunLaneSpec, sourceCommit: string, imageRef: stri
{ name: "AGENTRUN_RUNNER_SERVICE_ACCOUNT", value: spec.deployment.runner.serviceAccount },
{ name: "AGENTRUN_RUNNER_JOB_NAME_PREFIX", value: spec.deployment.runner.jobNamePrefix },
{ name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(spec.deployment.runner.idleTimeoutMs) },
{ name: "AGENTRUN_RUNNER_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS", value: String(spec.deployment.runner.missingTerminalAfterToolTimeoutMs) },
{ name: "AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", value: String(spec.deployment.runner.retention.maxRunners) },
{ name: "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER", value: spec.deployment.runner.retention.cleanupOrder },
{ name: "AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", value: String(spec.deployment.runner.retention.activeHeartbeatMaxAgeMs) },
scripts/src/hwlab-node-impl.ts
+352 -2
View File
@@ -7666,6 +7666,356 @@ function webObserveNextCommands(id: string): Record<string, string> {
};
}
function webObserveCell(value: unknown, maxLength = 96): string {
if (value === null || value === undefined) return "-";
const text = typeof value === "string" ? value : String(value);
const compact = text.replace(/\s+/gu, " ").trim();
if (compact.length === 0) return "-";
if (compact.length <= maxLength) return compact;
return `${compact.slice(0, Math.max(1, maxLength - 1))}...`;
}
function webObserveTable(headers: string[], rows: unknown[][]): string {
const stringRows = rows.map((row) => row.map((value) => webObserveCell(value)));
const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0)));
const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd();
return [renderRow(headers), ...stringRows.map(renderRow)].join("\n");
}
function webObserveArray(value: unknown): unknown[] {
return Array.isArray(value) ? value : [];
}
function withWebObserveRendered(result: Record<string, unknown>, renderedText: string): Record<string, unknown> {
return {
...result,
renderedText,
contentType: "text/plain",
};
}
function renderWebObserveStatusResult(result: Record<string, unknown>): Record<string, unknown> {
const observer = record(result.observer);
const commandResult = record(result.result);
const manifest = record(observer.manifest);
const heartbeat = record(observer.heartbeat);
const tails = record(observer.tails);
const samples = webObserveArray(tails.samples);
const network = webObserveArray(tails.network);
const control = webObserveArray(tails.control);
const artifacts = webObserveArray(tails.artifacts);
const lastSample = record(samples[samples.length - 1]);
const failedNetwork = network
.map((item) => record(item))
.filter((item) => item.type === "requestfailed" || typeof item.status === "number" && item.status >= 500);
const id = result.id ?? observer.id ?? manifest.jobId ?? heartbeat.jobId ?? "-";
const resultSection = result.ok === true ? [] : [
"",
webObserveTable(
["RESULT", "VALUE"],
[
["exitCode", commandResult.exitCode ?? "-"],
["timedOut", commandResult.timedOut ?? "-"],
["stdoutTail", commandResult.stdoutTail ?? commandResult.stdout ?? "-"],
["stderrTail", commandResult.stderrTail ?? commandResult.stderr ?? "-"],
],
),
];
const renderedText = [
webObserveTable(
["OBSERVER", "NODE", "LANE", "STATUS", "ALIVE", "PID", "SAMPLE", "COMMAND", "UPDATED"],
[[id, result.node, result.lane, manifest.status ?? heartbeat.status ?? result.status, observer.processAlive, observer.pid, heartbeat.sampleSeq, heartbeat.commandSeq, heartbeat.updatedAt]],
),
"",
webObserveTable(
["URL", "ROUTE_SESSION", "ACTIVE_SESSION", "MESSAGES", "TRACE_ROWS"],
[[heartbeat.currentUrl, lastSample.routeSessionId, lastSample.activeSessionId, lastSample.messageCount, lastSample.traceRowCount]],
),
"",
webObserveTable(
["TAIL", "COUNT", "DETAIL"],
[
["control", control.length, record(control[control.length - 1]).type ?? "-"],
["samples", samples.length, record(samples[samples.length - 1]).ts ?? "-"],
["network", network.length, failedNetwork.length === 0 ? "no recent failed/5xx tail" : `${failedNetwork.length} recent failed/5xx`],
["artifacts", artifacts.length, record(artifacts[artifacts.length - 1]).sha256 ?? "-"],
],
),
...resultSection,
"",
"NEXT",
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
` command: bun scripts/cli.ts hwlab nodes web-probe observe command ${id} --type mark --label checkpoint`,
].join("\n");
return withWebObserveRendered(result, renderedText);
}
function renderWebObserveCommandResult(result: Record<string, unknown>): Record<string, unknown> {
const observerCommand = record(result.observerCommand);
const observer = record(result.observer);
const id = result.id ?? "-";
const renderedText = [
webObserveTable(
["OBSERVER", "NODE", "LANE", "COMMAND_ID", "TYPE", "STATUS", "DETAIL"],
[[id, result.node, result.lane, result.commandId, observerCommand.type, result.status, observer.queued === true ? "queued" : observer.phase ?? observer.status ?? "-"]],
),
"",
webObserveTable(
["INPUT", "VALUE"],
[
["label", observerCommand.label ?? "-"],
["path", observerCommand.path ?? "-"],
["provider", observerCommand.provider ?? "-"],
["sessionId", observerCommand.sessionId ?? "-"],
["text", observerCommand.textHash === null || observerCommand.textHash === undefined ? "-" : `${observerCommand.textBytes ?? "-"}B ${observerCommand.textHash}`],
],
),
"",
"NEXT",
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
].join("\n");
return withWebObserveRendered(result, renderedText);
}
function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<string, unknown> {
const analysis = record(result.analysis);
const counts = record(analysis.counts);
const sampleMetrics = record(analysis.sampleMetrics);
const runtimeAlerts = record(analysis.runtimeAlerts);
const pagePerformance = record(analysis.pagePerformance);
const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item));
const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item));
const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item));
const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item));
const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item));
const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item));
const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item));
const id = result.id ?? "-";
const renderedText = [
webObserveTable(
["OBSERVER", "NODE", "LANE", "STATUS", "REPORT_JSON", "REPORT_MD"],
[[id, result.node, result.lane, result.status, analysis.reportJsonSha256, analysis.reportMdSha256]],
),
"",
webObserveTable(
["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ROUNDS", "ROWS"],
[[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]],
),
"",
jsonlReadIssues.length === 0
? "JSONL_READ_ISSUES\n-"
: webObserveTable(
["FILE", "CODE", "MESSAGE"],
jsonlReadIssues.map((item) => [item.file, item.code, item.message]),
),
"",
webObserveTable(
["TURN_TIMING", "VALUE"],
[
["nonMonotonic", sampleMetrics.turnTimingNonMonotonicCount],
["elapsedDecrease", sampleMetrics.turnTimingTotalElapsedDecreaseCount],
["recentJump", sampleMetrics.turnTimingRecentUpdateJumpCount],
["maxRecentStepSec", sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds],
],
),
"",
rounds.length === 0
? "ROUNDS\n-"
: webObserveTable(
["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "DIAG", "TERM", "NONMONO", "JUMP"],
rounds.map((item) => [
item.promptIndex,
item.promptCommandId,
item.sampleCount,
item.maxTotalElapsedSeconds,
item.lastTotalElapsedSeconds,
item.maxRecentUpdateSeconds,
item.lastRecentUpdateSeconds,
item.diagnosticSamples,
item.terminalSamples,
item.turnTimingNonMonotonicCount,
item.turnTimingRecentUpdateJumpCount,
]),
),
"",
turnColumns.length === 0
? "TURN_COLUMNS\n-"
: webObserveTable(
["TURN", "PROMPT", "TRACE", "MESSAGE", "FIRST_SEQ", "LAST_SEQ", "LAST_TS"],
turnColumns.map((item) => [item.label ?? item.id, item.lastPromptIndex ?? item.promptIndex, item.traceId, item.messageId, item.firstSeq, item.lastSeq, item.lastTs]),
),
"",
webObserveTable(
["ALERT", "COUNT"],
[
["httpError", runtimeAlerts.httpErrorCount],
["requestFailed", runtimeAlerts.requestFailedCount],
["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount],
["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount],
["consoleAlerts", runtimeAlerts.consoleAlertCount],
["executionErrors", runtimeAlerts.executionErrorCount],
],
),
"",
domDiagnosticGroups.length === 0
? "DOM_DIAGNOSTIC_GROUPS\n-"
: webObserveTable(
["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"],
domDiagnosticGroups.map((item) => [
item.diagnosticCode,
item.count,
item.firstSeq,
item.lastSeq,
item.firstAt,
item.lastAt,
webObserveArray(item.promptIndexes).join(",") || "-",
item.traceId,
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140),
]),
),
"",
domDiagnostics.length === 0
? "DOM_DIAGNOSTICS\n-"
: webObserveTable(
["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"],
domDiagnostics.map((item) => [
item.seq,
item.ts,
item.promptIndex,
item.diagnosticCode,
item.traceId,
item.httpStatus,
item.idleSeconds,
item.waitingFor,
item.lastEventLabel,
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180),
]),
),
"",
webObserveTable(
["PERF", "VALUE"],
[
["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount],
["slowPathCount", pagePerformance.slowPathCount],
["slowSampleCount", pagePerformance.slowSampleCount],
["worstP95Ms", pagePerformance.worstP95Ms],
],
),
"",
slowApis.length === 0
? "SLOW_API\n-"
: webObserveTable(
["SLOW_API", "P50", "P75", "P95", ">5S", "COUNT"],
slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overFiveSecondCount, item.sampleCount]),
),
"",
findings.length === 0
? "FINDINGS\n-"
: webObserveTable(
["FINDING", "SEVERITY", "COUNT", "SUMMARY"],
findings.map((item) => [item.id, item.severity, item.count, item.summary]),
),
"",
"REPORTS",
` json: ${analysis.reportJsonPath ?? "-"}`,
` md: ${analysis.reportMdPath ?? "-"}`,
"",
"NEXT",
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
].join("\n");
return withWebObserveRendered(result, renderedText);
}
function renderWebProbeScriptResult(result: Record<string, unknown>): Record<string, unknown> {
const summary = record(result.summary);
const issueEvidence = record(result.issueEvidence);
const probe = record(result.probe);
const script = record(probe.script);
const scriptResult = record(script.result);
const reportLoad = record(result.reportLoad);
const commandResult = record(result.result);
const recoveredArtifacts = record(result.recoveredArtifacts);
const recoveredArtifactSummary = record(recoveredArtifacts.artifacts);
const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item));
const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item));
const resultRows = webProbeScriptRecordRows(scriptResult, 12);
const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12);
const summaryRows = [
["ok", result.ok],
["status", result.status],
["degradedReason", result.degradedReason ?? "-"],
["failureKind", result.failureKind ?? "-"],
["scriptSource", result.scriptSource ?? "-"],
["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"],
["reportPath", reportLoad.path ?? probe.reportPath ?? "-"],
["reportSha256", probe.reportSha256 ?? "-"],
["scriptSha256", probe.scriptSha256 ?? "-"],
["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"],
["exitCode", commandResult.exitCode ?? "-"],
];
const renderedText = [
webObserveTable(
["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"],
[[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]],
),
"",
webObserveTable(["FIELD", "VALUE"], summaryRows),
"",
resultRows.length === 0
? "SCRIPT_RESULT\n-"
: webObserveTable(["KEY", "VALUE"], resultRows),
"",
evidenceRows.length === 0
? "ISSUE_EVIDENCE\n-"
: webObserveTable(["KEY", "VALUE"], evidenceRows),
"",
steps.length === 0
? "STEPS\n-"
: webObserveTable(
["STEP", "OK", "DETAIL"],
steps.map((item) => [
item.name ?? item.step ?? item.label ?? "-",
item.ok ?? item.status ?? "-",
webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item),
]),
),
"",
recoveredItems.length === 0
? "ARTIFACTS\n-"
: webObserveTable(
["KIND", "BYTES", "PATH"],
recoveredItems.map((item) => [item.kind, item.byteCount, item.path]),
),
"",
"NEXT",
` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`,
` rerun: ${result.command ?? `hwlab nodes web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`,
].join("\n");
return withWebObserveRendered(result, renderedText);
}
function webProbeScriptRecordRows(value: Record<string, unknown>, limit: number): unknown[][] {
return Object.entries(value)
.slice(0, limit)
.map(([key, nested]) => [key, webProbeScriptPreview(nested)]);
}
function webProbeScriptPreview(value: unknown): string {
if (value === null || value === undefined) return "-";
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim();
if (typeof value === "number" || typeof value === "boolean") return String(value);
if (Array.isArray(value)) {
const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", ");
return `[${value.length}] ${preview}`;
}
if (typeof value === "object") {
return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240);
}
return String(value);
}
function withWebObserveShortcuts(value: Record<string, unknown> | null, id: string | null): Record<string, unknown> | null {
if (value === null || id === null) return value;
return {
@@ -7838,7 +8188,7 @@ function runNodeWebProbeScript(
compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]);
compactResult.stderrTail = redactKnownSecrets(result.stderr.slice(-2000), [material.password ?? ""]);
}
return {
return renderWebProbeScriptResult({
ok: passed,
status: passed ? "pass" : "blocked",
command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`,
@@ -7867,7 +8217,7 @@ function runNodeWebProbeScript(
},
result: compactResult,
valuesRedacted: true,
};
});
}
function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string {
scripts/src/hwlab-node-web-observe-runner-source.ts
+9 -2
View File
@@ -1080,7 +1080,13 @@ const report = {
await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 });
await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 });
const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]);
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, readIssues: jsonlReadIssues.slice(0, 20), domDiagnosticGroups: runtimeAlerts.domDiagnosticsByText.slice(0, 20), turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
const compactRuntimeAlerts = {
...runtimeAlerts.summary,
domDiagnostics: runtimeAlerts.domDiagnostics.slice(-80),
domDiagnosticsByText: runtimeAlerts.domDiagnosticsByText.slice(0, 80),
domDiagnosticsByFingerprint: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 80),
};
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, jsonlReadIssues: jsonlReadIssues.slice(0, 20), readIssues: jsonlReadIssues.slice(0, 20), sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: compactRuntimeAlerts, domDiagnosticGroups: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 20), turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
async function readJson(file) {
try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; }
@@ -1725,8 +1731,9 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) {
},
networkHttpErrorsByPath: groupNetworkAlerts(httpErrors),
networkRequestFailedByPath: groupNetworkAlerts(requestFailed),
domDiagnostics: domDiagnostics.slice(0, 80),
domDiagnostics: domDiagnostics.slice(-80),
domDiagnosticsByText: groupDomDiagnostics(domDiagnostics),
domDiagnosticsByFingerprint: groupDomDiagnostics(domDiagnostics).slice(0, 80),
runtimeExecutionErrors: executionErrors.slice(0, 120),
runtimeExecutionErrorsByCode: groupExecutionErrors(executionErrors),
baselineRuntimeExecutionErrors: baselineExecutionErrors.slice(0, 80),
scripts/src/platform-infra-observability.ts
+43 -9
View File
@@ -679,17 +679,24 @@ async function trace(config: UniDeskConfig, options: TraceOptions): Promise<Reco
async function search(config: UniDeskConfig, options: SearchOptions): Promise<Record<string, unknown> | RenderedCliResult> {
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null;
const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080);
const effectiveTempoQuery = options.query ?? (businessTraceId === null ? inferSearchTempoQuery(options) : `{ .traceId = "${businessTraceId}" }`);
const effectiveOptions: SearchOptions = {
...options,
query: effectiveTempoQuery,
lookbackMinutes: effectiveLookbackMinutes,
};
const endSeconds = Math.floor(Date.now() / 1000);
const startSeconds = endSeconds - (options.lookbackMinutes * 60);
const tempoQuery = options.query ?? inferSearchTempoQuery(options);
const startSeconds = endSeconds - (effectiveLookbackMinutes * 60);
const params = new URLSearchParams({
start: String(startSeconds),
end: String(endSeconds),
limit: String(options.candidateLimit),
});
if (tempoQuery !== null) params.set("q", tempoQuery);
if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery);
const searchPath = `/api/search?${params.toString()}`;
const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, options));
const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions));
const parsed = parseJsonOutput(result.stdout);
const ok = result.exitCode === 0 && parsed?.ok === true;
const query = {
@@ -697,11 +704,14 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise<Re
service: observability.traceBackend.serviceName,
path: searchPath,
grep: options.grep,
tempoQuery,
tempoQuery: effectiveTempoQuery,
explicitTempoQuery: options.query,
pathFilter: options.path,
statusFilter: options.status,
lookbackMinutes: options.lookbackMinutes,
lookbackMinutes: effectiveLookbackMinutes,
requestedLookbackMinutes: options.lookbackMinutes,
businessTraceId,
mode: businessTraceId === null ? "candidate-grep" : "business-trace-exact",
candidateLimit: options.candidateLimit,
limit: options.limit,
};
@@ -733,6 +743,11 @@ function inferSearchTempoQuery(options: SearchOptions): string | null {
return null;
}
function businessTraceIdFromSearchText(value: string | null): string | null {
const match = value?.match(/\btrc_[A-Za-z0-9_-]+\b/u);
return match?.[0] ?? null;
}
async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise<Record<string, unknown> | RenderedCliResult> {
const observability = readObservabilityConfig();
const target = resolveTarget(observability, options.targetId);
@@ -876,7 +891,8 @@ function renderSearchTable(input: {
"Summary:",
` target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)}`,
` grep=${textValue(input.query.grep)} path=${textValue(input.query.pathFilter)} status=${textValue(input.query.statusFilter)} tempoQuery=${shortenMiddle(textValue(input.query.tempoQuery), 80)}`,
` lookbackMinutes=${textValue(input.query.lookbackMinutes)} candidateLimit=${textValue(input.query.candidateLimit)} limit=${textValue(input.query.limit)}`,
` mode=${textValue(input.query.mode)} businessTraceId=${textValue(input.query.businessTraceId)} lookbackMinutes=${textValue(input.query.lookbackMinutes)} requestedLookbackMinutes=${textValue(input.query.requestedLookbackMinutes)}`,
` candidateLimit=${textValue(input.query.candidateLimit)} limit=${textValue(input.query.limit)}`,
` candidates=${textValue(input.result.candidateTraceCount)} scanned=${textValue(input.result.scannedTraceCount)} matched=${textValue(input.result.matchedTraceCount)} stopped=${textValue(input.result.scanStopped)}`,
];
const firstTraceId = traces.length > 0 ? textValue(traces[0].traceId) : "";
@@ -1009,6 +1025,7 @@ function spanDetail(span: Record<string, unknown>, maxLength: number): string {
attrPart(attrs, "outputBytes"),
attrPart(attrs, "durationMs"),
attrPart(attrs, "cwd"),
attrPart(attrs, "command"),
attrPart(attrs, "waitingFor"),
attrPart(attrs, "lastEventLabel"),
attrPart(attrs, "idleMs"),
@@ -2192,7 +2209,7 @@ function searchScript(observability: ObservabilityConfig, target: ObservabilityT
return `
set -u
python3 - <<'PY'
import collections, json, subprocess, time
import collections, json, re, subprocess, time
SEARCH_PROXY_PATH = ${JSON.stringify(searchProxyPath)}
TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)}
@@ -2206,6 +2223,7 @@ STATUS_FILTER = ${statusLiteral}
LIMIT = ${options.limit}
CANDIDATE_LIMIT = ${options.candidateLimit}
DEADLINE = time.time() + 50
BUSINESS_TRACE_GREP = bool(GREP is not None and re.search(r"\\btrc_[A-Za-z0-9_-]+\\b", GREP))
IMPORTANT_ATTRS = [
"traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
"sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
@@ -2498,6 +2516,19 @@ def trace_summary(trace_id, meta, body, rc, stderr):
"stderrTail": (stderr or "")[-1000:],
}
def rich_trace_rank(summary):
services = set(summary.get("services") or [])
service_score = 0
for service in ("agentrun-runner", "agentrun-manager", "hwlab-cloud-api"):
if service in services:
service_score += 1
return (
service_score,
int(summary.get("spanCount") or 0),
int(summary.get("errorSpanCount") or 0),
int(summary.get("matchedSpanCount") or 0),
)
search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15)
search_parsed = parse_json(search_body)
search_parse_ok = isinstance(search_parsed, dict)
@@ -2526,8 +2557,10 @@ for trace_id, meta in candidate_trace_ids:
scanned.append(summary)
if summary.get("matchingActive") is not True or summary.get("rawMatched") is True or (summary.get("matchedSpanCount") or 0) > 0:
matched.append(summary)
if len(matched) >= LIMIT and summary.get("matchingActive") is True:
if len(matched) >= LIMIT and GREP is not None and not BUSINESS_TRACE_GREP:
break
if BUSINESS_TRACE_GREP:
matched.sort(key=rich_trace_rank, reverse=True)
payload = {
"ok": search_rc == 0 and search_parse_ok,
@@ -2537,6 +2570,7 @@ payload = {
"tempoQuery": QUERY,
"pathFilter": PATH_FILTER,
"statusFilter": STATUS_FILTER,
"businessTraceSearch": BUSINESS_TRACE_GREP,
"limit": LIMIT,
"candidateLimit": CANDIDATE_LIMIT,
"searchParseOk": search_parse_ok,