fix: add web probe otel agentrun timeout diagnostics
This commit is contained in:
Codex
@@ -7666,6 +7666,356 @@ function webObserveNextCommands(id: string): Record<string, string> {
|
||||
};
|
||||
}
|
||||
|
||||
function webObserveCell(value: unknown, maxLength = 96): string {
|
||||
if (value === null || value === undefined) return "-";
|
||||
const text = typeof value === "string" ? value : String(value);
|
||||
const compact = text.replace(/\s+/gu, " ").trim();
|
||||
if (compact.length === 0) return "-";
|
||||
if (compact.length <= maxLength) return compact;
|
||||
return `${compact.slice(0, Math.max(1, maxLength - 1))}...`;
|
||||
}
|
||||
|
||||
function webObserveTable(headers: string[], rows: unknown[][]): string {
|
||||
const stringRows = rows.map((row) => row.map((value) => webObserveCell(value)));
|
||||
const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0)));
|
||||
const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd();
|
||||
return [renderRow(headers), ...stringRows.map(renderRow)].join("\n");
|
||||
}
|
||||
|
||||
function webObserveArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function withWebObserveRendered(result: Record<string, unknown>, renderedText: string): Record<string, unknown> {
|
||||
return {
|
||||
...result,
|
||||
renderedText,
|
||||
contentType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
function renderWebObserveStatusResult(result: Record<string, unknown>): Record<string, unknown> {
|
||||
const observer = record(result.observer);
|
||||
const commandResult = record(result.result);
|
||||
const manifest = record(observer.manifest);
|
||||
const heartbeat = record(observer.heartbeat);
|
||||
const tails = record(observer.tails);
|
||||
const samples = webObserveArray(tails.samples);
|
||||
const network = webObserveArray(tails.network);
|
||||
const control = webObserveArray(tails.control);
|
||||
const artifacts = webObserveArray(tails.artifacts);
|
||||
const lastSample = record(samples[samples.length - 1]);
|
||||
const failedNetwork = network
|
||||
.map((item) => record(item))
|
||||
.filter((item) => item.type === "requestfailed" || typeof item.status === "number" && item.status >= 500);
|
||||
const id = result.id ?? observer.id ?? manifest.jobId ?? heartbeat.jobId ?? "-";
|
||||
const resultSection = result.ok === true ? [] : [
|
||||
"",
|
||||
webObserveTable(
|
||||
["RESULT", "VALUE"],
|
||||
[
|
||||
["exitCode", commandResult.exitCode ?? "-"],
|
||||
["timedOut", commandResult.timedOut ?? "-"],
|
||||
["stdoutTail", commandResult.stdoutTail ?? commandResult.stdout ?? "-"],
|
||||
["stderrTail", commandResult.stderrTail ?? commandResult.stderr ?? "-"],
|
||||
],
|
||||
),
|
||||
];
|
||||
const renderedText = [
|
||||
webObserveTable(
|
||||
["OBSERVER", "NODE", "LANE", "STATUS", "ALIVE", "PID", "SAMPLE", "COMMAND", "UPDATED"],
|
||||
[[id, result.node, result.lane, manifest.status ?? heartbeat.status ?? result.status, observer.processAlive, observer.pid, heartbeat.sampleSeq, heartbeat.commandSeq, heartbeat.updatedAt]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["URL", "ROUTE_SESSION", "ACTIVE_SESSION", "MESSAGES", "TRACE_ROWS"],
|
||||
[[heartbeat.currentUrl, lastSample.routeSessionId, lastSample.activeSessionId, lastSample.messageCount, lastSample.traceRowCount]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["TAIL", "COUNT", "DETAIL"],
|
||||
[
|
||||
["control", control.length, record(control[control.length - 1]).type ?? "-"],
|
||||
["samples", samples.length, record(samples[samples.length - 1]).ts ?? "-"],
|
||||
["network", network.length, failedNetwork.length === 0 ? "no recent failed/5xx tail" : `${failedNetwork.length} recent failed/5xx`],
|
||||
["artifacts", artifacts.length, record(artifacts[artifacts.length - 1]).sha256 ?? "-"],
|
||||
],
|
||||
),
|
||||
...resultSection,
|
||||
"",
|
||||
"NEXT",
|
||||
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
||||
` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
|
||||
` command: bun scripts/cli.ts hwlab nodes web-probe observe command ${id} --type mark --label checkpoint`,
|
||||
].join("\n");
|
||||
return withWebObserveRendered(result, renderedText);
|
||||
}
|
||||
|
||||
function renderWebObserveCommandResult(result: Record<string, unknown>): Record<string, unknown> {
|
||||
const observerCommand = record(result.observerCommand);
|
||||
const observer = record(result.observer);
|
||||
const id = result.id ?? "-";
|
||||
const renderedText = [
|
||||
webObserveTable(
|
||||
["OBSERVER", "NODE", "LANE", "COMMAND_ID", "TYPE", "STATUS", "DETAIL"],
|
||||
[[id, result.node, result.lane, result.commandId, observerCommand.type, result.status, observer.queued === true ? "queued" : observer.phase ?? observer.status ?? "-"]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["INPUT", "VALUE"],
|
||||
[
|
||||
["label", observerCommand.label ?? "-"],
|
||||
["path", observerCommand.path ?? "-"],
|
||||
["provider", observerCommand.provider ?? "-"],
|
||||
["sessionId", observerCommand.sessionId ?? "-"],
|
||||
["text", observerCommand.textHash === null || observerCommand.textHash === undefined ? "-" : `${observerCommand.textBytes ?? "-"}B ${observerCommand.textHash}`],
|
||||
],
|
||||
),
|
||||
"",
|
||||
"NEXT",
|
||||
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
||||
` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
|
||||
].join("\n");
|
||||
return withWebObserveRendered(result, renderedText);
|
||||
}
|
||||
|
||||
function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<string, unknown> {
|
||||
const analysis = record(result.analysis);
|
||||
const counts = record(analysis.counts);
|
||||
const sampleMetrics = record(analysis.sampleMetrics);
|
||||
const runtimeAlerts = record(analysis.runtimeAlerts);
|
||||
const pagePerformance = record(analysis.pagePerformance);
|
||||
const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item));
|
||||
const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item));
|
||||
const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item));
|
||||
const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item));
|
||||
const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item));
|
||||
const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item));
|
||||
const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item));
|
||||
const id = result.id ?? "-";
|
||||
const renderedText = [
|
||||
webObserveTable(
|
||||
["OBSERVER", "NODE", "LANE", "STATUS", "REPORT_JSON", "REPORT_MD"],
|
||||
[[id, result.node, result.lane, result.status, analysis.reportJsonSha256, analysis.reportMdSha256]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ROUNDS", "ROWS"],
|
||||
[[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]],
|
||||
),
|
||||
"",
|
||||
jsonlReadIssues.length === 0
|
||||
? "JSONL_READ_ISSUES\n-"
|
||||
: webObserveTable(
|
||||
["FILE", "CODE", "MESSAGE"],
|
||||
jsonlReadIssues.map((item) => [item.file, item.code, item.message]),
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["TURN_TIMING", "VALUE"],
|
||||
[
|
||||
["nonMonotonic", sampleMetrics.turnTimingNonMonotonicCount],
|
||||
["elapsedDecrease", sampleMetrics.turnTimingTotalElapsedDecreaseCount],
|
||||
["recentJump", sampleMetrics.turnTimingRecentUpdateJumpCount],
|
||||
["maxRecentStepSec", sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds],
|
||||
],
|
||||
),
|
||||
"",
|
||||
rounds.length === 0
|
||||
? "ROUNDS\n-"
|
||||
: webObserveTable(
|
||||
["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "DIAG", "TERM", "NONMONO", "JUMP"],
|
||||
rounds.map((item) => [
|
||||
item.promptIndex,
|
||||
item.promptCommandId,
|
||||
item.sampleCount,
|
||||
item.maxTotalElapsedSeconds,
|
||||
item.lastTotalElapsedSeconds,
|
||||
item.maxRecentUpdateSeconds,
|
||||
item.lastRecentUpdateSeconds,
|
||||
item.diagnosticSamples,
|
||||
item.terminalSamples,
|
||||
item.turnTimingNonMonotonicCount,
|
||||
item.turnTimingRecentUpdateJumpCount,
|
||||
]),
|
||||
),
|
||||
"",
|
||||
turnColumns.length === 0
|
||||
? "TURN_COLUMNS\n-"
|
||||
: webObserveTable(
|
||||
["TURN", "PROMPT", "TRACE", "MESSAGE", "FIRST_SEQ", "LAST_SEQ", "LAST_TS"],
|
||||
turnColumns.map((item) => [item.label ?? item.id, item.lastPromptIndex ?? item.promptIndex, item.traceId, item.messageId, item.firstSeq, item.lastSeq, item.lastTs]),
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["ALERT", "COUNT"],
|
||||
[
|
||||
["httpError", runtimeAlerts.httpErrorCount],
|
||||
["requestFailed", runtimeAlerts.requestFailedCount],
|
||||
["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount],
|
||||
["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount],
|
||||
["consoleAlerts", runtimeAlerts.consoleAlertCount],
|
||||
["executionErrors", runtimeAlerts.executionErrorCount],
|
||||
],
|
||||
),
|
||||
"",
|
||||
domDiagnosticGroups.length === 0
|
||||
? "DOM_DIAGNOSTIC_GROUPS\n-"
|
||||
: webObserveTable(
|
||||
["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"],
|
||||
domDiagnosticGroups.map((item) => [
|
||||
item.diagnosticCode,
|
||||
item.count,
|
||||
item.firstSeq,
|
||||
item.lastSeq,
|
||||
item.firstAt,
|
||||
item.lastAt,
|
||||
webObserveArray(item.promptIndexes).join(",") || "-",
|
||||
item.traceId,
|
||||
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140),
|
||||
]),
|
||||
),
|
||||
"",
|
||||
domDiagnostics.length === 0
|
||||
? "DOM_DIAGNOSTICS\n-"
|
||||
: webObserveTable(
|
||||
["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"],
|
||||
domDiagnostics.map((item) => [
|
||||
item.seq,
|
||||
item.ts,
|
||||
item.promptIndex,
|
||||
item.diagnosticCode,
|
||||
item.traceId,
|
||||
item.httpStatus,
|
||||
item.idleSeconds,
|
||||
item.waitingFor,
|
||||
item.lastEventLabel,
|
||||
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180),
|
||||
]),
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["PERF", "VALUE"],
|
||||
[
|
||||
["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount],
|
||||
["slowPathCount", pagePerformance.slowPathCount],
|
||||
["slowSampleCount", pagePerformance.slowSampleCount],
|
||||
["worstP95Ms", pagePerformance.worstP95Ms],
|
||||
],
|
||||
),
|
||||
"",
|
||||
slowApis.length === 0
|
||||
? "SLOW_API\n-"
|
||||
: webObserveTable(
|
||||
["SLOW_API", "P50", "P75", "P95", ">5S", "COUNT"],
|
||||
slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overFiveSecondCount, item.sampleCount]),
|
||||
),
|
||||
"",
|
||||
findings.length === 0
|
||||
? "FINDINGS\n-"
|
||||
: webObserveTable(
|
||||
["FINDING", "SEVERITY", "COUNT", "SUMMARY"],
|
||||
findings.map((item) => [item.id, item.severity, item.count, item.summary]),
|
||||
),
|
||||
"",
|
||||
"REPORTS",
|
||||
` json: ${analysis.reportJsonPath ?? "-"}`,
|
||||
` md: ${analysis.reportMdPath ?? "-"}`,
|
||||
"",
|
||||
"NEXT",
|
||||
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
||||
].join("\n");
|
||||
return withWebObserveRendered(result, renderedText);
|
||||
}
|
||||
|
||||
function renderWebProbeScriptResult(result: Record<string, unknown>): Record<string, unknown> {
|
||||
const summary = record(result.summary);
|
||||
const issueEvidence = record(result.issueEvidence);
|
||||
const probe = record(result.probe);
|
||||
const script = record(probe.script);
|
||||
const scriptResult = record(script.result);
|
||||
const reportLoad = record(result.reportLoad);
|
||||
const commandResult = record(result.result);
|
||||
const recoveredArtifacts = record(result.recoveredArtifacts);
|
||||
const recoveredArtifactSummary = record(recoveredArtifacts.artifacts);
|
||||
const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item));
|
||||
const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item));
|
||||
const resultRows = webProbeScriptRecordRows(scriptResult, 12);
|
||||
const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12);
|
||||
const summaryRows = [
|
||||
["ok", result.ok],
|
||||
["status", result.status],
|
||||
["degradedReason", result.degradedReason ?? "-"],
|
||||
["failureKind", result.failureKind ?? "-"],
|
||||
["scriptSource", result.scriptSource ?? "-"],
|
||||
["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"],
|
||||
["reportPath", reportLoad.path ?? probe.reportPath ?? "-"],
|
||||
["reportSha256", probe.reportSha256 ?? "-"],
|
||||
["scriptSha256", probe.scriptSha256 ?? "-"],
|
||||
["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"],
|
||||
["exitCode", commandResult.exitCode ?? "-"],
|
||||
];
|
||||
const renderedText = [
|
||||
webObserveTable(
|
||||
["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"],
|
||||
[[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(["FIELD", "VALUE"], summaryRows),
|
||||
"",
|
||||
resultRows.length === 0
|
||||
? "SCRIPT_RESULT\n-"
|
||||
: webObserveTable(["KEY", "VALUE"], resultRows),
|
||||
"",
|
||||
evidenceRows.length === 0
|
||||
? "ISSUE_EVIDENCE\n-"
|
||||
: webObserveTable(["KEY", "VALUE"], evidenceRows),
|
||||
"",
|
||||
steps.length === 0
|
||||
? "STEPS\n-"
|
||||
: webObserveTable(
|
||||
["STEP", "OK", "DETAIL"],
|
||||
steps.map((item) => [
|
||||
item.name ?? item.step ?? item.label ?? "-",
|
||||
item.ok ?? item.status ?? "-",
|
||||
webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item),
|
||||
]),
|
||||
),
|
||||
"",
|
||||
recoveredItems.length === 0
|
||||
? "ARTIFACTS\n-"
|
||||
: webObserveTable(
|
||||
["KIND", "BYTES", "PATH"],
|
||||
recoveredItems.map((item) => [item.kind, item.byteCount, item.path]),
|
||||
),
|
||||
"",
|
||||
"NEXT",
|
||||
` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`,
|
||||
` rerun: ${result.command ?? `hwlab nodes web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`,
|
||||
].join("\n");
|
||||
return withWebObserveRendered(result, renderedText);
|
||||
}
|
||||
|
||||
function webProbeScriptRecordRows(value: Record<string, unknown>, limit: number): unknown[][] {
|
||||
return Object.entries(value)
|
||||
.slice(0, limit)
|
||||
.map(([key, nested]) => [key, webProbeScriptPreview(nested)]);
|
||||
}
|
||||
|
||||
function webProbeScriptPreview(value: unknown): string {
|
||||
if (value === null || value === undefined) return "-";
|
||||
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim();
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
||||
if (Array.isArray(value)) {
|
||||
const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", ");
|
||||
return `[${value.length}] ${preview}`;
|
||||
}
|
||||
if (typeof value === "object") {
|
||||
return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240);
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function withWebObserveShortcuts(value: Record<string, unknown> | null, id: string | null): Record<string, unknown> | null {
|
||||
if (value === null || id === null) return value;
|
||||
return {
|
||||
@@ -7838,7 +8188,7 @@ function runNodeWebProbeScript(
|
||||
compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]);
|
||||
compactResult.stderrTail = redactKnownSecrets(result.stderr.slice(-2000), [material.password ?? ""]);
|
||||
}
|
||||
return {
|
||||
return renderWebProbeScriptResult({
|
||||
ok: passed,
|
||||
status: passed ? "pass" : "blocked",
|
||||
command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`,
|
||||
@@ -7867,7 +8217,7 @@ function runNodeWebProbeScript(
|
||||
},
|
||||
result: compactResult,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string {
|
||||
|
||||
Reference in New Issue
Block a user