diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index 913184ae..a6d0c5d1 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -84,6 +84,7 @@ export function hwlabNodeWebProbeHelp(): Record { "bun scripts/cli.ts hwlab nodes web-probe observe command webobs-xxxx --type steer --text '继续观察当前 trace'", "bun scripts/cli.ts hwlab nodes web-probe observe command webobs-xxxx --type cancel", "bun scripts/cli.ts hwlab nodes web-probe observe status webobs-xxxx", + "bun scripts/cli.ts hwlab nodes web-probe observe stop webobs-xxxx --force", "bun scripts/cli.ts hwlab nodes web-probe observe collect webobs-xxxx --view turn-summary", "bun scripts/cli.ts hwlab nodes web-probe observe collect webobs-xxxx --view trace-frame --trace-id trc_xxx --sample-seq 42", "bun scripts/cli.ts hwlab nodes web-probe observe analyze webobs-xxxx", @@ -100,9 +101,11 @@ export function hwlabNodeWebProbeHelp(): Record { "Issue-ready evidence is available under issueEvidence and summary.issueEvidence; full script report is persisted under probe.reportPath with a SHA-256 fingerprint.", "observe sampling is passive by default: it records DOM summaries and natural page request/response/requestfailed events with observerInitiated=false; it does not actively fetch Workbench APIs, reload, switch sessions, route/intercept, or call repair helpers.", "observe start registers a local UniDesk-side observer id under .state/web-observe/index.json; after start, prefer observe status|command|stop|collect|analyze instead of repeating --node/--lane/--state-dir.", + "observe status reports runner liveness separately from process existence, including heartbeat age, stale threshold, command backlog, and abandoned command counts.", "observe command actions are explicit user/control actions and are appended to control.jsonl; use --type newSession/selectProvider/sendPrompt/steer/cancel/goto/screenshot/mark/stop. steer/cancel reuse the Workbench composer path, not private backend APIs. For long prompts use sendPrompt/steer --text-stdin; keep prompt text out of issue comments by citing textHash/textBytes.", + "observe stop --force first checks heartbeat/backlog health; if the runner is stale or commands are not being consumed, it kills the recorded PID from outside the command queue and marks pending/processing commands abandoned so analyze classifies them as tooling findings.", "observe collect --view turn-summary renders the multi-turn CLI reading layer from samples/control artifacts; --view trace-frame renders one sampled trace frame with a fixed Final Response block. collect views do not save a second source of truth.", - "observe analyze is offline-only: it reads artifact JSONL and writes analysis/report.md plus analysis/report.json without accessing Workbench APIs or driving the browser.", + "observe analyze is offline-only: it reads artifact JSONL plus observer command/heartbeat artifacts and writes analysis/report.md plus analysis/report.json without accessing Workbench APIs or driving the browser.", "observe analyze scans every sampled DOM point, extracts Workbench timing text such as 总耗时/total and 最近 N 秒/分前, and writes a sample point vs turn timing report: each Markdown table row starts with the timestamp, followed by each turn's 总耗时(s) and 最近更新(s). Timing series are reported for post-processing/manual analysis instead of auto-judged from status tail output.", "observe analyze also reports visible “加载中” count, owner attribution, concurrent loading owners, and continuous visible segments; fixes must reduce real loading latency, not reveal incomplete content early to make this metric disappear.", "script/observe support --browser-proxy-mode auto|direct; use direct for A/B evidence when frontend RUM is slow but OTel server spans are absent or fast, instead of falling back to raw Playwright.", diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 792ee4df..62327309 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -8074,13 +8074,7 @@ function runNodeWebProbeObserveStart( } function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record | RenderedCliResult { - const script = [ - "set -eu", - nodeWebObserveResolveStateDirShell(options), - nodeWebObserveStatusNodeScript(options.tailLines, options.node, options.lane), - ].join("\n"); - const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); - const status = parseJsonObject(result.stdout); + const { result, status } = readNodeWebProbeObserveRemoteStatus(options, spec, options.tailLines, options.commandTimeoutSeconds); const observerId = webObserveIdFromStatus(status, options); const statusReadable = status !== null; const ok = result.exitCode === 0 && statusReadable && status.ok !== false; @@ -8113,6 +8107,21 @@ function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: }); } +function readNodeWebProbeObserveRemoteStatus( + options: NodeWebProbeObserveOptions, + spec: HwlabRuntimeLaneSpec, + tailLines: number, + timeoutSeconds: number, +): { result: ReturnType; status: Record | null } { + const script = [ + "set -eu", + nodeWebObserveResolveStateDirShell(options), + nodeWebObserveStatusNodeScript(tailLines, options.node, options.lane), + ].join("\n"); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, timeoutSeconds); + return { result, status: parseJsonObject(result.stdout) }; +} + function webObserveText(value: unknown): string { if (value === null || value === undefined || value === "") return "-"; if (typeof value === "string") return value; @@ -8141,42 +8150,45 @@ function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec sessionId: options.commandSessionId, provider: options.commandProvider, }; + const preStopStatus = options.force && stopCommand + ? readNodeWebProbeObserveRemoteStatus(options, spec, 1, Math.min(options.commandTimeoutSeconds, 30)) + : null; + const preStopDiagnostics = record(preStopStatus?.status?.diagnostics); + const preStopCommands = record(preStopStatus?.status?.commands); + const preStopPending = Number(preStopCommands?.pendingCount ?? 0); + const preStopProcessing = Number(preStopCommands?.processingCount ?? 0); + const forceBeforeQueueReason = options.force && stopCommand + ? preStopDiagnostics?.heartbeatStale === true + ? "heartbeat-stale" + : preStopPending > 0 || preStopProcessing > 0 + ? "command-backlog" + : null + : null; + if (forceBeforeQueueReason !== null) { + return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, forceBeforeQueueReason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, null); + } const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); + const waitMs = options.force && stopCommand ? Math.max(options.waitMs, 5000) : options.waitMs; const script = [ "set -eu", nodeWebObserveResolveStateDirShell(options), "mkdir -p \"$state_dir/commands/pending\"", `node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`, - nodeWebObserveWaitCommandShell(commandId, options.waitMs), + nodeWebObserveWaitCommandShell(commandId, waitMs), ].join("\n"); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const commandResult = parseJsonObject(result.stdout); - if (options.force && stopCommand && result.exitCode !== 0) { - const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [ - "set -eu", - nodeWebObserveResolveStateDirShell(options), - "if [ -f \"$state_dir/pid\" ]; then kill \"$(cat \"$state_dir/pid\")\" >/dev/null 2>&1 || true; fi", - "printf '{\"ok\":true,\"forced\":true,\"stateDir\":\"%s\"}\\n' \"$state_dir\"", - ].join("\n"), 55); - return withWebObserveCommandRendered({ - ok: killResult.exitCode === 0, - status: killResult.exitCode === 0 ? "forced-stop-requested" : "blocked", - command: webObserveCommandLabel("stop", options), - id: webObserveIdFromOptions(options), - node: options.node, - lane: options.lane, - workspace: spec.workspace, - commandId, - observerCommand: commandSummaryForOutput(payload), - gracefulResult: compactCommandResult(result), - forceResult: compactCommandResult(killResult), - full: options.full, - valuesRedacted: true, - }); + if (options.force && stopCommand && (result.exitCode !== 0 || commandResult?.waitTimedOut === true || commandResult?.queued === true)) { + const reason = result.exitCode !== 0 + ? "graceful-stop-failed" + : commandResult?.waitTimedOut === true + ? "graceful-stop-not-consumed" + : "graceful-stop-queued"; + return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, reason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, result); } return withWebObserveCommandRendered({ ok: result.exitCode === 0 && commandResult?.ok !== false, - status: result.exitCode === 0 ? (options.waitMs > 0 ? "completed-or-queued" : "queued") : "blocked", + status: result.exitCode === 0 ? (waitMs > 0 ? "completed-or-queued" : "queued") : "blocked", command: webObserveCommandLabel(stopCommand ? "stop" : "command", options), id: webObserveIdFromOptions(options), node: options.node, @@ -8191,6 +8203,43 @@ function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec }); } +function runNodeWebProbeObserveForceStop( + options: NodeWebProbeObserveOptions, + spec: HwlabRuntimeLaneSpec, + payload: Record, + commandId: string, + reason: string, + preflightResult: ReturnType | null, + preflightStatus: Record | null, + gracefulResult: ReturnType | null, +): Record | RenderedCliResult { + const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [ + "set -eu", + nodeWebObserveResolveStateDirShell(options), + nodeWebObserveForceStopNodeScript(reason, commandId), + ].join("\n"), 55); + const forcePayload = parseJsonObject(killResult.stdout); + return withWebObserveCommandRendered({ + ok: killResult.exitCode === 0 && forcePayload?.ok !== false, + status: killResult.exitCode === 0 && forcePayload?.ok !== false ? "forced-stopped" : "blocked", + command: webObserveCommandLabel("stop", options), + id: webObserveIdFromOptions(options), + node: options.node, + lane: options.lane, + workspace: spec.workspace, + commandId, + observerCommand: commandSummaryForOutput(payload), + observer: forcePayload, + forceReason: reason, + preflightObserver: preflightStatus, + preflightResult: preflightResult === null ? null : compactCommandResult(preflightResult), + gracefulResult: gracefulResult === null ? null : compactCommandResult(gracefulResult), + forceResult: compactCommandResultWithStdoutTail(killResult), + full: options.full, + valuesRedacted: true, + }); +} + function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record | RenderedCliResult { const collectScript = options.collectView === "files" ? nodeWebObserveCollectNodeScript(options.maxFiles, options.collectFile, options.collectFinding, options.collectGrep) @@ -8344,6 +8393,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec "const sourceSseStreams = Array.isArray(source?.pagePerformanceSseStreams) ? source.pagePerformanceSseStreams : (Array.isArray(fullPagePerformance?.sameOriginApiByPath) ? fullPagePerformance.sameOriginApiByPath.filter((item) => isLongLivedApi(item)) : []);", "const combinedFindingSource = firstNonEmptyArray(source?.findings, fullSource?.findings, fullRecentWindow?.findings, source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings);", "const combinedFindings = sortFindings(combinedFindingSource);", + "const toolFindings = sortFindings([...firstArray(source?.toolFindings), ...firstArray(fullSource?.toolFindings), ...combinedFindingSource.filter((item) => String(item?.id ?? item?.kind ?? item?.code ?? '').startsWith('tool-'))]);", "const archiveRedFindings = sortFindings(firstNonEmptyArray(source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings, fullSource?.findings).filter((item) => String(item?.severity ?? item?.level ?? '').toLowerCase() === 'red'));", "const allFindingsForCommands = [...firstArray(source?.findings), ...firstArray(fullSource?.findings), ...firstArray(fullRecentWindow?.findings), ...firstArray(source?.archiveSummary?.redFindings), ...firstArray(fullSource?.archiveSummary?.redFindings)];", "const findingSamplesById = (id) => { for (const item of allFindingsForCommands) { if (String(item?.id ?? item?.kind ?? item?.code ?? '') !== id) continue; if (Array.isArray(item?.samples) && item.samples.length > 0) return item.samples; } return []; };", @@ -8364,6 +8414,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec " pagePerformanceSlowApi: takeHead(sourceSlowApi, 4).map(slimSlowApi),", " archivePagePerformanceSlowApi: takeHead(archiveSlowApi, 8).map(slimSlowApi),", " pagePerformanceSseStreams: takeHead(sourceSseStreams, 4).map((item) => ({ path: item?.path ?? item?.route ?? null, route: item?.route ?? null, sampleCount: item?.sampleCount ?? null, streamOpenSampleCount: item?.streamOpenSampleCount ?? null, streamOpenP95Ms: item?.streamOpenP95Ms ?? null, streamOpenMaxMs: item?.streamOpenMaxMs ?? null, streamOpenBudgetMs: item?.streamOpenBudgetMs ?? null, streamOpenOverBudgetCount: item?.streamOpenOverBudgetCount ?? null, streamOpenOverFiveSecondCount: item?.streamOpenOverFiveSecondCount ?? null, streamLifetimeOverFiveSecondCount: item?.streamLifetimeOverFiveSecondCount ?? null })),", + " toolFindings: takeHead(toolFindings, 8).map(slimFinding),", + " commandState: objectOrNull(source.commandState) || objectOrNull(fullSource?.commandState) || null,", " findings: takeHead(combinedFindings, 8).map(slimFinding),", " archiveRedFindings: takeHead(archiveRedFindings, 8).map(slimFinding),", " httpErrorGroups: takeHead(firstArray(source.httpErrorGroups, fullRecentWindow?.runtimeAlerts?.networkHttpErrorsByPath, fullSource?.httpErrorGroups, fullSource?.runtimeAlerts?.networkHttpErrorsByPath), 4).map(slimNetworkGroup),", @@ -8401,7 +8453,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec " stderrTail: tail(readText(stderrPath))", " }", "};", - "const compactStdoutLimitBytes = 10000;", + "const compactStdoutLimitBytes = 8000;", "const compactOutput = (value) => JSON.stringify(value);", "let output = compactOutput(compact);", "if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {", @@ -8447,6 +8499,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec " runtimeAlerts: compact.runtimeAlerts ?? null,", " pagePerformance: compact.pagePerformance ?? null,", " promptNetwork: compact.promptNetwork ?? null,", + " toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [],", + " commandState: compact.commandState ?? null,", " findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4) : [],", " archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4) : [],", " turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 5) : [],", @@ -8507,6 +8561,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec " } : null,", " runtimeAlerts: compact.runtimeAlerts ?? null,", " pagePerformance: compact.pagePerformance ?? null,", + " toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [],", + " commandState: compact.commandState ?? null,", " findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4) : [],", " archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4) : [],", " pagePerformanceSlowApi: Array.isArray(compact.pagePerformanceSlowApi) ? compact.pagePerformanceSlowApi.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, route: item.route ?? item.path ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? item.p95 ?? null, maxMs: item.maxMs ?? item.max ?? null, budgetMs: item.budgetMs ?? null, overBudgetCount: item.overBudgetCount ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null, slowSamples: Array.isArray(item.slowSamples) ? item.slowSamples.slice(0, 1) : [] })) : [],", @@ -8545,6 +8601,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec " } : null,", " runtimeAlerts: compact.runtimeAlerts ?? null,", " pagePerformance: compact.pagePerformance ?? null,", + " toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [],", + " commandState: compact.commandState ?? null,", " findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],", " archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],", " httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, status: item.status ?? null, path: item.path ?? null, lastAt: item.lastAt ?? null })) : [],", @@ -8566,7 +8624,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec " output = compactOutput(ultratiny);", " }", " if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {", - " output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, jsonlScope: compact.jsonlScope ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ?? null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });", + " output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, jsonlScope: compact.jsonlScope ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ? { redFindingCount: compact.archiveSummary.redFindingCount ?? null, findingCount: compact.archiveSummary.findingCount ?? null, sampleCount: compact.archiveSummary.sampleMetrics?.sampleCount ?? null, slowPathCount: compact.archiveSummary.pagePerformance?.slowPathCount ?? null } : null, toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [], commandState: compact.commandState ? { pendingCount: compact.commandState.pendingCount ?? null, processingCount: compact.commandState.processingCount ?? null, abandonedCount: compact.commandState.abandonedCount ?? null, failedCount: compact.commandState.failedCount ?? null } : null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });", " }", " }", "}", @@ -8692,7 +8750,7 @@ function recoverWebObserveAnalyzeFromArtifacts(options: NodeWebProbeObserveOptio "const archiveSummary = objectOrNull(source.archiveSummary) || {};", "const archiveSampleMetrics = objectOrNull(archiveSummary.sampleMetrics) || objectOrNull(source.sampleMetrics?.summary) || objectOrNull(srcMetrics.summary) || {};", "const slowApis = arr(source.pagePerformanceSlowApi).length > 0 ? arr(source.pagePerformanceSlowApi) : arr(pagePerformance.sameOriginApiByPath).filter((item) => Number(item?.overBudgetCount ?? item?.overFiveSecondCount ?? 0) > 0);", - "const compact = { ok: source.ok === true, command: source.command ?? 'web-probe-observe analyze', stateDir: source.stateDir ?? stateDir, jsonlScope: source.jsonlScope ?? null, alertThresholds: source.alertThresholds ?? null, counts: source.counts ?? null, archiveSummary: { ...archiveSummary, sampleMetrics: archiveSampleMetrics, pagePerformance: objectOrNull(archiveSummary.pagePerformance) || objectOrNull(pagePerformance.summary) || {}, runtimeAlerts: objectOrNull(archiveSummary.runtimeAlerts) || objectOrNull(runtimeAlerts.summary) || {}, redFindings: arr(archiveSummary.redFindings).slice(0, 12).map(slimFinding) }, analysisWindow: source.analysisWindow ?? objectOrNull(recent.summary), sampleMetrics: compactMetrics(srcMetrics), pageProvenance: objectOrNull(source.pageProvenance?.summary) || source.pageProvenance ?? null, pagePerformance: objectOrNull(pagePerformance.summary) || pagePerformance, promptNetwork: objectOrNull(promptNetwork.summary) || promptNetwork, runtimeAlerts: objectOrNull(runtimeAlerts.summary) || runtimeAlerts, runnerErrors: arr(source.runnerErrors).slice(-8), commandFailures: arr(source.commandFailures).slice(-8), httpErrorGroups: arr(source.httpErrorGroups ?? runtimeAlerts.networkHttpErrorsByPath).slice(0, 8).map(slimGroup), requestFailedGroups: arr(source.requestFailedGroups ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 8).map(slimGroup), domDiagnosticGroups: arr(source.domDiagnosticGroups ?? runtimeAlerts.domDiagnosticsByText).slice(0, 5).map(slimGroup), domDiagnosticSamples: arr(source.domDiagnosticSamples ?? runtimeAlerts.domDiagnostics).slice(0, 8).map(slimGroup), consoleAlertGroups: arr(source.consoleAlertGroups ?? runtimeAlerts.consoleAlertsByPath).slice(0, 8).map(slimGroup), consoleAlertSamples: arr(source.consoleAlertSamples ?? runtimeAlerts.consoleAlerts).slice(0, 8).map(slimGroup), turnTimingRecentUpdateJumps: arr(source.turnTimingRecentUpdateJumps ?? srcMetrics.turnTimingRecentUpdateSawtoothJumps).slice(0, 8), turnTimingElapsedZeroResets: arr(source.turnTimingElapsedZeroResets ?? srcMetrics.turnTimingElapsedZeroResets).slice(0, 8), turnTimingTotalElapsedForwardJumps: arr(source.turnTimingTotalElapsedForwardJumps ?? srcMetrics.turnTimingTotalElapsedForwardJumps).slice(0, 8), pagePerformanceSlowApi: slowApis.slice(0, 8).map(slimSlowApi), archivePagePerformanceSlowApi: arr(source.archivePagePerformanceSlowApi).slice(0, 8).map(slimSlowApi), pagePerformancePartialApi: arr(source.pagePerformancePartialApi).slice(0, 8), pagePerformanceSseStreams: arr(source.pagePerformanceSseStreams).slice(0, 8), findings: arr(source.findings).slice(0, 12).map(slimFinding), archiveRedFindings: arr(source.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 12).map(slimFinding), reportJsonPath: source.reportJsonPath ?? reportJsonPath, reportJsonSha256: source.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: source.reportMdPath ?? reportMdPath, reportMdSha256: source.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(objectOrNull(source.analyzer) || {}), recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true };", + "const compact = { ok: source.ok === true, command: source.command ?? 'web-probe-observe analyze', stateDir: source.stateDir ?? stateDir, jsonlScope: source.jsonlScope ?? null, alertThresholds: source.alertThresholds ?? null, counts: source.counts ?? null, archiveSummary: { ...archiveSummary, sampleMetrics: archiveSampleMetrics, pagePerformance: objectOrNull(archiveSummary.pagePerformance) || objectOrNull(pagePerformance.summary) || {}, runtimeAlerts: objectOrNull(archiveSummary.runtimeAlerts) || objectOrNull(runtimeAlerts.summary) || {}, redFindings: arr(archiveSummary.redFindings).slice(0, 12).map(slimFinding) }, analysisWindow: source.analysisWindow ?? objectOrNull(recent.summary), sampleMetrics: compactMetrics(srcMetrics), pageProvenance: objectOrNull(source.pageProvenance?.summary) || source.pageProvenance ?? null, pagePerformance: objectOrNull(pagePerformance.summary) || pagePerformance, promptNetwork: objectOrNull(promptNetwork.summary) || promptNetwork, runtimeAlerts: objectOrNull(runtimeAlerts.summary) || runtimeAlerts, runnerErrors: arr(source.runnerErrors).slice(-8), commandFailures: arr(source.commandFailures).slice(-8), commandState: objectOrNull(source.commandState) || null, toolFindings: arr(source.toolFindings).slice(0, 8).map(slimFinding), httpErrorGroups: arr(source.httpErrorGroups ?? runtimeAlerts.networkHttpErrorsByPath).slice(0, 8).map(slimGroup), requestFailedGroups: arr(source.requestFailedGroups ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 8).map(slimGroup), domDiagnosticGroups: arr(source.domDiagnosticGroups ?? runtimeAlerts.domDiagnosticsByText).slice(0, 5).map(slimGroup), domDiagnosticSamples: arr(source.domDiagnosticSamples ?? runtimeAlerts.domDiagnostics).slice(0, 8).map(slimGroup), consoleAlertGroups: arr(source.consoleAlertGroups ?? runtimeAlerts.consoleAlertsByPath).slice(0, 8).map(slimGroup), consoleAlertSamples: arr(source.consoleAlertSamples ?? runtimeAlerts.consoleAlerts).slice(0, 8).map(slimGroup), turnTimingRecentUpdateJumps: arr(source.turnTimingRecentUpdateJumps ?? srcMetrics.turnTimingRecentUpdateSawtoothJumps).slice(0, 8), turnTimingElapsedZeroResets: arr(source.turnTimingElapsedZeroResets ?? srcMetrics.turnTimingElapsedZeroResets).slice(0, 8), turnTimingTotalElapsedForwardJumps: arr(source.turnTimingTotalElapsedForwardJumps ?? srcMetrics.turnTimingTotalElapsedForwardJumps).slice(0, 8), pagePerformanceSlowApi: slowApis.slice(0, 8).map(slimSlowApi), archivePagePerformanceSlowApi: arr(source.archivePagePerformanceSlowApi).slice(0, 8).map(slimSlowApi), pagePerformancePartialApi: arr(source.pagePerformancePartialApi).slice(0, 8), pagePerformanceSseStreams: arr(source.pagePerformanceSseStreams).slice(0, 8), findings: arr(source.findings).slice(0, 12).map(slimFinding), archiveRedFindings: arr(source.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 12).map(slimFinding), reportJsonPath: source.reportJsonPath ?? reportJsonPath, reportJsonSha256: source.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: source.reportMdPath ?? reportMdPath, reportMdSha256: source.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(objectOrNull(source.analyzer) || {}), recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true };", "console.log(JSON.stringify(compact));", "UNIDESK_WEB_OBSERVE_RECOVER_ANALYZE_ARTIFACT", ].join("\n"); @@ -8825,7 +8883,12 @@ function renderWebObserveAnalyzeTable(value: Record): string { const archiveSlowApis = Array.isArray(analysis?.archivePagePerformanceSlowApi) ? analysis.archivePagePerformanceSlowApi.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const partialApis = Array.isArray(analysis?.pagePerformancePartialApi) ? analysis.pagePerformancePartialApi.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const sseStreams = Array.isArray(analysis?.pagePerformanceSseStreams) ? analysis.pagePerformanceSseStreams.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; - const findings = Array.isArray(analysis?.findings) ? analysis.findings.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; + const allFindingRecords = Array.isArray(analysis?.findings) ? analysis.findings.map(record).filter((item): item is Record => item !== null) : []; + const explicitToolFindings = Array.isArray(analysis?.toolFindings) ? analysis.toolFindings.map(record).filter((item): item is Record => item !== null) : []; + const inferredToolFindings = allFindingRecords.filter((item) => String(item.id ?? item.kind ?? item.code ?? "").startsWith("tool-")); + const toolFindings = dedupeWebObserveRecords(explicitToolFindings.length > 0 ? explicitToolFindings : inferredToolFindings, (item) => String(item.id ?? item.kind ?? item.code ?? "")).slice(0, 8); + const commandState = record(analysis?.commandState); + const findings = allFindingRecords.slice(0, 8); const archiveRedFindings = Array.isArray(analysis?.archiveRedFindings) ? analysis.archiveRedFindings.map(record).filter((item): item is Record => item !== null).slice(0, 8) : Array.isArray(archiveSummary?.redFindings) @@ -9163,6 +9226,18 @@ function renderWebObserveAnalyzeTable(value: Record): string { webObserveShort(webObserveText(item.message || item.failureKind || item.name), 96), ]) : [["-", "-", "-", "-", "-", "-", "-"]]), "", + "Tool findings:", + webObserveTable(["KIND", "SEVERITY", "COUNT", "SUMMARY"], toolFindings.length > 0 ? toolFindings.map((item) => [ + webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 36), + webObserveText(item.severity ?? item.level), + webObserveText(item.count ?? item.sampleCount), + webObserveShort(webObserveText(item.summary ?? item.message), 110), + ]) : [["-", "-", "-", "-"]]), + webObserveTable(["TOOL_COMMANDS", "COUNT"], [[ + `pending=${webObserveText(commandState?.pendingCount)}`, + `processing=${webObserveText(commandState?.processingCount)} abandoned=${webObserveText(commandState?.abandonedCount)} failed=${webObserveText(commandState?.failedCount)}`, + ]]), + "", "HTTP error groups:", webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], httpErrorRows.length > 0 ? httpErrorRows : [["-", "-", "-", "-", "-", "-"]]), "", @@ -9296,6 +9371,18 @@ function renderWebObserveAnalyzeTable(value: Record): string { return lines.join("\n"); } +function dedupeWebObserveRecords(items: Record[], keyOf: (item: Record) => string): Record[] { + const out: Record[] = []; + const seen = new Set(); + for (const item of items) { + const key = keyOf(item); + if (!key || key === "-" || seen.has(key)) continue; + seen.add(key); + out.push(item); + } + return out; +} + function nodeWebObserveResolveStateDirShell(options: Pick): string { if (options.stateDir !== null) { return [ @@ -9781,6 +9868,7 @@ function renderWebObserveAnalyzeResult(result: Record): Record< const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item)); const sseStreams = webObserveArray(analysis.pagePerformanceSseStreams).slice(0, 8).map((item) => record(item)); const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item)); + const commandState = record(analysis.commandState); const id = result.id ?? "-"; const blockerRows = String(result.status ?? "") === "blocked" ? [ @@ -9867,6 +9955,16 @@ function renderWebObserveAnalyzeResult(result: Record): Record< ], ), "", + webObserveTable( + ["TOOL_COMMANDS", "COUNT"], + [ + ["pending", commandState.pendingCount], + ["processing", commandState.processingCount], + ["abandoned", commandState.abandonedCount], + ["failed", commandState.failedCount], + ], + ), + "", domDiagnosticGroups.length === 0 ? "DOM_DIAGNOSTIC_GROUPS\n-" : webObserveTable( @@ -9932,7 +10030,7 @@ function renderWebObserveAnalyzeResult(result: Record): Record< ? "FINDINGS\n-" : webObserveTable( ["FINDING", "SEVERITY", "COUNT", "SUMMARY"], - findings.map((item) => [item.id, item.severity, item.count, item.summary]), + findings.map((item) => [item.id ?? item.kind ?? item.code, item.severity ?? item.level, item.count ?? item.sampleCount, item.summary ?? item.message]), ), "", "REPORTS", @@ -10044,26 +10142,102 @@ function withWebObserveShortcuts(value: Record | null, id: stri function nodeWebObserveStatusNodeScript(tailLines: number, node: string, lane: string): string { return `node -e ${shellQuote(` -const fs=require('fs'),path=require('path'); -const dir=process.env.state_dir||process.argv[1]; -const node=process.argv[2]; -const lane=process.argv[3]; -const tailN=Math.min(${tailLines},3); -const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}}; -const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}}; -const short=(value)=>String(value||'').slice(0,160); -const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null; -const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null; -const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null; -const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':''; -const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):''; -const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)}); -const compactSample=(item)=>({seq:item.seq,ts:item.ts,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,loadingCount:Array.isArray(item.loadings)?item.loadings.length:0,loadingOwners:Array.isArray(item.loadings)?Array.from(new Set(item.loadings.map((loading)=>loading.ownerLabel||loading.ownerKind||loading.ownerKey||'loading'))).slice(0,4):[],sessionRail:item.sessionRail?{visibleCount:item.sessionRail.visibleCount??null,fallbackTitleCount:item.sessionRail.fallbackTitleCount??null,fallbackTitleRatio:item.sessionRail.fallbackTitleRatio??null,fallbackItems:Array.isArray(item.sessionRail.fallbackItems)?item.sessionRail.fallbackItems.slice(0,4).map((fallback)=>({sessionIdPrefix:fallback.sessionIdPrefix??null,titlePreview:fallback.titlePreview??null,active:fallback.active===true})):[]}:null}); -const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:short(item.url),status:item.status||null,failure:item.failure?short(item.failure):null}); -const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null; -let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}} -console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(readJson('manifest.json')),heartbeat:compactHeartbeat(readJson('heartbeat.json')),tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork)},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true})); -`)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`; + const fs=require('fs'),path=require('path'); + const dir=process.env.state_dir||process.argv[1]; + const node=process.argv[2]; + const lane=process.argv[3]; + const tailN=Math.min(${tailLines},3); + const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}}; + const readJsonPath=(file)=>{try{return JSON.parse(fs.readFileSync(file,'utf8'))}catch{return null}}; + const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}}; + const short=(value)=>String(value||'').slice(0,160); + const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null; + const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null; + const compactHeartbeat=(item,diag)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,ageSeconds:diag.heartbeatAgeSeconds,stale:diag.heartbeatStale,staleAfterSeconds:diag.heartbeatStaleAfterSeconds,effectiveLiveness:diag.effectiveLiveness,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null; + const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':''; + const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):''; + const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)}); + const compactSample=(item)=>({seq:item.seq,ts:item.ts,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,loadingCount:Array.isArray(item.loadings)?item.loadings.length:0,loadingOwners:Array.isArray(item.loadings)?Array.from(new Set(item.loadings.map((loading)=>loading.ownerLabel||loading.ownerKind||loading.ownerKey||'loading'))).slice(0,4):[],sessionRail:item.sessionRail?{visibleCount:item.sessionRail.visibleCount??null,fallbackTitleCount:item.sessionRail.fallbackTitleCount??null,fallbackTitleRatio:item.sessionRail.fallbackTitleRatio??null,fallbackItems:Array.isArray(item.sessionRail.fallbackItems)?item.sessionRail.fallbackItems.slice(0,4).map((fallback)=>({sessionIdPrefix:fallback.sessionIdPrefix??null,titlePreview:fallback.titlePreview??null,active:fallback.active===true})):[]}:null}); + const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:short(item.url),status:item.status||null,failure:item.failure?short(item.failure):null}); + const commandDir=(name)=>path.join(dir,'commands',name); + const commandSummary=(name)=>{ + const root=commandDir(name); let entries=[]; + try{entries=fs.readdirSync(root).filter((item)=>item.endsWith('.json')).sort();}catch{} + const items=entries.slice(0,12).map((entry)=>{const file=path.join(root,entry); const parsed=readJsonPath(file)||{}; let stat=null; try{stat=fs.statSync(file)}catch{}; const createdAt=parsed.createdAt||parsed.abandonedAt||parsed.completedAt||parsed.failedAt||null; const createdMs=Date.parse(String(createdAt||'')); const ageSeconds=Number.isFinite(createdMs)?Math.max(0,Math.round((Date.now()-createdMs)/1000)):null; return {id:parsed.id||parsed.commandId||entry.replace(/[.]json$/,''),type:parsed.type||null,createdAt,ageSeconds,mtime:stat?stat.mtime.toISOString():null};}); + const ages=items.map((item)=>item.ageSeconds).filter((value)=>Number.isFinite(value)); + return {count:entries.length,oldestAgeSeconds:ages.length?Math.max(...ages):null,items}; + }; + const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null; + let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}} + const manifest=readJson('manifest.json'); + const heartbeat=readJson('heartbeat.json'); + const terminal=/^(completed|failed|force-stopped|stopped|abandoned)$/u.test(String((heartbeat&&heartbeat.status)||(manifest&&manifest.status)||'')); + const sampleIntervalMs=Number(manifest&&manifest.sampling&&manifest.sampling.sampleIntervalMs)||5000; + const staleAfterMs=Math.max(60000,sampleIntervalMs*3); + const updatedRaw=heartbeat&&(heartbeat.updatedAt||heartbeat.lastSampleAt)||null; + const updatedMs=Date.parse(String(updatedRaw||'')); + const heartbeatAgeSeconds=Number.isFinite(updatedMs)?Math.max(0,Math.round((Date.now()-updatedMs)/1000)):null; + const heartbeatStale=alive&&!terminal&&(!Number.isFinite(updatedMs)||Date.now()-updatedMs>staleAfterMs); + const commandsPending=commandSummary('pending'); + const commandsProcessing=commandSummary('processing'); + const commandsAbandoned=commandSummary('abandoned'); + const commandsFailed=commandSummary('failed'); + const diagnostics={heartbeatAgeSeconds,heartbeatStale,heartbeatStaleAfterSeconds:Math.round(staleAfterMs/1000),heartbeatUpdatedAt:updatedRaw,terminal,processAlive:alive,effectiveLiveness:heartbeatStale?'stale':alive?'alive':'not-running',commandBacklog:commandsPending.count+commandsProcessing.count,oldestPendingAgeSeconds:commandsPending.oldestAgeSeconds,oldestProcessingAgeSeconds:commandsProcessing.oldestAgeSeconds,valuesRedacted:true}; + const commands={pendingCount:commandsPending.count,processingCount:commandsProcessing.count,abandonedCount:commandsAbandoned.count,failedCount:commandsFailed.count,pending:commandsPending.items,processing:commandsProcessing.items,abandoned:commandsAbandoned.items.slice(0,6),failed:commandsFailed.items.slice(0,6),valuesRedacted:true}; + console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(manifest),heartbeat:compactHeartbeat(heartbeat,diagnostics),diagnostics,commands,tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork)},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,forceStop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir+' --force',analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true})); + `)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`; +} + +function nodeWebObserveForceStopNodeScript(reason: string, forcedByCommandId: string): string { + return `node -e ${shellQuote(` + const fs=require('fs'),path=require('path'),crypto=require('crypto'); + const dir=process.argv[1],reason=process.argv[2],forcedByCommandId=process.argv[3]; + const now=new Date().toISOString(); + const short=(value,max=160)=>value===undefined||value===null?null:String(value).replace(/\\s+/g,' ').slice(0,max); + const shaText=(value)=>'sha256:'+crypto.createHash('sha256').update(String(value||'')).digest('hex'); + const readJson=(file)=>{try{return JSON.parse(fs.readFileSync(file,'utf8'))}catch{return null}}; + const writeJson=(file,value)=>{fs.mkdirSync(path.dirname(file),{recursive:true,mode:0o700}); fs.writeFileSync(file,JSON.stringify(value,null,2)+'\\n',{mode:0o600});}; + const appendJsonl=(file,value)=>{fs.appendFileSync(file,JSON.stringify(value)+'\\n',{mode:0o600});}; + const summarizeCommand=(command,id)=>({id,type:command&&command.type||null,createdAt:command&&command.createdAt||null,source:command&&command.source||null,path:command&&command.path||null,label:command&&command.label||null,sessionId:command&&command.sessionId||null,provider:command&&command.provider||null,textHash:typeof(command&&command.text)==='string'?shaText(command.text):null,textBytes:typeof(command&&command.text)==='string'?Buffer.byteLength(command.text):null,valuesRedacted:true}); + const moveCommands=(bucket)=>{ + const source=path.join(dir,'commands',bucket); let names=[]; + try{names=fs.readdirSync(source).filter((name)=>name.endsWith('.json')).sort();}catch{} + const moved=[]; + for(const name of names){ + const file=path.join(source,name); + const parsed=readJson(file)||{}; + const id=String(parsed.id||parsed.commandId||name.replace(/[.]json$/,'')).replace(/[^A-Za-z0-9_.-]/g,'-')||name.replace(/[.]json$/,''); + const record={ok:false,status:'abandoned',commandId:id,type:parsed.type||null,abandonedAt:now,reason,sourceBucket:bucket,forcedByCommandId,command:summarizeCommand(parsed,id),valuesRedacted:true}; + writeJson(path.join(dir,'commands','abandoned',id+'.json'),record); + try{fs.unlinkSync(file)}catch{} + appendJsonl(path.join(dir,'control.jsonl'),{ts:now,seq:null,phase:'abandoned',type:parsed.type||null,commandId:id,detail:{reason,sourceBucket:bucket,forcedByCommandId,valuesRedacted:true},valuesRedacted:true}); + moved.push({id,type:parsed.type||null,sourceBucket:bucket,createdAt:parsed.createdAt||null}); + } + return moved; + }; + const pidFile=path.join(dir,'pid'); + const pidText=fs.existsSync(pidFile)?fs.readFileSync(pidFile,'utf8').trim():''; + const pid=Number(pidText); + const alive=()=>Number.isFinite(pid)&&pid>0?(()=>{try{process.kill(pid,0); return true;}catch{return false;}})():false; + const sleep=(ms)=>Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms); + const aliveBefore=alive(); + let termSent=false,killSent=false; + if(aliveBefore){try{process.kill(pid,'SIGTERM'); termSent=true;}catch{}} + for(let i=0;i<10&&alive();i+=1) sleep(250); + if(alive()){try{process.kill(pid,'SIGKILL'); killSent=true;}catch{}} + for(let i=0;i<8&&alive();i+=1) sleep(250); + const pending=moveCommands('pending'); + const processing=moveCommands('processing'); + const aliveAfter=alive(); + const manifestPath=path.join(dir,'manifest.json'); + const heartbeatPath=path.join(dir,'heartbeat.json'); + const manifest=readJson(manifestPath)||{}; + const heartbeat=readJson(heartbeatPath)||{}; + writeJson(manifestPath,{...manifest,ok:false,status:'force-stopped',forceStoppedAt:now,forceStop:{reason,forcedByCommandId,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,valuesRedacted:true}}); + writeJson(heartbeatPath,{...heartbeat,ok:false,status:'force-stopped',updatedAt:now,forceStoppedAt:now,forceStop:{reason,forcedByCommandId,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,valuesRedacted:true}}); + appendJsonl(path.join(dir,'control.jsonl'),{ts:now,seq:null,phase:'completed',type:'forceStop',commandId:forcedByCommandId,detail:{reason,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,valuesRedacted:true},valuesRedacted:true}); + console.log(JSON.stringify({ok:!aliveAfter,command:'web-probe-observe force-stop',stateDir:dir,forced:true,reason,forcedByCommandId,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,abandoned:[...pending,...processing].slice(0,20),valuesRedacted:true})); + `)} "$state_dir" ${shellQuote(reason)} ${shellQuote(forcedByCommandId)}`; } function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null, collectGrep: string | null): string { diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index 749fb764..2db92e86 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -44,6 +44,7 @@ const errors = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFil const artifacts = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("artifacts.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); const manifest = await readJson(path.join(stateDir, "manifest.json")); const heartbeat = await readJson(path.join(stateDir, "heartbeat.json")); +const commandState = await readCommandState(stateDir); await mkdir(analysisDir, { recursive: true, mode: 0o700 }); const transitions = buildTransitions(samples); @@ -93,7 +94,8 @@ const runnerErrors = errors.slice(-8).map((item) => { }; }); const commandFailures = summarizeCommandFailures(control); -const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures); +const toolFindings = buildToolFindings({ manifest, heartbeat, commandState }); +const findings = [...toolFindings, ...buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures)]; if (jsonlReadIssues.length > 0) findings.unshift({ id: "jsonl-read-issues", severity: "red", summary: "observer analyzer hit JSONL read/parse issues", count: jsonlReadIssues.length, issues: jsonlReadIssues.slice(0, 20) }); const recentWindow = buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest }); const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl })); @@ -116,6 +118,8 @@ const report = { runtimeAlerts, runnerErrors, commandFailures, + commandState, + toolFindings, findings, windows: { recent: recentWindow }, readIssues: jsonlReadIssues, @@ -159,6 +163,8 @@ console.log(JSON.stringify({ runtimeAlerts: recentWindow.runtimeAlerts.summary, runnerErrors, commandFailures: commandFailures.slice(-8), + commandState, + toolFindings: toolFindings.slice(0, 8).map((item) => ({ kind: item.id, severity: item.severity, count: item.count ?? null, summary: String(item.summary ?? "").slice(0, 180) })), httpErrorGroups: recentWindow.runtimeAlerts.networkHttpErrorsByPath.slice(0, 8).map((item) => ({ method: item.method ?? null, status: item.status ?? null, @@ -356,6 +362,135 @@ async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } } +async function readCommandState(rootDir) { + const buckets = {}; + for (const bucket of ["pending", "processing", "done", "failed", "abandoned"]) { + buckets[bucket] = await readCommandBucket(path.join(rootDir, "commands", bucket), bucket); + } + return { + pendingCount: buckets.pending.count, + processingCount: buckets.processing.count, + doneCount: buckets.done.count, + failedCount: buckets.failed.count, + abandonedCount: buckets.abandoned.count, + pending: buckets.pending.items, + processing: buckets.processing.items, + failed: buckets.failed.items.slice(0, 12), + abandoned: buckets.abandoned.items.slice(0, 20), + summary: { + backlogCount: buckets.pending.count + buckets.processing.count, + oldestPendingAgeSeconds: buckets.pending.oldestAgeSeconds, + oldestProcessingAgeSeconds: buckets.processing.oldestAgeSeconds, + valuesRedacted: true + }, + valuesRedacted: true + }; +} + +async function readCommandBucket(dir, bucket) { + let names = []; + try { + names = (await readdir(dir)).filter((name) => name.endsWith(".json")).sort(); + } catch (error) { + if (!error || error.code !== "ENOENT") jsonlReadIssues.push({ file: path.basename(dir), kind: "command-dir-read-error", code: error && error.code ? String(error.code) : null, errorMessage: limitText(error && error.message ? error.message : String(error), 240) }); + return { bucket, count: 0, oldestAgeSeconds: null, items: [] }; + } + const items = []; + for (const name of names.slice(0, 200)) { + const file = path.join(dir, name); + const parsed = await readJson(file) || {}; + let meta = null; + try { meta = await stat(file); } catch {} + const timestamp = parsed.createdAt || parsed.abandonedAt || parsed.completedAt || parsed.failedAt || (meta ? meta.mtime.toISOString() : null); + const timestampMs = Date.parse(String(timestamp || "")); + items.push({ + bucket, + id: parsed.id || parsed.commandId || name.replace(/[.]json$/u, ""), + type: parsed.type || parsed.command?.type || null, + createdAt: parsed.createdAt || null, + completedAt: parsed.completedAt || null, + failedAt: parsed.failedAt || null, + abandonedAt: parsed.abandonedAt || null, + reason: parsed.reason || parsed.error?.message || null, + ageSeconds: Number.isFinite(timestampMs) ? Math.max(0, Math.round((Date.now() - timestampMs) / 1000)) : null, + mtime: meta ? meta.mtime.toISOString() : null, + valuesRedacted: true + }); + } + const ages = items.map((item) => item.ageSeconds).filter((value) => Number.isFinite(value)); + return { bucket, count: names.length, oldestAgeSeconds: ages.length > 0 ? Math.max(...ages) : null, items }; +} + +function buildToolFindings({ manifest, heartbeat, commandState }) { + const findings = []; + const diagnostics = heartbeatDiagnostics(manifest, heartbeat); + if (diagnostics.heartbeatStale) { + findings.push({ + id: "tool-runner-heartbeat-stale", + severity: "red", + summary: "web-probe observe runner heartbeat is stale; treat this as observer tooling failure, not Workbench behavior", + count: 1, + diagnostics, + valuesRedacted: true + }); + } + if ((commandState?.pendingCount ?? 0) > 0 || (commandState?.processingCount ?? 0) > 0) { + findings.push({ + id: "tool-pending-commands-unconsumed", + severity: "red", + summary: "web-probe observe has pending/processing control commands that were not consumed by the runner", + count: (commandState?.pendingCount ?? 0) + (commandState?.processingCount ?? 0), + pending: (commandState?.pending ?? []).slice(0, 12), + processing: (commandState?.processing ?? []).slice(0, 12), + valuesRedacted: true + }); + } + if ((commandState?.abandonedCount ?? 0) > 0) { + findings.push({ + id: "tool-commands-abandoned", + severity: "info", + summary: "web-probe observe force-stop abandoned queued commands; do not interpret these as Workbench command failures", + count: commandState.abandonedCount, + commands: (commandState.abandoned ?? []).slice(0, 20), + valuesRedacted: true + }); + } + if (heartbeat?.forceStop || manifest?.forceStop) { + findings.push({ + id: "tool-runner-force-stopped", + severity: "info", + summary: "web-probe observe runner was stopped by the CLI outside the command queue", + count: 1, + forceStop: heartbeat?.forceStop || manifest?.forceStop, + valuesRedacted: true + }); + } + return findings; +} + +function heartbeatDiagnostics(manifest, heartbeat) { + const status = String(heartbeat?.status || manifest?.status || ""); + const terminal = /^(completed|failed|force-stopped|stopped|abandoned)$/u.test(status); + const sampleIntervalMs = Number(manifest?.sampling?.sampleIntervalMs) || 5000; + const staleAfterMs = Math.max(60000, sampleIntervalMs * 3); + const updatedAt = heartbeat?.updatedAt || heartbeat?.lastSampleAt || null; + const updatedMs = Date.parse(String(updatedAt || "")); + const ageSeconds = Number.isFinite(updatedMs) ? Math.max(0, Math.round((Date.now() - updatedMs) / 1000)) : null; + const heartbeatStale = !terminal && (!Number.isFinite(updatedMs) || Date.now() - updatedMs > staleAfterMs); + return { + status: status || null, + terminal, + updatedAt, + heartbeatAgeSeconds: ageSeconds, + heartbeatStale, + heartbeatStaleAfterSeconds: Math.round(staleAfterMs / 1000), + sampleSeq: heartbeat?.sampleSeq ?? null, + commandSeq: heartbeat?.commandSeq ?? null, + activeCommandId: heartbeat?.activeCommandId ?? null, + valuesRedacted: true + }; +} + function safeArchivePrefix(value) { const text = String(value || "").trim(); if (!text) return ""; diff --git a/scripts/src/hwlab-node-web-observe-render.ts b/scripts/src/hwlab-node-web-observe-render.ts index 2836b5dc..f945f100 100644 --- a/scripts/src/hwlab-node-web-observe-render.ts +++ b/scripts/src/hwlab-node-web-observe-render.ts @@ -33,6 +33,8 @@ function renderWebObserveStatusTable(value: Record): string { const observer = record(value.observer); const manifest = record(observer?.manifest); const heartbeat = record(observer?.heartbeat); + const diagnostics = record(observer?.diagnostics) ?? record(value.diagnostics); + const commands = record(observer?.commands); const tails = record(observer?.tails); const result = record(value.result); const samples = Array.isArray(tails?.samples) ? tails.samples.map(record).filter((item): item is Record => item !== null).slice(-5) : []; @@ -55,20 +57,38 @@ function renderWebObserveStatusTable(value: Record): string { const manifestNetwork = record(manifest?.network); const manifestBrowser = record(manifestNetwork?.browser); const manifestProxy = record(manifestNetwork?.proxy); + const heartbeatAge = diagnostics?.heartbeatAgeSeconds ?? heartbeat?.ageSeconds; + const heartbeatStale = diagnostics?.heartbeatStale ?? heartbeat?.stale; + const heartbeatLiveness = diagnostics?.effectiveLiveness ?? heartbeat?.effectiveLiveness; const lines = [ `hwlab nodes web-probe observe status (${webObserveText(value.status)})`, "", - webObserveTable(["ID", "NODE", "LANE", "ALIVE", "PID", "SAMPLE", "UPDATED", "TARGET"], [[ + webObserveTable(["ID", "NODE", "LANE", "LIVENESS", "ALIVE", "PID", "SAMPLE", "CMD_SEQ", "HB_AGE_S", "STALE", "UPDATED", "TARGET"], [[ value.id, value.node, value.lane, + heartbeatLiveness, observer?.processAlive, observer?.pid, heartbeat?.sampleSeq, + heartbeat?.commandSeq, + heartbeatAge, + heartbeatStale, webObserveShort(webObserveText(heartbeat?.updatedAt ?? heartbeat?.lastSampleAt), 24), webObserveShort(`${webObserveText(manifest?.baseUrl)}${webObserveText(manifest?.targetPath) === "-" ? "" : webObserveText(manifest?.targetPath)}`, 52), ]]), "", + "Runner diagnostics:", + webObserveTable(["HEARTBEAT_STALE_AFTER_S", "COMMAND_BACKLOG", "PENDING", "PROCESSING", "ABANDONED", "FAILED", "OLDEST_PENDING_S"], [[ + diagnostics?.heartbeatStaleAfterSeconds, + diagnostics?.commandBacklog, + commands?.pendingCount, + commands?.processingCount, + commands?.abandonedCount, + commands?.failedCount, + diagnostics?.oldestPendingAgeSeconds, + ]]), + "", ...(value.ok === false ? [ "Blocked detail:", webObserveTable(["REASON", "EXIT", "TIMEOUT", "STDOUT", "STDERR"], [[ @@ -153,6 +173,8 @@ function renderWebObserveStatusTable(value: Record): string { const command = webObserveText(next?.[key]); if (command !== "-") lines.push(` ${command}`); } + const forceStop = webObserveText(next?.forceStop); + if (forceStop !== "-") lines.push(` ${forceStop}`); lines.push("", "Disclosure:"); lines.push(" default view is a bounded table; use observe collect/analyze for artifact-level drill-down."); return lines.join("\n");