fix: handle stale web observe runners (#875)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
+230
-56
@@ -8074,13 +8074,7 @@ function runNodeWebProbeObserveStart(
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveStatusNodeScript(options.tailLines, options.node, options.lane),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const status = parseJsonObject(result.stdout);
|
||||
const { result, status } = readNodeWebProbeObserveRemoteStatus(options, spec, options.tailLines, options.commandTimeoutSeconds);
|
||||
const observerId = webObserveIdFromStatus(status, options);
|
||||
const statusReadable = status !== null;
|
||||
const ok = result.exitCode === 0 && statusReadable && status.ok !== false;
|
||||
@@ -8113,6 +8107,21 @@ function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec:
|
||||
});
|
||||
}
|
||||
|
||||
function readNodeWebProbeObserveRemoteStatus(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
tailLines: number,
|
||||
timeoutSeconds: number,
|
||||
): { result: ReturnType<typeof runTransWorkspaceStdinScript>; status: Record<string, unknown> | null } {
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveStatusNodeScript(tailLines, options.node, options.lane),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, timeoutSeconds);
|
||||
return { result, status: parseJsonObject(result.stdout) };
|
||||
}
|
||||
|
||||
function webObserveText(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
if (typeof value === "string") return value;
|
||||
@@ -8141,42 +8150,45 @@ function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec
|
||||
sessionId: options.commandSessionId,
|
||||
provider: options.commandProvider,
|
||||
};
|
||||
const preStopStatus = options.force && stopCommand
|
||||
? readNodeWebProbeObserveRemoteStatus(options, spec, 1, Math.min(options.commandTimeoutSeconds, 30))
|
||||
: null;
|
||||
const preStopDiagnostics = record(preStopStatus?.status?.diagnostics);
|
||||
const preStopCommands = record(preStopStatus?.status?.commands);
|
||||
const preStopPending = Number(preStopCommands?.pendingCount ?? 0);
|
||||
const preStopProcessing = Number(preStopCommands?.processingCount ?? 0);
|
||||
const forceBeforeQueueReason = options.force && stopCommand
|
||||
? preStopDiagnostics?.heartbeatStale === true
|
||||
? "heartbeat-stale"
|
||||
: preStopPending > 0 || preStopProcessing > 0
|
||||
? "command-backlog"
|
||||
: null
|
||||
: null;
|
||||
if (forceBeforeQueueReason !== null) {
|
||||
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, forceBeforeQueueReason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, null);
|
||||
}
|
||||
const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
||||
const waitMs = options.force && stopCommand ? Math.max(options.waitMs, 5000) : options.waitMs;
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
"mkdir -p \"$state_dir/commands/pending\"",
|
||||
`node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`,
|
||||
nodeWebObserveWaitCommandShell(commandId, options.waitMs),
|
||||
nodeWebObserveWaitCommandShell(commandId, waitMs),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const commandResult = parseJsonObject(result.stdout);
|
||||
if (options.force && stopCommand && result.exitCode !== 0) {
|
||||
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
"if [ -f \"$state_dir/pid\" ]; then kill \"$(cat \"$state_dir/pid\")\" >/dev/null 2>&1 || true; fi",
|
||||
"printf '{\"ok\":true,\"forced\":true,\"stateDir\":\"%s\"}\\n' \"$state_dir\"",
|
||||
].join("\n"), 55);
|
||||
return withWebObserveCommandRendered({
|
||||
ok: killResult.exitCode === 0,
|
||||
status: killResult.exitCode === 0 ? "forced-stop-requested" : "blocked",
|
||||
command: webObserveCommandLabel("stop", options),
|
||||
id: webObserveIdFromOptions(options),
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
gracefulResult: compactCommandResult(result),
|
||||
forceResult: compactCommandResult(killResult),
|
||||
full: options.full,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (options.force && stopCommand && (result.exitCode !== 0 || commandResult?.waitTimedOut === true || commandResult?.queued === true)) {
|
||||
const reason = result.exitCode !== 0
|
||||
? "graceful-stop-failed"
|
||||
: commandResult?.waitTimedOut === true
|
||||
? "graceful-stop-not-consumed"
|
||||
: "graceful-stop-queued";
|
||||
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, reason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, result);
|
||||
}
|
||||
return withWebObserveCommandRendered({
|
||||
ok: result.exitCode === 0 && commandResult?.ok !== false,
|
||||
status: result.exitCode === 0 ? (options.waitMs > 0 ? "completed-or-queued" : "queued") : "blocked",
|
||||
status: result.exitCode === 0 ? (waitMs > 0 ? "completed-or-queued" : "queued") : "blocked",
|
||||
command: webObserveCommandLabel(stopCommand ? "stop" : "command", options),
|
||||
id: webObserveIdFromOptions(options),
|
||||
node: options.node,
|
||||
@@ -8191,6 +8203,43 @@ function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec
|
||||
});
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveForceStop(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
payload: Record<string, unknown>,
|
||||
commandId: string,
|
||||
reason: string,
|
||||
preflightResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
||||
preflightStatus: Record<string, unknown> | null,
|
||||
gracefulResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
||||
): Record<string, unknown> | RenderedCliResult {
|
||||
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveForceStopNodeScript(reason, commandId),
|
||||
].join("\n"), 55);
|
||||
const forcePayload = parseJsonObject(killResult.stdout);
|
||||
return withWebObserveCommandRendered({
|
||||
ok: killResult.exitCode === 0 && forcePayload?.ok !== false,
|
||||
status: killResult.exitCode === 0 && forcePayload?.ok !== false ? "forced-stopped" : "blocked",
|
||||
command: webObserveCommandLabel("stop", options),
|
||||
id: webObserveIdFromOptions(options),
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
observer: forcePayload,
|
||||
forceReason: reason,
|
||||
preflightObserver: preflightStatus,
|
||||
preflightResult: preflightResult === null ? null : compactCommandResult(preflightResult),
|
||||
gracefulResult: gracefulResult === null ? null : compactCommandResult(gracefulResult),
|
||||
forceResult: compactCommandResultWithStdoutTail(killResult),
|
||||
full: options.full,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
||||
const collectScript = options.collectView === "files"
|
||||
? nodeWebObserveCollectNodeScript(options.maxFiles, options.collectFile, options.collectFinding, options.collectGrep)
|
||||
@@ -8344,6 +8393,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
|
||||
"const sourceSseStreams = Array.isArray(source?.pagePerformanceSseStreams) ? source.pagePerformanceSseStreams : (Array.isArray(fullPagePerformance?.sameOriginApiByPath) ? fullPagePerformance.sameOriginApiByPath.filter((item) => isLongLivedApi(item)) : []);",
|
||||
"const combinedFindingSource = firstNonEmptyArray(source?.findings, fullSource?.findings, fullRecentWindow?.findings, source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings);",
|
||||
"const combinedFindings = sortFindings(combinedFindingSource);",
|
||||
"const toolFindings = sortFindings([...firstArray(source?.toolFindings), ...firstArray(fullSource?.toolFindings), ...combinedFindingSource.filter((item) => String(item?.id ?? item?.kind ?? item?.code ?? '').startsWith('tool-'))]);",
|
||||
"const archiveRedFindings = sortFindings(firstNonEmptyArray(source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings, fullSource?.findings).filter((item) => String(item?.severity ?? item?.level ?? '').toLowerCase() === 'red'));",
|
||||
"const allFindingsForCommands = [...firstArray(source?.findings), ...firstArray(fullSource?.findings), ...firstArray(fullRecentWindow?.findings), ...firstArray(source?.archiveSummary?.redFindings), ...firstArray(fullSource?.archiveSummary?.redFindings)];",
|
||||
"const findingSamplesById = (id) => { for (const item of allFindingsForCommands) { if (String(item?.id ?? item?.kind ?? item?.code ?? '') !== id) continue; if (Array.isArray(item?.samples) && item.samples.length > 0) return item.samples; } return []; };",
|
||||
@@ -8364,6 +8414,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
|
||||
" pagePerformanceSlowApi: takeHead(sourceSlowApi, 4).map(slimSlowApi),",
|
||||
" archivePagePerformanceSlowApi: takeHead(archiveSlowApi, 8).map(slimSlowApi),",
|
||||
" pagePerformanceSseStreams: takeHead(sourceSseStreams, 4).map((item) => ({ path: item?.path ?? item?.route ?? null, route: item?.route ?? null, sampleCount: item?.sampleCount ?? null, streamOpenSampleCount: item?.streamOpenSampleCount ?? null, streamOpenP95Ms: item?.streamOpenP95Ms ?? null, streamOpenMaxMs: item?.streamOpenMaxMs ?? null, streamOpenBudgetMs: item?.streamOpenBudgetMs ?? null, streamOpenOverBudgetCount: item?.streamOpenOverBudgetCount ?? null, streamOpenOverFiveSecondCount: item?.streamOpenOverFiveSecondCount ?? null, streamLifetimeOverFiveSecondCount: item?.streamLifetimeOverFiveSecondCount ?? null })),",
|
||||
" toolFindings: takeHead(toolFindings, 8).map(slimFinding),",
|
||||
" commandState: objectOrNull(source.commandState) || objectOrNull(fullSource?.commandState) || null,",
|
||||
" findings: takeHead(combinedFindings, 8).map(slimFinding),",
|
||||
" archiveRedFindings: takeHead(archiveRedFindings, 8).map(slimFinding),",
|
||||
" httpErrorGroups: takeHead(firstArray(source.httpErrorGroups, fullRecentWindow?.runtimeAlerts?.networkHttpErrorsByPath, fullSource?.httpErrorGroups, fullSource?.runtimeAlerts?.networkHttpErrorsByPath), 4).map(slimNetworkGroup),",
|
||||
@@ -8401,7 +8453,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
|
||||
" stderrTail: tail(readText(stderrPath))",
|
||||
" }",
|
||||
"};",
|
||||
"const compactStdoutLimitBytes = 10000;",
|
||||
"const compactStdoutLimitBytes = 8000;",
|
||||
"const compactOutput = (value) => JSON.stringify(value);",
|
||||
"let output = compactOutput(compact);",
|
||||
"if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {",
|
||||
@@ -8447,6 +8499,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
|
||||
" runtimeAlerts: compact.runtimeAlerts ?? null,",
|
||||
" pagePerformance: compact.pagePerformance ?? null,",
|
||||
" promptNetwork: compact.promptNetwork ?? null,",
|
||||
" toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [],",
|
||||
" commandState: compact.commandState ?? null,",
|
||||
" findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4) : [],",
|
||||
" archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4) : [],",
|
||||
" turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 5) : [],",
|
||||
@@ -8507,6 +8561,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
|
||||
" } : null,",
|
||||
" runtimeAlerts: compact.runtimeAlerts ?? null,",
|
||||
" pagePerformance: compact.pagePerformance ?? null,",
|
||||
" toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [],",
|
||||
" commandState: compact.commandState ?? null,",
|
||||
" findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4) : [],",
|
||||
" archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4) : [],",
|
||||
" pagePerformanceSlowApi: Array.isArray(compact.pagePerformanceSlowApi) ? compact.pagePerformanceSlowApi.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, route: item.route ?? item.path ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? item.p95 ?? null, maxMs: item.maxMs ?? item.max ?? null, budgetMs: item.budgetMs ?? null, overBudgetCount: item.overBudgetCount ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null, slowSamples: Array.isArray(item.slowSamples) ? item.slowSamples.slice(0, 1) : [] })) : [],",
|
||||
@@ -8545,6 +8601,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
|
||||
" } : null,",
|
||||
" runtimeAlerts: compact.runtimeAlerts ?? null,",
|
||||
" pagePerformance: compact.pagePerformance ?? null,",
|
||||
" toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [],",
|
||||
" commandState: compact.commandState ?? null,",
|
||||
" findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],",
|
||||
" archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],",
|
||||
" httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, status: item.status ?? null, path: item.path ?? null, lastAt: item.lastAt ?? null })) : [],",
|
||||
@@ -8566,7 +8624,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
|
||||
" output = compactOutput(ultratiny);",
|
||||
" }",
|
||||
" if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {",
|
||||
" output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, jsonlScope: compact.jsonlScope ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ?? null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });",
|
||||
" output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, jsonlScope: compact.jsonlScope ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ? { redFindingCount: compact.archiveSummary.redFindingCount ?? null, findingCount: compact.archiveSummary.findingCount ?? null, sampleCount: compact.archiveSummary.sampleMetrics?.sampleCount ?? null, slowPathCount: compact.archiveSummary.pagePerformance?.slowPathCount ?? null } : null, toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [], commandState: compact.commandState ? { pendingCount: compact.commandState.pendingCount ?? null, processingCount: compact.commandState.processingCount ?? null, abandonedCount: compact.commandState.abandonedCount ?? null, failedCount: compact.commandState.failedCount ?? null } : null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });",
|
||||
" }",
|
||||
" }",
|
||||
"}",
|
||||
@@ -8692,7 +8750,7 @@ function recoverWebObserveAnalyzeFromArtifacts(options: NodeWebProbeObserveOptio
|
||||
"const archiveSummary = objectOrNull(source.archiveSummary) || {};",
|
||||
"const archiveSampleMetrics = objectOrNull(archiveSummary.sampleMetrics) || objectOrNull(source.sampleMetrics?.summary) || objectOrNull(srcMetrics.summary) || {};",
|
||||
"const slowApis = arr(source.pagePerformanceSlowApi).length > 0 ? arr(source.pagePerformanceSlowApi) : arr(pagePerformance.sameOriginApiByPath).filter((item) => Number(item?.overBudgetCount ?? item?.overFiveSecondCount ?? 0) > 0);",
|
||||
"const compact = { ok: source.ok === true, command: source.command ?? 'web-probe-observe analyze', stateDir: source.stateDir ?? stateDir, jsonlScope: source.jsonlScope ?? null, alertThresholds: source.alertThresholds ?? null, counts: source.counts ?? null, archiveSummary: { ...archiveSummary, sampleMetrics: archiveSampleMetrics, pagePerformance: objectOrNull(archiveSummary.pagePerformance) || objectOrNull(pagePerformance.summary) || {}, runtimeAlerts: objectOrNull(archiveSummary.runtimeAlerts) || objectOrNull(runtimeAlerts.summary) || {}, redFindings: arr(archiveSummary.redFindings).slice(0, 12).map(slimFinding) }, analysisWindow: source.analysisWindow ?? objectOrNull(recent.summary), sampleMetrics: compactMetrics(srcMetrics), pageProvenance: objectOrNull(source.pageProvenance?.summary) || source.pageProvenance ?? null, pagePerformance: objectOrNull(pagePerformance.summary) || pagePerformance, promptNetwork: objectOrNull(promptNetwork.summary) || promptNetwork, runtimeAlerts: objectOrNull(runtimeAlerts.summary) || runtimeAlerts, runnerErrors: arr(source.runnerErrors).slice(-8), commandFailures: arr(source.commandFailures).slice(-8), httpErrorGroups: arr(source.httpErrorGroups ?? runtimeAlerts.networkHttpErrorsByPath).slice(0, 8).map(slimGroup), requestFailedGroups: arr(source.requestFailedGroups ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 8).map(slimGroup), domDiagnosticGroups: arr(source.domDiagnosticGroups ?? runtimeAlerts.domDiagnosticsByText).slice(0, 5).map(slimGroup), domDiagnosticSamples: arr(source.domDiagnosticSamples ?? runtimeAlerts.domDiagnostics).slice(0, 8).map(slimGroup), consoleAlertGroups: arr(source.consoleAlertGroups ?? runtimeAlerts.consoleAlertsByPath).slice(0, 8).map(slimGroup), consoleAlertSamples: arr(source.consoleAlertSamples ?? runtimeAlerts.consoleAlerts).slice(0, 8).map(slimGroup), turnTimingRecentUpdateJumps: arr(source.turnTimingRecentUpdateJumps ?? srcMetrics.turnTimingRecentUpdateSawtoothJumps).slice(0, 8), turnTimingElapsedZeroResets: arr(source.turnTimingElapsedZeroResets ?? srcMetrics.turnTimingElapsedZeroResets).slice(0, 8), turnTimingTotalElapsedForwardJumps: arr(source.turnTimingTotalElapsedForwardJumps ?? srcMetrics.turnTimingTotalElapsedForwardJumps).slice(0, 8), pagePerformanceSlowApi: slowApis.slice(0, 8).map(slimSlowApi), archivePagePerformanceSlowApi: arr(source.archivePagePerformanceSlowApi).slice(0, 8).map(slimSlowApi), pagePerformancePartialApi: arr(source.pagePerformancePartialApi).slice(0, 8), pagePerformanceSseStreams: arr(source.pagePerformanceSseStreams).slice(0, 8), findings: arr(source.findings).slice(0, 12).map(slimFinding), archiveRedFindings: arr(source.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 12).map(slimFinding), reportJsonPath: source.reportJsonPath ?? reportJsonPath, reportJsonSha256: source.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: source.reportMdPath ?? reportMdPath, reportMdSha256: source.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(objectOrNull(source.analyzer) || {}), recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true };",
|
||||
"const compact = { ok: source.ok === true, command: source.command ?? 'web-probe-observe analyze', stateDir: source.stateDir ?? stateDir, jsonlScope: source.jsonlScope ?? null, alertThresholds: source.alertThresholds ?? null, counts: source.counts ?? null, archiveSummary: { ...archiveSummary, sampleMetrics: archiveSampleMetrics, pagePerformance: objectOrNull(archiveSummary.pagePerformance) || objectOrNull(pagePerformance.summary) || {}, runtimeAlerts: objectOrNull(archiveSummary.runtimeAlerts) || objectOrNull(runtimeAlerts.summary) || {}, redFindings: arr(archiveSummary.redFindings).slice(0, 12).map(slimFinding) }, analysisWindow: source.analysisWindow ?? objectOrNull(recent.summary), sampleMetrics: compactMetrics(srcMetrics), pageProvenance: objectOrNull(source.pageProvenance?.summary) || source.pageProvenance ?? null, pagePerformance: objectOrNull(pagePerformance.summary) || pagePerformance, promptNetwork: objectOrNull(promptNetwork.summary) || promptNetwork, runtimeAlerts: objectOrNull(runtimeAlerts.summary) || runtimeAlerts, runnerErrors: arr(source.runnerErrors).slice(-8), commandFailures: arr(source.commandFailures).slice(-8), commandState: objectOrNull(source.commandState) || null, toolFindings: arr(source.toolFindings).slice(0, 8).map(slimFinding), httpErrorGroups: arr(source.httpErrorGroups ?? runtimeAlerts.networkHttpErrorsByPath).slice(0, 8).map(slimGroup), requestFailedGroups: arr(source.requestFailedGroups ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 8).map(slimGroup), domDiagnosticGroups: arr(source.domDiagnosticGroups ?? runtimeAlerts.domDiagnosticsByText).slice(0, 5).map(slimGroup), domDiagnosticSamples: arr(source.domDiagnosticSamples ?? runtimeAlerts.domDiagnostics).slice(0, 8).map(slimGroup), consoleAlertGroups: arr(source.consoleAlertGroups ?? runtimeAlerts.consoleAlertsByPath).slice(0, 8).map(slimGroup), consoleAlertSamples: arr(source.consoleAlertSamples ?? runtimeAlerts.consoleAlerts).slice(0, 8).map(slimGroup), turnTimingRecentUpdateJumps: arr(source.turnTimingRecentUpdateJumps ?? srcMetrics.turnTimingRecentUpdateSawtoothJumps).slice(0, 8), turnTimingElapsedZeroResets: arr(source.turnTimingElapsedZeroResets ?? srcMetrics.turnTimingElapsedZeroResets).slice(0, 8), turnTimingTotalElapsedForwardJumps: arr(source.turnTimingTotalElapsedForwardJumps ?? srcMetrics.turnTimingTotalElapsedForwardJumps).slice(0, 8), pagePerformanceSlowApi: slowApis.slice(0, 8).map(slimSlowApi), archivePagePerformanceSlowApi: arr(source.archivePagePerformanceSlowApi).slice(0, 8).map(slimSlowApi), pagePerformancePartialApi: arr(source.pagePerformancePartialApi).slice(0, 8), pagePerformanceSseStreams: arr(source.pagePerformanceSseStreams).slice(0, 8), findings: arr(source.findings).slice(0, 12).map(slimFinding), archiveRedFindings: arr(source.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 12).map(slimFinding), reportJsonPath: source.reportJsonPath ?? reportJsonPath, reportJsonSha256: source.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: source.reportMdPath ?? reportMdPath, reportMdSha256: source.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(objectOrNull(source.analyzer) || {}), recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true };",
|
||||
"console.log(JSON.stringify(compact));",
|
||||
"UNIDESK_WEB_OBSERVE_RECOVER_ANALYZE_ARTIFACT",
|
||||
].join("\n");
|
||||
@@ -8825,7 +8883,12 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
|
||||
const archiveSlowApis = Array.isArray(analysis?.archivePagePerformanceSlowApi) ? analysis.archivePagePerformanceSlowApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
||||
const partialApis = Array.isArray(analysis?.pagePerformancePartialApi) ? analysis.pagePerformancePartialApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
||||
const sseStreams = Array.isArray(analysis?.pagePerformanceSseStreams) ? analysis.pagePerformanceSseStreams.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
||||
const findings = Array.isArray(analysis?.findings) ? analysis.findings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
||||
const allFindingRecords = Array.isArray(analysis?.findings) ? analysis.findings.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
|
||||
const explicitToolFindings = Array.isArray(analysis?.toolFindings) ? analysis.toolFindings.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
|
||||
const inferredToolFindings = allFindingRecords.filter((item) => String(item.id ?? item.kind ?? item.code ?? "").startsWith("tool-"));
|
||||
const toolFindings = dedupeWebObserveRecords(explicitToolFindings.length > 0 ? explicitToolFindings : inferredToolFindings, (item) => String(item.id ?? item.kind ?? item.code ?? "")).slice(0, 8);
|
||||
const commandState = record(analysis?.commandState);
|
||||
const findings = allFindingRecords.slice(0, 8);
|
||||
const archiveRedFindings = Array.isArray(analysis?.archiveRedFindings)
|
||||
? analysis.archiveRedFindings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8)
|
||||
: Array.isArray(archiveSummary?.redFindings)
|
||||
@@ -9163,6 +9226,18 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
|
||||
webObserveShort(webObserveText(item.message || item.failureKind || item.name), 96),
|
||||
]) : [["-", "-", "-", "-", "-", "-", "-"]]),
|
||||
"",
|
||||
"Tool findings:",
|
||||
webObserveTable(["KIND", "SEVERITY", "COUNT", "SUMMARY"], toolFindings.length > 0 ? toolFindings.map((item) => [
|
||||
webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 36),
|
||||
webObserveText(item.severity ?? item.level),
|
||||
webObserveText(item.count ?? item.sampleCount),
|
||||
webObserveShort(webObserveText(item.summary ?? item.message), 110),
|
||||
]) : [["-", "-", "-", "-"]]),
|
||||
webObserveTable(["TOOL_COMMANDS", "COUNT"], [[
|
||||
`pending=${webObserveText(commandState?.pendingCount)}`,
|
||||
`processing=${webObserveText(commandState?.processingCount)} abandoned=${webObserveText(commandState?.abandonedCount)} failed=${webObserveText(commandState?.failedCount)}`,
|
||||
]]),
|
||||
"",
|
||||
"HTTP error groups:",
|
||||
webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], httpErrorRows.length > 0 ? httpErrorRows : [["-", "-", "-", "-", "-", "-"]]),
|
||||
"",
|
||||
@@ -9296,6 +9371,18 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function dedupeWebObserveRecords(items: Record<string, unknown>[], keyOf: (item: Record<string, unknown>) => string): Record<string, unknown>[] {
|
||||
const out: Record<string, unknown>[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const item of items) {
|
||||
const key = keyOf(item);
|
||||
if (!key || key === "-" || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(item);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function nodeWebObserveResolveStateDirShell(options: Pick<NodeWebProbeObserveOptions, "stateDir" | "jobId" | "node" | "lane">): string {
|
||||
if (options.stateDir !== null) {
|
||||
return [
|
||||
@@ -9781,6 +9868,7 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
|
||||
const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item));
|
||||
const sseStreams = webObserveArray(analysis.pagePerformanceSseStreams).slice(0, 8).map((item) => record(item));
|
||||
const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item));
|
||||
const commandState = record(analysis.commandState);
|
||||
const id = result.id ?? "-";
|
||||
const blockerRows = String(result.status ?? "") === "blocked"
|
||||
? [
|
||||
@@ -9867,6 +9955,16 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
|
||||
],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["TOOL_COMMANDS", "COUNT"],
|
||||
[
|
||||
["pending", commandState.pendingCount],
|
||||
["processing", commandState.processingCount],
|
||||
["abandoned", commandState.abandonedCount],
|
||||
["failed", commandState.failedCount],
|
||||
],
|
||||
),
|
||||
"",
|
||||
domDiagnosticGroups.length === 0
|
||||
? "DOM_DIAGNOSTIC_GROUPS\n-"
|
||||
: webObserveTable(
|
||||
@@ -9932,7 +10030,7 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
|
||||
? "FINDINGS\n-"
|
||||
: webObserveTable(
|
||||
["FINDING", "SEVERITY", "COUNT", "SUMMARY"],
|
||||
findings.map((item) => [item.id, item.severity, item.count, item.summary]),
|
||||
findings.map((item) => [item.id ?? item.kind ?? item.code, item.severity ?? item.level, item.count ?? item.sampleCount, item.summary ?? item.message]),
|
||||
),
|
||||
"",
|
||||
"REPORTS",
|
||||
@@ -10044,26 +10142,102 @@ function withWebObserveShortcuts(value: Record<string, unknown> | null, id: stri
|
||||
|
||||
function nodeWebObserveStatusNodeScript(tailLines: number, node: string, lane: string): string {
|
||||
return `node -e ${shellQuote(`
|
||||
const fs=require('fs'),path=require('path');
|
||||
const dir=process.env.state_dir||process.argv[1];
|
||||
const node=process.argv[2];
|
||||
const lane=process.argv[3];
|
||||
const tailN=Math.min(${tailLines},3);
|
||||
const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}};
|
||||
const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}};
|
||||
const short=(value)=>String(value||'').slice(0,160);
|
||||
const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null;
|
||||
const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null;
|
||||
const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null;
|
||||
const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':'';
|
||||
const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):'';
|
||||
const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)});
|
||||
const compactSample=(item)=>({seq:item.seq,ts:item.ts,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,loadingCount:Array.isArray(item.loadings)?item.loadings.length:0,loadingOwners:Array.isArray(item.loadings)?Array.from(new Set(item.loadings.map((loading)=>loading.ownerLabel||loading.ownerKind||loading.ownerKey||'loading'))).slice(0,4):[],sessionRail:item.sessionRail?{visibleCount:item.sessionRail.visibleCount??null,fallbackTitleCount:item.sessionRail.fallbackTitleCount??null,fallbackTitleRatio:item.sessionRail.fallbackTitleRatio??null,fallbackItems:Array.isArray(item.sessionRail.fallbackItems)?item.sessionRail.fallbackItems.slice(0,4).map((fallback)=>({sessionIdPrefix:fallback.sessionIdPrefix??null,titlePreview:fallback.titlePreview??null,active:fallback.active===true})):[]}:null});
|
||||
const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:short(item.url),status:item.status||null,failure:item.failure?short(item.failure):null});
|
||||
const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null;
|
||||
let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}}
|
||||
console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(readJson('manifest.json')),heartbeat:compactHeartbeat(readJson('heartbeat.json')),tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork)},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true}));
|
||||
`)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`;
|
||||
const fs=require('fs'),path=require('path');
|
||||
const dir=process.env.state_dir||process.argv[1];
|
||||
const node=process.argv[2];
|
||||
const lane=process.argv[3];
|
||||
const tailN=Math.min(${tailLines},3);
|
||||
const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}};
|
||||
const readJsonPath=(file)=>{try{return JSON.parse(fs.readFileSync(file,'utf8'))}catch{return null}};
|
||||
const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}};
|
||||
const short=(value)=>String(value||'').slice(0,160);
|
||||
const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null;
|
||||
const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null;
|
||||
const compactHeartbeat=(item,diag)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,ageSeconds:diag.heartbeatAgeSeconds,stale:diag.heartbeatStale,staleAfterSeconds:diag.heartbeatStaleAfterSeconds,effectiveLiveness:diag.effectiveLiveness,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null;
|
||||
const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':'';
|
||||
const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):'';
|
||||
const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)});
|
||||
const compactSample=(item)=>({seq:item.seq,ts:item.ts,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,loadingCount:Array.isArray(item.loadings)?item.loadings.length:0,loadingOwners:Array.isArray(item.loadings)?Array.from(new Set(item.loadings.map((loading)=>loading.ownerLabel||loading.ownerKind||loading.ownerKey||'loading'))).slice(0,4):[],sessionRail:item.sessionRail?{visibleCount:item.sessionRail.visibleCount??null,fallbackTitleCount:item.sessionRail.fallbackTitleCount??null,fallbackTitleRatio:item.sessionRail.fallbackTitleRatio??null,fallbackItems:Array.isArray(item.sessionRail.fallbackItems)?item.sessionRail.fallbackItems.slice(0,4).map((fallback)=>({sessionIdPrefix:fallback.sessionIdPrefix??null,titlePreview:fallback.titlePreview??null,active:fallback.active===true})):[]}:null});
|
||||
const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:short(item.url),status:item.status||null,failure:item.failure?short(item.failure):null});
|
||||
const commandDir=(name)=>path.join(dir,'commands',name);
|
||||
const commandSummary=(name)=>{
|
||||
const root=commandDir(name); let entries=[];
|
||||
try{entries=fs.readdirSync(root).filter((item)=>item.endsWith('.json')).sort();}catch{}
|
||||
const items=entries.slice(0,12).map((entry)=>{const file=path.join(root,entry); const parsed=readJsonPath(file)||{}; let stat=null; try{stat=fs.statSync(file)}catch{}; const createdAt=parsed.createdAt||parsed.abandonedAt||parsed.completedAt||parsed.failedAt||null; const createdMs=Date.parse(String(createdAt||'')); const ageSeconds=Number.isFinite(createdMs)?Math.max(0,Math.round((Date.now()-createdMs)/1000)):null; return {id:parsed.id||parsed.commandId||entry.replace(/[.]json$/,''),type:parsed.type||null,createdAt,ageSeconds,mtime:stat?stat.mtime.toISOString():null};});
|
||||
const ages=items.map((item)=>item.ageSeconds).filter((value)=>Number.isFinite(value));
|
||||
return {count:entries.length,oldestAgeSeconds:ages.length?Math.max(...ages):null,items};
|
||||
};
|
||||
const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null;
|
||||
let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}}
|
||||
const manifest=readJson('manifest.json');
|
||||
const heartbeat=readJson('heartbeat.json');
|
||||
const terminal=/^(completed|failed|force-stopped|stopped|abandoned)$/u.test(String((heartbeat&&heartbeat.status)||(manifest&&manifest.status)||''));
|
||||
const sampleIntervalMs=Number(manifest&&manifest.sampling&&manifest.sampling.sampleIntervalMs)||5000;
|
||||
const staleAfterMs=Math.max(60000,sampleIntervalMs*3);
|
||||
const updatedRaw=heartbeat&&(heartbeat.updatedAt||heartbeat.lastSampleAt)||null;
|
||||
const updatedMs=Date.parse(String(updatedRaw||''));
|
||||
const heartbeatAgeSeconds=Number.isFinite(updatedMs)?Math.max(0,Math.round((Date.now()-updatedMs)/1000)):null;
|
||||
const heartbeatStale=alive&&!terminal&&(!Number.isFinite(updatedMs)||Date.now()-updatedMs>staleAfterMs);
|
||||
const commandsPending=commandSummary('pending');
|
||||
const commandsProcessing=commandSummary('processing');
|
||||
const commandsAbandoned=commandSummary('abandoned');
|
||||
const commandsFailed=commandSummary('failed');
|
||||
const diagnostics={heartbeatAgeSeconds,heartbeatStale,heartbeatStaleAfterSeconds:Math.round(staleAfterMs/1000),heartbeatUpdatedAt:updatedRaw,terminal,processAlive:alive,effectiveLiveness:heartbeatStale?'stale':alive?'alive':'not-running',commandBacklog:commandsPending.count+commandsProcessing.count,oldestPendingAgeSeconds:commandsPending.oldestAgeSeconds,oldestProcessingAgeSeconds:commandsProcessing.oldestAgeSeconds,valuesRedacted:true};
|
||||
const commands={pendingCount:commandsPending.count,processingCount:commandsProcessing.count,abandonedCount:commandsAbandoned.count,failedCount:commandsFailed.count,pending:commandsPending.items,processing:commandsProcessing.items,abandoned:commandsAbandoned.items.slice(0,6),failed:commandsFailed.items.slice(0,6),valuesRedacted:true};
|
||||
console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(manifest),heartbeat:compactHeartbeat(heartbeat,diagnostics),diagnostics,commands,tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork)},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,forceStop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir+' --force',analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true}));
|
||||
`)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`;
|
||||
}
|
||||
|
||||
function nodeWebObserveForceStopNodeScript(reason: string, forcedByCommandId: string): string {
|
||||
return `node -e ${shellQuote(`
|
||||
const fs=require('fs'),path=require('path'),crypto=require('crypto');
|
||||
const dir=process.argv[1],reason=process.argv[2],forcedByCommandId=process.argv[3];
|
||||
const now=new Date().toISOString();
|
||||
const short=(value,max=160)=>value===undefined||value===null?null:String(value).replace(/\\s+/g,' ').slice(0,max);
|
||||
const shaText=(value)=>'sha256:'+crypto.createHash('sha256').update(String(value||'')).digest('hex');
|
||||
const readJson=(file)=>{try{return JSON.parse(fs.readFileSync(file,'utf8'))}catch{return null}};
|
||||
const writeJson=(file,value)=>{fs.mkdirSync(path.dirname(file),{recursive:true,mode:0o700}); fs.writeFileSync(file,JSON.stringify(value,null,2)+'\\n',{mode:0o600});};
|
||||
const appendJsonl=(file,value)=>{fs.appendFileSync(file,JSON.stringify(value)+'\\n',{mode:0o600});};
|
||||
const summarizeCommand=(command,id)=>({id,type:command&&command.type||null,createdAt:command&&command.createdAt||null,source:command&&command.source||null,path:command&&command.path||null,label:command&&command.label||null,sessionId:command&&command.sessionId||null,provider:command&&command.provider||null,textHash:typeof(command&&command.text)==='string'?shaText(command.text):null,textBytes:typeof(command&&command.text)==='string'?Buffer.byteLength(command.text):null,valuesRedacted:true});
|
||||
const moveCommands=(bucket)=>{
|
||||
const source=path.join(dir,'commands',bucket); let names=[];
|
||||
try{names=fs.readdirSync(source).filter((name)=>name.endsWith('.json')).sort();}catch{}
|
||||
const moved=[];
|
||||
for(const name of names){
|
||||
const file=path.join(source,name);
|
||||
const parsed=readJson(file)||{};
|
||||
const id=String(parsed.id||parsed.commandId||name.replace(/[.]json$/,'')).replace(/[^A-Za-z0-9_.-]/g,'-')||name.replace(/[.]json$/,'');
|
||||
const record={ok:false,status:'abandoned',commandId:id,type:parsed.type||null,abandonedAt:now,reason,sourceBucket:bucket,forcedByCommandId,command:summarizeCommand(parsed,id),valuesRedacted:true};
|
||||
writeJson(path.join(dir,'commands','abandoned',id+'.json'),record);
|
||||
try{fs.unlinkSync(file)}catch{}
|
||||
appendJsonl(path.join(dir,'control.jsonl'),{ts:now,seq:null,phase:'abandoned',type:parsed.type||null,commandId:id,detail:{reason,sourceBucket:bucket,forcedByCommandId,valuesRedacted:true},valuesRedacted:true});
|
||||
moved.push({id,type:parsed.type||null,sourceBucket:bucket,createdAt:parsed.createdAt||null});
|
||||
}
|
||||
return moved;
|
||||
};
|
||||
const pidFile=path.join(dir,'pid');
|
||||
const pidText=fs.existsSync(pidFile)?fs.readFileSync(pidFile,'utf8').trim():'';
|
||||
const pid=Number(pidText);
|
||||
const alive=()=>Number.isFinite(pid)&&pid>0?(()=>{try{process.kill(pid,0); return true;}catch{return false;}})():false;
|
||||
const sleep=(ms)=>Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms);
|
||||
const aliveBefore=alive();
|
||||
let termSent=false,killSent=false;
|
||||
if(aliveBefore){try{process.kill(pid,'SIGTERM'); termSent=true;}catch{}}
|
||||
for(let i=0;i<10&&alive();i+=1) sleep(250);
|
||||
if(alive()){try{process.kill(pid,'SIGKILL'); killSent=true;}catch{}}
|
||||
for(let i=0;i<8&&alive();i+=1) sleep(250);
|
||||
const pending=moveCommands('pending');
|
||||
const processing=moveCommands('processing');
|
||||
const aliveAfter=alive();
|
||||
const manifestPath=path.join(dir,'manifest.json');
|
||||
const heartbeatPath=path.join(dir,'heartbeat.json');
|
||||
const manifest=readJson(manifestPath)||{};
|
||||
const heartbeat=readJson(heartbeatPath)||{};
|
||||
writeJson(manifestPath,{...manifest,ok:false,status:'force-stopped',forceStoppedAt:now,forceStop:{reason,forcedByCommandId,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,valuesRedacted:true}});
|
||||
writeJson(heartbeatPath,{...heartbeat,ok:false,status:'force-stopped',updatedAt:now,forceStoppedAt:now,forceStop:{reason,forcedByCommandId,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,valuesRedacted:true}});
|
||||
appendJsonl(path.join(dir,'control.jsonl'),{ts:now,seq:null,phase:'completed',type:'forceStop',commandId:forcedByCommandId,detail:{reason,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,valuesRedacted:true},valuesRedacted:true});
|
||||
console.log(JSON.stringify({ok:!aliveAfter,command:'web-probe-observe force-stop',stateDir:dir,forced:true,reason,forcedByCommandId,pid:Number.isFinite(pid)?pid:null,termSent,killSent,aliveBefore,aliveAfter,pendingAbandoned:pending.length,processingAbandoned:processing.length,abandoned:[...pending,...processing].slice(0,20),valuesRedacted:true}));
|
||||
`)} "$state_dir" ${shellQuote(reason)} ${shellQuote(forcedByCommandId)}`;
|
||||
}
|
||||
|
||||
function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null, collectGrep: string | null): string {
|
||||
|
||||
Reference in New Issue
Block a user