diff --git a/scripts/src/cli-child-json-recovery.test.ts b/scripts/src/cli-child-json-recovery.test.ts index d3d0bb83..cdfd4cee 100644 --- a/scripts/src/cli-child-json-recovery.test.ts +++ b/scripts/src/cli-child-json-recovery.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "bun:test"; -import { resolveCliChildJsonObject } from "./cli-child-json-recovery"; +import { resolveCliChildJsonCommandResult, resolveCliChildJsonObject } from "./cli-child-json-recovery"; test("child JSON recovery reads full JSON from CLI stdout dump wrapper", async () => { const dir = await mkdtemp(join(tmpdir(), "unidesk-cli-child-json-dump-")); @@ -34,6 +34,34 @@ test("child JSON recovery reads full JSON from CLI stdout dump wrapper", async ( assert.equal(resolved.diagnostics.dumpJsonOk, true); }); +test("child JSON recovery command-result helper keeps dump diagnostics", async () => { + const dir = await mkdtemp(join(tmpdir(), "unidesk-cli-child-json-result-dump-")); + const dumpPath = join(dir, "dump.json"); + await writeFile(dumpPath, JSON.stringify({ ok: true, verified: true, bytes: 123 }) + "\n"); + const resolved = resolveCliChildJsonCommandResult({ + result: { + stdout: JSON.stringify({ + ok: true, + data: { + outputTruncated: true, + reason: "stdout-json-bytes-exceeded-threshold", + dump: { path: dumpPath }, + }, + }), + stderr: "", + exitCode: 0, + timedOut: false, + }, + requestedStdoutType: "trans download verification JSON", + acceptParsed: (value) => value.ok === true && value.verified === true, + }); + + assert.equal(resolved.source, "dump"); + assert.equal(resolved.parsed?.verified, true); + assert.equal(resolved.diagnostics.requestedStdoutType, "trans download verification JSON"); + assert.equal(resolved.diagnostics.dumpPath, dumpPath); +}); + test("child JSON recovery falls back to artifact after trans truncation summary", () => { const summary = { code: "ssh-truncation-summary", diff --git a/scripts/src/cli-child-json-recovery.ts b/scripts/src/cli-child-json-recovery.ts index 9a750206..e0a1e46b 100644 --- a/scripts/src/cli-child-json-recovery.ts +++ b/scripts/src/cli-child-json-recovery.ts @@ -19,6 +19,23 @@ export interface CliChildJsonResolution { diagnostics: Record; } +export function resolveCliChildJsonCommandResult(options: { + result: { stdout: string; stderr?: string; exitCode?: number | null; timedOut?: boolean }; + requestedStdoutType: string; + acceptParsed?: (value: Record) => boolean; + artifactFallback?: CliChildJsonArtifactFallback | null; + forceArtifactFallbackReason?: string | null; +}): CliChildJsonResolution { + return resolveCliChildJsonObject({ + stdout: options.result.stdout, + stderr: options.result.stderr ?? "", + requestedStdoutType: options.requestedStdoutType, + acceptParsed: options.acceptParsed, + artifactFallback: options.artifactFallback, + forceArtifactFallbackReason: options.forceArtifactFallbackReason, + }); +} + export function resolveCliChildJsonObject(options: { stdout: string; stderr?: string; diff --git a/scripts/src/hwlab-fake-model-provider.ts b/scripts/src/hwlab-fake-model-provider.ts index 58b27ade..2a157006 100644 --- a/scripts/src/hwlab-fake-model-provider.ts +++ b/scripts/src/hwlab-fake-model-provider.ts @@ -5,6 +5,7 @@ import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "n import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "./config"; import { runCommand, type CommandResult } from "./command"; +import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; import { resolveAgentRunLaneTarget, type AgentRunLaneSpec } from "./agentrun-lanes"; import { resolveSecretSourceRoot } from "./agentrun/secrets"; import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; @@ -353,7 +354,8 @@ function applyFakeModelProvider(state: FakeModelProviderState): Record 0 ? applyPayload : compactCommand(applyResult, state.options.full), + apply: Object.keys(applyPayload).length > 0 ? { ...applyPayload, stdoutRecovery: applyResolution.diagnostics, valuesRedacted: true } : { ...compactCommand(applyResult, state.options.full), stdoutRecovery: applyResolution.diagnostics }, status, next: { smoke: `bun scripts/cli.ts hwlab nodes fake-model-provider smoke --node ${state.options.node} --lane ${state.options.lane} --provider ${state.options.provider}`, @@ -548,7 +550,8 @@ function remoteFakeModelProviderStatus(state: FakeModelProviderState): Record 0 ? payload : { result: compactCommand(result, state.options.full) }), + stdoutRecovery: payloadResolution.diagnostics, valuesPrinted: false, }; } @@ -570,7 +574,8 @@ function remoteFakeModelProviderSmoke(state: FakeModelProviderState): Record 0 ? payload : { result: compactCommand(result, state.options.full) }), + stdoutRecovery: payloadResolution.diagnostics, valuesPrinted: false, }; } @@ -905,22 +911,13 @@ function tomlString(value: string): string { return JSON.stringify(value); } -function parseJsonObject(text: string): Record { - const trimmed = text.trim(); - if (trimmed.length > 0) { - try { - return record(JSON.parse(trimmed) as unknown, "json"); - } catch { - const start = trimmed.indexOf("{"); - const end = trimmed.lastIndexOf("}"); - if (start >= 0 && end > start) { - try { - return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown, "json"); - } catch {} - } - } - } - return {}; +function resolveFakeModelProviderRemotePayload(result: CommandResult, requestedStdoutType: string): { parsed: Record | null; diagnostics: Record } { + const resolved = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType, + acceptParsed: (value) => typeof value.ok === "boolean" || typeof value.error === "string" || typeof value.failureKind === "string" || Object.keys(value).length > 1, + }); + return { parsed: resolved.parsed, diagnostics: resolved.diagnostics }; } function compactCommand(result: CommandResult, full = false): Record { diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index 62cc20ee..d8d0da5b 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -6,6 +6,7 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import type { CommandResult } from "./command"; import { runCommand } from "./command"; +import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; import { repoRoot, rootPath } from "./config"; import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; import type { ChildCliResult, CompactCommandResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; @@ -809,10 +810,13 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p let parsed: Record | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 }); - parsed = parseJsonObject(result.stdout); - const recovered = parsed === null ? recoverTruncatedSshStdoutJson(result) : null; - if (recovered !== null) parsed = recovered.parsed; - attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, parsedFromDump: recovered !== null, dumpPath: recovered?.dumpPath ?? null, valuesRedacted: true }); + const parsedResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "web-probe sentinel service proxy response JSON", + acceptParsed: isSentinelServiceResponseContract, + }); + parsed = parsedResolution.parsed; + attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, stdoutRecovery: parsedResolution.diagnostics, valuesRedacted: true }); if (result.exitCode === 0) break; } return { @@ -832,27 +836,15 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p }; } -function recoverTruncatedSshStdoutJson(result: CommandResult): { parsed: Record; dumpPath: string } | null { - const dumpPath = sshStdoutDumpPathFromStderr(result.stderr); - if (dumpPath === null || !existsSync(dumpPath)) return null; - const parsed = parseJsonObject(readFileSync(dumpPath, "utf8")); - return parsed === null ? null : { parsed, dumpPath }; -} - -function sshStdoutDumpPathFromStderr(value: string): string | null { - for (const rawLine of value.split(/\r?\n/u)) { - const line = rawLine.trim(); - const prefix = "UNIDESK_SSH_STDOUT_TRUNCATED "; - if (!line.startsWith(prefix)) continue; - try { - const payload = JSON.parse(line.slice(prefix.length)) as unknown; - const dumpPath = stringAtNullable(record(payload), "dumpPath"); - if (dumpPath !== null) return dumpPath; - } catch { - return null; - } - } - return null; +function isSentinelServiceResponseContract(value: Record): boolean { + return value.ok === false + || typeof value.view === "string" + || typeof value.renderedText === "string" + || typeof value.error === "string" + || Array.isArray(value.availableViews) + || Object.keys(record(value.run)).length > 0 + || Object.keys(record(value.summary)).length > 0 + || Array.isArray(value.findings); } function compactSentinelServiceBodyJson(value: Record | null): unknown { @@ -1392,7 +1384,12 @@ function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: st "NODE", ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const parsed = parseJsonObject(result.stdout); + const parsedResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "web-probe sentinel quick-verify analysis summary JSON", + acceptParsed: (value) => value.ok === true || value.ok === false || typeof value.reason === "string" || typeof value.reportJsonPath === "string" || Object.keys(record(value.counts)).length > 0, + }); + const parsed = parsedResolution.parsed; const parsedRecord = record(parsed); const reason = stringAtNullable(parsedRecord, "reason") ?? (result.timedOut ? "workspace-artifact-read-timeout" @@ -1406,6 +1403,7 @@ function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: st reason, stateDir: stringAtNullable(parsedRecord, "stateDir") ?? stateDir, result: compactCommand(result), + stdoutRecovery: parsedResolution.diagnostics, valuesRedacted: true, }; } @@ -1426,7 +1424,12 @@ function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now())); const script = quickVerifyObserverStartupWaitScript(indexEntry.stateDir, waitMs, pollSleepMs); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 }); - const payload = parseJsonObject(result.stdout); + const payloadResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "quick-verify observer startup wait JSON", + acceptParsed: (value) => typeof value.status === "string" || typeof value.heartbeatStatus === "string" || Array.isArray(value.observations) || Object.keys(record(value.startup)).length > 0, + }); + const payload = payloadResolution.parsed; if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); const terminalPayload = { observerId, @@ -1435,7 +1438,7 @@ function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: heartbeatStatus: typeof payload?.heartbeatStatus === "string" ? payload.heartbeatStatus : null, startup: record(payload?.startup), observations: observations.slice(-6), - waitResult: compactCommand(result), + waitResult: { ...compactCommand(result), stdoutRecovery: payloadResolution.diagnostics }, valuesRedacted: true, }; if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-startup-wait-chunk-timeout") { @@ -1574,7 +1577,10 @@ export function runChildCli(args: string[], timeoutSeconds: number, input?: stri }); return { ok: result.exitCode === 0 && !result.timedOut, - parsed: parseJsonObject(result.stdout), + parsed: resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: `child CLI ${args.join(" ")} JSON`, + }).parsed, result: compactCommandWithTail(result), }; } @@ -1596,7 +1602,12 @@ function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: stri const waitMs = Math.max(1000, Math.min(Math.max(1000, Math.trunc(chunkSeconds * 1000)), deadline - Date.now())); const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 8000 }); - const payload = parseJsonObject(result.stdout); + const payloadResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "quick-verify prompt wait JSON", + acceptParsed: (value) => typeof value.status === "string" || typeof value.traceId === "string" || Array.isArray(value.observations) || value.finalResponseEmpty === true || value.composerReadyForTurn === true, + }); + const payload = payloadResolution.parsed; if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); const status = typeof payload?.status === "string" ? payload.status : null; const terminalPayload = { @@ -1607,7 +1618,7 @@ function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: stri composerReadyForTurn: payload?.composerReadyForTurn === true, composerAction: typeof payload?.composerAction === "string" ? payload.composerAction : null, observations: observations.slice(-6), - waitResult: compactCommand(result), + waitResult: { ...compactCommand(result), stdoutRecovery: payloadResolution.diagnostics }, valuesRedacted: true, }; if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-wait-chunk-timeout") { @@ -1840,18 +1851,7 @@ function normalizeQuickVerifyStatus(value: string | null): string { function cliDataPayload(parsed: Record | null): Record { const root = record(parsed); - const payload = isRecord(root.data) ? root.data : root; - return cliDumpPayload(payload) ?? payload; -} - -function cliDumpPayload(payload: Record): Record | null { - if (payload.outputTruncated !== true) return null; - const dumpPath = stringAtNullable(record(payload.dump), "path"); - if (dumpPath === null || !existsSync(dumpPath)) return null; - const dumped = parseJsonObject(readFileSync(dumpPath, "utf8")); - if (dumped === null) return null; - const dumpedRoot = record(dumped); - return isRecord(dumpedRoot.data) ? dumpedRoot.data : dumpedRoot; + return isRecord(root.data) ? root.data : root; } function findScenario(state: SentinelCicdState, scenarioId: string): Record | null { @@ -2256,7 +2256,7 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number, return [quickVerifyControlFinding( "quick-verify-diagnostics-inconclusive", "快速验证诊断信息不足", - "quick verify did not prove a durable completed turn, but structured diagnostics did not match any specific failure code.", + "quick verify did not prove a durable completed turn, but structured diagnostics did not match a specific failure code.", `quick verify has rowCount=${rowCount} scopedRowCount=${scopedRowCount}, traceIdPresent=${traceDiagnostics.traceIdPresent === true}, finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`, "Improve turn-summary/trace-frame diagnostics before making a business recovery decision.", promptIndex, diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index d4bec958..796de27f 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -1,9 +1,9 @@ // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel. // Responsibility: P5 web-probe sentinel service validation, maintenance, report and dashboard commands. -import { existsSync, readFileSync } from "node:fs"; import type { CommandResult } from "./command"; import { runCommand } from "./command"; +import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; import { repoRoot } from "./config"; import { startJob } from "./jobs"; import type { RenderedCliResult } from "./output"; @@ -513,8 +513,13 @@ function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Recor "NODE", ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 55)) * 1000 }); - const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; + const parsedResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "web-probe sentinel report summary JSON", + acceptParsed: (value) => value.ok === true || value.ok === false || typeof value.reason === "string" || typeof value.reportJsonPath === "string" || Object.keys(record(value.counts)).length > 0, + }); + const parsed = parsedResolution.parsed; + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), stdoutRecovery: parsedResolution.diagnostics, valuesRedacted: true }; } function isSafeSentinelReportStateDir(value: string): boolean { @@ -1468,10 +1473,13 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p let parsed: Record | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 }); - parsed = parseJsonObject(result.stdout); - const recovered = parsed === null ? recoverTruncatedSshStdoutJson(result) : null; - if (recovered !== null) parsed = recovered.parsed; - attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, parsedFromDump: recovered !== null, dumpPath: recovered?.dumpPath ?? null, valuesRedacted: true }); + const parsedResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "web-probe sentinel service proxy response JSON", + acceptParsed: isSentinelServiceResponseContract, + }); + parsed = parsedResolution.parsed; + attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, stdoutRecovery: parsedResolution.diagnostics, valuesRedacted: true }); if (result.exitCode === 0) break; } const compactBodyJson = compactSentinelServiceBodyJson(parsed); @@ -1492,27 +1500,15 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p }; } -function recoverTruncatedSshStdoutJson(result: CommandResult): { parsed: Record; dumpPath: string } | null { - const dumpPath = sshStdoutDumpPathFromStderr(result.stderr); - if (dumpPath === null || !existsSync(dumpPath)) return null; - const parsed = parseJsonObject(readFileSync(dumpPath, "utf8")); - return parsed === null ? null : { parsed, dumpPath }; -} - -function sshStdoutDumpPathFromStderr(value: string): string | null { - for (const rawLine of value.split(/\r?\n/u)) { - const line = rawLine.trim(); - const prefix = "UNIDESK_SSH_STDOUT_TRUNCATED "; - if (!line.startsWith(prefix)) continue; - try { - const payload = JSON.parse(line.slice(prefix.length)) as unknown; - const dumpPath = stringAtNullable(record(payload), "dumpPath"); - if (dumpPath !== null) return dumpPath; - } catch { - return null; - } - } - return null; +function isSentinelServiceResponseContract(value: Record): boolean { + return value.ok === false + || typeof value.view === "string" + || typeof value.renderedText === "string" + || typeof value.error === "string" + || Array.isArray(value.availableViews) + || Object.keys(record(value.run)).length > 0 + || Object.keys(record(value.summary)).length > 0 + || Array.isArray(value.findings); } function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number, url: string): Record { diff --git a/scripts/src/hwlab-node/cleanup.ts b/scripts/src/hwlab-node/cleanup.ts index 4f6bdfd7..ffa550d5 100644 --- a/scripts/src/hwlab-node/cleanup.ts +++ b/scripts/src/hwlab-node/cleanup.ts @@ -10,6 +10,7 @@ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from " import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; +import { resolveCliChildJsonCommandResult } from "../cli-child-json-recovery"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; @@ -36,15 +37,6 @@ import { parseNodeScopedDelegatedOptions } from "./plan"; import { compactRuntimeCommand, compactRuntimeCommandStats, nodeRuntimeUnsupportedAction, runNodeHostScript, transPath } from "./runtime-common"; import { optionValue, positiveIntegerOption, shellQuote, statusText } from "./utils"; -export function parseJsonObject(text: string): Record { - try { - const parsed = JSON.parse(text) as unknown; - return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; - } catch { - return {}; - } -} - export function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record, statusPath: string, timedOut: boolean): CommandResult { return { command: [transPath(), spec.nodeRoute, "sh", "--", ""], @@ -102,7 +94,7 @@ export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string { return spec.gitopsRoot; } -export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { +export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult; stdoutRecovery?: Record } { const mirror = nodeRuntimeSourceMirrorTarget(spec); const script = [ "set +e", @@ -135,10 +127,25 @@ export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { source "NODE", ].join("\n"); const result = runNodeK3sScript(spec, script, 45); - const payload = parseJsonObject(result.stdout); + const payloadResolution = resolveRuntimeCleanupJson(result, "node runtime source snapshot JSON"); + const payload = payloadResolution.parsed ?? {}; const payloadCommit = typeof payload.sourceCommit === "string" && /^[0-9a-f]{40}$/iu.test(payload.sourceCommit) ? payload.sourceCommit.toLowerCase() : null; const match = payloadCommit ?? /[0-9a-f]{40}/iu.exec(statusText(result))?.[0].toLowerCase() ?? null; - return { sourceCommit: result.exitCode === 0 ? match : null, result }; + return { sourceCommit: result.exitCode === 0 ? match : null, result, stdoutRecovery: payloadResolution.diagnostics }; +} + +function resolveRuntimeCleanupJson(result: CommandResult, requestedStdoutType: string): { parsed: Record | null; diagnostics: Record } { + const resolved = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType, + acceptParsed: (value) => typeof value.ok === "boolean" + || typeof value.mode === "string" + || typeof value.sourceCommit === "string" + || typeof value.degradedReason === "string" + || typeof value.error === "string" + || Object.keys(value).length > 1, + }); + return { parsed: resolved.parsed, diagnostics: resolved.diagnostics }; } interface NodeRuntimeSourceMirrorTarget { @@ -688,7 +695,8 @@ export function nodeRuntimeCleanupLegacyDockerImages(scoped: ReturnType value.ok === true && (Array.isArray(value.rounds) || Array.isArray(value.turnColumns)), + }).parsed; + const recoveredRounds = Array.isArray(recovered?.rounds) ? recovered.rounds : []; + const recoveredColumns = Array.isArray(recovered?.turnColumns) ? recovered.turnColumns : []; if (recoveredRounds.length === 0 && recoveredColumns.length === 0) return analysis; const nextMetrics = { ...(sampleMetrics ?? {}) }; if (!hasRounds && recoveredRounds.length > 0) { @@ -1083,10 +1089,16 @@ export function discoverWebObserveIndexEntryOnTarget(id: string, node: string, l "NODE", ].join("\n"); const result = runTransWorkspaceStdinScript(node, spec.workspace, script, 55); - const payload = parseJsonObject(result.stdout); + const discovery = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "web-probe observe index discovery JSON", + acceptParsed: (value) => value.ok === true && (value.found === true || value.found === false || value.ambiguous === true), + }); + const payload = discovery.parsed; if (result.exitCode !== 0 || result.timedOut || payload?.ok !== true) { const reason = result.timedOut ? "timeout" : payload?.ambiguous === true ? "ambiguous" : `exit=${result.exitCode}`; - throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)}`); + const recovery = JSON.stringify(discovery.diagnostics).slice(0, 360); + throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)} recovery=${recovery}`); } if (payload.found !== true) return null; const entry = record(payload.entry); diff --git a/scripts/src/hwlab-node/web-observe-scripts.ts b/scripts/src/hwlab-node/web-observe-scripts.ts index 9400d99c..5442ecfc 100644 --- a/scripts/src/hwlab-node/web-observe-scripts.ts +++ b/scripts/src/hwlab-node/web-observe-scripts.ts @@ -10,6 +10,7 @@ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from " import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; +import { resolveCliChildJsonCommandResult } from "../cli-child-json-recovery"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; @@ -31,7 +32,7 @@ import type { RenderedCliResult } from "../output"; import type { BootstrapAdminPasswordMaterial, NodeWebProbeScriptOptions, RuntimeSecretSpec } from "./entry"; import type { NodeWebProbeHostProxyEnv } from "./web-probe-observe"; import { isSafeWebProbeScriptArtifactPath, isSafeWebProbeScriptReportPath, isSafeWebProbeScriptRunDir, runTransWorkspaceStdinScript } from "./public-exposure"; -import { compactCommandResultRedacted, nullableRecord, parseJsonObject, redactKnownSecrets, shellQuote } from "./utils"; +import { compactCommandResultRedacted, nullableRecord, redactKnownSecrets, shellQuote } from "./utils"; import { renderWebProbeScriptResult } from "./web-observe-render"; import { nodeWebProbeHostProxyEnv } from "./web-probe-observe"; @@ -481,11 +482,26 @@ export function runNodeWebProbeScript( const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const commandTimedOut = result.timedOut || result.exitCode === 124; - const stdoutReport = parseJsonObject(result.stdout); const runPaths = webProbeScriptRunPathsFromStderr(result.stderr); - const recoveredReport = stdoutReport === null ? readNodeWebProbeScriptReport(options, spec, runPaths.reportPath) : null; - const recoveredArtifacts = stdoutReport === null || commandTimedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null; - const parsedReport = stdoutReport ?? recoveredReport?.report ?? null; + let recoveredReport: WebProbeScriptReportReadResult | null = null; + const stdoutResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "web-probe script report JSON", + acceptParsed: isWebProbeScriptReportContract, + forceArtifactFallbackReason: commandTimedOut ? "remote-command-timeout" : null, + artifactFallback: { + path: runPaths.reportPath, + nextCommand: runPaths.reportPath === null ? null : `trans ${options.node}:${spec.workspace} cat ${shellQuote(runPaths.reportPath)}`, + read: () => { + recoveredReport = readNodeWebProbeScriptReport(options, spec, runPaths.reportPath); + if (recoveredReport?.report) return { ok: true, value: recoveredReport.report, path: recoveredReport.path, reason: recoveredReport.source }; + return { ok: false, reason: recoveredReport?.degradedReason ?? "web-probe-report-artifact-missing", path: runPaths.reportPath }; + }, + }, + }); + const stdoutReport = stdoutResolution.source === "stdout" || stdoutResolution.source === "dump" ? stdoutResolution.parsed : null; + const recoveredArtifacts = stdoutResolution.parsed === null || commandTimedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null; + const parsedReport = stdoutResolution.parsed; const report = compactWebProbeScriptResult(parsedReport); const passed = result.exitCode === 0 && report?.ok === true; const summary = nullableRecord(report?.summary); @@ -493,9 +509,10 @@ export function runNodeWebProbeScript( const outputFailureKind = parsedReport === null ? commandTimedOut ? "web-probe-command-timeout" - : stdoutBytes > 64 * 1024 - ? "web-probe-output-too-large" - : "web-probe-report-parse-failed" + : stringOrNullValue(stdoutResolution.diagnostics.fallbackReason) + ?? (stdoutBytes > 64 * 1024 + ? "web-probe-output-too-large" + : "web-probe-report-parse-failed") : null; const degradedReason = commandTimedOut ? "web-probe-command-timeout" @@ -518,7 +535,8 @@ export function runNodeWebProbeScript( const effectiveSummary = summary !== null ? { ...summary, transportTimedOut: commandTimedOut, - recoveredFrom: stdoutReport !== null ? "stdout" : recoveredReport?.source ?? null, + recoveredFrom: stdoutReport !== null ? stdoutResolution.source : recoveredReport?.source ?? null, + stdoutRecovery: stdoutResolution.diagnostics, } : (outputFailureKind === null ? null : { ok: false, status: "blocked", @@ -539,8 +557,9 @@ export function runNodeWebProbeScript( }, screenshots: recoveredArtifacts?.screenshots ?? [], artifacts: recoveredArtifacts?.artifacts ?? null, - stdoutBytes, - exitCode: result.exitCode, + stdoutBytes, + stdoutRecovery: stdoutResolution.diagnostics, + exitCode: result.exitCode, stderrTail: result.stderr.trim().slice(-2000), valuesRedacted: true, }); @@ -566,12 +585,14 @@ export function runNodeWebProbeScript( summary: effectiveSummary, issueEvidence, probe: report, - reportLoad: stdoutReport !== null ? { source: "stdout", path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : { + reportLoad: stdoutReport !== null ? { source: stdoutResolution.source, path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : { source: recoveredReport.source, path: recoveredReport.path, degradedReason: recoveredReport.degradedReason, result: recoveredReport.result === null ? null : compactCommandResultRedacted(recoveredReport.result, [material.password ?? ""]), + stdoutRecovery: recoveredReport.stdoutRecovery ?? null, }, + stdoutRecovery: stdoutResolution.diagnostics, warnings: webProbeScriptGovernanceWarnings(options), hints: webProbeScriptGovernanceHints(options), preferredCommands: webProbeScriptPreferredCommands(options), @@ -667,11 +688,20 @@ export function lineValueFromText(text: string, name: string): string | null { return match ? match[1].trim() : null; } +interface WebProbeScriptReportReadResult { + source: string; + report: Record | null; + result: CommandResult | null; + degradedReason: string | null; + path: string | null; + stdoutRecovery?: Record | null; +} + export function readNodeWebProbeScriptReport( options: NodeWebProbeScriptOptions, spec: HwlabRuntimeLaneSpec, reportPath: string | null, -): { source: string; report: Record | null; result: CommandResult | null; degradedReason: string | null; path: string | null } | null { +): WebProbeScriptReportReadResult | null { if (!reportPath) return null; if (!isSafeWebProbeScriptReportPath(reportPath)) return { source: "unsafe-path", report: null, result: null, degradedReason: "web-probe-report-path-invalid", path: reportPath }; const script = [ @@ -753,16 +783,38 @@ export function readNodeWebProbeScriptReport( if (result.exitCode !== 0 || result.timedOut) { return { source: "report-file", report: null, result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-report-read-failed", path: reportPath }; } - const report = parseJsonObject(result.stdout); + const reportResolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: "web-probe script report artifact compact JSON", + acceptParsed: isWebProbeScriptReportContract, + }); + const report = reportResolution.parsed; return { source: "report-file", report, result, - degradedReason: report === null ? "web-probe-report-parse-failed" : null, + degradedReason: report === null ? stringOrNullValue(reportResolution.diagnostics.fallbackReason) ?? "web-probe-report-parse-failed" : null, path: reportPath, + stdoutRecovery: reportResolution.diagnostics, }; } +function isWebProbeScriptReportContract(value: Record): boolean { + return value.ok === true + || value.ok === false + || typeof value.status === "string" + || typeof value.reportPath === "string" + || typeof value.error === "string" + || typeof value.failureKind === "string" + || Object.keys(nullableRecord(value.summary) ?? {}).length > 0 + || Object.keys(nullableRecord(value.script) ?? {}).length > 0 + || Array.isArray(value.steps); +} + +function stringOrNullValue(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + export function readNodeWebProbeScriptArtifacts( options: NodeWebProbeScriptOptions, spec: HwlabRuntimeLaneSpec, diff --git a/scripts/src/web-probe-remote-artifact.ts b/scripts/src/web-probe-remote-artifact.ts index a05de8b1..2a424f7b 100644 --- a/scripts/src/web-probe-remote-artifact.ts +++ b/scripts/src/web-probe-remote-artifact.ts @@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto"; import { basename, join, resolve } from "node:path"; import { repoRoot } from "./config"; import { runCommand, type CommandResult } from "./command"; +import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; export interface WebProbeRemoteArtifactJobOptions { route: string; @@ -99,7 +100,12 @@ export function runWebProbeRemoteArtifactJob(options: WebProbeRemoteArtifactJobO const download = runCommand(downloadArgs, repoRoot, { timeoutMs: commandTimeoutMs }); commandResults.push(download); lastDownload = download; - const parsed = parseJsonObject(download.stdout); + const downloadJson = resolveCliChildJsonCommandResult({ + result: download, + requestedStdoutType: "trans download verification JSON", + acceptParsed: (value) => typeof value.ok === "boolean" || typeof value.verified === "boolean" || typeof value.bytes === "number" || typeof value.sha256 === "string", + }); + const parsed = downloadJson.parsed ?? {}; if (download.exitCode === 0 && parsed.ok === true && parsed.verified === true) { artifacts.push({ remotePath: artifact.remotePath, @@ -109,6 +115,7 @@ export function runWebProbeRemoteArtifactJob(options: WebProbeRemoteArtifactJobO verified: true, verification: parsed.verification ?? null, transfer: parsed.transfer ?? null, + stdoutRecovery: downloadJson.diagnostics, manifestBytes: artifact.bytes, manifestSha256: artifact.sha256, }); @@ -124,6 +131,11 @@ export function runWebProbeRemoteArtifactJob(options: WebProbeRemoteArtifactJobO exitCode: lastDownload?.exitCode ?? null, stdoutTail: lastDownload?.stdout.slice(-1000) ?? "", stderrTail: lastDownload?.stderr.slice(-1000) ?? "", + stdoutRecovery: lastDownload === null ? null : resolveCliChildJsonCommandResult({ + result: lastDownload, + requestedStdoutType: "trans download verification JSON", + acceptParsed: (value) => typeof value.ok === "boolean" || typeof value.verified === "boolean" || typeof value.bytes === "number" || typeof value.sha256 === "string", + }).diagnostics, }; break; } @@ -468,17 +480,6 @@ function wrapBase64(value: string): string { return value.replace(/.{1,76}/gu, "$&\n").trimEnd(); } -function parseJsonObject(text: string): Record { - const trimmed = text.trim(); - if (trimmed.length === 0) return {}; - try { - const parsed = JSON.parse(trimmed) as unknown; - return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; - } catch { - return {}; - } -} - function parseTabStatus(text: string): Record { const out: Record = {}; for (const rawLine of text.split(/\r?\n/u)) {