mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cli): migrate child json recovery callers

Browse Source
This commit is contained in:
Codex
2026-07-02 19:25:27 +00:00
parent 51512f3d7b
commit 146ecc5e82
9 changed files with 254 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node/cleanup.ts
+25 -16
View File
@@ -10,6 +10,7 @@ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { resolveCliChildJsonCommandResult } from "../cli-child-json-recovery";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
@@ -36,15 +37,6 @@ import { parseNodeScopedDelegatedOptions } from "./plan";
import { compactRuntimeCommand, compactRuntimeCommandStats, nodeRuntimeUnsupportedAction, runNodeHostScript, transPath } from "./runtime-common";
import { optionValue, positiveIntegerOption, shellQuote, statusText } from "./utils";
export function parseJsonObject(text: string): Record<string, unknown> {
try {
const parsed = JSON.parse(text) as unknown;
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
} catch {
return {};
}
}
export function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record<string, unknown>, statusPath: string, timedOut: boolean): CommandResult {
return {
command: [transPath(), spec.nodeRoute, "sh", "--", "<remote async script>"],
@@ -102,7 +94,7 @@ export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
return spec.gitopsRoot;
}
export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult; stdoutRecovery?: Record<string, unknown> } {
const mirror = nodeRuntimeSourceMirrorTarget(spec);
const script = [
"set +e",
@@ -135,10 +127,25 @@ export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { source
"NODE",
].join("\n");
const result = runNodeK3sScript(spec, script, 45);
const payload = parseJsonObject(result.stdout);
const payloadResolution = resolveRuntimeCleanupJson(result, "node runtime source snapshot JSON");
const payload = payloadResolution.parsed ?? {};
const payloadCommit = typeof payload.sourceCommit === "string" && /^[0-9a-f]{40}$/iu.test(payload.sourceCommit) ? payload.sourceCommit.toLowerCase() : null;
const match = payloadCommit ?? /[0-9a-f]{40}/iu.exec(statusText(result))?.[0].toLowerCase() ?? null;
return { sourceCommit: result.exitCode === 0 ? match : null, result };
return { sourceCommit: result.exitCode === 0 ? match : null, result, stdoutRecovery: payloadResolution.diagnostics };
}
function resolveRuntimeCleanupJson(result: CommandResult, requestedStdoutType: string): { parsed: Record<string, unknown> | null; diagnostics: Record<string, unknown> } {
const resolved = resolveCliChildJsonCommandResult({
result,
requestedStdoutType,
acceptParsed: (value) => typeof value.ok === "boolean"
|| typeof value.mode === "string"
|| typeof value.sourceCommit === "string"
|| typeof value.degradedReason === "string"
|| typeof value.error === "string"
|| Object.keys(value).length > 1,
});
return { parsed: resolved.parsed, diagnostics: resolved.diagnostics };
}
interface NodeRuntimeSourceMirrorTarget {
@@ -688,7 +695,8 @@ export function nodeRuntimeCleanupLegacyDockerImages(scoped: ReturnType<typeof p
"UNIDESK_LEGACY_DOCKER_IMAGE_GC",
].join("\n");
const result = runNodeHostScript(scoped.spec, script, scoped.timeoutSeconds);
const payload = parseJsonObject(result.stdout);
const payloadResolution = resolveRuntimeCleanupJson(result, "legacy docker image cleanup JSON");
const payload = payloadResolution.parsed ?? {};
const ok = isCommandSuccess(result) && payload.ok !== false;
return {
ok,
@@ -711,7 +719,7 @@ export function nodeRuntimeCleanupLegacyDockerImages(scoped: ReturnType<typeof p
valuesRedacted: true,
},
cleanup: options.dryRun || ok ? compactNodeRuntimeLegacyDockerImageCleanupPayload(payload) : payload,
result: ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result),
result: { ...(ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result)), stdoutRecovery: payloadResolution.diagnostics },
degradedReason: ok ? undefined : "legacy-docker-image-cleanup-failed",
next: options.dryRun
? { confirm: `${command} --min-age-hours ${options.minAgeHours} --keep-per-repository ${options.keepPerRepository} --limit ${options.limit} --confirm` }
@@ -769,7 +777,8 @@ export function nodeRuntimeCleanupLegacyDockerRegistryVolume(scoped: ReturnType<
"UNIDESK_LEGACY_DOCKER_REGISTRY_VOLUME_GC",
].join("\n");
const result = runNodeHostScript(scoped.spec, script, scoped.timeoutSeconds);
const payload = parseJsonObject(result.stdout);
const payloadResolution = resolveRuntimeCleanupJson(result, "legacy docker registry volume cleanup JSON");
const payload = payloadResolution.parsed ?? {};
const ok = isCommandSuccess(result) && payload.ok !== false;
return {
ok,
@@ -784,7 +793,7 @@ export function nodeRuntimeCleanupLegacyDockerRegistryVolume(scoped: ReturnType<
},
target,
cleanup: compactNodeRuntimeLegacyDockerRegistryVolumeCleanupPayload(payload),
result: ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result),
result: { ...(ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result)), stdoutRecovery: payloadResolution.diagnostics },
degradedReason: ok ? undefined : "legacy-docker-registry-volume-cleanup-failed",
next: dryRun
? { confirm: `${command} --confirm --wait` }
scripts/src/hwlab-node/web-observe-render.ts
+18 -6
View File
@@ -10,6 +10,7 @@ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { resolveCliChildJsonCommandResult } from "../cli-child-json-recovery";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
@@ -30,7 +31,7 @@ import type { RenderedCliResult } from "../output";
import type { NodeWebProbeObserveAction, NodeWebProbeObserveOptions, WebObserveIndexEntry } from "./entry";
import { runTransWorkspaceStdinScript } from "./public-exposure";
import { parseJsonObject, record, shellQuote } from "./utils";
import { record, shellQuote } from "./utils";
import { isSafeWebObserveJobId, isSafeWebObserveStateDir, safeWebObserveSegment } from "./web-observe-scripts";
import { webObserveShort, webObserveText } from "./web-probe-observe";
@@ -68,9 +69,14 @@ export function recoverWebObserveAnalyzeTurnDetails(options: NodeWebProbeObserve
"console.log(JSON.stringify({ ok: true, rounds, turnColumns }));",
"UNIDESK_WEB_OBSERVE_RECOVER_TURN_DETAILS",
].join("\n");
const recovered = parseJsonObject(runTransWorkspaceStdinScript(options.node, spec.workspace, recoverScript, Math.min(options.commandTimeoutSeconds, 30)).stdout);
const recoveredRounds = Array.isArray(recovered.rounds) ? recovered.rounds : [];
const recoveredColumns = Array.isArray(recovered.turnColumns) ? recovered.turnColumns : [];
const recoverResult = runTransWorkspaceStdinScript(options.node, spec.workspace, recoverScript, Math.min(options.commandTimeoutSeconds, 30));
const recovered = resolveCliChildJsonCommandResult({
result: recoverResult,
requestedStdoutType: "web-probe observe analyze turn-detail recovery JSON",
acceptParsed: (value) => value.ok === true && (Array.isArray(value.rounds) || Array.isArray(value.turnColumns)),
}).parsed;
const recoveredRounds = Array.isArray(recovered?.rounds) ? recovered.rounds : [];
const recoveredColumns = Array.isArray(recovered?.turnColumns) ? recovered.turnColumns : [];
if (recoveredRounds.length === 0 && recoveredColumns.length === 0) return analysis;
const nextMetrics = { ...(sampleMetrics ?? {}) };
if (!hasRounds && recoveredRounds.length > 0) {
@@ -1083,10 +1089,16 @@ export function discoverWebObserveIndexEntryOnTarget(id: string, node: string, l
"NODE",
].join("\n");
const result = runTransWorkspaceStdinScript(node, spec.workspace, script, 55);
const payload = parseJsonObject(result.stdout);
const discovery = resolveCliChildJsonCommandResult({
result,
requestedStdoutType: "web-probe observe index discovery JSON",
acceptParsed: (value) => value.ok === true && (value.found === true || value.found === false || value.ambiguous === true),
});
const payload = discovery.parsed;
if (result.exitCode !== 0 || result.timedOut || payload?.ok !== true) {
const reason = result.timedOut ? "timeout" : payload?.ambiguous === true ? "ambiguous" : `exit=${result.exitCode}`;
throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)}`);
const recovery = JSON.stringify(discovery.diagnostics).slice(0, 360);
throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)} recovery=${recovery}`);
}
if (payload.found !== true) return null;
const entry = record(payload.entry);
scripts/src/hwlab-node/web-observe-scripts.ts
+67 -15
View File
@@ -10,6 +10,7 @@ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { resolveCliChildJsonCommandResult } from "../cli-child-json-recovery";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
@@ -31,7 +32,7 @@ import type { RenderedCliResult } from "../output";
import type { BootstrapAdminPasswordMaterial, NodeWebProbeScriptOptions, RuntimeSecretSpec } from "./entry";
import type { NodeWebProbeHostProxyEnv } from "./web-probe-observe";
import { isSafeWebProbeScriptArtifactPath, isSafeWebProbeScriptReportPath, isSafeWebProbeScriptRunDir, runTransWorkspaceStdinScript } from "./public-exposure";
import { compactCommandResultRedacted, nullableRecord, parseJsonObject, redactKnownSecrets, shellQuote } from "./utils";
import { compactCommandResultRedacted, nullableRecord, redactKnownSecrets, shellQuote } from "./utils";
import { renderWebProbeScriptResult } from "./web-observe-render";
import { nodeWebProbeHostProxyEnv } from "./web-probe-observe";
@@ -481,11 +482,26 @@ export function runNodeWebProbeScript(
const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath);
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const commandTimedOut = result.timedOut || result.exitCode === 124;
const stdoutReport = parseJsonObject(result.stdout);
const runPaths = webProbeScriptRunPathsFromStderr(result.stderr);
const recoveredReport = stdoutReport === null ? readNodeWebProbeScriptReport(options, spec, runPaths.reportPath) : null;
const recoveredArtifacts = stdoutReport === null || commandTimedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null;
const parsedReport = stdoutReport ?? recoveredReport?.report ?? null;
let recoveredReport: WebProbeScriptReportReadResult | null = null;
const stdoutResolution = resolveCliChildJsonCommandResult({
result,
requestedStdoutType: "web-probe script report JSON",
acceptParsed: isWebProbeScriptReportContract,
forceArtifactFallbackReason: commandTimedOut ? "remote-command-timeout" : null,
artifactFallback: {
path: runPaths.reportPath,
nextCommand: runPaths.reportPath === null ? null : `trans ${options.node}:${spec.workspace} cat ${shellQuote(runPaths.reportPath)}`,
read: () => {
recoveredReport = readNodeWebProbeScriptReport(options, spec, runPaths.reportPath);
if (recoveredReport?.report) return { ok: true, value: recoveredReport.report, path: recoveredReport.path, reason: recoveredReport.source };
return { ok: false, reason: recoveredReport?.degradedReason ?? "web-probe-report-artifact-missing", path: runPaths.reportPath };
},
},
});
const stdoutReport = stdoutResolution.source === "stdout" || stdoutResolution.source === "dump" ? stdoutResolution.parsed : null;
const recoveredArtifacts = stdoutResolution.parsed === null || commandTimedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null;
const parsedReport = stdoutResolution.parsed;
const report = compactWebProbeScriptResult(parsedReport);
const passed = result.exitCode === 0 && report?.ok === true;
const summary = nullableRecord(report?.summary);
@@ -493,9 +509,10 @@ export function runNodeWebProbeScript(
const outputFailureKind = parsedReport === null
? commandTimedOut
? "web-probe-command-timeout"
: stdoutBytes > 64 * 1024
? "web-probe-output-too-large"
: "web-probe-report-parse-failed"
: stringOrNullValue(stdoutResolution.diagnostics.fallbackReason)
?? (stdoutBytes > 64 * 1024
? "web-probe-output-too-large"
: "web-probe-report-parse-failed")
: null;
const degradedReason = commandTimedOut
? "web-probe-command-timeout"
@@ -518,7 +535,8 @@ export function runNodeWebProbeScript(
const effectiveSummary = summary !== null ? {
...summary,
transportTimedOut: commandTimedOut,
recoveredFrom: stdoutReport !== null ? "stdout" : recoveredReport?.source ?? null,
recoveredFrom: stdoutReport !== null ? stdoutResolution.source : recoveredReport?.source ?? null,
stdoutRecovery: stdoutResolution.diagnostics,
} : (outputFailureKind === null ? null : {
ok: false,
status: "blocked",
@@ -539,8 +557,9 @@ export function runNodeWebProbeScript(
},
screenshots: recoveredArtifacts?.screenshots ?? [],
artifacts: recoveredArtifacts?.artifacts ?? null,
stdoutBytes,
exitCode: result.exitCode,
stdoutBytes,
stdoutRecovery: stdoutResolution.diagnostics,
exitCode: result.exitCode,
stderrTail: result.stderr.trim().slice(-2000),
valuesRedacted: true,
});
@@ -566,12 +585,14 @@ export function runNodeWebProbeScript(
summary: effectiveSummary,
issueEvidence,
probe: report,
reportLoad: stdoutReport !== null ? { source: "stdout", path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : {
reportLoad: stdoutReport !== null ? { source: stdoutResolution.source, path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : {
source: recoveredReport.source,
path: recoveredReport.path,
degradedReason: recoveredReport.degradedReason,
result: recoveredReport.result === null ? null : compactCommandResultRedacted(recoveredReport.result, [material.password ?? ""]),
stdoutRecovery: recoveredReport.stdoutRecovery ?? null,
},
stdoutRecovery: stdoutResolution.diagnostics,
warnings: webProbeScriptGovernanceWarnings(options),
hints: webProbeScriptGovernanceHints(options),
preferredCommands: webProbeScriptPreferredCommands(options),
@@ -667,11 +688,20 @@ export function lineValueFromText(text: string, name: string): string | null {
return match ? match[1].trim() : null;
}
interface WebProbeScriptReportReadResult {
source: string;
report: Record<string, unknown> | null;
result: CommandResult | null;
degradedReason: string | null;
path: string | null;
stdoutRecovery?: Record<string, unknown> | null;
}
export function readNodeWebProbeScriptReport(
options: NodeWebProbeScriptOptions,
spec: HwlabRuntimeLaneSpec,
reportPath: string | null,
): { source: string; report: Record<string, unknown> | null; result: CommandResult | null; degradedReason: string | null; path: string | null } | null {
): WebProbeScriptReportReadResult | null {
if (!reportPath) return null;
if (!isSafeWebProbeScriptReportPath(reportPath)) return { source: "unsafe-path", report: null, result: null, degradedReason: "web-probe-report-path-invalid", path: reportPath };
const script = [
@@ -753,16 +783,38 @@ export function readNodeWebProbeScriptReport(
if (result.exitCode !== 0 || result.timedOut) {
return { source: "report-file", report: null, result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-report-read-failed", path: reportPath };
}
const report = parseJsonObject(result.stdout);
const reportResolution = resolveCliChildJsonCommandResult({
result,
requestedStdoutType: "web-probe script report artifact compact JSON",
acceptParsed: isWebProbeScriptReportContract,
});
const report = reportResolution.parsed;
return {
source: "report-file",
report,
result,
degradedReason: report === null ? "web-probe-report-parse-failed" : null,
degradedReason: report === null ? stringOrNullValue(reportResolution.diagnostics.fallbackReason) ?? "web-probe-report-parse-failed" : null,
path: reportPath,
stdoutRecovery: reportResolution.diagnostics,
};
}
function isWebProbeScriptReportContract(value: Record<string, unknown>): boolean {
return value.ok === true
|| value.ok === false
|| typeof value.status === "string"
|| typeof value.reportPath === "string"
|| typeof value.error === "string"
|| typeof value.failureKind === "string"
|| Object.keys(nullableRecord(value.summary) ?? {}).length > 0
|| Object.keys(nullableRecord(value.script) ?? {}).length > 0
|| Array.isArray(value.steps);
}
function stringOrNullValue(value: unknown): string | null {
return typeof value === "string" && value.length > 0 ? value : null;
}
export function readNodeWebProbeScriptArtifacts(
options: NodeWebProbeScriptOptions,
spec: HwlabRuntimeLaneSpec,