diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 4c2e6256..0f0bf2f3 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -8,7 +8,7 @@ import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { classifySshTcpPoolFailure } from "./ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; -import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes"; +import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource, nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "./hwlab-node-help"; @@ -5860,11 +5860,19 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions { if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "observe") throw new Error("web-probe usage: run|script|observe --node NODE --lane vNN [--url URL]"); if (actionRaw === "observe") { const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1)); - const indexed = normalized.id === null ? null : readWebObserveIndexEntry(normalized.id); - const node = optionValue(args, "--node") ?? indexed?.node; - const lane = optionValue(args, "--lane") ?? indexed?.lane; + const explicitNode = optionValue(args, "--node"); + const explicitLane = optionValue(args, "--lane"); + const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null; + const indexed = requestedId === null + ? null + : readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null); + const node = explicitNode ?? indexed?.node; + const lane = explicitLane ?? indexed?.lane; if (node === undefined || lane === undefined) { - throw new Error("web-probe observe requires --node/--lane for start, or a known observer id from observe start for status|command|stop|collect|analyze"); + if (requestedId !== null) { + throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`); + } + throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id"); } assertNodeId(node); assertLane(lane); @@ -6650,6 +6658,114 @@ function readWebObserveIndexEntry(id: string): WebObserveIndexEntry | null { return readWebObserveIndex()[id] ?? null; } +function discoverWebObserveIndexEntry(id: string): WebObserveIndexEntry | null { + const matches: WebObserveIndexEntry[] = []; + const errors: string[] = []; + for (const lane of hwlabRuntimeLaneIds()) { + for (const node of hwlabRuntimeNodeIds()) { + let spec: HwlabRuntimeLaneSpec; + try { + spec = hwlabRuntimeLaneSpecForNode(lane, node); + } catch { + continue; + } + try { + const entry = discoverWebObserveIndexEntryOnTarget(id, node, lane, spec); + if (entry !== null) matches.push(entry); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + errors.push(`${node}/${lane}: ${message.slice(0, 240)}`); + } + } + } + if (matches.length > 1) { + const candidates = matches.map((entry) => `${entry.node}/${entry.lane}:${entry.stateDir}`).join(", "); + throw new Error(`web-probe observe observer-id-ambiguous: ${id}; candidates=${candidates}`); + } + if (matches.length === 0) { + if (errors.length > 0) { + throw new Error(`web-probe observe observer-id-unknown: ${id}; discoveryErrors=${errors.join(" | ")}`); + } + return null; + } + upsertWebObserveIndexEntry(matches[0]!); + return matches[0]!; +} + +function discoverWebObserveIndexEntryOnTarget(id: string, node: string, lane: HwlabRuntimeLane, spec: HwlabRuntimeLaneSpec): WebObserveIndexEntry | null { + const observeRoot = `.state/web-observe/${safeWebObserveSegment(node)}/${safeWebObserveSegment(lane)}`; + const script = [ + "set -eu", + `observe_root=${shellQuote(observeRoot)}`, + `job_id=${shellQuote(id)}`, + "node - \"$observe_root\" \"$job_id\" <<'NODE'", + "const fs = require('fs');", + "const path = require('path');", + "const [root, jobId] = process.argv.slice(2);", + "const matches = [];", + "function readJson(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } }", + "function walk(dir, depth) {", + " if (depth > 10 || matches.length > 1) return;", + " let entries = [];", + " try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }", + " for (const entry of entries) {", + " if (!entry.isDirectory()) continue;", + " const full = path.join(dir, entry.name);", + " if (entry.name.endsWith('_' + jobId)) matches.push(full);", + " else walk(full, depth + 1);", + " if (matches.length > 1) return;", + " }", + "}", + "walk(root, 0);", + "if (matches.length === 0) { console.log(JSON.stringify({ ok: true, found: false, root, jobId })); process.exit(0); }", + "if (matches.length > 1) { console.log(JSON.stringify({ ok: false, found: true, ambiguous: true, root, jobId, matches })); process.exit(3); }", + "const stateDir = matches[0];", + "const manifest = readJson(path.join(stateDir, 'manifest.json')) || {};", + "const heartbeat = readJson(path.join(stateDir, 'heartbeat.json')) || {};", + "let pid = null;", + "try { const raw = fs.readFileSync(path.join(stateDir, 'pid'), 'utf8').trim(); const value = Number(raw); if (Number.isFinite(value)) pid = value; } catch {}", + "console.log(JSON.stringify({", + " ok: true,", + " found: true,", + " ambiguous: false,", + " entry: {", + " id: jobId,", + " stateDir,", + " url: typeof manifest.baseUrl === 'string' ? manifest.baseUrl : '',", + " targetPath: typeof manifest.targetPath === 'string' ? manifest.targetPath : '',", + " status: typeof manifest.status === 'string' ? manifest.status : null,", + " pid,", + " startedAt: typeof manifest.startedAt === 'string' ? manifest.startedAt : null,", + " updatedAt: typeof heartbeat.updatedAt === 'string' ? heartbeat.updatedAt : new Date().toISOString()", + " }", + "}));", + "NODE", + ].join("\n"); + const result = runTransWorkspaceStdinScript(node, spec.workspace, script, 55); + const payload = parseJsonObject(result.stdout); + if (result.exitCode !== 0 || result.timedOut || payload?.ok !== true) { + const reason = result.timedOut ? "timeout" : payload?.ambiguous === true ? "ambiguous" : `exit=${result.exitCode}`; + throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)}`); + } + if (payload.found !== true) return null; + const entry = record(payload.entry); + const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : ""; + if (!isSafeWebObserveStateDir(stateDir)) throw new Error(`observer discovery returned unsafe stateDir: ${stateDir}`); + return { + id, + node, + lane, + workspace: spec.workspace, + stateDir, + url: typeof entry.url === "string" ? entry.url : "", + targetPath: typeof entry.targetPath === "string" ? entry.targetPath : "", + status: typeof entry.status === "string" ? entry.status : null, + pid: typeof entry.pid === "number" ? entry.pid : null, + startedAt: typeof entry.startedAt === "string" ? entry.startedAt : null, + updatedAt: typeof entry.updatedAt === "string" ? entry.updatedAt : new Date().toISOString(), + }; +} + function upsertWebObserveIndexEntry(entry: WebObserveIndexEntry): Record { const path = webObserveIndexPath(); try {