fix(web-probe): resolve observe id from state (#557)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
@@ -8,7 +8,7 @@ import { runCommand, type CommandResult } from "./command";
|
||||
import { startJob } from "./jobs";
|
||||
import { classifySshTcpPoolFailure } from "./ssh";
|
||||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
|
||||
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
|
||||
import { nodeWebObserveAnalyzerSource, nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "./hwlab-node-help";
|
||||
@@ -5860,11 +5860,19 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "observe") throw new Error("web-probe usage: run|script|observe --node NODE --lane vNN [--url URL]");
|
||||
if (actionRaw === "observe") {
|
||||
const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1));
|
||||
const indexed = normalized.id === null ? null : readWebObserveIndexEntry(normalized.id);
|
||||
const node = optionValue(args, "--node") ?? indexed?.node;
|
||||
const lane = optionValue(args, "--lane") ?? indexed?.lane;
|
||||
const explicitNode = optionValue(args, "--node");
|
||||
const explicitLane = optionValue(args, "--lane");
|
||||
const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null;
|
||||
const indexed = requestedId === null
|
||||
? null
|
||||
: readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null);
|
||||
const node = explicitNode ?? indexed?.node;
|
||||
const lane = explicitLane ?? indexed?.lane;
|
||||
if (node === undefined || lane === undefined) {
|
||||
throw new Error("web-probe observe requires --node/--lane for start, or a known observer id from observe start for status|command|stop|collect|analyze");
|
||||
if (requestedId !== null) {
|
||||
throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`);
|
||||
}
|
||||
throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id");
|
||||
}
|
||||
assertNodeId(node);
|
||||
assertLane(lane);
|
||||
@@ -6650,6 +6658,114 @@ function readWebObserveIndexEntry(id: string): WebObserveIndexEntry | null {
|
||||
return readWebObserveIndex()[id] ?? null;
|
||||
}
|
||||
|
||||
function discoverWebObserveIndexEntry(id: string): WebObserveIndexEntry | null {
|
||||
const matches: WebObserveIndexEntry[] = [];
|
||||
const errors: string[] = [];
|
||||
for (const lane of hwlabRuntimeLaneIds()) {
|
||||
for (const node of hwlabRuntimeNodeIds()) {
|
||||
let spec: HwlabRuntimeLaneSpec;
|
||||
try {
|
||||
spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const entry = discoverWebObserveIndexEntryOnTarget(id, node, lane, spec);
|
||||
if (entry !== null) matches.push(entry);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push(`${node}/${lane}: ${message.slice(0, 240)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
const candidates = matches.map((entry) => `${entry.node}/${entry.lane}:${entry.stateDir}`).join(", ");
|
||||
throw new Error(`web-probe observe observer-id-ambiguous: ${id}; candidates=${candidates}`);
|
||||
}
|
||||
if (matches.length === 0) {
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`web-probe observe observer-id-unknown: ${id}; discoveryErrors=${errors.join(" | ")}`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
upsertWebObserveIndexEntry(matches[0]!);
|
||||
return matches[0]!;
|
||||
}
|
||||
|
||||
function discoverWebObserveIndexEntryOnTarget(id: string, node: string, lane: HwlabRuntimeLane, spec: HwlabRuntimeLaneSpec): WebObserveIndexEntry | null {
|
||||
const observeRoot = `.state/web-observe/${safeWebObserveSegment(node)}/${safeWebObserveSegment(lane)}`;
|
||||
const script = [
|
||||
"set -eu",
|
||||
`observe_root=${shellQuote(observeRoot)}`,
|
||||
`job_id=${shellQuote(id)}`,
|
||||
"node - \"$observe_root\" \"$job_id\" <<'NODE'",
|
||||
"const fs = require('fs');",
|
||||
"const path = require('path');",
|
||||
"const [root, jobId] = process.argv.slice(2);",
|
||||
"const matches = [];",
|
||||
"function readJson(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } }",
|
||||
"function walk(dir, depth) {",
|
||||
" if (depth > 10 || matches.length > 1) return;",
|
||||
" let entries = [];",
|
||||
" try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }",
|
||||
" for (const entry of entries) {",
|
||||
" if (!entry.isDirectory()) continue;",
|
||||
" const full = path.join(dir, entry.name);",
|
||||
" if (entry.name.endsWith('_' + jobId)) matches.push(full);",
|
||||
" else walk(full, depth + 1);",
|
||||
" if (matches.length > 1) return;",
|
||||
" }",
|
||||
"}",
|
||||
"walk(root, 0);",
|
||||
"if (matches.length === 0) { console.log(JSON.stringify({ ok: true, found: false, root, jobId })); process.exit(0); }",
|
||||
"if (matches.length > 1) { console.log(JSON.stringify({ ok: false, found: true, ambiguous: true, root, jobId, matches })); process.exit(3); }",
|
||||
"const stateDir = matches[0];",
|
||||
"const manifest = readJson(path.join(stateDir, 'manifest.json')) || {};",
|
||||
"const heartbeat = readJson(path.join(stateDir, 'heartbeat.json')) || {};",
|
||||
"let pid = null;",
|
||||
"try { const raw = fs.readFileSync(path.join(stateDir, 'pid'), 'utf8').trim(); const value = Number(raw); if (Number.isFinite(value)) pid = value; } catch {}",
|
||||
"console.log(JSON.stringify({",
|
||||
" ok: true,",
|
||||
" found: true,",
|
||||
" ambiguous: false,",
|
||||
" entry: {",
|
||||
" id: jobId,",
|
||||
" stateDir,",
|
||||
" url: typeof manifest.baseUrl === 'string' ? manifest.baseUrl : '',",
|
||||
" targetPath: typeof manifest.targetPath === 'string' ? manifest.targetPath : '',",
|
||||
" status: typeof manifest.status === 'string' ? manifest.status : null,",
|
||||
" pid,",
|
||||
" startedAt: typeof manifest.startedAt === 'string' ? manifest.startedAt : null,",
|
||||
" updatedAt: typeof heartbeat.updatedAt === 'string' ? heartbeat.updatedAt : new Date().toISOString()",
|
||||
" }",
|
||||
"}));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(node, spec.workspace, script, 55);
|
||||
const payload = parseJsonObject(result.stdout);
|
||||
if (result.exitCode !== 0 || result.timedOut || payload?.ok !== true) {
|
||||
const reason = result.timedOut ? "timeout" : payload?.ambiguous === true ? "ambiguous" : `exit=${result.exitCode}`;
|
||||
throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)}`);
|
||||
}
|
||||
if (payload.found !== true) return null;
|
||||
const entry = record(payload.entry);
|
||||
const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : "";
|
||||
if (!isSafeWebObserveStateDir(stateDir)) throw new Error(`observer discovery returned unsafe stateDir: ${stateDir}`);
|
||||
return {
|
||||
id,
|
||||
node,
|
||||
lane,
|
||||
workspace: spec.workspace,
|
||||
stateDir,
|
||||
url: typeof entry.url === "string" ? entry.url : "",
|
||||
targetPath: typeof entry.targetPath === "string" ? entry.targetPath : "",
|
||||
status: typeof entry.status === "string" ? entry.status : null,
|
||||
pid: typeof entry.pid === "number" ? entry.pid : null,
|
||||
startedAt: typeof entry.startedAt === "string" ? entry.startedAt : null,
|
||||
updatedAt: typeof entry.updatedAt === "string" ? entry.updatedAt : new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function upsertWebObserveIndexEntry(entry: WebObserveIndexEntry): Record<string, unknown> {
|
||||
const path = webObserveIndexPath();
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user