feat: add passive web probe observer
This commit is contained in:
Codex
@@ -10,6 +10,7 @@ import { classifySshTcpPoolFailure } from "./ssh";
|
||||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
|
||||
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
|
||||
import { nodeWebObserveAnalyzerSource, nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "./hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "./hwlab-node-web-probe-summary";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "./hwlab-node-observability-promql";
|
||||
@@ -56,7 +57,35 @@ interface NodeWebProbeScriptOptions {
|
||||
};
|
||||
}
|
||||
|
||||
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions;
|
||||
type NodeWebProbeObserveAction = "start" | "status" | "command" | "stop" | "collect" | "analyze";
|
||||
type NodeWebProbeObserveCommandType = "login" | "preflight" | "goto" | "sendPrompt" | "clickSession" | "screenshot" | "mark" | "stop";
|
||||
|
||||
interface NodeWebProbeObserveOptions {
|
||||
action: "observe";
|
||||
observeAction: NodeWebProbeObserveAction;
|
||||
node: string;
|
||||
lane: string;
|
||||
url: string;
|
||||
targetPath: string;
|
||||
viewport: string;
|
||||
sampleIntervalMs: number;
|
||||
screenshotIntervalMs: number;
|
||||
maxSamples: number;
|
||||
commandTimeoutSeconds: number;
|
||||
waitMs: number;
|
||||
tailLines: number;
|
||||
maxFiles: number;
|
||||
stateDir: string | null;
|
||||
jobId: string | null;
|
||||
force: boolean;
|
||||
commandType: NodeWebProbeObserveCommandType | null;
|
||||
commandText: string | null;
|
||||
commandPath: string | null;
|
||||
commandLabel: string | null;
|
||||
commandSessionId: string | null;
|
||||
}
|
||||
|
||||
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions | NodeWebProbeObserveOptions;
|
||||
|
||||
type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary" | "performance-summary";
|
||||
|
||||
@@ -5896,13 +5925,14 @@ function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof par
|
||||
|
||||
function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "run" && actionRaw !== "script") throw new Error("web-probe usage: run|script --node NODE --lane vNN [--url URL]");
|
||||
if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "observe") throw new Error("web-probe usage: run|script|observe --node NODE --lane vNN [--url URL]");
|
||||
const node = requiredOption(args, "--node");
|
||||
assertNodeId(node);
|
||||
const lane = requiredOption(args, "--lane");
|
||||
assertLane(lane);
|
||||
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
||||
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
||||
if (actionRaw === "observe") return parseNodeWebProbeObserveOptions(args.slice(1), node, lane, spec);
|
||||
if (actionRaw === "script") {
|
||||
assertKnownOptions(args.slice(1), new Set([
|
||||
"--node",
|
||||
@@ -6001,6 +6031,88 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
};
|
||||
}
|
||||
|
||||
function parseNodeWebProbeObserveOptions(args: string[], node: string, lane: string, spec: HwlabRuntimeLaneSpec): NodeWebProbeObserveOptions {
|
||||
const [observeActionRaw] = args;
|
||||
if (
|
||||
observeActionRaw !== "start"
|
||||
&& observeActionRaw !== "status"
|
||||
&& observeActionRaw !== "command"
|
||||
&& observeActionRaw !== "stop"
|
||||
&& observeActionRaw !== "collect"
|
||||
&& observeActionRaw !== "analyze"
|
||||
) {
|
||||
throw new Error("web-probe observe usage: observe start|status|command|stop|collect|analyze --node NODE --lane vNN [--state-dir DIR|--job-id ID]");
|
||||
}
|
||||
assertKnownOptions(args.slice(1), new Set([
|
||||
"--node",
|
||||
"--lane",
|
||||
"--url",
|
||||
"--target-path",
|
||||
"--viewport",
|
||||
"--sample-interval-ms",
|
||||
"--screenshot-interval-ms",
|
||||
"--max-samples",
|
||||
"--command-timeout-seconds",
|
||||
"--wait-ms",
|
||||
"--tail-lines",
|
||||
"--max-files",
|
||||
"--state-dir",
|
||||
"--job-id",
|
||||
"--type",
|
||||
"--text",
|
||||
"--path",
|
||||
"--label",
|
||||
"--session-id",
|
||||
]), new Set(["--force"]));
|
||||
const commandTypeRaw = optionValue(args, "--type") ?? null;
|
||||
const commandType = commandTypeRaw === null ? null : parseNodeWebProbeObserveCommandType(commandTypeRaw);
|
||||
const stateDir = optionValue(args, "--state-dir") ?? null;
|
||||
const jobId = optionValue(args, "--job-id") ?? null;
|
||||
if (stateDir !== null && !isSafeWebObserveStateDir(stateDir)) throw new Error(`unsafe web-probe observe --state-dir: ${stateDir}`);
|
||||
if (jobId !== null && !isSafeWebObserveJobId(jobId)) throw new Error(`unsafe web-probe observe --job-id: ${jobId}`);
|
||||
if (observeActionRaw !== "start" && stateDir === null && jobId === null) {
|
||||
throw new Error("web-probe observe status|command|stop|collect|analyze requires --state-dir or --job-id");
|
||||
}
|
||||
return {
|
||||
action: "observe",
|
||||
observeAction: observeActionRaw,
|
||||
node,
|
||||
lane,
|
||||
url: optionValue(args, "--url") ?? spec.publicWebUrl,
|
||||
targetPath: optionValue(args, "--target-path") ?? "/workbench",
|
||||
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
||||
sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000),
|
||||
screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000),
|
||||
maxSamples: positiveIntegerOption(args, "--max-samples", 0, 10_000_000),
|
||||
commandTimeoutSeconds: positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600),
|
||||
waitMs: positiveIntegerOption(args, "--wait-ms", observeActionRaw === "command" || observeActionRaw === "stop" ? 30000 : 0, 600000),
|
||||
tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200),
|
||||
maxFiles: positiveIntegerOption(args, "--max-files", 80, 5000),
|
||||
stateDir,
|
||||
jobId,
|
||||
force: args.includes("--force"),
|
||||
commandType,
|
||||
commandText: optionValue(args, "--text") ?? null,
|
||||
commandPath: optionValue(args, "--path") ?? null,
|
||||
commandLabel: optionValue(args, "--label") ?? null,
|
||||
commandSessionId: optionValue(args, "--session-id") ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserveCommandType {
|
||||
if (
|
||||
value === "login"
|
||||
|| value === "preflight"
|
||||
|| value === "goto"
|
||||
|| value === "sendPrompt"
|
||||
|| value === "clickSession"
|
||||
|| value === "screenshot"
|
||||
|| value === "mark"
|
||||
|| value === "stop"
|
||||
) return value;
|
||||
throw new Error(`web-probe observe command --type must be login, preflight, goto, sendPrompt, clickSession, screenshot, mark, or stop; got ${value}`);
|
||||
}
|
||||
|
||||
function nodeWebProbeAutoCommandTimeoutSeconds(input: {
|
||||
timeoutMs: number;
|
||||
waitAfterSubmitMs: number;
|
||||
@@ -6039,6 +6151,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
|
||||
const lane = options.lane;
|
||||
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
||||
const spec = hwlabRuntimeLaneSpecForNode(lane, options.node);
|
||||
if (options.action === "observe" && options.observeAction !== "start") return runNodeWebProbeObserve(options, spec, null, null, null);
|
||||
const secretSpec = runtimeSecretSpec({ node: options.node, lane });
|
||||
const material = readBootstrapAdminPasswordMaterial(secretSpec);
|
||||
const credential = webProbeCredential(secretSpec, material);
|
||||
@@ -6046,7 +6159,9 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
command: `hwlab nodes web-probe ${options.action} --node ${options.node} --lane ${options.lane}`,
|
||||
command: options.action === "observe"
|
||||
? `hwlab nodes web-probe observe ${options.observeAction} --node ${options.node} --lane ${options.lane}`
|
||||
: `hwlab nodes web-probe ${options.action} --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
@@ -6056,6 +6171,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
|
||||
next: { secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${secretSpec.bootstrapAdminSecret}` },
|
||||
};
|
||||
}
|
||||
if (options.action === "observe") return runNodeWebProbeObserve(options, spec, secretSpec, material, credential);
|
||||
if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
|
||||
if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
|
||||
const probeArgs = nodeWebProbeRunArgs(options, "run");
|
||||
@@ -6279,6 +6395,311 @@ function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAd
|
||||
};
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserve(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
secretSpec: RuntimeSecretSpec | null,
|
||||
material: BootstrapAdminPasswordMaterial | null,
|
||||
credential: Record<string, unknown> | null,
|
||||
): Record<string, unknown> {
|
||||
if (options.observeAction === "start") {
|
||||
if (secretSpec === null || material === null || credential === null || material.password === null) throw new Error("web-probe observe start requires bootstrap admin credential material");
|
||||
return runNodeWebProbeObserveStart(options, spec, secretSpec, material, credential);
|
||||
}
|
||||
if (options.observeAction === "status") return runNodeWebProbeObserveStatus(options, spec);
|
||||
if (options.observeAction === "command") return runNodeWebProbeObserveCommand(options, spec, false);
|
||||
if (options.observeAction === "stop") return runNodeWebProbeObserveCommand({ ...options, commandType: "stop" }, spec, true);
|
||||
if (options.observeAction === "collect") return runNodeWebProbeObserveCollect(options, spec);
|
||||
return runNodeWebProbeObserveAnalyze(options, spec);
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveStart(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
secretSpec: RuntimeSecretSpec,
|
||||
material: BootstrapAdminPasswordMaterial,
|
||||
credential: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
||||
const day = timestamp.slice(0, 8);
|
||||
const stateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
|
||||
const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
"mkdir -p \"$state_dir\"",
|
||||
"chmod 700 \"$state_dir\"",
|
||||
"runner=\"$state_dir/observer-runner.mjs\"",
|
||||
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
|
||||
"chmod 700 \"$runner\"",
|
||||
[
|
||||
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
|
||||
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
|
||||
`HWLAB_WEB_PASS=${shellQuote(material.password)}`,
|
||||
`UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`,
|
||||
`UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`,
|
||||
`UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`,
|
||||
`UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
"nohup node \"$runner\" >\"$state_dir/stdout.log\" 2>\"$state_dir/stderr.log\" </dev/null &",
|
||||
].join(" "),
|
||||
"pid=$!",
|
||||
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
||||
"sleep 1",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts hwlab nodes web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const started = parseJsonObject(result.stdout);
|
||||
return {
|
||||
ok: result.exitCode === 0 && started?.ok === true,
|
||||
status: result.exitCode === 0 && started?.ok === true ? "started" : "blocked",
|
||||
command: `hwlab nodes web-probe observe start --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
url: options.url,
|
||||
targetPath: options.targetPath,
|
||||
credential,
|
||||
observer: started,
|
||||
result: compactCommandResultRedacted(result, [material.password ?? ""]),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveStatusNodeScript(options.tailLines, options.node, options.lane),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const status = parseJsonObject(result.stdout);
|
||||
return {
|
||||
ok: result.exitCode === 0 && status?.ok !== false,
|
||||
status: result.exitCode === 0 ? "observed" : "blocked",
|
||||
command: `hwlab nodes web-probe observe status --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
observer: status,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record<string, unknown> {
|
||||
const type = options.commandType ?? (stopCommand ? "stop" : null);
|
||||
if (type === null) throw new Error("web-probe observe command requires --type");
|
||||
const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const payload = {
|
||||
id: commandId,
|
||||
type,
|
||||
createdAt: new Date().toISOString(),
|
||||
source: "cli",
|
||||
path: options.commandPath,
|
||||
text: options.commandText,
|
||||
label: options.commandLabel,
|
||||
sessionId: options.commandSessionId,
|
||||
};
|
||||
const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
"mkdir -p \"$state_dir/commands/pending\"",
|
||||
`node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`,
|
||||
nodeWebObserveWaitCommandShell(commandId, options.waitMs),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const commandResult = parseJsonObject(result.stdout);
|
||||
if (options.force && stopCommand && result.exitCode !== 0) {
|
||||
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
"if [ -f \"$state_dir/pid\" ]; then kill \"$(cat \"$state_dir/pid\")\" >/dev/null 2>&1 || true; fi",
|
||||
"printf '{\"ok\":true,\"forced\":true,\"stateDir\":\"%s\"}\\n' \"$state_dir\"",
|
||||
].join("\n"), 55);
|
||||
return {
|
||||
ok: killResult.exitCode === 0,
|
||||
status: killResult.exitCode === 0 ? "forced-stop-requested" : "blocked",
|
||||
command: `hwlab nodes web-probe observe stop --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
gracefulResult: compactCommandResult(result),
|
||||
forceResult: compactCommandResult(killResult),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: result.exitCode === 0 && commandResult?.ok !== false,
|
||||
status: result.exitCode === 0 ? (options.waitMs > 0 ? "completed-or-queued" : "queued") : "blocked",
|
||||
command: stopCommand
|
||||
? `hwlab nodes web-probe observe stop --node ${options.node} --lane ${options.lane}`
|
||||
: `hwlab nodes web-probe observe command --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
observer: commandResult,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveCollectNodeScript(options.maxFiles),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const collect = parseJsonObject(result.stdout);
|
||||
return {
|
||||
ok: result.exitCode === 0 && collect?.ok !== false,
|
||||
status: result.exitCode === 0 ? "collected" : "blocked",
|
||||
command: `hwlab nodes web-probe observe collect --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
collect,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
"analyzer=\"$state_dir/observer-analyzer.mjs\"",
|
||||
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$analyzer" ${shellQuote(analyzerB64)}`,
|
||||
"chmod 700 \"$analyzer\"",
|
||||
"UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" node \"$analyzer\"",
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const analysis = parseJsonObject(result.stdout);
|
||||
return {
|
||||
ok: result.exitCode === 0 && analysis?.ok === true,
|
||||
status: result.exitCode === 0 && analysis?.ok === true ? "analyzed" : "blocked",
|
||||
command: `hwlab nodes web-probe observe analyze --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
analysis,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeWebObserveResolveStateDirShell(options: Pick<NodeWebProbeObserveOptions, "stateDir" | "jobId" | "node" | "lane">): string {
|
||||
if (options.stateDir !== null) {
|
||||
return [
|
||||
`state_dir=${shellQuote(options.stateDir)}`,
|
||||
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"state-dir-missing\",\"stateDir\":\"%s\"}\\n' \"$state_dir\"; exit 2; }",
|
||||
].join("\n");
|
||||
}
|
||||
if (options.jobId === null) throw new Error("web-probe observe requires --state-dir or --job-id");
|
||||
const root = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`;
|
||||
return [
|
||||
`observe_root=${shellQuote(root)}`,
|
||||
`job_id=${shellQuote(options.jobId)}`,
|
||||
"state_dir=$(find \"$observe_root\" -type d -name \"*_${job_id}\" 2>/dev/null | sort | tail -n 1 || true)",
|
||||
"test -n \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"job-id-not-found\",\"jobId\":\"%s\",\"root\":\"%s\"}\\n' \"$job_id\" \"$observe_root\"; exit 2; }",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function nodeWebObserveStatusNodeScript(tailLines: number, node: string, lane: string): string {
|
||||
return `node -e ${shellQuote(`
|
||||
const fs=require('fs'),path=require('path');
|
||||
const dir=process.env.state_dir||process.argv[1];
|
||||
const node=process.argv[2];
|
||||
const lane=process.argv[3];
|
||||
const tailN=${tailLines};
|
||||
const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}};
|
||||
const tailJsonl=(name)=>{try{return fs.readFileSync(path.join(dir,name),'utf8').trim().split(/\\r?\\n/).filter(Boolean).slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}})}catch{return []}};
|
||||
const short=(value)=>String(value||'').slice(0,160);
|
||||
const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt}:null;
|
||||
const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,source:item.source,input:item.input,afterUrl:item.afterUrl,detail:item.detail&&typeof item.detail==='object'?{reason:item.detail.reason||null,mark:item.detail.mark||null,stopping:item.detail.stopping===true,pageId:item.detail.pageId||null}:null});
|
||||
const compactSample=(item)=>({seq:item.seq,ts:item.ts,reason:item.reason,url:item.url,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,observerInitiated:item.observerInitiated===true,scroll:item.scroll||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,messages:Array.isArray(item.messages)?item.messages.slice(-3).map((row)=>({index:row.index,tag:row.tag,status:row.status||null,textHash:row.textHash||null,textPreview:short(row.textPreview),textBytes:row.textBytes||0})):[],traceRows:Array.isArray(item.traceRows)?item.traceRows.slice(-3).map((row)=>({index:row.index,tag:row.tag,status:row.status||null,textHash:row.textHash||null,textPreview:short(row.textPreview),textBytes:row.textBytes||0})):[],performanceCount:Array.isArray(item.performance)?item.performance.length:0});
|
||||
const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:item.url,resourceType:item.resourceType,status:item.status||null,observerInitiated:item.observerInitiated===true,bodyRead:item.bodyRead===true,commandId:item.commandId||null,failure:item.failure||null});
|
||||
const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null;
|
||||
let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}}
|
||||
console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(readJson('manifest.json')),heartbeat:readJson('heartbeat.json'),tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork),errors:tailJsonl('errors.jsonl'),artifacts:tailJsonl('artifacts.jsonl')},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true},null,2));
|
||||
`)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`;
|
||||
}
|
||||
|
||||
function nodeWebObserveCollectNodeScript(maxFiles: number): string {
|
||||
return `node -e ${shellQuote(`
|
||||
const fs=require('fs'),path=require('path'),crypto=require('crypto');
|
||||
const dir=process.argv[1]; const maxFiles=${maxFiles};
|
||||
const out=[]; const walk=(p)=>{for(const ent of fs.readdirSync(p,{withFileTypes:true})){const full=path.join(p,ent.name); if(ent.isDirectory()) walk(full); else out.push(full); if(out.length>=maxFiles) return;}};
|
||||
walk(dir);
|
||||
const files=out.slice(0,maxFiles).map(file=>{const st=fs.statSync(file); const hash=crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); return {path:file,relative:path.relative(dir,file),byteCount:st.size,sha256:'sha256:'+hash}});
|
||||
const totalBytes=files.reduce((sum,item)=>sum+item.byteCount,0);
|
||||
console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,fileCount:files.length,totalBytes,files,valuesRedacted:true},null,2));
|
||||
`)} "$state_dir"`;
|
||||
}
|
||||
|
||||
function nodeWebObserveWaitCommandShell(commandId: string, waitMs: number): string {
|
||||
if (waitMs <= 0) {
|
||||
return [
|
||||
`printf '{"ok":true,"queued":true,"commandId":%s,"stateDir":%s}\\n' ${shellQuote(JSON.stringify(commandId))} "$(node -e "console.log(JSON.stringify(process.argv[1]))" "$state_dir")"`,
|
||||
].join("\n");
|
||||
}
|
||||
const waitSeconds = Math.max(1, Math.ceil(waitMs / 1000));
|
||||
return [
|
||||
`command_id=${shellQuote(commandId)}`,
|
||||
`deadline=$(( $(date +%s) + ${waitSeconds} ))`,
|
||||
"while [ \"$(date +%s)\" -le \"$deadline\" ]; do",
|
||||
" if [ -f \"$state_dir/commands/done/${command_id}.json\" ]; then cat \"$state_dir/commands/done/${command_id}.json\"; exit 0; fi",
|
||||
" if [ -f \"$state_dir/commands/failed/${command_id}.json\" ]; then cat \"$state_dir/commands/failed/${command_id}.json\"; exit 2; fi",
|
||||
" sleep 1",
|
||||
"done",
|
||||
"printf '{\"ok\":true,\"queued\":true,\"waitTimedOut\":true,\"commandId\":\"%s\",\"stateDir\":\"%s\"}\\n' \"$command_id\" \"$state_dir\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function commandSummaryForOutput(payload: Record<string, unknown>): Record<string, unknown> {
|
||||
const text = typeof payload.text === "string" ? payload.text : null;
|
||||
return {
|
||||
id: payload.id,
|
||||
type: payload.type,
|
||||
path: payload.path ?? null,
|
||||
label: payload.label ?? null,
|
||||
sessionId: payload.sessionId ?? null,
|
||||
textHash: text === null ? null : `sha256:${createHash("sha256").update(text).digest("hex")}`,
|
||||
textBytes: text === null ? null : Buffer.byteLength(text),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function isSafeWebObserveStateDir(value: string): boolean {
|
||||
return value.length > 0
|
||||
&& !value.includes("\0")
|
||||
&& !value.includes("..")
|
||||
&& /^\.state\/web-observe\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/\d{4}\/\d{2}\/\d{2}\/[A-Za-z0-9_.TZ-]+_[A-Za-z0-9_.-]+_webobs-[A-Za-z0-9_.-]+$/u.test(value);
|
||||
}
|
||||
|
||||
function isSafeWebObserveJobId(value: string): boolean {
|
||||
return /^webobs-[A-Za-z0-9_.-]+$/u.test(value);
|
||||
}
|
||||
|
||||
function safeWebObserveSegment(value: string): string {
|
||||
return value.replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "") || "item";
|
||||
}
|
||||
|
||||
function safeWebObserveTargetSegment(value: string): string {
|
||||
const segment = value.replace(/^https?:\/\//u, "").replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
||||
return segment.slice(0, 48) || "workbench";
|
||||
}
|
||||
|
||||
function runNodeWebProbeScript(
|
||||
options: NodeWebProbeScriptOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
|
||||
Reference in New Issue
Block a user