From d4575f6ea3c32e9e14a5986326c182407dc3c795 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 2 Jul 2026 20:25:22 +0000 Subject: [PATCH] feat(web-probe): migrate observe actions to child json recovery --- .../web-probe-observe-action-config.ts | 174 ++++ .../web-probe-observe-actions.test.ts | 104 +++ .../hwlab-node/web-probe-observe-actions.ts | 744 ++++++++++++++++++ scripts/src/hwlab-node/web-probe-observe.ts | 682 +--------------- 4 files changed, 1025 insertions(+), 679 deletions(-) create mode 100644 scripts/src/hwlab-node/web-probe-observe-action-config.ts create mode 100644 scripts/src/hwlab-node/web-probe-observe-actions.test.ts create mode 100644 scripts/src/hwlab-node/web-probe-observe-actions.ts diff --git a/scripts/src/hwlab-node/web-probe-observe-action-config.ts b/scripts/src/hwlab-node/web-probe-observe-action-config.ts new file mode 100644 index 00000000..6dc2d7e0 --- /dev/null +++ b/scripts/src/hwlab-node/web-probe-observe-action-config.ts @@ -0,0 +1,174 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: shared WebProbe observe action configuration helpers. +import { readFileSync } from "node:fs"; +import { repoRoot, rootPath } from "../config"; +import { runCommand } from "../command"; +import { hwlabRuntimeLaneConfigPath, type HwlabRuntimeLaneSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeBrowserFreezePolicySpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; + +import type { WebProbeBrowserProxyMode } from "./entry"; +import { transPath } from "./runtime-common"; +import { record, shellQuote } from "./utils"; + +export function webObserveGcDefaultKeepHours(): number { + const configPath = "config/unidesk-cli.yaml"; + const parsed = record(Bun.YAML.parse(readFileSync(rootPath(configPath), "utf8")) as unknown); + const gc = record(parsed.gc); + const scratch = record(gc.stateStaleScratch); + const keepHours = scratch.keepHours; + if (!Number.isInteger(keepHours) || keepHours < 0) { + throw new Error(`${configPath}#gc.stateStaleScratch.keepHours must be a non-negative integer`); + } + return keepHours; +} + +export function nodeWebProbeAlertThresholds(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeAlertThresholdsSpec { + const thresholds = spec.webProbe?.alertThresholds; + if (thresholds === undefined) { + throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.alertThresholds for web-probe observe`); + } + return thresholds; +} + +export function nodeWebProbeBrowserFreezePolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeBrowserFreezePolicySpec { + const policy = spec.webProbe?.browserFreezePolicy; + if (policy === undefined) { + throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.browserFreezePolicy for web-probe observe`); + } + return policy; +} + +export function nodeWebProbeProjectManagementConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeProjectManagementSpec | null { + return spec.webProbe?.projectManagement ?? null; +} + +export function nodeWebProbeAuthLoginConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeAuthLoginSpec | null { + return spec.webProbe?.authLogin ?? null; +} + +export interface NodeWebProbeHostProxyEnv { + readonly envAssignments: string[]; + readonly summary: Record; +} + +export function nodeWebProbeHostProxyEnv(spec: HwlabRuntimeLaneSpec, browserProxyMode: WebProbeBrowserProxyMode = "auto"): NodeWebProbeHostProxyEnv { + if (browserProxyMode === "direct") { + return { + envAssignments: [], + summary: { + source: "option", + mode: "direct", + networkProfileId: spec.networkProfileId, + proxy: { enabled: false }, + valuesPrinted: false, + }, + }; + } + const proxy = spec.networkProfile.proxy; + const serviceCache = new Map(); + const http = resolveNodeWebProbeHostProxyUrl(spec, proxy.http, serviceCache); + const https = resolveNodeWebProbeHostProxyUrl(spec, proxy.https, serviceCache); + const all = resolveNodeWebProbeHostProxyUrl(spec, proxy.all, serviceCache); + const noProxy = proxy.noProxy.join(","); + return { + envAssignments: [ + ["HTTP_PROXY", http.url], + ["HTTPS_PROXY", https.url], + ["ALL_PROXY", all.url], + ["http_proxy", http.url], + ["https_proxy", https.url], + ["all_proxy", all.url], + ["NO_PROXY", noProxy], + ["no_proxy", noProxy], + ].map(([key, value]) => `${key}=${shellQuote(value)}`), + summary: { + source: "yaml", + mode: "host-env", + networkProfileId: spec.networkProfileId, + proxy: { + http: http.summary, + https: https.summary, + all: all.summary, + noProxyCount: proxy.noProxy.length, + }, + valuesPrinted: false, + }, + }; +} + +export function resolveNodeWebProbeHostProxyUrl( + spec: HwlabRuntimeLaneSpec, + rawUrl: string, + serviceCache: Map, +): { url: string; summary: Record } { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch (error) { + throw new Error(`config/hwlab-node-lanes.yaml networkProfiles.${spec.networkProfileId}.proxy contains invalid proxy URL: ${error instanceof Error ? error.message : String(error)}`); + } + const service = parseKubernetesServiceDnsHost(parsed.hostname); + if (service === null) { + return { + url: rawUrl, + summary: { + mode: "host-url", + host: parsed.hostname, + port: parsed.port || null, + valuesPrinted: false, + }, + }; + } + const clusterIp = resolveKubernetesServiceClusterIp(spec, service.namespace, service.name, serviceCache); + const originalHost = parsed.hostname; + parsed.hostname = clusterIp; + const resolvedUrl = normalizedProxyUrl(parsed); + return { + url: resolvedUrl, + summary: { + mode: "k8s-service-cluster-ip", + service: service.name, + namespace: service.namespace, + originalHost, + resolvedHost: clusterIp, + port: parsed.port || null, + valuesPrinted: false, + }, + }; +} + +export function parseKubernetesServiceDnsHost(hostname: string): { name: string; namespace: string } | null { + const match = hostname.toLowerCase().match(/^([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.svc(?:\.cluster\.local)?$/u); + if (match === null) return null; + return { name: match[1] ?? "", namespace: match[3] ?? "" }; +} + +export function resolveKubernetesServiceClusterIp( + spec: HwlabRuntimeLaneSpec, + namespace: string, + serviceName: string, + serviceCache: Map, +): string { + const cacheKey = `${namespace}/${serviceName}`; + const cached = serviceCache.get(cacheKey); + if (cached !== undefined) return cached; + const result = runCommand([transPath(), spec.nodeKubeRoute, "get", "svc", "-n", namespace, serviceName, "-o", "jsonpath={.spec.clusterIP}"], repoRoot, { timeoutMs: 20_000 }); + const clusterIp = result.stdout.trim(); + if (result.exitCode !== 0 || clusterIp.length === 0) { + const reason = result.stderr.trim().slice(-500) || result.stdout.trim().slice(-500) || `exitCode=${result.exitCode}`; + throw new Error(`web-probe proxy service resolution failed for ${spec.nodeId}/${spec.lane} ${namespace}/${serviceName}: ${reason}`); + } + serviceCache.set(cacheKey, clusterIp); + return clusterIp; +} + +export function normalizedProxyUrl(parsed: URL): string { + const value = parsed.toString(); + if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, ""); + return value; +} + +export function webProbeAccountEnvAssignments(): string[] { + return Object.entries(process.env) + .filter(([key, value]) => value !== undefined && /^HWLAB_WEB_(?:ACCOUNT_)?[A-Z0-9_]+_(?:JSON|USER|PASS)$/u.test(key)) + .map(([key, value]) => `${key}=${shellQuote(value ?? "")}`); +} diff --git a/scripts/src/hwlab-node/web-probe-observe-actions.test.ts b/scripts/src/hwlab-node/web-probe-observe-actions.test.ts new file mode 100644 index 00000000..8d202c1f --- /dev/null +++ b/scripts/src/hwlab-node/web-probe-observe-actions.test.ts @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "bun:test"; + +import { resolveWebObserveActionJson } from "./web-probe-observe-actions"; + +test("web observe action JSON recovery accepts concrete start contract", () => { + const resolved = resolveWebObserveActionJson({ + stdout: JSON.stringify({ + ok: true, + command: "web-probe-observe start", + jobId: "webobs-fixture", + stateDir: ".state/web-observe/JD01/v03/fixture", + valuesRedacted: true, + }), + stderr: "", + exitCode: 0, + timedOut: false, + }, "start"); + + assert.equal(resolved.source, "stdout"); + assert.equal(resolved.parsed?.jobId, "webobs-fixture"); + assert.equal(resolved.diagnostics.stdoutContractAccepted, true); +}); + +test("web observe action JSON recovery reads dump wrapper for status contract", async () => { + const dir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-status-dump-")); + const dumpPath = join(dir, "status.json"); + await writeFile(dumpPath, JSON.stringify({ + ok: true, + status: "running", + jobId: "webobs-fixture", + heartbeat: { status: "running" }, + diagnostics: { heartbeatStale: false }, + }) + "\n"); + + const resolved = resolveWebObserveActionJson({ + stdout: JSON.stringify({ + ok: true, + data: { + outputTruncated: true, + reason: "stdout-json-bytes-exceeded-threshold", + dump: { path: dumpPath }, + }, + }), + stderr: "", + exitCode: 0, + timedOut: false, + }, "status"); + + assert.equal(resolved.source, "dump"); + assert.equal(resolved.parsed?.jobId, "webobs-fixture"); + assert.equal(resolved.diagnostics.stdoutKind, "dump-wrapper"); + assert.equal(resolved.diagnostics.dumpPath, dumpPath); +}); + +test("web observe action JSON recovery reads SSH truncation summary dump for command contract", async () => { + const dir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-command-dump-")); + const dumpPath = join(dir, "command.json"); + await writeFile(dumpPath, JSON.stringify({ + ok: true, + queued: false, + waitTimedOut: false, + commandId: "cmd-fixture", + }) + "\n"); + const summary = { + code: "ssh-truncation-summary", + stdout: { + stream: "stdout", + dumpPath, + forwardedBytes: 10240, + thresholdBytes: 10240, + }, + }; + + const resolved = resolveWebObserveActionJson({ + stdout: `UNIDESK_SSH_TRUNCATION_SUMMARY ${JSON.stringify(summary)}\n`, + stderr: "", + exitCode: 0, + timedOut: false, + }, "command"); + + assert.equal(resolved.source, "dump"); + assert.equal(resolved.parsed?.commandId, "cmd-fixture"); + assert.equal(resolved.diagnostics.stdoutKind, "ssh-truncation-summary"); + assert.equal(resolved.diagnostics.dumpPath, dumpPath); +}); + +test("web observe action JSON recovery rejects ok-only action contract", () => { + const resolved = resolveWebObserveActionJson({ + stdout: JSON.stringify({ ok: true }), + stderr: "", + exitCode: 0, + timedOut: false, + }, "start"); + + assert.equal(resolved.parsed, null); + assert.equal(resolved.source, null); + assert.equal(resolved.diagnostics.stdoutKind, "json"); + assert.equal(resolved.diagnostics.fallbackReason, "stdout-json-contract-invalid"); + assert.equal(resolved.diagnostics.stdoutContractAccepted, false); +}); diff --git a/scripts/src/hwlab-node/web-probe-observe-actions.ts b/scripts/src/hwlab-node/web-probe-observe-actions.ts new file mode 100644 index 00000000..25603cae --- /dev/null +++ b/scripts/src/hwlab-node/web-probe-observe-actions.ts @@ -0,0 +1,744 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. WebProbe observe action handlers. +// Responsibility: start/status/command/stop/gc remote action execution and child JSON recovery. +import { randomBytes } from "node:crypto"; +import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery"; +import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes"; +import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; +import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; +import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; +import type { RenderedCliResult } from "../output"; + +import type { BootstrapAdminPasswordMaterial, NodeWebProbeObserveOptions, RuntimeSecretSpec } from "./entry"; +import { runTransWorkspaceStdinScript } from "./public-exposure"; +import { compactCommandResult, compactCommandResultRedacted, compactCommandResultWithStdoutTail, record, shellQuote } from "./utils"; +import { nodeWebProbeAlertThresholds, nodeWebProbeAuthLoginConfig, nodeWebProbeBrowserFreezePolicy, nodeWebProbeHostProxyEnv, nodeWebProbeProjectManagementConfig, webObserveGcDefaultKeepHours, webProbeAccountEnvAssignments } from "./web-probe-observe-action-config"; +import { nodeWebObserveResolveStateDirShell, renderWebObserveStartResult, upsertWebObserveIndexEntry, webObserveCommandLabel, webObserveIdFromOptions, webObserveIdFromStatus, webObserveIndexEntryFromOptions, webObserveNextCommands, withWebObserveShortcuts } from "./web-observe-render"; +import { commandSummaryForOutput, nodeWebObserveForceStopNodeScript, nodeWebObserveStatusNodeScript, nodeWebObserveWaitCommandShell, safeWebObserveSegment, safeWebObserveTargetSegment } from "./web-observe-scripts"; + +type WebObserveActionJsonContract = "gc" | "start" | "status" | "command" | "force-stop"; + +export function resolveWebObserveActionJson( + result: { stdout: string; stderr?: string; exitCode?: number | null; timedOut?: boolean }, + contract: WebObserveActionJsonContract, +): CliChildJsonResolution { + return resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: `web-probe observe ${contract} contract`, + acceptParsed: (value) => isWebObserveActionJsonContract(value, contract), + }); +} + +function isWebObserveActionJsonContract(value: Record, contract: WebObserveActionJsonContract): boolean { + if (acceptFailureContract(value)) return true; + if (contract === "gc") { + return value.command === "web-probe observe gc" && hasAnyKey(value, ["mode", "stateRoot", "scannedRuns", "candidateCount", "selectedCount", "protectedCount"]); + } + if (contract === "start") { + return value.command === "web-probe-observe start" && hasAnyKey(value, ["jobId", "stateDir", "pid", "manifestPath", "heartbeat", "manifest"]); + } + if (contract === "status") { + return hasAnyKey(value, ["jobId", "stateDir", "manifest", "heartbeat", "diagnostics", "commands", "samples", "status"]); + } + if (contract === "command") { + return hasAnyKey(value, ["ok", "queued", "waitTimedOut", "commandId", "status", "path", "error", "reason"]); + } + return hasAnyKey(value, ["ok", "forceStopped", "killed", "status", "pid", "reason", "error", "commandId"]); +} + +function acceptFailureContract(value: Record): boolean { + return value.ok === false && hasAnyKey(value, ["error", "reason", "degradedReason", "status", "diagnostics", "result", "command"]); +} + +function hasAnyKey(value: Record, keys: string[]): boolean { + return keys.some((key) => value[key] !== undefined && value[key] !== null); +} + +export function runNodeWebProbeObserveGc(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record { + const stateRoot = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`; + const script = [ + "set -eu", + `node - ${shellQuote(stateRoot)} ${shellQuote(String(options.gcKeepHours))} ${shellQuote(String(options.gcLimit))} ${shellQuote(options.confirm ? "run" : "plan")} ${shellQuote(options.node)} ${shellQuote(options.lane)} <<'UNIDESK_WEB_OBSERVE_GC'`, + nodeWebObserveGcNodeScript(), + "UNIDESK_WEB_OBSERVE_GC", + ].join("\n"); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); + const payloadResolution = resolveWebObserveActionJson(result, "gc"); + const payload = payloadResolution.parsed ?? {}; + const ok = result.exitCode === 0 && payloadResolution.parsed !== null && payload.ok !== false; + return { + ok, + status: ok ? (options.confirm ? "cleaned" : "planned") : "blocked", + command: `web-probe observe gc --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + stateRoot, + mode: options.confirm ? "confirmed-run" : "dry-run", + retention: { + source: "config/unidesk-cli.yaml#gc.stateStaleScratch.keepHours", + keepHours: options.gcKeepHours, + userOverride: options.gcKeepHours !== webObserveGcDefaultKeepHours(), + }, + gc: options.full || options.raw ? payload : compactWebObserveGcPayload(payload), + stdoutJson: payloadResolution.diagnostics, + result: ok + ? { + exitCode: result.exitCode, + timedOut: result.timedOut, + stdoutBytes: result.stdout.length, + stderrBytes: result.stderr.length, + } + : compactCommandResultWithStdoutTail(result), + valuesRedacted: true, + }; +} + +function compactWebObserveGcPayload(payload: Record): Record { + const candidates = Array.isArray(payload.candidates) ? payload.candidates.map(record) : []; + const protectedRuns = Array.isArray(payload.protected) ? payload.protected.map(record) : []; + const deleted = Array.isArray(payload.deleted) ? payload.deleted.map(record) : []; + const failures = Array.isArray(payload.failures) ? payload.failures.map(record) : []; + return { + ok: payload.ok ?? null, + mode: payload.mode ?? null, + mutation: payload.mutation ?? null, + keepHours: payload.keepHours ?? null, + limit: payload.limit ?? null, + diskBefore: payload.diskBefore ?? null, + diskAfter: payload.diskAfter ?? null, + scannedRuns: payload.scannedRuns ?? null, + candidateCount: payload.candidateCount ?? null, + selectedCount: payload.selectedCount ?? null, + deferredCount: payload.deferredCount ?? null, + protectedCount: payload.protectedCount ?? null, + estimatedReclaimBytes: payload.estimatedReclaimBytes ?? null, + estimatedReclaimHuman: payload.estimatedReclaimHuman ?? null, + reclaimedBytes: payload.reclaimedBytes ?? null, + reclaimedHuman: payload.reclaimedHuman ?? null, + candidates: candidates.slice(0, 20).map((item) => ({ + id: item.id ?? null, + runDir: item.runDir ?? null, + status: item.status ?? null, + ageHours: item.ageHours ?? null, + rawHuman: item.rawHuman ?? null, + artifactCount: item.artifactCount ?? null, + })), + deleted: deleted.slice(0, 20).map((item) => ({ + id: item.id ?? null, + runDir: item.runDir ?? null, + rawHuman: item.rawHuman ?? null, + artifactCount: item.artifactCount ?? null, + })), + failures: failures.slice(0, 20), + protected: protectedRuns.slice(0, 20).map((item) => ({ + id: item.id ?? null, + runDir: item.runDir ?? null, + status: item.status ?? null, + ageHours: item.ageHours ?? null, + rawHuman: item.rawHuman ?? null, + reasons: item.reasons ?? null, + })), + disclosure: { + candidateRowsShown: Math.min(20, candidates.length), + deletedRowsShown: Math.min(20, deleted.length), + protectedRowsShown: Math.min(20, protectedRuns.length), + full: "rerun with --full or --raw for all candidates/artifacts", + }, + valuesRedacted: true, + }; +} + +function nodeWebObserveGcNodeScript(): string { + return String.raw` +const fs = require("fs"); +const path = require("path"); +const childProcess = require("child_process"); + +const stateRoot = process.argv[2]; +const keepHours = Number(process.argv[3]); +const limit = Number(process.argv[4]); +const mode = process.argv[5] === "run" ? "run" : "plan"; +const nodeId = process.argv[6]; +const lane = process.argv[7]; +const keepMs = keepHours * 60 * 60 * 1000; +const nowMs = Date.now(); +const rawNames = ["samples.jsonl", "browser-process.jsonl", "network.jsonl", "console.jsonl", "artifacts.jsonl", "performance-events.jsonl", "screenshots", "performance"]; + +function jsonRead(file) { + try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; } +} + +function parseTime(value) { + if (typeof value !== "string") return 0; + const ms = Date.parse(value); + return Number.isFinite(ms) ? ms : 0; +} + +function dirTimeMs(runDir) { + const base = path.basename(runDir); + const match = base.match(/^(\d{8})T(\d{6})Z_/); + if (!match) return 0; + const raw = match[1].slice(0, 4) + "-" + match[1].slice(4, 6) + "-" + match[1].slice(6, 8) + "T" + match[2].slice(0, 2) + ":" + match[2].slice(2, 4) + ":" + match[2].slice(4, 6) + "Z"; + return Date.parse(raw) || 0; +} + +function newestKnownTimeMs(runDir, manifest, heartbeat) { + return Math.max( + dirTimeMs(runDir), + parseTime(manifest && manifest.completedAt), + parseTime(manifest && manifest.updatedAt), + parseTime(manifest && manifest.startedAt), + parseTime(manifest && manifest.pageProvenance && manifest.pageProvenance.observedAt), + parseTime(heartbeat && heartbeat.completedAt), + parseTime(heartbeat && heartbeat.forceStoppedAt), + parseTime(heartbeat && heartbeat.updatedAt), + parseTime(heartbeat && heartbeat.pageProvenance && heartbeat.pageProvenance.observedAt), + ); +} + +function sizePath(target) { + let total = 0; + const stack = [target]; + while (stack.length > 0) { + const current = stack.pop(); + let st; + try { st = fs.lstatSync(current); } catch { continue; } + if (st.isSymbolicLink()) continue; + total += st.blocks ? st.blocks * 512 : st.size; + if (!st.isDirectory()) continue; + let entries; + try { entries = fs.readdirSync(current); } catch { continue; } + for (const entry of entries) stack.push(path.join(current, entry)); + } + return total; +} + +function human(bytes) { + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit += 1; } + return value.toFixed(unit === 0 ? 0 : 1) + units[unit]; +} + +function disk() { + try { + const line = childProcess.execFileSync("df", ["-Pk", "/"], { encoding: "utf8" }).trim().split("\n").slice(-1)[0]; + const parts = line.trim().split(/\s+/); + const sizeBytes = Number(parts[1]) * 1024; + const usedBytes = Number(parts[2]) * 1024; + const availableBytes = Number(parts[3]) * 1024; + const usePercent = Number(String(parts[4]).replace("%", "")); + return { filesystem: parts[0], sizeBytes, usedBytes, availableBytes, usePercent, usedHuman: human(usedBytes), availableHuman: human(availableBytes) }; + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } +} + +function findRunDirs(root) { + const runs = []; + for (const year of safeReadDir(root)) { + const y = path.join(root, year.name); + if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) continue; + for (const month of safeReadDir(y)) { + const m = path.join(y, month.name); + if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) continue; + for (const day of safeReadDir(m)) { + const d = path.join(m, day.name); + if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) continue; + for (const run of safeReadDir(d)) { + const r = path.join(d, run.name); + if (run.isDirectory() && /_webobs-[A-Za-z0-9_.-]+$/.test(run.name)) runs.push(r); + } + } + } + } + return runs; +} + +function safeReadDir(dir) { + try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; } +} + +function pidObserverAlive(pid, runAbs) { + if (!Number.isInteger(pid) || pid <= 1) return { alive: false, reason: null }; + try { + const cmdline = fs.readFileSync("/proc/" + pid + "/cmdline", "utf8").replace(/\0/g, " "); + if (cmdline.includes("observer-runner") || cmdline.includes(runAbs)) return { alive: true, reason: "pid-alive" }; + return { alive: false, reason: null }; + } catch (error) { + if (fs.existsSync("/proc/" + pid)) return { alive: true, reason: "pid-alive-unreadable" }; + return { alive: false, reason: null }; + } +} + +function openRunDirs(rootAbs) { + const active = new Set(); + const procEntries = safeReadDir("/proc").filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name)); + for (const proc of procEntries) { + const fdDir = path.join("/proc", proc.name, "fd"); + for (const fd of safeReadDir(fdDir)) { + let target; + try { target = fs.readlinkSync(path.join(fdDir, fd.name)).replace(/ \(deleted\)$/, ""); } catch { continue; } + if (!target.startsWith(rootAbs + path.sep)) continue; + const rel = path.relative(rootAbs, target).split(path.sep); + if (rel.length >= 4) active.add(path.join(rootAbs, rel[0], rel[1], rel[2], rel[3])); + } + } + return active; +} + +function rawArtifacts(runDir) { + const artifacts = []; + for (const name of rawNames) { + const target = path.join(runDir, name); + if (!fs.existsSync(target)) continue; + const bytes = sizePath(target); + if (bytes > 0) artifacts.push({ name, path: target, bytes, human: human(bytes) }); + } + return artifacts; +} + +function removeArtifact(artifact) { + const st = fs.lstatSync(artifact.path); + if (st.isSymbolicLink()) throw new Error("refusing symlink: " + artifact.path); + if (st.isDirectory()) fs.rmSync(artifact.path, { recursive: true, force: false }); + else fs.unlinkSync(artifact.path); +} + +const rootAbs = path.resolve(stateRoot); +const diskBefore = disk(); +const openDirs = openRunDirs(rootAbs); +const candidates = []; +const protectedRuns = []; + +for (const runDir of findRunDirs(rootAbs)) { + const manifest = jsonRead(path.join(runDir, "manifest.json")); + const heartbeat = jsonRead(path.join(runDir, "heartbeat.json")); + const pid = Number((heartbeat && heartbeat.pid) || (manifest && manifest.pid) || 0); + const pidState = pidObserverAlive(pid, runDir); + const hasOpenFd = openDirs.has(runDir); + const newestMs = newestKnownTimeMs(runDir, manifest, heartbeat); + const ageMs = newestMs > 0 ? nowMs - newestMs : 0; + const artifacts = rawArtifacts(runDir); + const rawBytes = artifacts.reduce((sum, item) => sum + item.bytes, 0); + const rel = path.relative(rootAbs, runDir); + const reasons = []; + if (!manifest) reasons.push("manifest-missing"); + if (pidState.alive) reasons.push(pidState.reason || "pid-alive"); + if (hasOpenFd) reasons.push("open-fd"); + if (newestMs <= 0) reasons.push("time-unknown"); + if (ageMs < keepMs) reasons.push("retention-window"); + if (rawBytes <= 0) reasons.push("no-raw-artifacts"); + const summary = { + id: (manifest && manifest.jobId) || path.basename(runDir).match(/(webobs-[A-Za-z0-9_.-]+)$/)?.[1] || null, + runDir: rel, + status: heartbeat && heartbeat.status || null, + pid: Number.isInteger(pid) && pid > 0 ? pid : null, + newestAt: newestMs > 0 ? new Date(newestMs).toISOString() : null, + ageHours: newestMs > 0 ? Number((ageMs / 3600000).toFixed(2)) : null, + rawBytes, + rawHuman: human(rawBytes), + artifacts, + }; + if (reasons.length > 0) protectedRuns.push({ ...summary, reasons }); + else candidates.push(summary); +} + +candidates.sort((a, b) => b.rawBytes - a.rawBytes); +const selected = candidates.slice(0, limit); +const selectedSet = new Set(selected.map((item) => item.runDir)); +const deferred = candidates.filter((item) => !selectedSet.has(item.runDir)); +let reclaimedBytes = 0; +const deleted = []; +const failures = []; + +if (mode === "run") { + for (const item of selected) { + const runDir = path.join(rootAbs, item.runDir); + try { + for (const artifact of item.artifacts) removeArtifact(artifact); + reclaimedBytes += item.rawBytes; + deleted.push({ id: item.id, runDir: item.runDir, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length }); + } catch (error) { + failures.push({ id: item.id, runDir: item.runDir, error: error instanceof Error ? error.message : String(error) }); + } + } +} + +const estimatedReclaimBytes = selected.reduce((sum, item) => sum + item.rawBytes, 0); +const diskAfter = mode === "run" ? disk() : null; +console.log(JSON.stringify({ + ok: failures.length === 0, + command: "web-probe observe gc", + node: nodeId, + lane, + stateRoot, + mode, + mutation: mode === "run", + keepHours, + limit, + diskBefore, + diskAfter, + scannedRuns: candidates.length + protectedRuns.length, + candidateCount: candidates.length, + selectedCount: selected.length, + deferredCount: deferred.length, + protectedCount: protectedRuns.length, + estimatedReclaimBytes, + estimatedReclaimHuman: human(estimatedReclaimBytes), + reclaimedBytes, + reclaimedHuman: human(reclaimedBytes), + candidates: selected.map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length, artifacts: item.artifacts.map((artifact) => ({ name: artifact.name, bytes: artifact.bytes, human: artifact.human })) })), + deleted, + failures, + protected: protectedRuns.slice(0, Math.min(50, protectedRuns.length)).map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, reasons: item.reasons })), + valuesRedacted: true, +}, null, 2)); +`; +} + +export function runNodeWebProbeObserveStart( + options: NodeWebProbeObserveOptions, + spec: HwlabRuntimeLaneSpec, + secretSpec: RuntimeSecretSpec, + material: BootstrapAdminPasswordMaterial, + credential: Record, +): Record | RenderedCliResult { + const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`; + const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z"); + const day = timestamp.slice(0, 8); + const defaultStateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`; + const stateDir = options.stateDir ?? defaultStateDir; + const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64"); + const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64; + const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode); + const alertThresholds = nodeWebProbeAlertThresholds(spec); + const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec); + const projectManagement = nodeWebProbeProjectManagementConfig(spec); + const authLogin = nodeWebProbeAuthLoginConfig(spec); + const runnerEnvAssignments = [ + ...webProbeProxy.envAssignments, + ...webProbeAccountEnvAssignments(), + ...(spec.webProbe?.playwrightBrowsersPath === undefined ? [] : [ + `PLAYWRIGHT_BROWSERS_PATH=${shellQuote(spec.webProbe.playwrightBrowsersPath)}`, + ]), + `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`, + `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, + `HWLAB_WEB_PASS=${shellQuote(material.password)}`, + `UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`, + `UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`, + `UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`, + `UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`, + `UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`, + `UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`, + `UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`, + `UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`, + `UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`, + `UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`, + `UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`, + `UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`, + `UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`, + ...(authLogin === null + ? [] + : [ + `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS=${shellQuote(String(authLogin.maxAttempts))}`, + `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS=${shellQuote(String(authLogin.requestTimeoutMs))}`, + `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS=${shellQuote(String(authLogin.initialDelayMs))}`, + `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`, + ]), + ].join(" "); + const script = [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + "mkdir -p \"$state_dir\"", + "chmod 700 \"$state_dir\"", + "runner=\"$state_dir/observer-runner.mjs\"", + "runner_b64=\"$state_dir/observer-runner.mjs.b64\"", + "cat >\"$runner_b64\" <<'UNIDESK_WEB_OBSERVE_RUNNER_B64'", + runnerB64Body, + "UNIDESK_WEB_OBSERVE_RUNNER_B64", + "node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"", + "rm -f \"$runner_b64\"", + "chmod 700 \"$runner\"", + `if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" "$state_dir/stdout.log" 2>"$state_dir/stderr.log" \"$state_dir/pid\"", + "sleep 1", + `node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`, + ].join("\n"); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); + const startResolution = resolveWebObserveActionJson(result, "start"); + const started = startResolution.parsed; + const startOk = result.exitCode === 0 && started?.ok === true; + const recovery = startOk + ? null + : readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30)); + const recoveredStatus = !startOk && recovery !== null && recovery.result.exitCode === 0 && recovery.status !== null && recovery.status.ok !== false + ? recovery.status + : null; + const effectiveOk = startOk || recoveredStatus !== null; + const observer = startOk ? started : recoveredStatus; + const observerId = typeof started?.jobId === "string" + ? started.jobId + : webObserveIdFromStatus(recoveredStatus, { ...options, id: jobId }) ?? jobId; + const degradedReason = !startOk && recoveredStatus !== null + ? "web-probe-start-transport-timeout-recovered" + : result.timedOut + ? "web-probe-command-timeout" + : result.exitCode !== 0 + ? "web-probe-observe-start-failed" + : null; + const index = effectiveOk + ? upsertWebObserveIndexEntry(startOk ? { + id: observerId, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + stateDir, + url: options.url, + targetPath: options.targetPath, + status: "running", + pid: typeof started?.pid === "number" ? started.pid : null, + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } : webObserveIndexEntryFromOptions({ ...options, id: observerId, stateDir }, spec, observerId, recoveredStatus ?? {})) + : null; + return renderWebObserveStartResult({ + ok: effectiveOk, + status: effectiveOk ? "started" : "blocked", + command: `web-probe observe start --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + network: webProbeProxy.summary, + alertThresholds, + browserFreezePolicy, + projectManagement, + targetPath: options.targetPath, + id: observerId, + degradedReason, + credential, + observer: withWebObserveShortcuts(observer, observerId), + wrapper: buildWebObserveWrapperForObserveOptions("start", options, spec.workspace, { id: observerId, jobId: observerId, stateDir }), + index, + next: webObserveNextCommands(observerId), + stdoutJson: startResolution.diagnostics, + result: compactCommandResultRedacted(result, [material.password ?? ""]), + recovery: recovery === null ? null : { + attempted: true, + ok: recoveredStatus !== null, + reason: degradedReason, + stdoutJson: recovery.stdoutJson, + result: compactCommandResultWithStdoutTail(recovery.result), + valuesRedacted: true, + }, + valuesRedacted: true, + }); +} + +export function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record | RenderedCliResult { + const { result, status, stdoutJson } = readNodeWebProbeObserveRemoteStatus(options, spec, options.tailLines, options.commandTimeoutSeconds); + const observerId = webObserveIdFromStatus(status, options); + const statusReadable = status !== null; + const ok = result.exitCode === 0 && statusReadable && status.ok !== false; + const degradedReason = result.timedOut + ? "web-probe-command-timeout" + : result.exitCode !== 0 + ? "web-probe-observe-status-failed" + : !statusReadable + ? "web-probe-observe-status-unreadable" + : typeof status.degradedReason === "string" + ? status.degradedReason + : null; + const index = ok && observerId !== null && options.stateDir !== null + ? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status)) + : null; + return withWebObserveStatusRendered({ + ok, + status: ok ? "observed" : "blocked", + command: webObserveCommandLabel("status", options), + id: observerId, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + degradedReason, + observer: withWebObserveShortcuts(status, observerId), + wrapper: buildWebObserveWrapperForObserveOptions("status", options, spec.workspace, { id: observerId, stateDir: webObserveWrapperStateDirFromStatus(status, options.stateDir) }), + index, + next: observerId === null ? null : webObserveNextCommands(observerId), + stdoutJson, + result: compactCommandResultWithStdoutTail(result), + valuesRedacted: true, + }); +} + +export function readNodeWebProbeObserveRemoteStatus( + options: NodeWebProbeObserveOptions, + spec: HwlabRuntimeLaneSpec, + tailLines: number, + timeoutSeconds: number, +): { result: ReturnType; status: Record | null; stdoutJson: Record } { + const script = [ + "set -eu", + nodeWebObserveResolveStateDirShell(options), + nodeWebObserveStatusNodeScript(tailLines, options.node, options.lane), + ].join("\n"); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, timeoutSeconds); + const statusResolution = resolveWebObserveActionJson(result, "status"); + return { result, status: statusResolution.parsed, stdoutJson: statusResolution.diagnostics }; +} + +export function webObserveText(value: unknown): string { + if (value === null || value === undefined || value === "") return "-"; + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return JSON.stringify(value); +} + +export function webObserveShort(value: string, maxLength: number): string { + if (value.length <= maxLength) return value; + if (maxLength <= 1) return value.slice(0, maxLength); + return `${value.slice(0, maxLength - 1)}~`; +} + +export function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record | RenderedCliResult { + const type = options.commandType ?? (stopCommand ? "stop" : null); + if (type === null) throw new Error("web-probe observe command requires --type"); + const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`; + const payload = { + id: commandId, + type, + createdAt: new Date().toISOString(), + source: "cli", + path: options.commandPath, + text: options.commandText, + label: options.commandLabel, + sessionId: options.commandSessionId, + provider: options.commandProvider, + afterRound: options.commandAfterRound, + severity: options.commandSeverity, + alternateSessionStrategy: options.commandAlternateSessionStrategy, + expectedSentinelRange: options.commandExpectedSentinelRange, + expectedActionWaitMs: options.commandExpectedActionWaitMs, + durationMs: options.commandDurationMs, + requireComposerReady: options.commandRequireComposerReady, + waitProjectManagementReady: options.commandWaitProjectManagementReady, + findingId: options.commandFindingId, + blocking: options.commandBlocking, + accountId: options.commandAccountId, + fromAccountId: options.commandFromAccountId, + toAccountId: options.commandToAccountId, + sourceId: options.commandSourceId, + fileRef: options.commandFileRef, + filename: options.commandFilename, + taskRef: options.commandTaskRef, + taskId: options.commandTaskId, + field: options.commandField, + link: options.commandLink, + title: options.commandTitle, + body: options.commandBody, + status: options.commandStatus, + hwpodId: options.commandHwpodId, + nodeId: options.commandNodeId, + workspaceRoot: options.commandWorkspaceRoot, + root: options.commandRoot, + }; + const preStopStatus = options.force && stopCommand + ? readNodeWebProbeObserveRemoteStatus(options, spec, 1, Math.min(options.commandTimeoutSeconds, 30)) + : null; + const preStopDiagnostics = record(preStopStatus?.status?.diagnostics); + const preStopCommands = record(preStopStatus?.status?.commands); + const preStopPending = Number(preStopCommands?.pendingCount ?? 0); + const preStopProcessing = Number(preStopCommands?.processingCount ?? 0); + const forceBeforeQueueReason = options.force && stopCommand + ? preStopDiagnostics?.heartbeatStale === true + ? "heartbeat-stale" + : preStopPending > 0 || preStopProcessing > 0 + ? "command-backlog" + : null + : null; + if (forceBeforeQueueReason !== null) { + return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, forceBeforeQueueReason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, null); + } + const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); + const waitMs = options.force && stopCommand ? Math.max(options.waitMs, 5000) : options.waitMs; + const script = [ + "set -eu", + nodeWebObserveResolveStateDirShell(options), + "mkdir -p \"$state_dir/commands/pending\"", + `node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`, + nodeWebObserveWaitCommandShell(commandId, waitMs), + ].join("\n"); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); + const commandResolution = resolveWebObserveActionJson(result, "command"); + const commandResult = commandResolution.parsed; + if (options.force && stopCommand && (result.exitCode !== 0 || commandResult?.waitTimedOut === true || commandResult?.queued === true)) { + const reason = result.exitCode !== 0 + ? "graceful-stop-failed" + : commandResult?.waitTimedOut === true + ? "graceful-stop-not-consumed" + : "graceful-stop-queued"; + return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, reason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, result); + } + const payloadResult = { + ok: result.exitCode === 0 && commandResult !== null && commandResult.ok !== false, + status: result.exitCode === 0 && commandResult !== null ? (waitMs > 0 ? "completed-or-queued" : "queued") : "blocked", + command: webObserveCommandLabel(stopCommand ? "stop" : "command", options), + id: webObserveIdFromOptions(options), + node: options.node, + lane: options.lane, + workspace: spec.workspace, + commandId, + observerCommand: commandSummaryForOutput(payload), + observer: commandResult, + wrapper: buildWebObserveWrapperForObserveOptions(stopCommand ? "stop" : "command", options, spec.workspace, { commandType: type }), + stdoutJson: commandResolution.diagnostics, + result: compactCommandResult(result), + full: options.full, + valuesRedacted: true, + }; + return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult); +} + +export function runNodeWebProbeObserveForceStop( + options: NodeWebProbeObserveOptions, + spec: HwlabRuntimeLaneSpec, + payload: Record, + commandId: string, + reason: string, + preflightResult: ReturnType | null, + preflightStatus: Record | null, + gracefulResult: ReturnType | null, +): Record | RenderedCliResult { + const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [ + "set -eu", + nodeWebObserveResolveStateDirShell(options), + nodeWebObserveForceStopNodeScript(reason, commandId), + ].join("\n"), 55); + const forceResolution = resolveWebObserveActionJson(killResult, "force-stop"); + const forcePayload = forceResolution.parsed; + const payloadResult = { + ok: killResult.exitCode === 0 && forcePayload !== null && forcePayload.ok !== false, + status: killResult.exitCode === 0 && forcePayload !== null && forcePayload.ok !== false ? "forced-stopped" : "blocked", + command: webObserveCommandLabel("stop", options), + id: webObserveIdFromOptions(options), + node: options.node, + lane: options.lane, + workspace: spec.workspace, + commandId, + observerCommand: commandSummaryForOutput(payload), + observer: forcePayload, + wrapper: buildWebObserveWrapperForObserveOptions("stop", options, spec.workspace, { commandType: "stop" }), + forceReason: reason, + preflightObserver: preflightStatus, + preflightResult: preflightResult === null ? null : compactCommandResult(preflightResult), + gracefulResult: gracefulResult === null ? null : compactCommandResult(gracefulResult), + stdoutJson: forceResolution.diagnostics, + forceResult: compactCommandResultWithStdoutTail(killResult), + full: options.full, + valuesRedacted: true, + }; + return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult); +} diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 2a594492..5cf01423 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -35,11 +35,14 @@ import type { BootstrapAdminPasswordMaterial, NodeWebProbeObserveCommandType, No import { runTransWorkspaceStdinScript, runtimeSecretSpec } from "./public-exposure"; import { transPath } from "./runtime-common"; import { assertLane, assertNodeId, compactCommandResult, compactCommandResultRedacted, compactCommandResultWithStdoutTail, nullableRecord, optionValue, parseJsonObject, positiveIntegerOption, record, requiredOption, shellQuote } from "./utils"; +import { runNodeWebProbeObserveCommand, runNodeWebProbeObserveGc, runNodeWebProbeObserveStart, runNodeWebProbeObserveStatus } from "./web-probe-observe-actions"; import { runNodeWebProbeObserveCollect } from "./web-probe-observe-collect"; import { nodeWebObserveResolveStateDirShell, recoverWebObserveAnalyzeTurnDetails, renderWebObserveStartResult, renderWebProbeRunResult, upsertWebObserveIndexEntry, webObserveCommandLabel, webObserveIdFromOptions, webObserveIdFromStatus, webObserveIndexEntryFromOptions, webObserveNextCommands, withWebObserveAnalyzeRendered, withWebObserveShortcuts } from "./web-observe-render"; import { commandSummaryForOutput, isSafeWebObserveArchivePrefix, isSafeWebObserveCollectFile, isSafeWebObserveFindingId, isSafeWebObserveJobId, isSafeWebObserveStateDir, isSafeWebObserveTraceId, nodeWebObserveForceStopNodeScript, nodeWebObserveStatusNodeScript, nodeWebObserveWaitCommandShell, runNodeWebProbeScript, safeWebObserveSegment, safeWebObserveTargetSegment } from "./web-observe-scripts"; import { displayRepoPath, readBootstrapAdminPasswordMaterial, sleepSync } from "./web-probe"; +export { webObserveShort, webObserveText } from "./web-probe-observe-actions"; + export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSentinelOptions { const [sentinelActionRaw] = args; if ( @@ -1343,685 +1346,6 @@ export function runNodeWebProbeObserve( return runNodeWebProbeObserveAnalyze(options, spec); } -export function runNodeWebProbeObserveGc(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record { - const stateRoot = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`; - const script = [ - "set -eu", - `node - ${shellQuote(stateRoot)} ${shellQuote(String(options.gcKeepHours))} ${shellQuote(String(options.gcLimit))} ${shellQuote(options.confirm ? "run" : "plan")} ${shellQuote(options.node)} ${shellQuote(options.lane)} <<'UNIDESK_WEB_OBSERVE_GC'`, - nodeWebObserveGcNodeScript(), - "UNIDESK_WEB_OBSERVE_GC", - ].join("\n"); - const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); - const payload = parseJsonObject(result.stdout); - const ok = result.exitCode === 0 && payload.ok !== false; - return { - ok, - status: ok ? (options.confirm ? "cleaned" : "planned") : "blocked", - command: `web-probe observe gc --node ${options.node} --lane ${options.lane}`, - node: options.node, - lane: options.lane, - workspace: spec.workspace, - stateRoot, - mode: options.confirm ? "confirmed-run" : "dry-run", - retention: { - source: "config/unidesk-cli.yaml#gc.stateStaleScratch.keepHours", - keepHours: options.gcKeepHours, - userOverride: options.gcKeepHours !== webObserveGcDefaultKeepHours(), - }, - gc: options.full || options.raw ? payload : compactWebObserveGcPayload(payload), - result: ok - ? { - exitCode: result.exitCode, - timedOut: result.timedOut, - stdoutBytes: result.stdout.length, - stderrBytes: result.stderr.length, - } - : compactCommandResultWithStdoutTail(result), - valuesRedacted: true, - }; -} - -function compactWebObserveGcPayload(payload: Record): Record { - const candidates = Array.isArray(payload.candidates) ? payload.candidates.map(record) : []; - const protectedRuns = Array.isArray(payload.protected) ? payload.protected.map(record) : []; - const deleted = Array.isArray(payload.deleted) ? payload.deleted.map(record) : []; - const failures = Array.isArray(payload.failures) ? payload.failures.map(record) : []; - return { - ok: payload.ok ?? null, - mode: payload.mode ?? null, - mutation: payload.mutation ?? null, - keepHours: payload.keepHours ?? null, - limit: payload.limit ?? null, - diskBefore: payload.diskBefore ?? null, - diskAfter: payload.diskAfter ?? null, - scannedRuns: payload.scannedRuns ?? null, - candidateCount: payload.candidateCount ?? null, - selectedCount: payload.selectedCount ?? null, - deferredCount: payload.deferredCount ?? null, - protectedCount: payload.protectedCount ?? null, - estimatedReclaimBytes: payload.estimatedReclaimBytes ?? null, - estimatedReclaimHuman: payload.estimatedReclaimHuman ?? null, - reclaimedBytes: payload.reclaimedBytes ?? null, - reclaimedHuman: payload.reclaimedHuman ?? null, - candidates: candidates.slice(0, 20).map((item) => ({ - id: item.id ?? null, - runDir: item.runDir ?? null, - status: item.status ?? null, - ageHours: item.ageHours ?? null, - rawHuman: item.rawHuman ?? null, - artifactCount: item.artifactCount ?? null, - })), - deleted: deleted.slice(0, 20).map((item) => ({ - id: item.id ?? null, - runDir: item.runDir ?? null, - rawHuman: item.rawHuman ?? null, - artifactCount: item.artifactCount ?? null, - })), - failures: failures.slice(0, 20), - protected: protectedRuns.slice(0, 20).map((item) => ({ - id: item.id ?? null, - runDir: item.runDir ?? null, - status: item.status ?? null, - ageHours: item.ageHours ?? null, - rawHuman: item.rawHuman ?? null, - reasons: item.reasons ?? null, - })), - disclosure: { - candidateRowsShown: Math.min(20, candidates.length), - deletedRowsShown: Math.min(20, deleted.length), - protectedRowsShown: Math.min(20, protectedRuns.length), - full: "rerun with --full or --raw for all candidates/artifacts", - }, - valuesRedacted: true, - }; -} - -function nodeWebObserveGcNodeScript(): string { - return String.raw` -const fs = require("fs"); -const path = require("path"); -const childProcess = require("child_process"); - -const stateRoot = process.argv[2]; -const keepHours = Number(process.argv[3]); -const limit = Number(process.argv[4]); -const mode = process.argv[5] === "run" ? "run" : "plan"; -const nodeId = process.argv[6]; -const lane = process.argv[7]; -const keepMs = keepHours * 60 * 60 * 1000; -const nowMs = Date.now(); -const rawNames = ["samples.jsonl", "browser-process.jsonl", "network.jsonl", "console.jsonl", "artifacts.jsonl", "performance-events.jsonl", "screenshots", "performance"]; - -function jsonRead(file) { - try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; } -} - -function parseTime(value) { - if (typeof value !== "string") return 0; - const ms = Date.parse(value); - return Number.isFinite(ms) ? ms : 0; -} - -function dirTimeMs(runDir) { - const base = path.basename(runDir); - const match = base.match(/^(\d{8})T(\d{6})Z_/); - if (!match) return 0; - const raw = match[1].slice(0, 4) + "-" + match[1].slice(4, 6) + "-" + match[1].slice(6, 8) + "T" + match[2].slice(0, 2) + ":" + match[2].slice(2, 4) + ":" + match[2].slice(4, 6) + "Z"; - return Date.parse(raw) || 0; -} - -function newestKnownTimeMs(runDir, manifest, heartbeat) { - return Math.max( - dirTimeMs(runDir), - parseTime(manifest && manifest.completedAt), - parseTime(manifest && manifest.updatedAt), - parseTime(manifest && manifest.startedAt), - parseTime(manifest && manifest.pageProvenance && manifest.pageProvenance.observedAt), - parseTime(heartbeat && heartbeat.completedAt), - parseTime(heartbeat && heartbeat.forceStoppedAt), - parseTime(heartbeat && heartbeat.updatedAt), - parseTime(heartbeat && heartbeat.pageProvenance && heartbeat.pageProvenance.observedAt), - ); -} - -function sizePath(target) { - let total = 0; - const stack = [target]; - while (stack.length > 0) { - const current = stack.pop(); - let st; - try { st = fs.lstatSync(current); } catch { continue; } - if (st.isSymbolicLink()) continue; - total += st.blocks ? st.blocks * 512 : st.size; - if (!st.isDirectory()) continue; - let entries; - try { entries = fs.readdirSync(current); } catch { continue; } - for (const entry of entries) stack.push(path.join(current, entry)); - } - return total; -} - -function human(bytes) { - const units = ["B", "KiB", "MiB", "GiB", "TiB"]; - let value = bytes; - let unit = 0; - while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit += 1; } - return value.toFixed(unit === 0 ? 0 : 1) + units[unit]; -} - -function disk() { - try { - const line = childProcess.execFileSync("df", ["-Pk", "/"], { encoding: "utf8" }).trim().split("\n").slice(-1)[0]; - const parts = line.trim().split(/\s+/); - const sizeBytes = Number(parts[1]) * 1024; - const usedBytes = Number(parts[2]) * 1024; - const availableBytes = Number(parts[3]) * 1024; - const usePercent = Number(String(parts[4]).replace("%", "")); - return { filesystem: parts[0], sizeBytes, usedBytes, availableBytes, usePercent, usedHuman: human(usedBytes), availableHuman: human(availableBytes) }; - } catch (error) { - return { error: error instanceof Error ? error.message : String(error) }; - } -} - -function findRunDirs(root) { - const runs = []; - for (const year of safeReadDir(root)) { - const y = path.join(root, year.name); - if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) continue; - for (const month of safeReadDir(y)) { - const m = path.join(y, month.name); - if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) continue; - for (const day of safeReadDir(m)) { - const d = path.join(m, day.name); - if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) continue; - for (const run of safeReadDir(d)) { - const r = path.join(d, run.name); - if (run.isDirectory() && /_webobs-[A-Za-z0-9_.-]+$/.test(run.name)) runs.push(r); - } - } - } - } - return runs; -} - -function safeReadDir(dir) { - try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; } -} - -function pidObserverAlive(pid, runAbs) { - if (!Number.isInteger(pid) || pid <= 1) return { alive: false, reason: null }; - try { - const cmdline = fs.readFileSync("/proc/" + pid + "/cmdline", "utf8").replace(/\0/g, " "); - if (cmdline.includes("observer-runner") || cmdline.includes(runAbs)) return { alive: true, reason: "pid-alive" }; - return { alive: false, reason: null }; - } catch (error) { - if (fs.existsSync("/proc/" + pid)) return { alive: true, reason: "pid-alive-unreadable" }; - return { alive: false, reason: null }; - } -} - -function openRunDirs(rootAbs) { - const active = new Set(); - const procEntries = safeReadDir("/proc").filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name)); - for (const proc of procEntries) { - const fdDir = path.join("/proc", proc.name, "fd"); - for (const fd of safeReadDir(fdDir)) { - let target; - try { target = fs.readlinkSync(path.join(fdDir, fd.name)).replace(/ \(deleted\)$/, ""); } catch { continue; } - if (!target.startsWith(rootAbs + path.sep)) continue; - const rel = path.relative(rootAbs, target).split(path.sep); - if (rel.length >= 4) active.add(path.join(rootAbs, rel[0], rel[1], rel[2], rel[3])); - } - } - return active; -} - -function rawArtifacts(runDir) { - const artifacts = []; - for (const name of rawNames) { - const target = path.join(runDir, name); - if (!fs.existsSync(target)) continue; - const bytes = sizePath(target); - if (bytes > 0) artifacts.push({ name, path: target, bytes, human: human(bytes) }); - } - return artifacts; -} - -function removeArtifact(artifact) { - const st = fs.lstatSync(artifact.path); - if (st.isSymbolicLink()) throw new Error("refusing symlink: " + artifact.path); - if (st.isDirectory()) fs.rmSync(artifact.path, { recursive: true, force: false }); - else fs.unlinkSync(artifact.path); -} - -const rootAbs = path.resolve(stateRoot); -const diskBefore = disk(); -const openDirs = openRunDirs(rootAbs); -const candidates = []; -const protectedRuns = []; - -for (const runDir of findRunDirs(rootAbs)) { - const manifest = jsonRead(path.join(runDir, "manifest.json")); - const heartbeat = jsonRead(path.join(runDir, "heartbeat.json")); - const pid = Number((heartbeat && heartbeat.pid) || (manifest && manifest.pid) || 0); - const pidState = pidObserverAlive(pid, runDir); - const hasOpenFd = openDirs.has(runDir); - const newestMs = newestKnownTimeMs(runDir, manifest, heartbeat); - const ageMs = newestMs > 0 ? nowMs - newestMs : 0; - const artifacts = rawArtifacts(runDir); - const rawBytes = artifacts.reduce((sum, item) => sum + item.bytes, 0); - const rel = path.relative(rootAbs, runDir); - const reasons = []; - if (!manifest) reasons.push("manifest-missing"); - if (pidState.alive) reasons.push(pidState.reason || "pid-alive"); - if (hasOpenFd) reasons.push("open-fd"); - if (newestMs <= 0) reasons.push("time-unknown"); - if (ageMs < keepMs) reasons.push("retention-window"); - if (rawBytes <= 0) reasons.push("no-raw-artifacts"); - const summary = { - id: (manifest && manifest.jobId) || path.basename(runDir).match(/(webobs-[A-Za-z0-9_.-]+)$/)?.[1] || null, - runDir: rel, - status: heartbeat && heartbeat.status || null, - pid: Number.isInteger(pid) && pid > 0 ? pid : null, - newestAt: newestMs > 0 ? new Date(newestMs).toISOString() : null, - ageHours: newestMs > 0 ? Number((ageMs / 3600000).toFixed(2)) : null, - rawBytes, - rawHuman: human(rawBytes), - artifacts, - }; - if (reasons.length > 0) protectedRuns.push({ ...summary, reasons }); - else candidates.push(summary); -} - -candidates.sort((a, b) => b.rawBytes - a.rawBytes); -const selected = candidates.slice(0, limit); -const selectedSet = new Set(selected.map((item) => item.runDir)); -const deferred = candidates.filter((item) => !selectedSet.has(item.runDir)); -let reclaimedBytes = 0; -const deleted = []; -const failures = []; - -if (mode === "run") { - for (const item of selected) { - const runDir = path.join(rootAbs, item.runDir); - try { - for (const artifact of item.artifacts) removeArtifact(artifact); - reclaimedBytes += item.rawBytes; - deleted.push({ id: item.id, runDir: item.runDir, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length }); - } catch (error) { - failures.push({ id: item.id, runDir: item.runDir, error: error instanceof Error ? error.message : String(error) }); - } - } -} - -const estimatedReclaimBytes = selected.reduce((sum, item) => sum + item.rawBytes, 0); -const diskAfter = mode === "run" ? disk() : null; -console.log(JSON.stringify({ - ok: failures.length === 0, - command: "web-probe observe gc", - node: nodeId, - lane, - stateRoot, - mode, - mutation: mode === "run", - keepHours, - limit, - diskBefore, - diskAfter, - scannedRuns: candidates.length + protectedRuns.length, - candidateCount: candidates.length, - selectedCount: selected.length, - deferredCount: deferred.length, - protectedCount: protectedRuns.length, - estimatedReclaimBytes, - estimatedReclaimHuman: human(estimatedReclaimBytes), - reclaimedBytes, - reclaimedHuman: human(reclaimedBytes), - candidates: selected.map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length, artifacts: item.artifacts.map((artifact) => ({ name: artifact.name, bytes: artifact.bytes, human: artifact.human })) })), - deleted, - failures, - protected: protectedRuns.slice(0, Math.min(50, protectedRuns.length)).map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, reasons: item.reasons })), - valuesRedacted: true, -}, null, 2)); -`; -} - -export function runNodeWebProbeObserveStart( - options: NodeWebProbeObserveOptions, - spec: HwlabRuntimeLaneSpec, - secretSpec: RuntimeSecretSpec, - material: BootstrapAdminPasswordMaterial, - credential: Record, -): Record | RenderedCliResult { - const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`; - const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z"); - const day = timestamp.slice(0, 8); - const defaultStateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`; - const stateDir = options.stateDir ?? defaultStateDir; - const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64"); - const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64; - const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode); - const alertThresholds = nodeWebProbeAlertThresholds(spec); - const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec); - const projectManagement = nodeWebProbeProjectManagementConfig(spec); - const authLogin = nodeWebProbeAuthLoginConfig(spec); - const runnerEnvAssignments = [ - ...webProbeProxy.envAssignments, - ...webProbeAccountEnvAssignments(), - ...(spec.webProbe?.playwrightBrowsersPath === undefined ? [] : [ - `PLAYWRIGHT_BROWSERS_PATH=${shellQuote(spec.webProbe.playwrightBrowsersPath)}`, - ]), - `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`, - `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, - `HWLAB_WEB_PASS=${shellQuote(material.password)}`, - `UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`, - `UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`, - `UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`, - `UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`, - `UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`, - `UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`, - `UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`, - `UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`, - `UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`, - `UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`, - `UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`, - `UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`, - `UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`, - ...(authLogin === null - ? [] - : [ - `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS=${shellQuote(String(authLogin.maxAttempts))}`, - `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS=${shellQuote(String(authLogin.requestTimeoutMs))}`, - `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS=${shellQuote(String(authLogin.initialDelayMs))}`, - `UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`, - ]), - ].join(" "); - const script = [ - "set -eu", - `state_dir=${shellQuote(stateDir)}`, - "mkdir -p \"$state_dir\"", - "chmod 700 \"$state_dir\"", - "runner=\"$state_dir/observer-runner.mjs\"", - "runner_b64=\"$state_dir/observer-runner.mjs.b64\"", - "cat >\"$runner_b64\" <<'UNIDESK_WEB_OBSERVE_RUNNER_B64'", - runnerB64Body, - "UNIDESK_WEB_OBSERVE_RUNNER_B64", - "node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"", - "rm -f \"$runner_b64\"", - "chmod 700 \"$runner\"", - `if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" "$state_dir/stdout.log" 2>"$state_dir/stderr.log" \"$state_dir/pid\"", - "sleep 1", - `node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`, - ].join("\n"); - const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); - const started = parseJsonObject(result.stdout); - const startOk = result.exitCode === 0 && started?.ok === true; - const recovery = startOk - ? null - : readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30)); - const recoveredStatus = !startOk && recovery !== null && recovery.result.exitCode === 0 && recovery.status !== null && recovery.status.ok !== false - ? recovery.status - : null; - const effectiveOk = startOk || recoveredStatus !== null; - const observer = startOk ? started : recoveredStatus; - const observerId = typeof started?.jobId === "string" - ? started.jobId - : webObserveIdFromStatus(recoveredStatus, { ...options, id: jobId }) ?? jobId; - const degradedReason = !startOk && recoveredStatus !== null - ? "web-probe-start-transport-timeout-recovered" - : result.timedOut - ? "web-probe-command-timeout" - : result.exitCode !== 0 - ? "web-probe-observe-start-failed" - : null; - const index = effectiveOk - ? upsertWebObserveIndexEntry(startOk ? { - id: observerId, - node: options.node, - lane: options.lane, - workspace: spec.workspace, - stateDir, - url: options.url, - targetPath: options.targetPath, - status: "running", - pid: typeof started.pid === "number" ? started.pid : null, - startedAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - } : webObserveIndexEntryFromOptions({ ...options, id: observerId, stateDir }, spec, observerId, recoveredStatus ?? {})) - : null; - return renderWebObserveStartResult({ - ok: effectiveOk, - status: effectiveOk ? "started" : "blocked", - command: `web-probe observe start --node ${options.node} --lane ${options.lane}`, - node: options.node, - lane: options.lane, - workspace: spec.workspace, - url: options.url, - network: webProbeProxy.summary, - alertThresholds, - browserFreezePolicy, - projectManagement, - targetPath: options.targetPath, - id: observerId, - degradedReason, - credential, - observer: withWebObserveShortcuts(observer, observerId), - wrapper: buildWebObserveWrapperForObserveOptions("start", options, spec.workspace, { id: observerId, jobId: observerId, stateDir }), - index, - next: webObserveNextCommands(observerId), - result: compactCommandResultRedacted(result, [material.password ?? ""]), - recovery: recovery === null ? null : { - attempted: true, - ok: recoveredStatus !== null, - reason: degradedReason, - result: compactCommandResultWithStdoutTail(recovery.result), - valuesRedacted: true, - }, - valuesRedacted: true, - }); -} - -export function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record | RenderedCliResult { - const { result, status } = readNodeWebProbeObserveRemoteStatus(options, spec, options.tailLines, options.commandTimeoutSeconds); - const observerId = webObserveIdFromStatus(status, options); - const statusReadable = status !== null; - const ok = result.exitCode === 0 && statusReadable && status.ok !== false; - const degradedReason = result.timedOut - ? "web-probe-command-timeout" - : result.exitCode !== 0 - ? "web-probe-observe-status-failed" - : !statusReadable - ? "web-probe-observe-status-unreadable" - : typeof status.degradedReason === "string" - ? status.degradedReason - : null; - const index = ok && observerId !== null && options.stateDir !== null - ? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status)) - : null; - return withWebObserveStatusRendered({ - ok, - status: ok ? "observed" : "blocked", - command: webObserveCommandLabel("status", options), - id: observerId, - node: options.node, - lane: options.lane, - workspace: spec.workspace, - degradedReason, - observer: withWebObserveShortcuts(status, observerId), - wrapper: buildWebObserveWrapperForObserveOptions("status", options, spec.workspace, { id: observerId, stateDir: webObserveWrapperStateDirFromStatus(status, options.stateDir) }), - index, - next: observerId === null ? null : webObserveNextCommands(observerId), - result: compactCommandResultWithStdoutTail(result), - valuesRedacted: true, - }); -} - -export function readNodeWebProbeObserveRemoteStatus( - options: NodeWebProbeObserveOptions, - spec: HwlabRuntimeLaneSpec, - tailLines: number, - timeoutSeconds: number, -): { result: ReturnType; status: Record | null } { - const script = [ - "set -eu", - nodeWebObserveResolveStateDirShell(options), - nodeWebObserveStatusNodeScript(tailLines, options.node, options.lane), - ].join("\n"); - const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, timeoutSeconds); - return { result, status: parseJsonObject(result.stdout) }; -} - -export function webObserveText(value: unknown): string { - if (value === null || value === undefined || value === "") return "-"; - if (typeof value === "string") return value; - if (typeof value === "number" || typeof value === "boolean") return String(value); - return JSON.stringify(value); -} - -export function webObserveShort(value: string, maxLength: number): string { - if (value.length <= maxLength) return value; - if (maxLength <= 1) return value.slice(0, maxLength); - return `${value.slice(0, maxLength - 1)}~`; -} - -export function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record | RenderedCliResult { - const type = options.commandType ?? (stopCommand ? "stop" : null); - if (type === null) throw new Error("web-probe observe command requires --type"); - const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`; - const payload = { - id: commandId, - type, - createdAt: new Date().toISOString(), - source: "cli", - path: options.commandPath, - text: options.commandText, - label: options.commandLabel, - sessionId: options.commandSessionId, - provider: options.commandProvider, - afterRound: options.commandAfterRound, - severity: options.commandSeverity, - alternateSessionStrategy: options.commandAlternateSessionStrategy, - expectedSentinelRange: options.commandExpectedSentinelRange, - expectedActionWaitMs: options.commandExpectedActionWaitMs, - durationMs: options.commandDurationMs, - requireComposerReady: options.commandRequireComposerReady, - waitProjectManagementReady: options.commandWaitProjectManagementReady, - findingId: options.commandFindingId, - blocking: options.commandBlocking, - accountId: options.commandAccountId, - fromAccountId: options.commandFromAccountId, - toAccountId: options.commandToAccountId, - sourceId: options.commandSourceId, - fileRef: options.commandFileRef, - filename: options.commandFilename, - taskRef: options.commandTaskRef, - taskId: options.commandTaskId, - field: options.commandField, - link: options.commandLink, - title: options.commandTitle, - body: options.commandBody, - status: options.commandStatus, - hwpodId: options.commandHwpodId, - nodeId: options.commandNodeId, - workspaceRoot: options.commandWorkspaceRoot, - root: options.commandRoot, - }; - const preStopStatus = options.force && stopCommand - ? readNodeWebProbeObserveRemoteStatus(options, spec, 1, Math.min(options.commandTimeoutSeconds, 30)) - : null; - const preStopDiagnostics = record(preStopStatus?.status?.diagnostics); - const preStopCommands = record(preStopStatus?.status?.commands); - const preStopPending = Number(preStopCommands?.pendingCount ?? 0); - const preStopProcessing = Number(preStopCommands?.processingCount ?? 0); - const forceBeforeQueueReason = options.force && stopCommand - ? preStopDiagnostics?.heartbeatStale === true - ? "heartbeat-stale" - : preStopPending > 0 || preStopProcessing > 0 - ? "command-backlog" - : null - : null; - if (forceBeforeQueueReason !== null) { - return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, forceBeforeQueueReason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, null); - } - const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); - const waitMs = options.force && stopCommand ? Math.max(options.waitMs, 5000) : options.waitMs; - const script = [ - "set -eu", - nodeWebObserveResolveStateDirShell(options), - "mkdir -p \"$state_dir/commands/pending\"", - `node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`, - nodeWebObserveWaitCommandShell(commandId, waitMs), - ].join("\n"); - const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); - const commandResult = parseJsonObject(result.stdout); - if (options.force && stopCommand && (result.exitCode !== 0 || commandResult?.waitTimedOut === true || commandResult?.queued === true)) { - const reason = result.exitCode !== 0 - ? "graceful-stop-failed" - : commandResult?.waitTimedOut === true - ? "graceful-stop-not-consumed" - : "graceful-stop-queued"; - return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, reason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, result); - } - const payloadResult = { - ok: result.exitCode === 0 && commandResult?.ok !== false, - status: result.exitCode === 0 ? (waitMs > 0 ? "completed-or-queued" : "queued") : "blocked", - command: webObserveCommandLabel(stopCommand ? "stop" : "command", options), - id: webObserveIdFromOptions(options), - node: options.node, - lane: options.lane, - workspace: spec.workspace, - commandId, - observerCommand: commandSummaryForOutput(payload), - observer: commandResult, - wrapper: buildWebObserveWrapperForObserveOptions(stopCommand ? "stop" : "command", options, spec.workspace, { commandType: type }), - result: compactCommandResult(result), - full: options.full, - valuesRedacted: true, - }; - return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult); -} - -export function runNodeWebProbeObserveForceStop( - options: NodeWebProbeObserveOptions, - spec: HwlabRuntimeLaneSpec, - payload: Record, - commandId: string, - reason: string, - preflightResult: ReturnType | null, - preflightStatus: Record | null, - gracefulResult: ReturnType | null, -): Record | RenderedCliResult { - const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [ - "set -eu", - nodeWebObserveResolveStateDirShell(options), - nodeWebObserveForceStopNodeScript(reason, commandId), - ].join("\n"), 55); - const forcePayload = parseJsonObject(killResult.stdout); - const payloadResult = { - ok: killResult.exitCode === 0 && forcePayload?.ok !== false, - status: killResult.exitCode === 0 && forcePayload?.ok !== false ? "forced-stopped" : "blocked", - command: webObserveCommandLabel("stop", options), - id: webObserveIdFromOptions(options), - node: options.node, - lane: options.lane, - workspace: spec.workspace, - commandId, - observerCommand: commandSummaryForOutput(payload), - observer: forcePayload, - wrapper: buildWebObserveWrapperForObserveOptions("stop", options, spec.workspace, { commandType: "stop" }), - forceReason: reason, - preflightObserver: preflightStatus, - preflightResult: preflightResult === null ? null : compactCommandResult(preflightResult), - gracefulResult: gracefulResult === null ? null : compactCommandResult(gracefulResult), - forceResult: compactCommandResultWithStdoutTail(killResult), - full: options.full, - valuesRedacted: true, - }; - return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult); -} - export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record | RenderedCliResult { const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64"); const alertThresholds = nodeWebProbeAlertThresholds(spec);