Codex
814 lines
38 KiB
TypeScript
814 lines
38 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. WebProbe observe action handlers.
|
|
// Responsibility: start/status/command/stop/gc remote action execution and child JSON recovery.
|
|
import { randomBytes } from "node:crypto";
|
|
import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery";
|
|
import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { BootstrapAdminPasswordMaterial, NodeWebProbeObserveOptions, RuntimeSecretSpec } from "./entry";
|
|
import { runTransWorkspaceStdinScript } from "./public-exposure";
|
|
import { compactCommandResult, compactCommandResultRedacted, compactCommandResultWithStdoutTail, record, shellQuote } from "./utils";
|
|
import { nodeWebProbeAlertThresholds, nodeWebProbeAuthLoginConfig, nodeWebProbeBrowserFreezePolicy, nodeWebProbeHostProxyEnv, nodeWebProbeProjectManagementConfig, webObserveGcDefaultKeepHours, webProbeAccountEnvAssignments } from "./web-probe-observe-action-config";
|
|
import { nodeWebObserveResolveStateDirShell, renderWebObserveStartResult, upsertWebObserveIndexEntry, webObserveCommandLabel, webObserveIdFromOptions, webObserveIdFromStatus, webObserveIndexEntryFromOptions, webObserveNextCommands, withWebObserveShortcuts } from "./web-observe-render";
|
|
import { commandSummaryForOutput, nodeWebObserveForceStopNodeScript, nodeWebObserveStatusNodeScript, nodeWebObserveWaitCommandShell, safeWebObserveSegment, safeWebObserveTargetSegment } from "./web-observe-scripts";
|
|
|
|
type WebObserveActionJsonContract = "gc" | "start" | "status" | "command" | "force-stop";
|
|
|
|
export function resolveWebObserveActionJson(
|
|
result: { stdout: string; stderr?: string; exitCode?: number | null; timedOut?: boolean },
|
|
contract: WebObserveActionJsonContract,
|
|
): CliChildJsonResolution {
|
|
return resolveCliChildJsonCommandResult({
|
|
result,
|
|
requestedStdoutType: `web-probe observe ${contract} contract`,
|
|
acceptParsed: (value) => isWebObserveActionJsonContract(value, contract),
|
|
});
|
|
}
|
|
|
|
function isWebObserveActionJsonContract(value: Record<string, unknown>, contract: WebObserveActionJsonContract): boolean {
|
|
if (acceptFailureContract(value)) return true;
|
|
if (contract === "gc") {
|
|
return value.command === "web-probe observe gc" && hasAnyKey(value, ["mode", "stateRoot", "scannedRuns", "candidateCount", "selectedCount", "protectedCount"]);
|
|
}
|
|
if (contract === "start") {
|
|
return value.command === "web-probe-observe start" && hasAnyKey(value, ["jobId", "stateDir", "pid", "manifestPath", "heartbeat", "manifest"]);
|
|
}
|
|
if (contract === "status") {
|
|
return hasAnyKey(value, ["jobId", "stateDir", "manifest", "heartbeat", "diagnostics", "commands", "samples", "status"]);
|
|
}
|
|
if (contract === "command") {
|
|
return hasAnyKey(value, ["ok", "queued", "waitTimedOut", "commandId", "status", "path", "error", "reason"]);
|
|
}
|
|
return hasAnyKey(value, ["ok", "forceStopped", "killed", "status", "pid", "reason", "error", "commandId"]);
|
|
}
|
|
|
|
function acceptFailureContract(value: Record<string, unknown>): boolean {
|
|
return value.ok === false && hasAnyKey(value, ["error", "reason", "degradedReason", "status", "diagnostics", "result", "command"]);
|
|
}
|
|
|
|
function hasAnyKey(value: Record<string, unknown>, keys: string[]): boolean {
|
|
return keys.some((key) => value[key] !== undefined && value[key] !== null);
|
|
}
|
|
|
|
export function runNodeWebProbeObserveGc(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const stateRoot = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`;
|
|
const script = [
|
|
"set -eu",
|
|
`node - ${shellQuote(stateRoot)} ${shellQuote(String(options.gcKeepHours))} ${shellQuote(String(options.gcLimit))} ${shellQuote(options.confirm ? "run" : "plan")} ${shellQuote(options.node)} ${shellQuote(options.lane)} <<'UNIDESK_WEB_OBSERVE_GC'`,
|
|
nodeWebObserveGcNodeScript(),
|
|
"UNIDESK_WEB_OBSERVE_GC",
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const payloadResolution = resolveWebObserveActionJson(result, "gc");
|
|
const payload = payloadResolution.parsed ?? {};
|
|
const ok = result.exitCode === 0 && payloadResolution.parsed !== null && payload.ok !== false;
|
|
return {
|
|
ok,
|
|
status: ok ? (options.confirm ? "cleaned" : "planned") : "blocked",
|
|
command: `web-probe observe gc --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
stateRoot,
|
|
mode: options.confirm ? "confirmed-run" : "dry-run",
|
|
retention: {
|
|
source: "config/unidesk-cli.yaml#gc.stateStaleScratch.keepHours",
|
|
keepHours: options.gcKeepHours,
|
|
userOverride: options.gcKeepHours !== webObserveGcDefaultKeepHours(),
|
|
},
|
|
gc: options.full || options.raw ? payload : compactWebObserveGcPayload(payload),
|
|
stdoutJson: payloadResolution.diagnostics,
|
|
result: ok
|
|
? {
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: result.stdout.length,
|
|
stderrBytes: result.stderr.length,
|
|
}
|
|
: compactCommandResultWithStdoutTail(result),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactWebObserveGcPayload(payload: Record<string, unknown>): Record<string, unknown> {
|
|
const candidates = Array.isArray(payload.candidates) ? payload.candidates.map(record) : [];
|
|
const protectedRuns = Array.isArray(payload.protected) ? payload.protected.map(record) : [];
|
|
const deleted = Array.isArray(payload.deleted) ? payload.deleted.map(record) : [];
|
|
const failures = Array.isArray(payload.failures) ? payload.failures.map(record) : [];
|
|
return {
|
|
ok: payload.ok ?? null,
|
|
mode: payload.mode ?? null,
|
|
mutation: payload.mutation ?? null,
|
|
keepHours: payload.keepHours ?? null,
|
|
limit: payload.limit ?? null,
|
|
diskBefore: payload.diskBefore ?? null,
|
|
diskAfter: payload.diskAfter ?? null,
|
|
scannedRuns: payload.scannedRuns ?? null,
|
|
candidateCount: payload.candidateCount ?? null,
|
|
selectedCount: payload.selectedCount ?? null,
|
|
deferredCount: payload.deferredCount ?? null,
|
|
protectedCount: payload.protectedCount ?? null,
|
|
estimatedReclaimBytes: payload.estimatedReclaimBytes ?? null,
|
|
estimatedReclaimHuman: payload.estimatedReclaimHuman ?? null,
|
|
reclaimedBytes: payload.reclaimedBytes ?? null,
|
|
reclaimedHuman: payload.reclaimedHuman ?? null,
|
|
candidates: candidates.slice(0, 20).map((item) => ({
|
|
id: item.id ?? null,
|
|
runDir: item.runDir ?? null,
|
|
status: item.status ?? null,
|
|
ageHours: item.ageHours ?? null,
|
|
rawHuman: item.rawHuman ?? null,
|
|
artifactCount: item.artifactCount ?? null,
|
|
})),
|
|
deleted: deleted.slice(0, 20).map((item) => ({
|
|
id: item.id ?? null,
|
|
runDir: item.runDir ?? null,
|
|
rawHuman: item.rawHuman ?? null,
|
|
artifactCount: item.artifactCount ?? null,
|
|
})),
|
|
failures: failures.slice(0, 20),
|
|
protected: protectedRuns.slice(0, 20).map((item) => ({
|
|
id: item.id ?? null,
|
|
runDir: item.runDir ?? null,
|
|
status: item.status ?? null,
|
|
ageHours: item.ageHours ?? null,
|
|
rawHuman: item.rawHuman ?? null,
|
|
reasons: item.reasons ?? null,
|
|
})),
|
|
disclosure: {
|
|
candidateRowsShown: Math.min(20, candidates.length),
|
|
deletedRowsShown: Math.min(20, deleted.length),
|
|
protectedRowsShown: Math.min(20, protectedRuns.length),
|
|
full: "rerun with --full or --raw for all candidates/artifacts",
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function nodeWebObserveGcNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const childProcess = require("child_process");
|
|
|
|
const stateRoot = process.argv[2];
|
|
const keepHours = Number(process.argv[3]);
|
|
const limit = Number(process.argv[4]);
|
|
const mode = process.argv[5] === "run" ? "run" : "plan";
|
|
const nodeId = process.argv[6];
|
|
const lane = process.argv[7];
|
|
const keepMs = keepHours * 60 * 60 * 1000;
|
|
const nowMs = Date.now();
|
|
const rawNames = ["samples.jsonl", "browser-process.jsonl", "network.jsonl", "console.jsonl", "artifacts.jsonl", "performance-events.jsonl", "screenshots", "performance"];
|
|
|
|
function jsonRead(file) {
|
|
try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; }
|
|
}
|
|
|
|
function parseTime(value) {
|
|
if (typeof value !== "string") return 0;
|
|
const ms = Date.parse(value);
|
|
return Number.isFinite(ms) ? ms : 0;
|
|
}
|
|
|
|
function dirTimeMs(runDir) {
|
|
const base = path.basename(runDir);
|
|
const match = base.match(/^(\d{8})T(\d{6})Z_/);
|
|
if (!match) return 0;
|
|
const raw = match[1].slice(0, 4) + "-" + match[1].slice(4, 6) + "-" + match[1].slice(6, 8) + "T" + match[2].slice(0, 2) + ":" + match[2].slice(2, 4) + ":" + match[2].slice(4, 6) + "Z";
|
|
return Date.parse(raw) || 0;
|
|
}
|
|
|
|
function newestKnownTimeMs(runDir, manifest, heartbeat) {
|
|
return Math.max(
|
|
dirTimeMs(runDir),
|
|
parseTime(manifest && manifest.completedAt),
|
|
parseTime(manifest && manifest.updatedAt),
|
|
parseTime(manifest && manifest.startedAt),
|
|
parseTime(manifest && manifest.pageProvenance && manifest.pageProvenance.observedAt),
|
|
parseTime(heartbeat && heartbeat.completedAt),
|
|
parseTime(heartbeat && heartbeat.forceStoppedAt),
|
|
parseTime(heartbeat && heartbeat.updatedAt),
|
|
parseTime(heartbeat && heartbeat.pageProvenance && heartbeat.pageProvenance.observedAt),
|
|
);
|
|
}
|
|
|
|
function sizePath(target) {
|
|
let total = 0;
|
|
const stack = [target];
|
|
while (stack.length > 0) {
|
|
const current = stack.pop();
|
|
let st;
|
|
try { st = fs.lstatSync(current); } catch { continue; }
|
|
if (st.isSymbolicLink()) continue;
|
|
total += st.blocks ? st.blocks * 512 : st.size;
|
|
if (!st.isDirectory()) continue;
|
|
let entries;
|
|
try { entries = fs.readdirSync(current); } catch { continue; }
|
|
for (const entry of entries) stack.push(path.join(current, entry));
|
|
}
|
|
return total;
|
|
}
|
|
|
|
function human(bytes) {
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let value = bytes;
|
|
let unit = 0;
|
|
while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit += 1; }
|
|
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
|
}
|
|
|
|
function disk() {
|
|
try {
|
|
const line = childProcess.execFileSync("df", ["-Pk", "/"], { encoding: "utf8" }).trim().split("\n").slice(-1)[0];
|
|
const parts = line.trim().split(/\s+/);
|
|
const sizeBytes = Number(parts[1]) * 1024;
|
|
const usedBytes = Number(parts[2]) * 1024;
|
|
const availableBytes = Number(parts[3]) * 1024;
|
|
const usePercent = Number(String(parts[4]).replace("%", ""));
|
|
return { filesystem: parts[0], sizeBytes, usedBytes, availableBytes, usePercent, usedHuman: human(usedBytes), availableHuman: human(availableBytes) };
|
|
} catch (error) {
|
|
return { error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function findRunDirs(root) {
|
|
const runs = [];
|
|
for (const year of safeReadDir(root)) {
|
|
const y = path.join(root, year.name);
|
|
if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) continue;
|
|
for (const month of safeReadDir(y)) {
|
|
const m = path.join(y, month.name);
|
|
if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) continue;
|
|
for (const day of safeReadDir(m)) {
|
|
const d = path.join(m, day.name);
|
|
if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) continue;
|
|
for (const run of safeReadDir(d)) {
|
|
const r = path.join(d, run.name);
|
|
if (run.isDirectory() && /_webobs-[A-Za-z0-9_.-]+$/.test(run.name)) runs.push(r);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return runs;
|
|
}
|
|
|
|
function safeReadDir(dir) {
|
|
try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
|
|
}
|
|
|
|
function pidObserverAlive(pid, runAbs) {
|
|
if (!Number.isInteger(pid) || pid <= 1) return { alive: false, reason: null };
|
|
try {
|
|
const cmdline = fs.readFileSync("/proc/" + pid + "/cmdline", "utf8").replace(/\0/g, " ");
|
|
if (cmdline.includes("observer-runner") || cmdline.includes(runAbs)) return { alive: true, reason: "pid-alive" };
|
|
return { alive: false, reason: null };
|
|
} catch (error) {
|
|
if (fs.existsSync("/proc/" + pid)) return { alive: true, reason: "pid-alive-unreadable" };
|
|
return { alive: false, reason: null };
|
|
}
|
|
}
|
|
|
|
function openRunDirs(rootAbs) {
|
|
const active = new Set();
|
|
const procEntries = safeReadDir("/proc").filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name));
|
|
for (const proc of procEntries) {
|
|
const fdDir = path.join("/proc", proc.name, "fd");
|
|
for (const fd of safeReadDir(fdDir)) {
|
|
let target;
|
|
try { target = fs.readlinkSync(path.join(fdDir, fd.name)).replace(/ \(deleted\)$/, ""); } catch { continue; }
|
|
if (!target.startsWith(rootAbs + path.sep)) continue;
|
|
const rel = path.relative(rootAbs, target).split(path.sep);
|
|
if (rel.length >= 4) active.add(path.join(rootAbs, rel[0], rel[1], rel[2], rel[3]));
|
|
}
|
|
}
|
|
return active;
|
|
}
|
|
|
|
function rawArtifacts(runDir) {
|
|
const artifacts = [];
|
|
for (const name of rawNames) {
|
|
const target = path.join(runDir, name);
|
|
if (!fs.existsSync(target)) continue;
|
|
const bytes = sizePath(target);
|
|
if (bytes > 0) artifacts.push({ name, path: target, bytes, human: human(bytes) });
|
|
}
|
|
return artifacts;
|
|
}
|
|
|
|
function removeArtifact(artifact) {
|
|
const st = fs.lstatSync(artifact.path);
|
|
if (st.isSymbolicLink()) throw new Error("refusing symlink: " + artifact.path);
|
|
if (st.isDirectory()) fs.rmSync(artifact.path, { recursive: true, force: false });
|
|
else fs.unlinkSync(artifact.path);
|
|
}
|
|
|
|
const rootAbs = path.resolve(stateRoot);
|
|
const diskBefore = disk();
|
|
const openDirs = openRunDirs(rootAbs);
|
|
const candidates = [];
|
|
const protectedRuns = [];
|
|
|
|
for (const runDir of findRunDirs(rootAbs)) {
|
|
const manifest = jsonRead(path.join(runDir, "manifest.json"));
|
|
const heartbeat = jsonRead(path.join(runDir, "heartbeat.json"));
|
|
const pid = Number((heartbeat && heartbeat.pid) || (manifest && manifest.pid) || 0);
|
|
const pidState = pidObserverAlive(pid, runDir);
|
|
const hasOpenFd = openDirs.has(runDir);
|
|
const newestMs = newestKnownTimeMs(runDir, manifest, heartbeat);
|
|
const ageMs = newestMs > 0 ? nowMs - newestMs : 0;
|
|
const artifacts = rawArtifacts(runDir);
|
|
const rawBytes = artifacts.reduce((sum, item) => sum + item.bytes, 0);
|
|
const rel = path.relative(rootAbs, runDir);
|
|
const reasons = [];
|
|
if (!manifest) reasons.push("manifest-missing");
|
|
if (pidState.alive) reasons.push(pidState.reason || "pid-alive");
|
|
if (hasOpenFd) reasons.push("open-fd");
|
|
if (newestMs <= 0) reasons.push("time-unknown");
|
|
if (ageMs < keepMs) reasons.push("retention-window");
|
|
if (rawBytes <= 0) reasons.push("no-raw-artifacts");
|
|
const summary = {
|
|
id: (manifest && manifest.jobId) || path.basename(runDir).match(/(webobs-[A-Za-z0-9_.-]+)$/)?.[1] || null,
|
|
runDir: rel,
|
|
status: heartbeat && heartbeat.status || null,
|
|
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
newestAt: newestMs > 0 ? new Date(newestMs).toISOString() : null,
|
|
ageHours: newestMs > 0 ? Number((ageMs / 3600000).toFixed(2)) : null,
|
|
rawBytes,
|
|
rawHuman: human(rawBytes),
|
|
artifacts,
|
|
};
|
|
if (reasons.length > 0) protectedRuns.push({ ...summary, reasons });
|
|
else candidates.push(summary);
|
|
}
|
|
|
|
candidates.sort((a, b) => b.rawBytes - a.rawBytes);
|
|
const selected = candidates.slice(0, limit);
|
|
const selectedSet = new Set(selected.map((item) => item.runDir));
|
|
const deferred = candidates.filter((item) => !selectedSet.has(item.runDir));
|
|
let reclaimedBytes = 0;
|
|
const deleted = [];
|
|
const failures = [];
|
|
|
|
if (mode === "run") {
|
|
for (const item of selected) {
|
|
const runDir = path.join(rootAbs, item.runDir);
|
|
try {
|
|
for (const artifact of item.artifacts) removeArtifact(artifact);
|
|
reclaimedBytes += item.rawBytes;
|
|
deleted.push({ id: item.id, runDir: item.runDir, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length });
|
|
} catch (error) {
|
|
failures.push({ id: item.id, runDir: item.runDir, error: error instanceof Error ? error.message : String(error) });
|
|
}
|
|
}
|
|
}
|
|
|
|
const estimatedReclaimBytes = selected.reduce((sum, item) => sum + item.rawBytes, 0);
|
|
const diskAfter = mode === "run" ? disk() : null;
|
|
console.log(JSON.stringify({
|
|
ok: failures.length === 0,
|
|
command: "web-probe observe gc",
|
|
node: nodeId,
|
|
lane,
|
|
stateRoot,
|
|
mode,
|
|
mutation: mode === "run",
|
|
keepHours,
|
|
limit,
|
|
diskBefore,
|
|
diskAfter,
|
|
scannedRuns: candidates.length + protectedRuns.length,
|
|
candidateCount: candidates.length,
|
|
selectedCount: selected.length,
|
|
deferredCount: deferred.length,
|
|
protectedCount: protectedRuns.length,
|
|
estimatedReclaimBytes,
|
|
estimatedReclaimHuman: human(estimatedReclaimBytes),
|
|
reclaimedBytes,
|
|
reclaimedHuman: human(reclaimedBytes),
|
|
candidates: selected.map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length, artifacts: item.artifacts.map((artifact) => ({ name: artifact.name, bytes: artifact.bytes, human: artifact.human })) })),
|
|
deleted,
|
|
failures,
|
|
protected: protectedRuns.slice(0, Math.min(50, protectedRuns.length)).map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, reasons: item.reasons })),
|
|
valuesRedacted: true,
|
|
}, null, 2));
|
|
`;
|
|
}
|
|
|
|
export function runNodeWebProbeObserveStart(
|
|
options: NodeWebProbeObserveOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
secretSpec: RuntimeSecretSpec,
|
|
material: BootstrapAdminPasswordMaterial,
|
|
credential: Record<string, unknown>,
|
|
): Record<string, unknown> | RenderedCliResult {
|
|
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
|
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
|
const day = timestamp.slice(0, 8);
|
|
const defaultStateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
|
|
const stateDir = options.stateDir ?? defaultStateDir;
|
|
const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
|
|
const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64;
|
|
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
|
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
|
const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec);
|
|
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
|
|
const authLogin = nodeWebProbeAuthLoginConfig(spec);
|
|
const runnerEnvAssignments = [
|
|
...webProbeProxy.envAssignments,
|
|
...webProbeAccountEnvAssignments(),
|
|
...(spec.webProbe?.playwrightBrowsersPath === undefined ? [] : [
|
|
`PLAYWRIGHT_BROWSERS_PATH=${shellQuote(spec.webProbe.playwrightBrowsersPath)}`,
|
|
]),
|
|
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
|
|
`HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`,
|
|
`HWLAB_WEB_PASS=${shellQuote(material.password)}`,
|
|
`UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`,
|
|
`UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`,
|
|
`UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`,
|
|
`UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
|
|
`UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`,
|
|
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
|
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
|
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
|
`UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`,
|
|
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
|
|
...(authLogin === null
|
|
? []
|
|
: [
|
|
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS=${shellQuote(String(authLogin.maxAttempts))}`,
|
|
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS=${shellQuote(String(authLogin.requestTimeoutMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS=${shellQuote(String(authLogin.initialDelayMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
|
|
]),
|
|
].join(" ");
|
|
const script = [
|
|
"set -eu",
|
|
`state_dir=${shellQuote(stateDir)}`,
|
|
"mkdir -p \"$state_dir\"",
|
|
"chmod 700 \"$state_dir\"",
|
|
"runner=\"$state_dir/observer-runner.mjs\"",
|
|
"runner_b64=\"$state_dir/observer-runner.mjs.b64\"",
|
|
"cat >\"$runner_b64\" <<'UNIDESK_WEB_OBSERVE_RUNNER_B64'",
|
|
runnerB64Body,
|
|
"UNIDESK_WEB_OBSERVE_RUNNER_B64",
|
|
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
|
|
"rm -f \"$runner_b64\"",
|
|
"chmod 700 \"$runner\"",
|
|
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
|
|
"pid=$!",
|
|
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
|
"sleep 1",
|
|
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const startResolution = resolveWebObserveActionJson(result, "start");
|
|
const started = startResolution.parsed;
|
|
const startOk = result.exitCode === 0 && started?.ok === true;
|
|
const recovery = startOk
|
|
? null
|
|
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
|
|
const recoveredStatus = !startOk && recovery !== null && recovery.result.exitCode === 0 && recovery.status !== null && recovery.status.ok !== false
|
|
? recovery.status
|
|
: null;
|
|
const effectiveOk = startOk || recoveredStatus !== null;
|
|
const observer = startOk ? started : recoveredStatus;
|
|
const observerId = typeof started?.jobId === "string"
|
|
? started.jobId
|
|
: webObserveIdFromStatus(recoveredStatus, { ...options, id: jobId }) ?? jobId;
|
|
const degradedReason = !startOk && recoveredStatus !== null
|
|
? "web-probe-start-transport-timeout-recovered"
|
|
: result.timedOut
|
|
? "web-probe-command-timeout"
|
|
: result.exitCode !== 0
|
|
? "web-probe-observe-start-failed"
|
|
: null;
|
|
const index = effectiveOk
|
|
? upsertWebObserveIndexEntry(startOk ? {
|
|
id: observerId,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
stateDir,
|
|
url: options.url,
|
|
targetPath: options.targetPath,
|
|
status: "running",
|
|
pid: typeof started?.pid === "number" ? started.pid : null,
|
|
startedAt: new Date().toISOString(),
|
|
updatedAt: new Date().toISOString(),
|
|
} : webObserveIndexEntryFromOptions({ ...options, id: observerId, stateDir }, spec, observerId, recoveredStatus ?? {}))
|
|
: null;
|
|
return renderWebObserveStartResult({
|
|
ok: effectiveOk,
|
|
status: effectiveOk ? "started" : "blocked",
|
|
command: `web-probe observe start --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
url: options.url,
|
|
network: webProbeProxy.summary,
|
|
alertThresholds,
|
|
browserFreezePolicy,
|
|
projectManagement,
|
|
targetPath: options.targetPath,
|
|
id: observerId,
|
|
degradedReason,
|
|
credential,
|
|
observer: withWebObserveShortcuts(observer, observerId),
|
|
wrapper: buildWebObserveWrapperForObserveOptions("start", options, spec.workspace, { id: observerId, jobId: observerId, stateDir }),
|
|
index,
|
|
next: webObserveNextCommands(observerId),
|
|
stdoutJson: startResolution.diagnostics,
|
|
result: compactCommandResultRedacted(result, [material.password ?? ""]),
|
|
recovery: recovery === null ? null : {
|
|
attempted: true,
|
|
ok: recoveredStatus !== null,
|
|
reason: degradedReason,
|
|
stdoutJson: recovery.stdoutJson,
|
|
result: compactCommandResultWithStdoutTail(recovery.result),
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
export function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
|
const { result, status, stdoutJson } = readNodeWebProbeObserveRemoteStatus(options, spec, options.tailLines, options.commandTimeoutSeconds);
|
|
const observerId = webObserveIdFromStatus(status, options);
|
|
const statusReadable = status !== null;
|
|
const ok = result.exitCode === 0 && statusReadable && status.ok !== false;
|
|
const degradedReason = result.timedOut
|
|
? "web-probe-command-timeout"
|
|
: result.exitCode !== 0
|
|
? "web-probe-observe-status-failed"
|
|
: !statusReadable
|
|
? "web-probe-observe-status-unreadable"
|
|
: typeof status.degradedReason === "string"
|
|
? status.degradedReason
|
|
: null;
|
|
const index = ok && observerId !== null && options.stateDir !== null
|
|
? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status))
|
|
: null;
|
|
return withWebObserveStatusRendered({
|
|
ok,
|
|
status: ok ? "observed" : "blocked",
|
|
command: webObserveCommandLabel("status", options),
|
|
id: observerId,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
degradedReason,
|
|
observer: withWebObserveShortcuts(status, observerId),
|
|
wrapper: buildWebObserveWrapperForObserveOptions("status", options, spec.workspace, { id: observerId, stateDir: webObserveWrapperStateDirFromStatus(status, options.stateDir) }),
|
|
index,
|
|
next: observerId === null ? null : webObserveNextCommands(observerId),
|
|
stdoutJson,
|
|
result: compactCommandResultWithStdoutTail(result),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
export function readNodeWebProbeObserveRemoteStatus(
|
|
options: NodeWebProbeObserveOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
tailLines: number,
|
|
timeoutSeconds: number,
|
|
): { result: ReturnType<typeof runTransWorkspaceStdinScript>; status: Record<string, unknown> | null; stdoutJson: Record<string, unknown> } {
|
|
const script = [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
nodeWebObserveStatusNodeScript(tailLines, options.node, options.lane),
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, timeoutSeconds);
|
|
const statusResolution = resolveWebObserveActionJson(result, "status");
|
|
return { result, status: statusResolution.parsed, stdoutJson: statusResolution.diagnostics };
|
|
}
|
|
|
|
export function webObserveText(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
if (typeof value === "string") return value;
|
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
export function webObserveShort(value: string, maxLength: number): string {
|
|
if (value.length <= maxLength) return value;
|
|
if (maxLength <= 1) return value.slice(0, maxLength);
|
|
return `${value.slice(0, maxLength - 1)}~`;
|
|
}
|
|
|
|
export function buildWebObserveCommandVisibility(input: {
|
|
commandType: string;
|
|
commandResult: Record<string, unknown> | null;
|
|
exitCode: number | null;
|
|
}): {
|
|
status: string;
|
|
control: Record<string, unknown>;
|
|
asyncTurn: Record<string, unknown>;
|
|
} {
|
|
const commandResult = input.commandResult;
|
|
const executionStatus = input.exitCode !== 0 || commandResult === null
|
|
? "blocked"
|
|
: commandResult.ok === false
|
|
? "failed"
|
|
: commandResult.completedAt !== undefined && commandResult.completedAt !== null
|
|
? "completed"
|
|
: commandResult.queued === true || commandResult.waitTimedOut === true
|
|
? "queued"
|
|
: "accepted";
|
|
const actionResult = record(commandResult?.result);
|
|
const chatSubmit = record(actionResult.chatSubmit);
|
|
const traceId = typeof chatSubmit.traceId === "string" ? chatSubmit.traceId : null;
|
|
const otelTraceId = typeof chatSubmit.otelTraceId === "string" ? chatSubmit.otelTraceId : null;
|
|
const httpStatus = typeof chatSubmit.status === "number" ? chatSubmit.status : null;
|
|
const turnApplicable = input.commandType === "sendPrompt" || input.commandType === "steer";
|
|
const submissionAccepted = chatSubmit.sideEffectObserved === true
|
|
|| httpStatus !== null && httpStatus >= 200 && httpStatus < 300
|
|
|| traceId !== null;
|
|
const submissionStatus = !turnApplicable
|
|
? "not-applicable"
|
|
: executionStatus === "blocked"
|
|
? "not-submitted"
|
|
: executionStatus === "failed"
|
|
? "failed"
|
|
: submissionAccepted
|
|
? "accepted"
|
|
: executionStatus === "queued"
|
|
? "pending"
|
|
: "not-observed";
|
|
return {
|
|
status: "control-" + executionStatus,
|
|
control: {
|
|
executionStatus,
|
|
completedAt: commandResult?.completedAt ?? null,
|
|
completionMeaning: executionStatus === "completed" ? "observer-browser-control-action-finished" : null,
|
|
businessTurnTerminalImplied: false,
|
|
valuesRedacted: true,
|
|
},
|
|
asyncTurn: {
|
|
applicable: turnApplicable,
|
|
submissionStatus,
|
|
submissionHttpStatus: httpStatus,
|
|
terminalStatus: null,
|
|
terminalObserved: false,
|
|
terminalStatusSource: turnApplicable ? "not-evaluated-by-control-command" : "not-applicable",
|
|
traceId,
|
|
otelTraceId,
|
|
valuesRedacted: true,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record<string, unknown> | RenderedCliResult {
|
|
const type = options.commandType ?? (stopCommand ? "stop" : null);
|
|
if (type === null) throw new Error("web-probe observe command requires --type");
|
|
const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
|
const payload = {
|
|
id: commandId,
|
|
type,
|
|
createdAt: new Date().toISOString(),
|
|
source: "cli",
|
|
path: options.commandPath,
|
|
text: options.commandText,
|
|
label: options.commandLabel,
|
|
sessionId: options.commandSessionId,
|
|
provider: options.commandProvider,
|
|
afterRound: options.commandAfterRound,
|
|
severity: options.commandSeverity,
|
|
alternateSessionStrategy: options.commandAlternateSessionStrategy,
|
|
expectedSentinelRange: options.commandExpectedSentinelRange,
|
|
expectedActionWaitMs: options.commandExpectedActionWaitMs,
|
|
durationMs: options.commandDurationMs,
|
|
requireComposerReady: options.commandRequireComposerReady,
|
|
waitProjectManagementReady: options.commandWaitProjectManagementReady,
|
|
findingId: options.commandFindingId,
|
|
blocking: options.commandBlocking,
|
|
accountId: options.commandAccountId,
|
|
fromAccountId: options.commandFromAccountId,
|
|
toAccountId: options.commandToAccountId,
|
|
sourceId: options.commandSourceId,
|
|
fileRef: options.commandFileRef,
|
|
filename: options.commandFilename,
|
|
taskRef: options.commandTaskRef,
|
|
taskId: options.commandTaskId,
|
|
field: options.commandField,
|
|
link: options.commandLink,
|
|
title: options.commandTitle,
|
|
body: options.commandBody,
|
|
status: options.commandStatus,
|
|
hwpodId: options.commandHwpodId,
|
|
nodeId: options.commandNodeId,
|
|
workspaceRoot: options.commandWorkspaceRoot,
|
|
root: options.commandRoot,
|
|
};
|
|
const preStopStatus = options.force && stopCommand
|
|
? readNodeWebProbeObserveRemoteStatus(options, spec, 1, Math.min(options.commandTimeoutSeconds, 30))
|
|
: null;
|
|
const preStopDiagnostics = record(preStopStatus?.status?.diagnostics);
|
|
const preStopCommands = record(preStopStatus?.status?.commands);
|
|
const preStopPending = Number(preStopCommands?.pendingCount ?? 0);
|
|
const preStopProcessing = Number(preStopCommands?.processingCount ?? 0);
|
|
const forceBeforeQueueReason = options.force && stopCommand
|
|
? preStopDiagnostics?.heartbeatStale === true
|
|
? "heartbeat-stale"
|
|
: preStopPending > 0 || preStopProcessing > 0
|
|
? "command-backlog"
|
|
: null
|
|
: null;
|
|
if (forceBeforeQueueReason !== null) {
|
|
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, forceBeforeQueueReason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, null);
|
|
}
|
|
const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
|
const waitMs = options.force && stopCommand ? Math.max(options.waitMs, 5000) : options.waitMs;
|
|
const script = [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
"mkdir -p \"$state_dir/commands/pending\"",
|
|
`node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`,
|
|
nodeWebObserveWaitCommandShell(commandId, waitMs),
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const commandResolution = resolveWebObserveActionJson(result, "command");
|
|
const commandResult = commandResolution.parsed;
|
|
if (options.force && stopCommand && (result.exitCode !== 0 || commandResult?.waitTimedOut === true || commandResult?.queued === true)) {
|
|
const reason = result.exitCode !== 0
|
|
? "graceful-stop-failed"
|
|
: commandResult?.waitTimedOut === true
|
|
? "graceful-stop-not-consumed"
|
|
: "graceful-stop-queued";
|
|
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, reason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, result);
|
|
}
|
|
const visibility = buildWebObserveCommandVisibility({
|
|
commandType: type,
|
|
commandResult,
|
|
exitCode: result.exitCode,
|
|
});
|
|
const payloadResult = {
|
|
ok: result.exitCode === 0 && commandResult !== null && commandResult.ok !== false,
|
|
status: visibility.status,
|
|
command: webObserveCommandLabel(stopCommand ? "stop" : "command", options),
|
|
id: webObserveIdFromOptions(options),
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
commandId,
|
|
observerCommand: commandSummaryForOutput(payload),
|
|
observer: commandResult,
|
|
control: visibility.control,
|
|
asyncTurn: visibility.asyncTurn,
|
|
wrapper: buildWebObserveWrapperForObserveOptions(stopCommand ? "stop" : "command", options, spec.workspace, { commandType: type }),
|
|
stdoutJson: commandResolution.diagnostics,
|
|
result: compactCommandResult(result),
|
|
full: options.full,
|
|
valuesRedacted: true,
|
|
};
|
|
return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult);
|
|
}
|
|
|
|
export function runNodeWebProbeObserveForceStop(
|
|
options: NodeWebProbeObserveOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
payload: Record<string, unknown>,
|
|
commandId: string,
|
|
reason: string,
|
|
preflightResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
|
preflightStatus: Record<string, unknown> | null,
|
|
gracefulResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
|
): Record<string, unknown> | RenderedCliResult {
|
|
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
nodeWebObserveForceStopNodeScript(reason, commandId),
|
|
].join("\n"), 55);
|
|
const forceResolution = resolveWebObserveActionJson(killResult, "force-stop");
|
|
const forcePayload = forceResolution.parsed;
|
|
const payloadResult = {
|
|
ok: killResult.exitCode === 0 && forcePayload !== null && forcePayload.ok !== false,
|
|
status: killResult.exitCode === 0 && forcePayload !== null && forcePayload.ok !== false ? "forced-stopped" : "blocked",
|
|
command: webObserveCommandLabel("stop", options),
|
|
id: webObserveIdFromOptions(options),
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
commandId,
|
|
observerCommand: commandSummaryForOutput(payload),
|
|
observer: forcePayload,
|
|
wrapper: buildWebObserveWrapperForObserveOptions("stop", options, spec.workspace, { commandType: "stop" }),
|
|
forceReason: reason,
|
|
preflightObserver: preflightStatus,
|
|
preflightResult: preflightResult === null ? null : compactCommandResult(preflightResult),
|
|
gracefulResult: gracefulResult === null ? null : compactCommandResult(gracefulResult),
|
|
stdoutJson: forceResolution.diagnostics,
|
|
forceResult: compactCommandResultWithStdoutTail(killResult),
|
|
full: options.full,
|
|
valuesRedacted: true,
|
|
};
|
|
return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult);
|
|
}
|