Merge pull request #1472 from pikasTech/feat/1470-webprobe-observe-json-recovery
FEATURE: 拆分 WebProbe observe action handlers 并迁移 JSON recovery
This commit is contained in:
Lyon
@@ -0,0 +1,174 @@
|
||||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: shared WebProbe observe action configuration helpers.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { repoRoot, rootPath } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { hwlabRuntimeLaneConfigPath, type HwlabRuntimeLaneSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeBrowserFreezePolicySpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { WebProbeBrowserProxyMode } from "./entry";
|
||||
import { transPath } from "./runtime-common";
|
||||
import { record, shellQuote } from "./utils";
|
||||
|
||||
export function webObserveGcDefaultKeepHours(): number {
|
||||
const configPath = "config/unidesk-cli.yaml";
|
||||
const parsed = record(Bun.YAML.parse(readFileSync(rootPath(configPath), "utf8")) as unknown);
|
||||
const gc = record(parsed.gc);
|
||||
const scratch = record(gc.stateStaleScratch);
|
||||
const keepHours = scratch.keepHours;
|
||||
if (!Number.isInteger(keepHours) || keepHours < 0) {
|
||||
throw new Error(`${configPath}#gc.stateStaleScratch.keepHours must be a non-negative integer`);
|
||||
}
|
||||
return keepHours;
|
||||
}
|
||||
|
||||
export function nodeWebProbeAlertThresholds(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeAlertThresholdsSpec {
|
||||
const thresholds = spec.webProbe?.alertThresholds;
|
||||
if (thresholds === undefined) {
|
||||
throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.alertThresholds for web-probe observe`);
|
||||
}
|
||||
return thresholds;
|
||||
}
|
||||
|
||||
export function nodeWebProbeBrowserFreezePolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeBrowserFreezePolicySpec {
|
||||
const policy = spec.webProbe?.browserFreezePolicy;
|
||||
if (policy === undefined) {
|
||||
throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.browserFreezePolicy for web-probe observe`);
|
||||
}
|
||||
return policy;
|
||||
}
|
||||
|
||||
export function nodeWebProbeProjectManagementConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeProjectManagementSpec | null {
|
||||
return spec.webProbe?.projectManagement ?? null;
|
||||
}
|
||||
|
||||
export function nodeWebProbeAuthLoginConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeAuthLoginSpec | null {
|
||||
return spec.webProbe?.authLogin ?? null;
|
||||
}
|
||||
|
||||
export interface NodeWebProbeHostProxyEnv {
|
||||
readonly envAssignments: string[];
|
||||
readonly summary: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function nodeWebProbeHostProxyEnv(spec: HwlabRuntimeLaneSpec, browserProxyMode: WebProbeBrowserProxyMode = "auto"): NodeWebProbeHostProxyEnv {
|
||||
if (browserProxyMode === "direct") {
|
||||
return {
|
||||
envAssignments: [],
|
||||
summary: {
|
||||
source: "option",
|
||||
mode: "direct",
|
||||
networkProfileId: spec.networkProfileId,
|
||||
proxy: { enabled: false },
|
||||
valuesPrinted: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
const proxy = spec.networkProfile.proxy;
|
||||
const serviceCache = new Map<string, string>();
|
||||
const http = resolveNodeWebProbeHostProxyUrl(spec, proxy.http, serviceCache);
|
||||
const https = resolveNodeWebProbeHostProxyUrl(spec, proxy.https, serviceCache);
|
||||
const all = resolveNodeWebProbeHostProxyUrl(spec, proxy.all, serviceCache);
|
||||
const noProxy = proxy.noProxy.join(",");
|
||||
return {
|
||||
envAssignments: [
|
||||
["HTTP_PROXY", http.url],
|
||||
["HTTPS_PROXY", https.url],
|
||||
["ALL_PROXY", all.url],
|
||||
["http_proxy", http.url],
|
||||
["https_proxy", https.url],
|
||||
["all_proxy", all.url],
|
||||
["NO_PROXY", noProxy],
|
||||
["no_proxy", noProxy],
|
||||
].map(([key, value]) => `${key}=${shellQuote(value)}`),
|
||||
summary: {
|
||||
source: "yaml",
|
||||
mode: "host-env",
|
||||
networkProfileId: spec.networkProfileId,
|
||||
proxy: {
|
||||
http: http.summary,
|
||||
https: https.summary,
|
||||
all: all.summary,
|
||||
noProxyCount: proxy.noProxy.length,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveNodeWebProbeHostProxyUrl(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
rawUrl: string,
|
||||
serviceCache: Map<string, string>,
|
||||
): { url: string; summary: Record<string, unknown> } {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch (error) {
|
||||
throw new Error(`config/hwlab-node-lanes.yaml networkProfiles.${spec.networkProfileId}.proxy contains invalid proxy URL: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
const service = parseKubernetesServiceDnsHost(parsed.hostname);
|
||||
if (service === null) {
|
||||
return {
|
||||
url: rawUrl,
|
||||
summary: {
|
||||
mode: "host-url",
|
||||
host: parsed.hostname,
|
||||
port: parsed.port || null,
|
||||
valuesPrinted: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
const clusterIp = resolveKubernetesServiceClusterIp(spec, service.namespace, service.name, serviceCache);
|
||||
const originalHost = parsed.hostname;
|
||||
parsed.hostname = clusterIp;
|
||||
const resolvedUrl = normalizedProxyUrl(parsed);
|
||||
return {
|
||||
url: resolvedUrl,
|
||||
summary: {
|
||||
mode: "k8s-service-cluster-ip",
|
||||
service: service.name,
|
||||
namespace: service.namespace,
|
||||
originalHost,
|
||||
resolvedHost: clusterIp,
|
||||
port: parsed.port || null,
|
||||
valuesPrinted: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function parseKubernetesServiceDnsHost(hostname: string): { name: string; namespace: string } | null {
|
||||
const match = hostname.toLowerCase().match(/^([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.svc(?:\.cluster\.local)?$/u);
|
||||
if (match === null) return null;
|
||||
return { name: match[1] ?? "", namespace: match[3] ?? "" };
|
||||
}
|
||||
|
||||
export function resolveKubernetesServiceClusterIp(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
namespace: string,
|
||||
serviceName: string,
|
||||
serviceCache: Map<string, string>,
|
||||
): string {
|
||||
const cacheKey = `${namespace}/${serviceName}`;
|
||||
const cached = serviceCache.get(cacheKey);
|
||||
if (cached !== undefined) return cached;
|
||||
const result = runCommand([transPath(), spec.nodeKubeRoute, "get", "svc", "-n", namespace, serviceName, "-o", "jsonpath={.spec.clusterIP}"], repoRoot, { timeoutMs: 20_000 });
|
||||
const clusterIp = result.stdout.trim();
|
||||
if (result.exitCode !== 0 || clusterIp.length === 0) {
|
||||
const reason = result.stderr.trim().slice(-500) || result.stdout.trim().slice(-500) || `exitCode=${result.exitCode}`;
|
||||
throw new Error(`web-probe proxy service resolution failed for ${spec.nodeId}/${spec.lane} ${namespace}/${serviceName}: ${reason}`);
|
||||
}
|
||||
serviceCache.set(cacheKey, clusterIp);
|
||||
return clusterIp;
|
||||
}
|
||||
|
||||
export function normalizedProxyUrl(parsed: URL): string {
|
||||
const value = parsed.toString();
|
||||
if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
|
||||
return value;
|
||||
}
|
||||
|
||||
export function webProbeAccountEnvAssignments(): string[] {
|
||||
return Object.entries(process.env)
|
||||
.filter(([key, value]) => value !== undefined && /^HWLAB_WEB_(?:ACCOUNT_)?[A-Z0-9_]+_(?:JSON|USER|PASS)$/u.test(key))
|
||||
.map(([key, value]) => `${key}=${shellQuote(value ?? "")}`);
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { resolveWebObserveActionJson } from "./web-probe-observe-actions";
|
||||
|
||||
test("web observe action JSON recovery accepts concrete start contract", () => {
|
||||
const resolved = resolveWebObserveActionJson({
|
||||
stdout: JSON.stringify({
|
||||
ok: true,
|
||||
command: "web-probe-observe start",
|
||||
jobId: "webobs-fixture",
|
||||
stateDir: ".state/web-observe/JD01/v03/fixture",
|
||||
valuesRedacted: true,
|
||||
}),
|
||||
stderr: "",
|
||||
exitCode: 0,
|
||||
timedOut: false,
|
||||
}, "start");
|
||||
|
||||
assert.equal(resolved.source, "stdout");
|
||||
assert.equal(resolved.parsed?.jobId, "webobs-fixture");
|
||||
assert.equal(resolved.diagnostics.stdoutContractAccepted, true);
|
||||
});
|
||||
|
||||
test("web observe action JSON recovery reads dump wrapper for status contract", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-status-dump-"));
|
||||
const dumpPath = join(dir, "status.json");
|
||||
await writeFile(dumpPath, JSON.stringify({
|
||||
ok: true,
|
||||
status: "running",
|
||||
jobId: "webobs-fixture",
|
||||
heartbeat: { status: "running" },
|
||||
diagnostics: { heartbeatStale: false },
|
||||
}) + "\n");
|
||||
|
||||
const resolved = resolveWebObserveActionJson({
|
||||
stdout: JSON.stringify({
|
||||
ok: true,
|
||||
data: {
|
||||
outputTruncated: true,
|
||||
reason: "stdout-json-bytes-exceeded-threshold",
|
||||
dump: { path: dumpPath },
|
||||
},
|
||||
}),
|
||||
stderr: "",
|
||||
exitCode: 0,
|
||||
timedOut: false,
|
||||
}, "status");
|
||||
|
||||
assert.equal(resolved.source, "dump");
|
||||
assert.equal(resolved.parsed?.jobId, "webobs-fixture");
|
||||
assert.equal(resolved.diagnostics.stdoutKind, "dump-wrapper");
|
||||
assert.equal(resolved.diagnostics.dumpPath, dumpPath);
|
||||
});
|
||||
|
||||
test("web observe action JSON recovery reads SSH truncation summary dump for command contract", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-command-dump-"));
|
||||
const dumpPath = join(dir, "command.json");
|
||||
await writeFile(dumpPath, JSON.stringify({
|
||||
ok: true,
|
||||
queued: false,
|
||||
waitTimedOut: false,
|
||||
commandId: "cmd-fixture",
|
||||
}) + "\n");
|
||||
const summary = {
|
||||
code: "ssh-truncation-summary",
|
||||
stdout: {
|
||||
stream: "stdout",
|
||||
dumpPath,
|
||||
forwardedBytes: 10240,
|
||||
thresholdBytes: 10240,
|
||||
},
|
||||
};
|
||||
|
||||
const resolved = resolveWebObserveActionJson({
|
||||
stdout: `UNIDESK_SSH_TRUNCATION_SUMMARY ${JSON.stringify(summary)}\n`,
|
||||
stderr: "",
|
||||
exitCode: 0,
|
||||
timedOut: false,
|
||||
}, "command");
|
||||
|
||||
assert.equal(resolved.source, "dump");
|
||||
assert.equal(resolved.parsed?.commandId, "cmd-fixture");
|
||||
assert.equal(resolved.diagnostics.stdoutKind, "ssh-truncation-summary");
|
||||
assert.equal(resolved.diagnostics.dumpPath, dumpPath);
|
||||
});
|
||||
|
||||
test("web observe action JSON recovery rejects ok-only action contract", () => {
|
||||
const resolved = resolveWebObserveActionJson({
|
||||
stdout: JSON.stringify({ ok: true }),
|
||||
stderr: "",
|
||||
exitCode: 0,
|
||||
timedOut: false,
|
||||
}, "start");
|
||||
|
||||
assert.equal(resolved.parsed, null);
|
||||
assert.equal(resolved.source, null);
|
||||
assert.equal(resolved.diagnostics.stdoutKind, "json");
|
||||
assert.equal(resolved.diagnostics.fallbackReason, "stdout-json-contract-invalid");
|
||||
assert.equal(resolved.diagnostics.stdoutContractAccepted, false);
|
||||
});
|
||||
@@ -0,0 +1,744 @@
|
||||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. WebProbe observe action handlers.
|
||||
// Responsibility: start/status/command/stop/gc remote action execution and child JSON recovery.
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery";
|
||||
import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
import type { BootstrapAdminPasswordMaterial, NodeWebProbeObserveOptions, RuntimeSecretSpec } from "./entry";
|
||||
import { runTransWorkspaceStdinScript } from "./public-exposure";
|
||||
import { compactCommandResult, compactCommandResultRedacted, compactCommandResultWithStdoutTail, record, shellQuote } from "./utils";
|
||||
import { nodeWebProbeAlertThresholds, nodeWebProbeAuthLoginConfig, nodeWebProbeBrowserFreezePolicy, nodeWebProbeHostProxyEnv, nodeWebProbeProjectManagementConfig, webObserveGcDefaultKeepHours, webProbeAccountEnvAssignments } from "./web-probe-observe-action-config";
|
||||
import { nodeWebObserveResolveStateDirShell, renderWebObserveStartResult, upsertWebObserveIndexEntry, webObserveCommandLabel, webObserveIdFromOptions, webObserveIdFromStatus, webObserveIndexEntryFromOptions, webObserveNextCommands, withWebObserveShortcuts } from "./web-observe-render";
|
||||
import { commandSummaryForOutput, nodeWebObserveForceStopNodeScript, nodeWebObserveStatusNodeScript, nodeWebObserveWaitCommandShell, safeWebObserveSegment, safeWebObserveTargetSegment } from "./web-observe-scripts";
|
||||
|
||||
type WebObserveActionJsonContract = "gc" | "start" | "status" | "command" | "force-stop";
|
||||
|
||||
export function resolveWebObserveActionJson(
|
||||
result: { stdout: string; stderr?: string; exitCode?: number | null; timedOut?: boolean },
|
||||
contract: WebObserveActionJsonContract,
|
||||
): CliChildJsonResolution {
|
||||
return resolveCliChildJsonCommandResult({
|
||||
result,
|
||||
requestedStdoutType: `web-probe observe ${contract} contract`,
|
||||
acceptParsed: (value) => isWebObserveActionJsonContract(value, contract),
|
||||
});
|
||||
}
|
||||
|
||||
function isWebObserveActionJsonContract(value: Record<string, unknown>, contract: WebObserveActionJsonContract): boolean {
|
||||
if (acceptFailureContract(value)) return true;
|
||||
if (contract === "gc") {
|
||||
return value.command === "web-probe observe gc" && hasAnyKey(value, ["mode", "stateRoot", "scannedRuns", "candidateCount", "selectedCount", "protectedCount"]);
|
||||
}
|
||||
if (contract === "start") {
|
||||
return value.command === "web-probe-observe start" && hasAnyKey(value, ["jobId", "stateDir", "pid", "manifestPath", "heartbeat", "manifest"]);
|
||||
}
|
||||
if (contract === "status") {
|
||||
return hasAnyKey(value, ["jobId", "stateDir", "manifest", "heartbeat", "diagnostics", "commands", "samples", "status"]);
|
||||
}
|
||||
if (contract === "command") {
|
||||
return hasAnyKey(value, ["ok", "queued", "waitTimedOut", "commandId", "status", "path", "error", "reason"]);
|
||||
}
|
||||
return hasAnyKey(value, ["ok", "forceStopped", "killed", "status", "pid", "reason", "error", "commandId"]);
|
||||
}
|
||||
|
||||
function acceptFailureContract(value: Record<string, unknown>): boolean {
|
||||
return value.ok === false && hasAnyKey(value, ["error", "reason", "degradedReason", "status", "diagnostics", "result", "command"]);
|
||||
}
|
||||
|
||||
function hasAnyKey(value: Record<string, unknown>, keys: string[]): boolean {
|
||||
return keys.some((key) => value[key] !== undefined && value[key] !== null);
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveGc(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const stateRoot = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`;
|
||||
const script = [
|
||||
"set -eu",
|
||||
`node - ${shellQuote(stateRoot)} ${shellQuote(String(options.gcKeepHours))} ${shellQuote(String(options.gcLimit))} ${shellQuote(options.confirm ? "run" : "plan")} ${shellQuote(options.node)} ${shellQuote(options.lane)} <<'UNIDESK_WEB_OBSERVE_GC'`,
|
||||
nodeWebObserveGcNodeScript(),
|
||||
"UNIDESK_WEB_OBSERVE_GC",
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const payloadResolution = resolveWebObserveActionJson(result, "gc");
|
||||
const payload = payloadResolution.parsed ?? {};
|
||||
const ok = result.exitCode === 0 && payloadResolution.parsed !== null && payload.ok !== false;
|
||||
return {
|
||||
ok,
|
||||
status: ok ? (options.confirm ? "cleaned" : "planned") : "blocked",
|
||||
command: `web-probe observe gc --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
stateRoot,
|
||||
mode: options.confirm ? "confirmed-run" : "dry-run",
|
||||
retention: {
|
||||
source: "config/unidesk-cli.yaml#gc.stateStaleScratch.keepHours",
|
||||
keepHours: options.gcKeepHours,
|
||||
userOverride: options.gcKeepHours !== webObserveGcDefaultKeepHours(),
|
||||
},
|
||||
gc: options.full || options.raw ? payload : compactWebObserveGcPayload(payload),
|
||||
stdoutJson: payloadResolution.diagnostics,
|
||||
result: ok
|
||||
? {
|
||||
exitCode: result.exitCode,
|
||||
timedOut: result.timedOut,
|
||||
stdoutBytes: result.stdout.length,
|
||||
stderrBytes: result.stderr.length,
|
||||
}
|
||||
: compactCommandResultWithStdoutTail(result),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactWebObserveGcPayload(payload: Record<string, unknown>): Record<string, unknown> {
|
||||
const candidates = Array.isArray(payload.candidates) ? payload.candidates.map(record) : [];
|
||||
const protectedRuns = Array.isArray(payload.protected) ? payload.protected.map(record) : [];
|
||||
const deleted = Array.isArray(payload.deleted) ? payload.deleted.map(record) : [];
|
||||
const failures = Array.isArray(payload.failures) ? payload.failures.map(record) : [];
|
||||
return {
|
||||
ok: payload.ok ?? null,
|
||||
mode: payload.mode ?? null,
|
||||
mutation: payload.mutation ?? null,
|
||||
keepHours: payload.keepHours ?? null,
|
||||
limit: payload.limit ?? null,
|
||||
diskBefore: payload.diskBefore ?? null,
|
||||
diskAfter: payload.diskAfter ?? null,
|
||||
scannedRuns: payload.scannedRuns ?? null,
|
||||
candidateCount: payload.candidateCount ?? null,
|
||||
selectedCount: payload.selectedCount ?? null,
|
||||
deferredCount: payload.deferredCount ?? null,
|
||||
protectedCount: payload.protectedCount ?? null,
|
||||
estimatedReclaimBytes: payload.estimatedReclaimBytes ?? null,
|
||||
estimatedReclaimHuman: payload.estimatedReclaimHuman ?? null,
|
||||
reclaimedBytes: payload.reclaimedBytes ?? null,
|
||||
reclaimedHuman: payload.reclaimedHuman ?? null,
|
||||
candidates: candidates.slice(0, 20).map((item) => ({
|
||||
id: item.id ?? null,
|
||||
runDir: item.runDir ?? null,
|
||||
status: item.status ?? null,
|
||||
ageHours: item.ageHours ?? null,
|
||||
rawHuman: item.rawHuman ?? null,
|
||||
artifactCount: item.artifactCount ?? null,
|
||||
})),
|
||||
deleted: deleted.slice(0, 20).map((item) => ({
|
||||
id: item.id ?? null,
|
||||
runDir: item.runDir ?? null,
|
||||
rawHuman: item.rawHuman ?? null,
|
||||
artifactCount: item.artifactCount ?? null,
|
||||
})),
|
||||
failures: failures.slice(0, 20),
|
||||
protected: protectedRuns.slice(0, 20).map((item) => ({
|
||||
id: item.id ?? null,
|
||||
runDir: item.runDir ?? null,
|
||||
status: item.status ?? null,
|
||||
ageHours: item.ageHours ?? null,
|
||||
rawHuman: item.rawHuman ?? null,
|
||||
reasons: item.reasons ?? null,
|
||||
})),
|
||||
disclosure: {
|
||||
candidateRowsShown: Math.min(20, candidates.length),
|
||||
deletedRowsShown: Math.min(20, deleted.length),
|
||||
protectedRowsShown: Math.min(20, protectedRuns.length),
|
||||
full: "rerun with --full or --raw for all candidates/artifacts",
|
||||
},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeWebObserveGcNodeScript(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const childProcess = require("child_process");
|
||||
|
||||
const stateRoot = process.argv[2];
|
||||
const keepHours = Number(process.argv[3]);
|
||||
const limit = Number(process.argv[4]);
|
||||
const mode = process.argv[5] === "run" ? "run" : "plan";
|
||||
const nodeId = process.argv[6];
|
||||
const lane = process.argv[7];
|
||||
const keepMs = keepHours * 60 * 60 * 1000;
|
||||
const nowMs = Date.now();
|
||||
const rawNames = ["samples.jsonl", "browser-process.jsonl", "network.jsonl", "console.jsonl", "artifacts.jsonl", "performance-events.jsonl", "screenshots", "performance"];
|
||||
|
||||
function jsonRead(file) {
|
||||
try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; }
|
||||
}
|
||||
|
||||
function parseTime(value) {
|
||||
if (typeof value !== "string") return 0;
|
||||
const ms = Date.parse(value);
|
||||
return Number.isFinite(ms) ? ms : 0;
|
||||
}
|
||||
|
||||
function dirTimeMs(runDir) {
|
||||
const base = path.basename(runDir);
|
||||
const match = base.match(/^(\d{8})T(\d{6})Z_/);
|
||||
if (!match) return 0;
|
||||
const raw = match[1].slice(0, 4) + "-" + match[1].slice(4, 6) + "-" + match[1].slice(6, 8) + "T" + match[2].slice(0, 2) + ":" + match[2].slice(2, 4) + ":" + match[2].slice(4, 6) + "Z";
|
||||
return Date.parse(raw) || 0;
|
||||
}
|
||||
|
||||
function newestKnownTimeMs(runDir, manifest, heartbeat) {
|
||||
return Math.max(
|
||||
dirTimeMs(runDir),
|
||||
parseTime(manifest && manifest.completedAt),
|
||||
parseTime(manifest && manifest.updatedAt),
|
||||
parseTime(manifest && manifest.startedAt),
|
||||
parseTime(manifest && manifest.pageProvenance && manifest.pageProvenance.observedAt),
|
||||
parseTime(heartbeat && heartbeat.completedAt),
|
||||
parseTime(heartbeat && heartbeat.forceStoppedAt),
|
||||
parseTime(heartbeat && heartbeat.updatedAt),
|
||||
parseTime(heartbeat && heartbeat.pageProvenance && heartbeat.pageProvenance.observedAt),
|
||||
);
|
||||
}
|
||||
|
||||
function sizePath(target) {
|
||||
let total = 0;
|
||||
const stack = [target];
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
let st;
|
||||
try { st = fs.lstatSync(current); } catch { continue; }
|
||||
if (st.isSymbolicLink()) continue;
|
||||
total += st.blocks ? st.blocks * 512 : st.size;
|
||||
if (!st.isDirectory()) continue;
|
||||
let entries;
|
||||
try { entries = fs.readdirSync(current); } catch { continue; }
|
||||
for (const entry of entries) stack.push(path.join(current, entry));
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function human(bytes) {
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit += 1; }
|
||||
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
||||
}
|
||||
|
||||
function disk() {
|
||||
try {
|
||||
const line = childProcess.execFileSync("df", ["-Pk", "/"], { encoding: "utf8" }).trim().split("\n").slice(-1)[0];
|
||||
const parts = line.trim().split(/\s+/);
|
||||
const sizeBytes = Number(parts[1]) * 1024;
|
||||
const usedBytes = Number(parts[2]) * 1024;
|
||||
const availableBytes = Number(parts[3]) * 1024;
|
||||
const usePercent = Number(String(parts[4]).replace("%", ""));
|
||||
return { filesystem: parts[0], sizeBytes, usedBytes, availableBytes, usePercent, usedHuman: human(usedBytes), availableHuman: human(availableBytes) };
|
||||
} catch (error) {
|
||||
return { error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function findRunDirs(root) {
|
||||
const runs = [];
|
||||
for (const year of safeReadDir(root)) {
|
||||
const y = path.join(root, year.name);
|
||||
if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) continue;
|
||||
for (const month of safeReadDir(y)) {
|
||||
const m = path.join(y, month.name);
|
||||
if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) continue;
|
||||
for (const day of safeReadDir(m)) {
|
||||
const d = path.join(m, day.name);
|
||||
if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) continue;
|
||||
for (const run of safeReadDir(d)) {
|
||||
const r = path.join(d, run.name);
|
||||
if (run.isDirectory() && /_webobs-[A-Za-z0-9_.-]+$/.test(run.name)) runs.push(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
|
||||
function safeReadDir(dir) {
|
||||
try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
|
||||
}
|
||||
|
||||
function pidObserverAlive(pid, runAbs) {
|
||||
if (!Number.isInteger(pid) || pid <= 1) return { alive: false, reason: null };
|
||||
try {
|
||||
const cmdline = fs.readFileSync("/proc/" + pid + "/cmdline", "utf8").replace(/\0/g, " ");
|
||||
if (cmdline.includes("observer-runner") || cmdline.includes(runAbs)) return { alive: true, reason: "pid-alive" };
|
||||
return { alive: false, reason: null };
|
||||
} catch (error) {
|
||||
if (fs.existsSync("/proc/" + pid)) return { alive: true, reason: "pid-alive-unreadable" };
|
||||
return { alive: false, reason: null };
|
||||
}
|
||||
}
|
||||
|
||||
function openRunDirs(rootAbs) {
|
||||
const active = new Set();
|
||||
const procEntries = safeReadDir("/proc").filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name));
|
||||
for (const proc of procEntries) {
|
||||
const fdDir = path.join("/proc", proc.name, "fd");
|
||||
for (const fd of safeReadDir(fdDir)) {
|
||||
let target;
|
||||
try { target = fs.readlinkSync(path.join(fdDir, fd.name)).replace(/ \(deleted\)$/, ""); } catch { continue; }
|
||||
if (!target.startsWith(rootAbs + path.sep)) continue;
|
||||
const rel = path.relative(rootAbs, target).split(path.sep);
|
||||
if (rel.length >= 4) active.add(path.join(rootAbs, rel[0], rel[1], rel[2], rel[3]));
|
||||
}
|
||||
}
|
||||
return active;
|
||||
}
|
||||
|
||||
function rawArtifacts(runDir) {
|
||||
const artifacts = [];
|
||||
for (const name of rawNames) {
|
||||
const target = path.join(runDir, name);
|
||||
if (!fs.existsSync(target)) continue;
|
||||
const bytes = sizePath(target);
|
||||
if (bytes > 0) artifacts.push({ name, path: target, bytes, human: human(bytes) });
|
||||
}
|
||||
return artifacts;
|
||||
}
|
||||
|
||||
function removeArtifact(artifact) {
|
||||
const st = fs.lstatSync(artifact.path);
|
||||
if (st.isSymbolicLink()) throw new Error("refusing symlink: " + artifact.path);
|
||||
if (st.isDirectory()) fs.rmSync(artifact.path, { recursive: true, force: false });
|
||||
else fs.unlinkSync(artifact.path);
|
||||
}
|
||||
|
||||
const rootAbs = path.resolve(stateRoot);
|
||||
const diskBefore = disk();
|
||||
const openDirs = openRunDirs(rootAbs);
|
||||
const candidates = [];
|
||||
const protectedRuns = [];
|
||||
|
||||
for (const runDir of findRunDirs(rootAbs)) {
|
||||
const manifest = jsonRead(path.join(runDir, "manifest.json"));
|
||||
const heartbeat = jsonRead(path.join(runDir, "heartbeat.json"));
|
||||
const pid = Number((heartbeat && heartbeat.pid) || (manifest && manifest.pid) || 0);
|
||||
const pidState = pidObserverAlive(pid, runDir);
|
||||
const hasOpenFd = openDirs.has(runDir);
|
||||
const newestMs = newestKnownTimeMs(runDir, manifest, heartbeat);
|
||||
const ageMs = newestMs > 0 ? nowMs - newestMs : 0;
|
||||
const artifacts = rawArtifacts(runDir);
|
||||
const rawBytes = artifacts.reduce((sum, item) => sum + item.bytes, 0);
|
||||
const rel = path.relative(rootAbs, runDir);
|
||||
const reasons = [];
|
||||
if (!manifest) reasons.push("manifest-missing");
|
||||
if (pidState.alive) reasons.push(pidState.reason || "pid-alive");
|
||||
if (hasOpenFd) reasons.push("open-fd");
|
||||
if (newestMs <= 0) reasons.push("time-unknown");
|
||||
if (ageMs < keepMs) reasons.push("retention-window");
|
||||
if (rawBytes <= 0) reasons.push("no-raw-artifacts");
|
||||
const summary = {
|
||||
id: (manifest && manifest.jobId) || path.basename(runDir).match(/(webobs-[A-Za-z0-9_.-]+)$/)?.[1] || null,
|
||||
runDir: rel,
|
||||
status: heartbeat && heartbeat.status || null,
|
||||
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
||||
newestAt: newestMs > 0 ? new Date(newestMs).toISOString() : null,
|
||||
ageHours: newestMs > 0 ? Number((ageMs / 3600000).toFixed(2)) : null,
|
||||
rawBytes,
|
||||
rawHuman: human(rawBytes),
|
||||
artifacts,
|
||||
};
|
||||
if (reasons.length > 0) protectedRuns.push({ ...summary, reasons });
|
||||
else candidates.push(summary);
|
||||
}
|
||||
|
||||
candidates.sort((a, b) => b.rawBytes - a.rawBytes);
|
||||
const selected = candidates.slice(0, limit);
|
||||
const selectedSet = new Set(selected.map((item) => item.runDir));
|
||||
const deferred = candidates.filter((item) => !selectedSet.has(item.runDir));
|
||||
let reclaimedBytes = 0;
|
||||
const deleted = [];
|
||||
const failures = [];
|
||||
|
||||
if (mode === "run") {
|
||||
for (const item of selected) {
|
||||
const runDir = path.join(rootAbs, item.runDir);
|
||||
try {
|
||||
for (const artifact of item.artifacts) removeArtifact(artifact);
|
||||
reclaimedBytes += item.rawBytes;
|
||||
deleted.push({ id: item.id, runDir: item.runDir, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length });
|
||||
} catch (error) {
|
||||
failures.push({ id: item.id, runDir: item.runDir, error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const estimatedReclaimBytes = selected.reduce((sum, item) => sum + item.rawBytes, 0);
|
||||
const diskAfter = mode === "run" ? disk() : null;
|
||||
console.log(JSON.stringify({
|
||||
ok: failures.length === 0,
|
||||
command: "web-probe observe gc",
|
||||
node: nodeId,
|
||||
lane,
|
||||
stateRoot,
|
||||
mode,
|
||||
mutation: mode === "run",
|
||||
keepHours,
|
||||
limit,
|
||||
diskBefore,
|
||||
diskAfter,
|
||||
scannedRuns: candidates.length + protectedRuns.length,
|
||||
candidateCount: candidates.length,
|
||||
selectedCount: selected.length,
|
||||
deferredCount: deferred.length,
|
||||
protectedCount: protectedRuns.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: human(estimatedReclaimBytes),
|
||||
reclaimedBytes,
|
||||
reclaimedHuman: human(reclaimedBytes),
|
||||
candidates: selected.map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length, artifacts: item.artifacts.map((artifact) => ({ name: artifact.name, bytes: artifact.bytes, human: artifact.human })) })),
|
||||
deleted,
|
||||
failures,
|
||||
protected: protectedRuns.slice(0, Math.min(50, protectedRuns.length)).map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, reasons: item.reasons })),
|
||||
valuesRedacted: true,
|
||||
}, null, 2));
|
||||
`;
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveStart(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
secretSpec: RuntimeSecretSpec,
|
||||
material: BootstrapAdminPasswordMaterial,
|
||||
credential: Record<string, unknown>,
|
||||
): Record<string, unknown> | RenderedCliResult {
|
||||
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
||||
const day = timestamp.slice(0, 8);
|
||||
const defaultStateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
|
||||
const stateDir = options.stateDir ?? defaultStateDir;
|
||||
const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
|
||||
const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64;
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
||||
const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec);
|
||||
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
|
||||
const authLogin = nodeWebProbeAuthLoginConfig(spec);
|
||||
const runnerEnvAssignments = [
|
||||
...webProbeProxy.envAssignments,
|
||||
...webProbeAccountEnvAssignments(),
|
||||
...(spec.webProbe?.playwrightBrowsersPath === undefined ? [] : [
|
||||
`PLAYWRIGHT_BROWSERS_PATH=${shellQuote(spec.webProbe.playwrightBrowsersPath)}`,
|
||||
]),
|
||||
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
|
||||
`HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`,
|
||||
`HWLAB_WEB_PASS=${shellQuote(material.password)}`,
|
||||
`UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`,
|
||||
`UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`,
|
||||
`UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`,
|
||||
`UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`,
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`,
|
||||
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
|
||||
...(authLogin === null
|
||||
? []
|
||||
: [
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS=${shellQuote(String(authLogin.maxAttempts))}`,
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS=${shellQuote(String(authLogin.requestTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS=${shellQuote(String(authLogin.initialDelayMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
|
||||
]),
|
||||
].join(" ");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
"mkdir -p \"$state_dir\"",
|
||||
"chmod 700 \"$state_dir\"",
|
||||
"runner=\"$state_dir/observer-runner.mjs\"",
|
||||
"runner_b64=\"$state_dir/observer-runner.mjs.b64\"",
|
||||
"cat >\"$runner_b64\" <<'UNIDESK_WEB_OBSERVE_RUNNER_B64'",
|
||||
runnerB64Body,
|
||||
"UNIDESK_WEB_OBSERVE_RUNNER_B64",
|
||||
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
|
||||
"rm -f \"$runner_b64\"",
|
||||
"chmod 700 \"$runner\"",
|
||||
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
|
||||
"pid=$!",
|
||||
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
||||
"sleep 1",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const startResolution = resolveWebObserveActionJson(result, "start");
|
||||
const started = startResolution.parsed;
|
||||
const startOk = result.exitCode === 0 && started?.ok === true;
|
||||
const recovery = startOk
|
||||
? null
|
||||
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
|
||||
const recoveredStatus = !startOk && recovery !== null && recovery.result.exitCode === 0 && recovery.status !== null && recovery.status.ok !== false
|
||||
? recovery.status
|
||||
: null;
|
||||
const effectiveOk = startOk || recoveredStatus !== null;
|
||||
const observer = startOk ? started : recoveredStatus;
|
||||
const observerId = typeof started?.jobId === "string"
|
||||
? started.jobId
|
||||
: webObserveIdFromStatus(recoveredStatus, { ...options, id: jobId }) ?? jobId;
|
||||
const degradedReason = !startOk && recoveredStatus !== null
|
||||
? "web-probe-start-transport-timeout-recovered"
|
||||
: result.timedOut
|
||||
? "web-probe-command-timeout"
|
||||
: result.exitCode !== 0
|
||||
? "web-probe-observe-start-failed"
|
||||
: null;
|
||||
const index = effectiveOk
|
||||
? upsertWebObserveIndexEntry(startOk ? {
|
||||
id: observerId,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
stateDir,
|
||||
url: options.url,
|
||||
targetPath: options.targetPath,
|
||||
status: "running",
|
||||
pid: typeof started?.pid === "number" ? started.pid : null,
|
||||
startedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} : webObserveIndexEntryFromOptions({ ...options, id: observerId, stateDir }, spec, observerId, recoveredStatus ?? {}))
|
||||
: null;
|
||||
return renderWebObserveStartResult({
|
||||
ok: effectiveOk,
|
||||
status: effectiveOk ? "started" : "blocked",
|
||||
command: `web-probe observe start --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
url: options.url,
|
||||
network: webProbeProxy.summary,
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
targetPath: options.targetPath,
|
||||
id: observerId,
|
||||
degradedReason,
|
||||
credential,
|
||||
observer: withWebObserveShortcuts(observer, observerId),
|
||||
wrapper: buildWebObserveWrapperForObserveOptions("start", options, spec.workspace, { id: observerId, jobId: observerId, stateDir }),
|
||||
index,
|
||||
next: webObserveNextCommands(observerId),
|
||||
stdoutJson: startResolution.diagnostics,
|
||||
result: compactCommandResultRedacted(result, [material.password ?? ""]),
|
||||
recovery: recovery === null ? null : {
|
||||
attempted: true,
|
||||
ok: recoveredStatus !== null,
|
||||
reason: degradedReason,
|
||||
stdoutJson: recovery.stdoutJson,
|
||||
result: compactCommandResultWithStdoutTail(recovery.result),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
||||
const { result, status, stdoutJson } = readNodeWebProbeObserveRemoteStatus(options, spec, options.tailLines, options.commandTimeoutSeconds);
|
||||
const observerId = webObserveIdFromStatus(status, options);
|
||||
const statusReadable = status !== null;
|
||||
const ok = result.exitCode === 0 && statusReadable && status.ok !== false;
|
||||
const degradedReason = result.timedOut
|
||||
? "web-probe-command-timeout"
|
||||
: result.exitCode !== 0
|
||||
? "web-probe-observe-status-failed"
|
||||
: !statusReadable
|
||||
? "web-probe-observe-status-unreadable"
|
||||
: typeof status.degradedReason === "string"
|
||||
? status.degradedReason
|
||||
: null;
|
||||
const index = ok && observerId !== null && options.stateDir !== null
|
||||
? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status))
|
||||
: null;
|
||||
return withWebObserveStatusRendered({
|
||||
ok,
|
||||
status: ok ? "observed" : "blocked",
|
||||
command: webObserveCommandLabel("status", options),
|
||||
id: observerId,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
degradedReason,
|
||||
observer: withWebObserveShortcuts(status, observerId),
|
||||
wrapper: buildWebObserveWrapperForObserveOptions("status", options, spec.workspace, { id: observerId, stateDir: webObserveWrapperStateDirFromStatus(status, options.stateDir) }),
|
||||
index,
|
||||
next: observerId === null ? null : webObserveNextCommands(observerId),
|
||||
stdoutJson,
|
||||
result: compactCommandResultWithStdoutTail(result),
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
|
||||
export function readNodeWebProbeObserveRemoteStatus(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
tailLines: number,
|
||||
timeoutSeconds: number,
|
||||
): { result: ReturnType<typeof runTransWorkspaceStdinScript>; status: Record<string, unknown> | null; stdoutJson: Record<string, unknown> } {
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveStatusNodeScript(tailLines, options.node, options.lane),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, timeoutSeconds);
|
||||
const statusResolution = resolveWebObserveActionJson(result, "status");
|
||||
return { result, status: statusResolution.parsed, stdoutJson: statusResolution.diagnostics };
|
||||
}
|
||||
|
||||
export function webObserveText(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function webObserveShort(value: string, maxLength: number): string {
|
||||
if (value.length <= maxLength) return value;
|
||||
if (maxLength <= 1) return value.slice(0, maxLength);
|
||||
return `${value.slice(0, maxLength - 1)}~`;
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record<string, unknown> | RenderedCliResult {
|
||||
const type = options.commandType ?? (stopCommand ? "stop" : null);
|
||||
if (type === null) throw new Error("web-probe observe command requires --type");
|
||||
const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const payload = {
|
||||
id: commandId,
|
||||
type,
|
||||
createdAt: new Date().toISOString(),
|
||||
source: "cli",
|
||||
path: options.commandPath,
|
||||
text: options.commandText,
|
||||
label: options.commandLabel,
|
||||
sessionId: options.commandSessionId,
|
||||
provider: options.commandProvider,
|
||||
afterRound: options.commandAfterRound,
|
||||
severity: options.commandSeverity,
|
||||
alternateSessionStrategy: options.commandAlternateSessionStrategy,
|
||||
expectedSentinelRange: options.commandExpectedSentinelRange,
|
||||
expectedActionWaitMs: options.commandExpectedActionWaitMs,
|
||||
durationMs: options.commandDurationMs,
|
||||
requireComposerReady: options.commandRequireComposerReady,
|
||||
waitProjectManagementReady: options.commandWaitProjectManagementReady,
|
||||
findingId: options.commandFindingId,
|
||||
blocking: options.commandBlocking,
|
||||
accountId: options.commandAccountId,
|
||||
fromAccountId: options.commandFromAccountId,
|
||||
toAccountId: options.commandToAccountId,
|
||||
sourceId: options.commandSourceId,
|
||||
fileRef: options.commandFileRef,
|
||||
filename: options.commandFilename,
|
||||
taskRef: options.commandTaskRef,
|
||||
taskId: options.commandTaskId,
|
||||
field: options.commandField,
|
||||
link: options.commandLink,
|
||||
title: options.commandTitle,
|
||||
body: options.commandBody,
|
||||
status: options.commandStatus,
|
||||
hwpodId: options.commandHwpodId,
|
||||
nodeId: options.commandNodeId,
|
||||
workspaceRoot: options.commandWorkspaceRoot,
|
||||
root: options.commandRoot,
|
||||
};
|
||||
const preStopStatus = options.force && stopCommand
|
||||
? readNodeWebProbeObserveRemoteStatus(options, spec, 1, Math.min(options.commandTimeoutSeconds, 30))
|
||||
: null;
|
||||
const preStopDiagnostics = record(preStopStatus?.status?.diagnostics);
|
||||
const preStopCommands = record(preStopStatus?.status?.commands);
|
||||
const preStopPending = Number(preStopCommands?.pendingCount ?? 0);
|
||||
const preStopProcessing = Number(preStopCommands?.processingCount ?? 0);
|
||||
const forceBeforeQueueReason = options.force && stopCommand
|
||||
? preStopDiagnostics?.heartbeatStale === true
|
||||
? "heartbeat-stale"
|
||||
: preStopPending > 0 || preStopProcessing > 0
|
||||
? "command-backlog"
|
||||
: null
|
||||
: null;
|
||||
if (forceBeforeQueueReason !== null) {
|
||||
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, forceBeforeQueueReason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, null);
|
||||
}
|
||||
const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
||||
const waitMs = options.force && stopCommand ? Math.max(options.waitMs, 5000) : options.waitMs;
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
"mkdir -p \"$state_dir/commands/pending\"",
|
||||
`node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`,
|
||||
nodeWebObserveWaitCommandShell(commandId, waitMs),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const commandResolution = resolveWebObserveActionJson(result, "command");
|
||||
const commandResult = commandResolution.parsed;
|
||||
if (options.force && stopCommand && (result.exitCode !== 0 || commandResult?.waitTimedOut === true || commandResult?.queued === true)) {
|
||||
const reason = result.exitCode !== 0
|
||||
? "graceful-stop-failed"
|
||||
: commandResult?.waitTimedOut === true
|
||||
? "graceful-stop-not-consumed"
|
||||
: "graceful-stop-queued";
|
||||
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, reason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, result);
|
||||
}
|
||||
const payloadResult = {
|
||||
ok: result.exitCode === 0 && commandResult !== null && commandResult.ok !== false,
|
||||
status: result.exitCode === 0 && commandResult !== null ? (waitMs > 0 ? "completed-or-queued" : "queued") : "blocked",
|
||||
command: webObserveCommandLabel(stopCommand ? "stop" : "command", options),
|
||||
id: webObserveIdFromOptions(options),
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
observer: commandResult,
|
||||
wrapper: buildWebObserveWrapperForObserveOptions(stopCommand ? "stop" : "command", options, spec.workspace, { commandType: type }),
|
||||
stdoutJson: commandResolution.diagnostics,
|
||||
result: compactCommandResult(result),
|
||||
full: options.full,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult);
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveForceStop(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
payload: Record<string, unknown>,
|
||||
commandId: string,
|
||||
reason: string,
|
||||
preflightResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
||||
preflightStatus: Record<string, unknown> | null,
|
||||
gracefulResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
||||
): Record<string, unknown> | RenderedCliResult {
|
||||
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveForceStopNodeScript(reason, commandId),
|
||||
].join("\n"), 55);
|
||||
const forceResolution = resolveWebObserveActionJson(killResult, "force-stop");
|
||||
const forcePayload = forceResolution.parsed;
|
||||
const payloadResult = {
|
||||
ok: killResult.exitCode === 0 && forcePayload !== null && forcePayload.ok !== false,
|
||||
status: killResult.exitCode === 0 && forcePayload !== null && forcePayload.ok !== false ? "forced-stopped" : "blocked",
|
||||
command: webObserveCommandLabel("stop", options),
|
||||
id: webObserveIdFromOptions(options),
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
observer: forcePayload,
|
||||
wrapper: buildWebObserveWrapperForObserveOptions("stop", options, spec.workspace, { commandType: "stop" }),
|
||||
forceReason: reason,
|
||||
preflightObserver: preflightStatus,
|
||||
preflightResult: preflightResult === null ? null : compactCommandResult(preflightResult),
|
||||
gracefulResult: gracefulResult === null ? null : compactCommandResult(gracefulResult),
|
||||
stdoutJson: forceResolution.diagnostics,
|
||||
forceResult: compactCommandResultWithStdoutTail(killResult),
|
||||
full: options.full,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult);
|
||||
}
|
||||
@@ -35,11 +35,14 @@ import type { BootstrapAdminPasswordMaterial, NodeWebProbeObserveCommandType, No
|
||||
import { runTransWorkspaceStdinScript, runtimeSecretSpec } from "./public-exposure";
|
||||
import { transPath } from "./runtime-common";
|
||||
import { assertLane, assertNodeId, compactCommandResult, compactCommandResultRedacted, compactCommandResultWithStdoutTail, nullableRecord, optionValue, parseJsonObject, positiveIntegerOption, record, requiredOption, shellQuote } from "./utils";
|
||||
import { runNodeWebProbeObserveCommand, runNodeWebProbeObserveGc, runNodeWebProbeObserveStart, runNodeWebProbeObserveStatus } from "./web-probe-observe-actions";
|
||||
import { runNodeWebProbeObserveCollect } from "./web-probe-observe-collect";
|
||||
import { nodeWebObserveResolveStateDirShell, recoverWebObserveAnalyzeTurnDetails, renderWebObserveStartResult, renderWebProbeRunResult, upsertWebObserveIndexEntry, webObserveCommandLabel, webObserveIdFromOptions, webObserveIdFromStatus, webObserveIndexEntryFromOptions, webObserveNextCommands, withWebObserveAnalyzeRendered, withWebObserveShortcuts } from "./web-observe-render";
|
||||
import { commandSummaryForOutput, isSafeWebObserveArchivePrefix, isSafeWebObserveCollectFile, isSafeWebObserveFindingId, isSafeWebObserveJobId, isSafeWebObserveStateDir, isSafeWebObserveTraceId, nodeWebObserveForceStopNodeScript, nodeWebObserveStatusNodeScript, nodeWebObserveWaitCommandShell, runNodeWebProbeScript, safeWebObserveSegment, safeWebObserveTargetSegment } from "./web-observe-scripts";
|
||||
import { displayRepoPath, readBootstrapAdminPasswordMaterial, sleepSync } from "./web-probe";
|
||||
|
||||
export { webObserveShort, webObserveText } from "./web-probe-observe-actions";
|
||||
|
||||
export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSentinelOptions {
|
||||
const [sentinelActionRaw] = args;
|
||||
if (
|
||||
@@ -1343,685 +1346,6 @@ export function runNodeWebProbeObserve(
|
||||
return runNodeWebProbeObserveAnalyze(options, spec);
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveGc(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const stateRoot = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`;
|
||||
const script = [
|
||||
"set -eu",
|
||||
`node - ${shellQuote(stateRoot)} ${shellQuote(String(options.gcKeepHours))} ${shellQuote(String(options.gcLimit))} ${shellQuote(options.confirm ? "run" : "plan")} ${shellQuote(options.node)} ${shellQuote(options.lane)} <<'UNIDESK_WEB_OBSERVE_GC'`,
|
||||
nodeWebObserveGcNodeScript(),
|
||||
"UNIDESK_WEB_OBSERVE_GC",
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const payload = parseJsonObject(result.stdout);
|
||||
const ok = result.exitCode === 0 && payload.ok !== false;
|
||||
return {
|
||||
ok,
|
||||
status: ok ? (options.confirm ? "cleaned" : "planned") : "blocked",
|
||||
command: `web-probe observe gc --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
stateRoot,
|
||||
mode: options.confirm ? "confirmed-run" : "dry-run",
|
||||
retention: {
|
||||
source: "config/unidesk-cli.yaml#gc.stateStaleScratch.keepHours",
|
||||
keepHours: options.gcKeepHours,
|
||||
userOverride: options.gcKeepHours !== webObserveGcDefaultKeepHours(),
|
||||
},
|
||||
gc: options.full || options.raw ? payload : compactWebObserveGcPayload(payload),
|
||||
result: ok
|
||||
? {
|
||||
exitCode: result.exitCode,
|
||||
timedOut: result.timedOut,
|
||||
stdoutBytes: result.stdout.length,
|
||||
stderrBytes: result.stderr.length,
|
||||
}
|
||||
: compactCommandResultWithStdoutTail(result),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactWebObserveGcPayload(payload: Record<string, unknown>): Record<string, unknown> {
|
||||
const candidates = Array.isArray(payload.candidates) ? payload.candidates.map(record) : [];
|
||||
const protectedRuns = Array.isArray(payload.protected) ? payload.protected.map(record) : [];
|
||||
const deleted = Array.isArray(payload.deleted) ? payload.deleted.map(record) : [];
|
||||
const failures = Array.isArray(payload.failures) ? payload.failures.map(record) : [];
|
||||
return {
|
||||
ok: payload.ok ?? null,
|
||||
mode: payload.mode ?? null,
|
||||
mutation: payload.mutation ?? null,
|
||||
keepHours: payload.keepHours ?? null,
|
||||
limit: payload.limit ?? null,
|
||||
diskBefore: payload.diskBefore ?? null,
|
||||
diskAfter: payload.diskAfter ?? null,
|
||||
scannedRuns: payload.scannedRuns ?? null,
|
||||
candidateCount: payload.candidateCount ?? null,
|
||||
selectedCount: payload.selectedCount ?? null,
|
||||
deferredCount: payload.deferredCount ?? null,
|
||||
protectedCount: payload.protectedCount ?? null,
|
||||
estimatedReclaimBytes: payload.estimatedReclaimBytes ?? null,
|
||||
estimatedReclaimHuman: payload.estimatedReclaimHuman ?? null,
|
||||
reclaimedBytes: payload.reclaimedBytes ?? null,
|
||||
reclaimedHuman: payload.reclaimedHuman ?? null,
|
||||
candidates: candidates.slice(0, 20).map((item) => ({
|
||||
id: item.id ?? null,
|
||||
runDir: item.runDir ?? null,
|
||||
status: item.status ?? null,
|
||||
ageHours: item.ageHours ?? null,
|
||||
rawHuman: item.rawHuman ?? null,
|
||||
artifactCount: item.artifactCount ?? null,
|
||||
})),
|
||||
deleted: deleted.slice(0, 20).map((item) => ({
|
||||
id: item.id ?? null,
|
||||
runDir: item.runDir ?? null,
|
||||
rawHuman: item.rawHuman ?? null,
|
||||
artifactCount: item.artifactCount ?? null,
|
||||
})),
|
||||
failures: failures.slice(0, 20),
|
||||
protected: protectedRuns.slice(0, 20).map((item) => ({
|
||||
id: item.id ?? null,
|
||||
runDir: item.runDir ?? null,
|
||||
status: item.status ?? null,
|
||||
ageHours: item.ageHours ?? null,
|
||||
rawHuman: item.rawHuman ?? null,
|
||||
reasons: item.reasons ?? null,
|
||||
})),
|
||||
disclosure: {
|
||||
candidateRowsShown: Math.min(20, candidates.length),
|
||||
deletedRowsShown: Math.min(20, deleted.length),
|
||||
protectedRowsShown: Math.min(20, protectedRuns.length),
|
||||
full: "rerun with --full or --raw for all candidates/artifacts",
|
||||
},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeWebObserveGcNodeScript(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const childProcess = require("child_process");
|
||||
|
||||
const stateRoot = process.argv[2];
|
||||
const keepHours = Number(process.argv[3]);
|
||||
const limit = Number(process.argv[4]);
|
||||
const mode = process.argv[5] === "run" ? "run" : "plan";
|
||||
const nodeId = process.argv[6];
|
||||
const lane = process.argv[7];
|
||||
const keepMs = keepHours * 60 * 60 * 1000;
|
||||
const nowMs = Date.now();
|
||||
const rawNames = ["samples.jsonl", "browser-process.jsonl", "network.jsonl", "console.jsonl", "artifacts.jsonl", "performance-events.jsonl", "screenshots", "performance"];
|
||||
|
||||
function jsonRead(file) {
|
||||
try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; }
|
||||
}
|
||||
|
||||
function parseTime(value) {
|
||||
if (typeof value !== "string") return 0;
|
||||
const ms = Date.parse(value);
|
||||
return Number.isFinite(ms) ? ms : 0;
|
||||
}
|
||||
|
||||
function dirTimeMs(runDir) {
|
||||
const base = path.basename(runDir);
|
||||
const match = base.match(/^(\d{8})T(\d{6})Z_/);
|
||||
if (!match) return 0;
|
||||
const raw = match[1].slice(0, 4) + "-" + match[1].slice(4, 6) + "-" + match[1].slice(6, 8) + "T" + match[2].slice(0, 2) + ":" + match[2].slice(2, 4) + ":" + match[2].slice(4, 6) + "Z";
|
||||
return Date.parse(raw) || 0;
|
||||
}
|
||||
|
||||
function newestKnownTimeMs(runDir, manifest, heartbeat) {
|
||||
return Math.max(
|
||||
dirTimeMs(runDir),
|
||||
parseTime(manifest && manifest.completedAt),
|
||||
parseTime(manifest && manifest.updatedAt),
|
||||
parseTime(manifest && manifest.startedAt),
|
||||
parseTime(manifest && manifest.pageProvenance && manifest.pageProvenance.observedAt),
|
||||
parseTime(heartbeat && heartbeat.completedAt),
|
||||
parseTime(heartbeat && heartbeat.forceStoppedAt),
|
||||
parseTime(heartbeat && heartbeat.updatedAt),
|
||||
parseTime(heartbeat && heartbeat.pageProvenance && heartbeat.pageProvenance.observedAt),
|
||||
);
|
||||
}
|
||||
|
||||
function sizePath(target) {
|
||||
let total = 0;
|
||||
const stack = [target];
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
let st;
|
||||
try { st = fs.lstatSync(current); } catch { continue; }
|
||||
if (st.isSymbolicLink()) continue;
|
||||
total += st.blocks ? st.blocks * 512 : st.size;
|
||||
if (!st.isDirectory()) continue;
|
||||
let entries;
|
||||
try { entries = fs.readdirSync(current); } catch { continue; }
|
||||
for (const entry of entries) stack.push(path.join(current, entry));
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function human(bytes) {
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit += 1; }
|
||||
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
||||
}
|
||||
|
||||
function disk() {
|
||||
try {
|
||||
const line = childProcess.execFileSync("df", ["-Pk", "/"], { encoding: "utf8" }).trim().split("\n").slice(-1)[0];
|
||||
const parts = line.trim().split(/\s+/);
|
||||
const sizeBytes = Number(parts[1]) * 1024;
|
||||
const usedBytes = Number(parts[2]) * 1024;
|
||||
const availableBytes = Number(parts[3]) * 1024;
|
||||
const usePercent = Number(String(parts[4]).replace("%", ""));
|
||||
return { filesystem: parts[0], sizeBytes, usedBytes, availableBytes, usePercent, usedHuman: human(usedBytes), availableHuman: human(availableBytes) };
|
||||
} catch (error) {
|
||||
return { error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function findRunDirs(root) {
|
||||
const runs = [];
|
||||
for (const year of safeReadDir(root)) {
|
||||
const y = path.join(root, year.name);
|
||||
if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) continue;
|
||||
for (const month of safeReadDir(y)) {
|
||||
const m = path.join(y, month.name);
|
||||
if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) continue;
|
||||
for (const day of safeReadDir(m)) {
|
||||
const d = path.join(m, day.name);
|
||||
if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) continue;
|
||||
for (const run of safeReadDir(d)) {
|
||||
const r = path.join(d, run.name);
|
||||
if (run.isDirectory() && /_webobs-[A-Za-z0-9_.-]+$/.test(run.name)) runs.push(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
|
||||
function safeReadDir(dir) {
|
||||
try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
|
||||
}
|
||||
|
||||
function pidObserverAlive(pid, runAbs) {
|
||||
if (!Number.isInteger(pid) || pid <= 1) return { alive: false, reason: null };
|
||||
try {
|
||||
const cmdline = fs.readFileSync("/proc/" + pid + "/cmdline", "utf8").replace(/\0/g, " ");
|
||||
if (cmdline.includes("observer-runner") || cmdline.includes(runAbs)) return { alive: true, reason: "pid-alive" };
|
||||
return { alive: false, reason: null };
|
||||
} catch (error) {
|
||||
if (fs.existsSync("/proc/" + pid)) return { alive: true, reason: "pid-alive-unreadable" };
|
||||
return { alive: false, reason: null };
|
||||
}
|
||||
}
|
||||
|
||||
function openRunDirs(rootAbs) {
|
||||
const active = new Set();
|
||||
const procEntries = safeReadDir("/proc").filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name));
|
||||
for (const proc of procEntries) {
|
||||
const fdDir = path.join("/proc", proc.name, "fd");
|
||||
for (const fd of safeReadDir(fdDir)) {
|
||||
let target;
|
||||
try { target = fs.readlinkSync(path.join(fdDir, fd.name)).replace(/ \(deleted\)$/, ""); } catch { continue; }
|
||||
if (!target.startsWith(rootAbs + path.sep)) continue;
|
||||
const rel = path.relative(rootAbs, target).split(path.sep);
|
||||
if (rel.length >= 4) active.add(path.join(rootAbs, rel[0], rel[1], rel[2], rel[3]));
|
||||
}
|
||||
}
|
||||
return active;
|
||||
}
|
||||
|
||||
function rawArtifacts(runDir) {
|
||||
const artifacts = [];
|
||||
for (const name of rawNames) {
|
||||
const target = path.join(runDir, name);
|
||||
if (!fs.existsSync(target)) continue;
|
||||
const bytes = sizePath(target);
|
||||
if (bytes > 0) artifacts.push({ name, path: target, bytes, human: human(bytes) });
|
||||
}
|
||||
return artifacts;
|
||||
}
|
||||
|
||||
function removeArtifact(artifact) {
|
||||
const st = fs.lstatSync(artifact.path);
|
||||
if (st.isSymbolicLink()) throw new Error("refusing symlink: " + artifact.path);
|
||||
if (st.isDirectory()) fs.rmSync(artifact.path, { recursive: true, force: false });
|
||||
else fs.unlinkSync(artifact.path);
|
||||
}
|
||||
|
||||
const rootAbs = path.resolve(stateRoot);
|
||||
const diskBefore = disk();
|
||||
const openDirs = openRunDirs(rootAbs);
|
||||
const candidates = [];
|
||||
const protectedRuns = [];
|
||||
|
||||
for (const runDir of findRunDirs(rootAbs)) {
|
||||
const manifest = jsonRead(path.join(runDir, "manifest.json"));
|
||||
const heartbeat = jsonRead(path.join(runDir, "heartbeat.json"));
|
||||
const pid = Number((heartbeat && heartbeat.pid) || (manifest && manifest.pid) || 0);
|
||||
const pidState = pidObserverAlive(pid, runDir);
|
||||
const hasOpenFd = openDirs.has(runDir);
|
||||
const newestMs = newestKnownTimeMs(runDir, manifest, heartbeat);
|
||||
const ageMs = newestMs > 0 ? nowMs - newestMs : 0;
|
||||
const artifacts = rawArtifacts(runDir);
|
||||
const rawBytes = artifacts.reduce((sum, item) => sum + item.bytes, 0);
|
||||
const rel = path.relative(rootAbs, runDir);
|
||||
const reasons = [];
|
||||
if (!manifest) reasons.push("manifest-missing");
|
||||
if (pidState.alive) reasons.push(pidState.reason || "pid-alive");
|
||||
if (hasOpenFd) reasons.push("open-fd");
|
||||
if (newestMs <= 0) reasons.push("time-unknown");
|
||||
if (ageMs < keepMs) reasons.push("retention-window");
|
||||
if (rawBytes <= 0) reasons.push("no-raw-artifacts");
|
||||
const summary = {
|
||||
id: (manifest && manifest.jobId) || path.basename(runDir).match(/(webobs-[A-Za-z0-9_.-]+)$/)?.[1] || null,
|
||||
runDir: rel,
|
||||
status: heartbeat && heartbeat.status || null,
|
||||
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
||||
newestAt: newestMs > 0 ? new Date(newestMs).toISOString() : null,
|
||||
ageHours: newestMs > 0 ? Number((ageMs / 3600000).toFixed(2)) : null,
|
||||
rawBytes,
|
||||
rawHuman: human(rawBytes),
|
||||
artifacts,
|
||||
};
|
||||
if (reasons.length > 0) protectedRuns.push({ ...summary, reasons });
|
||||
else candidates.push(summary);
|
||||
}
|
||||
|
||||
candidates.sort((a, b) => b.rawBytes - a.rawBytes);
|
||||
const selected = candidates.slice(0, limit);
|
||||
const selectedSet = new Set(selected.map((item) => item.runDir));
|
||||
const deferred = candidates.filter((item) => !selectedSet.has(item.runDir));
|
||||
let reclaimedBytes = 0;
|
||||
const deleted = [];
|
||||
const failures = [];
|
||||
|
||||
if (mode === "run") {
|
||||
for (const item of selected) {
|
||||
const runDir = path.join(rootAbs, item.runDir);
|
||||
try {
|
||||
for (const artifact of item.artifacts) removeArtifact(artifact);
|
||||
reclaimedBytes += item.rawBytes;
|
||||
deleted.push({ id: item.id, runDir: item.runDir, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length });
|
||||
} catch (error) {
|
||||
failures.push({ id: item.id, runDir: item.runDir, error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const estimatedReclaimBytes = selected.reduce((sum, item) => sum + item.rawBytes, 0);
|
||||
const diskAfter = mode === "run" ? disk() : null;
|
||||
console.log(JSON.stringify({
|
||||
ok: failures.length === 0,
|
||||
command: "web-probe observe gc",
|
||||
node: nodeId,
|
||||
lane,
|
||||
stateRoot,
|
||||
mode,
|
||||
mutation: mode === "run",
|
||||
keepHours,
|
||||
limit,
|
||||
diskBefore,
|
||||
diskAfter,
|
||||
scannedRuns: candidates.length + protectedRuns.length,
|
||||
candidateCount: candidates.length,
|
||||
selectedCount: selected.length,
|
||||
deferredCount: deferred.length,
|
||||
protectedCount: protectedRuns.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: human(estimatedReclaimBytes),
|
||||
reclaimedBytes,
|
||||
reclaimedHuman: human(reclaimedBytes),
|
||||
candidates: selected.map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, artifactCount: item.artifacts.length, artifacts: item.artifacts.map((artifact) => ({ name: artifact.name, bytes: artifact.bytes, human: artifact.human })) })),
|
||||
deleted,
|
||||
failures,
|
||||
protected: protectedRuns.slice(0, Math.min(50, protectedRuns.length)).map((item) => ({ id: item.id, runDir: item.runDir, status: item.status, ageHours: item.ageHours, rawBytes: item.rawBytes, rawHuman: item.rawHuman, reasons: item.reasons })),
|
||||
valuesRedacted: true,
|
||||
}, null, 2));
|
||||
`;
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveStart(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
secretSpec: RuntimeSecretSpec,
|
||||
material: BootstrapAdminPasswordMaterial,
|
||||
credential: Record<string, unknown>,
|
||||
): Record<string, unknown> | RenderedCliResult {
|
||||
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
||||
const day = timestamp.slice(0, 8);
|
||||
const defaultStateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
|
||||
const stateDir = options.stateDir ?? defaultStateDir;
|
||||
const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
|
||||
const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64;
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
||||
const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec);
|
||||
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
|
||||
const authLogin = nodeWebProbeAuthLoginConfig(spec);
|
||||
const runnerEnvAssignments = [
|
||||
...webProbeProxy.envAssignments,
|
||||
...webProbeAccountEnvAssignments(),
|
||||
...(spec.webProbe?.playwrightBrowsersPath === undefined ? [] : [
|
||||
`PLAYWRIGHT_BROWSERS_PATH=${shellQuote(spec.webProbe.playwrightBrowsersPath)}`,
|
||||
]),
|
||||
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
|
||||
`HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`,
|
||||
`HWLAB_WEB_PASS=${shellQuote(material.password)}`,
|
||||
`UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`,
|
||||
`UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`,
|
||||
`UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`,
|
||||
`UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`,
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`,
|
||||
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
|
||||
...(authLogin === null
|
||||
? []
|
||||
: [
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS=${shellQuote(String(authLogin.maxAttempts))}`,
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS=${shellQuote(String(authLogin.requestTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS=${shellQuote(String(authLogin.initialDelayMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
|
||||
]),
|
||||
].join(" ");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
"mkdir -p \"$state_dir\"",
|
||||
"chmod 700 \"$state_dir\"",
|
||||
"runner=\"$state_dir/observer-runner.mjs\"",
|
||||
"runner_b64=\"$state_dir/observer-runner.mjs.b64\"",
|
||||
"cat >\"$runner_b64\" <<'UNIDESK_WEB_OBSERVE_RUNNER_B64'",
|
||||
runnerB64Body,
|
||||
"UNIDESK_WEB_OBSERVE_RUNNER_B64",
|
||||
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
|
||||
"rm -f \"$runner_b64\"",
|
||||
"chmod 700 \"$runner\"",
|
||||
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
|
||||
"pid=$!",
|
||||
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
||||
"sleep 1",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const started = parseJsonObject(result.stdout);
|
||||
const startOk = result.exitCode === 0 && started?.ok === true;
|
||||
const recovery = startOk
|
||||
? null
|
||||
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
|
||||
const recoveredStatus = !startOk && recovery !== null && recovery.result.exitCode === 0 && recovery.status !== null && recovery.status.ok !== false
|
||||
? recovery.status
|
||||
: null;
|
||||
const effectiveOk = startOk || recoveredStatus !== null;
|
||||
const observer = startOk ? started : recoveredStatus;
|
||||
const observerId = typeof started?.jobId === "string"
|
||||
? started.jobId
|
||||
: webObserveIdFromStatus(recoveredStatus, { ...options, id: jobId }) ?? jobId;
|
||||
const degradedReason = !startOk && recoveredStatus !== null
|
||||
? "web-probe-start-transport-timeout-recovered"
|
||||
: result.timedOut
|
||||
? "web-probe-command-timeout"
|
||||
: result.exitCode !== 0
|
||||
? "web-probe-observe-start-failed"
|
||||
: null;
|
||||
const index = effectiveOk
|
||||
? upsertWebObserveIndexEntry(startOk ? {
|
||||
id: observerId,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
stateDir,
|
||||
url: options.url,
|
||||
targetPath: options.targetPath,
|
||||
status: "running",
|
||||
pid: typeof started.pid === "number" ? started.pid : null,
|
||||
startedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
} : webObserveIndexEntryFromOptions({ ...options, id: observerId, stateDir }, spec, observerId, recoveredStatus ?? {}))
|
||||
: null;
|
||||
return renderWebObserveStartResult({
|
||||
ok: effectiveOk,
|
||||
status: effectiveOk ? "started" : "blocked",
|
||||
command: `web-probe observe start --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
url: options.url,
|
||||
network: webProbeProxy.summary,
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
targetPath: options.targetPath,
|
||||
id: observerId,
|
||||
degradedReason,
|
||||
credential,
|
||||
observer: withWebObserveShortcuts(observer, observerId),
|
||||
wrapper: buildWebObserveWrapperForObserveOptions("start", options, spec.workspace, { id: observerId, jobId: observerId, stateDir }),
|
||||
index,
|
||||
next: webObserveNextCommands(observerId),
|
||||
result: compactCommandResultRedacted(result, [material.password ?? ""]),
|
||||
recovery: recovery === null ? null : {
|
||||
attempted: true,
|
||||
ok: recoveredStatus !== null,
|
||||
reason: degradedReason,
|
||||
result: compactCommandResultWithStdoutTail(recovery.result),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
||||
const { result, status } = readNodeWebProbeObserveRemoteStatus(options, spec, options.tailLines, options.commandTimeoutSeconds);
|
||||
const observerId = webObserveIdFromStatus(status, options);
|
||||
const statusReadable = status !== null;
|
||||
const ok = result.exitCode === 0 && statusReadable && status.ok !== false;
|
||||
const degradedReason = result.timedOut
|
||||
? "web-probe-command-timeout"
|
||||
: result.exitCode !== 0
|
||||
? "web-probe-observe-status-failed"
|
||||
: !statusReadable
|
||||
? "web-probe-observe-status-unreadable"
|
||||
: typeof status.degradedReason === "string"
|
||||
? status.degradedReason
|
||||
: null;
|
||||
const index = ok && observerId !== null && options.stateDir !== null
|
||||
? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status))
|
||||
: null;
|
||||
return withWebObserveStatusRendered({
|
||||
ok,
|
||||
status: ok ? "observed" : "blocked",
|
||||
command: webObserveCommandLabel("status", options),
|
||||
id: observerId,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
degradedReason,
|
||||
observer: withWebObserveShortcuts(status, observerId),
|
||||
wrapper: buildWebObserveWrapperForObserveOptions("status", options, spec.workspace, { id: observerId, stateDir: webObserveWrapperStateDirFromStatus(status, options.stateDir) }),
|
||||
index,
|
||||
next: observerId === null ? null : webObserveNextCommands(observerId),
|
||||
result: compactCommandResultWithStdoutTail(result),
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
|
||||
export function readNodeWebProbeObserveRemoteStatus(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
tailLines: number,
|
||||
timeoutSeconds: number,
|
||||
): { result: ReturnType<typeof runTransWorkspaceStdinScript>; status: Record<string, unknown> | null } {
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveStatusNodeScript(tailLines, options.node, options.lane),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, timeoutSeconds);
|
||||
return { result, status: parseJsonObject(result.stdout) };
|
||||
}
|
||||
|
||||
export function webObserveText(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function webObserveShort(value: string, maxLength: number): string {
|
||||
if (value.length <= maxLength) return value;
|
||||
if (maxLength <= 1) return value.slice(0, maxLength);
|
||||
return `${value.slice(0, maxLength - 1)}~`;
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record<string, unknown> | RenderedCliResult {
|
||||
const type = options.commandType ?? (stopCommand ? "stop" : null);
|
||||
if (type === null) throw new Error("web-probe observe command requires --type");
|
||||
const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const payload = {
|
||||
id: commandId,
|
||||
type,
|
||||
createdAt: new Date().toISOString(),
|
||||
source: "cli",
|
||||
path: options.commandPath,
|
||||
text: options.commandText,
|
||||
label: options.commandLabel,
|
||||
sessionId: options.commandSessionId,
|
||||
provider: options.commandProvider,
|
||||
afterRound: options.commandAfterRound,
|
||||
severity: options.commandSeverity,
|
||||
alternateSessionStrategy: options.commandAlternateSessionStrategy,
|
||||
expectedSentinelRange: options.commandExpectedSentinelRange,
|
||||
expectedActionWaitMs: options.commandExpectedActionWaitMs,
|
||||
durationMs: options.commandDurationMs,
|
||||
requireComposerReady: options.commandRequireComposerReady,
|
||||
waitProjectManagementReady: options.commandWaitProjectManagementReady,
|
||||
findingId: options.commandFindingId,
|
||||
blocking: options.commandBlocking,
|
||||
accountId: options.commandAccountId,
|
||||
fromAccountId: options.commandFromAccountId,
|
||||
toAccountId: options.commandToAccountId,
|
||||
sourceId: options.commandSourceId,
|
||||
fileRef: options.commandFileRef,
|
||||
filename: options.commandFilename,
|
||||
taskRef: options.commandTaskRef,
|
||||
taskId: options.commandTaskId,
|
||||
field: options.commandField,
|
||||
link: options.commandLink,
|
||||
title: options.commandTitle,
|
||||
body: options.commandBody,
|
||||
status: options.commandStatus,
|
||||
hwpodId: options.commandHwpodId,
|
||||
nodeId: options.commandNodeId,
|
||||
workspaceRoot: options.commandWorkspaceRoot,
|
||||
root: options.commandRoot,
|
||||
};
|
||||
const preStopStatus = options.force && stopCommand
|
||||
? readNodeWebProbeObserveRemoteStatus(options, spec, 1, Math.min(options.commandTimeoutSeconds, 30))
|
||||
: null;
|
||||
const preStopDiagnostics = record(preStopStatus?.status?.diagnostics);
|
||||
const preStopCommands = record(preStopStatus?.status?.commands);
|
||||
const preStopPending = Number(preStopCommands?.pendingCount ?? 0);
|
||||
const preStopProcessing = Number(preStopCommands?.processingCount ?? 0);
|
||||
const forceBeforeQueueReason = options.force && stopCommand
|
||||
? preStopDiagnostics?.heartbeatStale === true
|
||||
? "heartbeat-stale"
|
||||
: preStopPending > 0 || preStopProcessing > 0
|
||||
? "command-backlog"
|
||||
: null
|
||||
: null;
|
||||
if (forceBeforeQueueReason !== null) {
|
||||
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, forceBeforeQueueReason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, null);
|
||||
}
|
||||
const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
||||
const waitMs = options.force && stopCommand ? Math.max(options.waitMs, 5000) : options.waitMs;
|
||||
const script = [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
"mkdir -p \"$state_dir/commands/pending\"",
|
||||
`node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`,
|
||||
nodeWebObserveWaitCommandShell(commandId, waitMs),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const commandResult = parseJsonObject(result.stdout);
|
||||
if (options.force && stopCommand && (result.exitCode !== 0 || commandResult?.waitTimedOut === true || commandResult?.queued === true)) {
|
||||
const reason = result.exitCode !== 0
|
||||
? "graceful-stop-failed"
|
||||
: commandResult?.waitTimedOut === true
|
||||
? "graceful-stop-not-consumed"
|
||||
: "graceful-stop-queued";
|
||||
return runNodeWebProbeObserveForceStop(options, spec, payload, commandId, reason, preStopStatus?.result ?? null, preStopStatus?.status ?? null, result);
|
||||
}
|
||||
const payloadResult = {
|
||||
ok: result.exitCode === 0 && commandResult?.ok !== false,
|
||||
status: result.exitCode === 0 ? (waitMs > 0 ? "completed-or-queued" : "queued") : "blocked",
|
||||
command: webObserveCommandLabel(stopCommand ? "stop" : "command", options),
|
||||
id: webObserveIdFromOptions(options),
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
observer: commandResult,
|
||||
wrapper: buildWebObserveWrapperForObserveOptions(stopCommand ? "stop" : "command", options, spec.workspace, { commandType: type }),
|
||||
result: compactCommandResult(result),
|
||||
full: options.full,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult);
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveForceStop(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
payload: Record<string, unknown>,
|
||||
commandId: string,
|
||||
reason: string,
|
||||
preflightResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
||||
preflightStatus: Record<string, unknown> | null,
|
||||
gracefulResult: ReturnType<typeof runTransWorkspaceStdinScript> | null,
|
||||
): Record<string, unknown> | RenderedCliResult {
|
||||
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
|
||||
"set -eu",
|
||||
nodeWebObserveResolveStateDirShell(options),
|
||||
nodeWebObserveForceStopNodeScript(reason, commandId),
|
||||
].join("\n"), 55);
|
||||
const forcePayload = parseJsonObject(killResult.stdout);
|
||||
const payloadResult = {
|
||||
ok: killResult.exitCode === 0 && forcePayload?.ok !== false,
|
||||
status: killResult.exitCode === 0 && forcePayload?.ok !== false ? "forced-stopped" : "blocked",
|
||||
command: webObserveCommandLabel("stop", options),
|
||||
id: webObserveIdFromOptions(options),
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
commandId,
|
||||
observerCommand: commandSummaryForOutput(payload),
|
||||
observer: forcePayload,
|
||||
wrapper: buildWebObserveWrapperForObserveOptions("stop", options, spec.workspace, { commandType: "stop" }),
|
||||
forceReason: reason,
|
||||
preflightObserver: preflightStatus,
|
||||
preflightResult: preflightResult === null ? null : compactCommandResult(preflightResult),
|
||||
gracefulResult: gracefulResult === null ? null : compactCommandResult(gracefulResult),
|
||||
forceResult: compactCommandResultWithStdoutTail(killResult),
|
||||
full: options.full,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return options.raw ? payloadResult : withWebObserveCommandRendered(payloadResult);
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
||||
const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64");
|
||||
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
||||
|
||||
Reference in New Issue
Block a user