Codex
370 lines
16 KiB
TypeScript
370 lines
16 KiB
TypeScript
import { readFileSync } from "node:fs";
|
|
import { runCommandObserved, type CommandProgress, type CommandResult } from "../command";
|
|
import { repoRoot, rootPath } from "../config";
|
|
import type { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { record } from "./utils";
|
|
|
|
export const NODE_RUNTIME_STATUS_WORKER_ENV = "UNIDESK_HWLAB_CONTROL_PLANE_STATUS_WORKER";
|
|
const WORKER_PROGRESS_PREFIX = "UNIDESK_HWLAB_CONTROL_PLANE_STATUS_PROGRESS ";
|
|
const STATUS_CONFIG_PATH = "config/hwlab-node-control-plane.yaml";
|
|
const WORKER_CAPTURE_CHARS = 2 * 1024 * 1024;
|
|
const WORKER_STDERR_TAIL_CHARS = 1600;
|
|
const WORKER_STDOUT_TAIL_CHARS = 1200;
|
|
|
|
export const NODE_RUNTIME_STATUS_PROBES = [
|
|
"source",
|
|
"control-plane",
|
|
"pipeline-run",
|
|
"argo",
|
|
"runtime",
|
|
"public",
|
|
"git-mirror",
|
|
] as const;
|
|
|
|
export type NodeRuntimeStatusProbe = typeof NODE_RUNTIME_STATUS_PROBES[number];
|
|
export type NodeRuntimeStatusProbeState = "started" | "completed" | "failed" | "skipped";
|
|
|
|
export interface NodeRuntimeStatusWorkerEvent {
|
|
probe: NodeRuntimeStatusProbe;
|
|
state: NodeRuntimeStatusProbeState;
|
|
at: string;
|
|
durationMs?: number;
|
|
reason?: string;
|
|
}
|
|
|
|
export interface NodeRuntimeStatusPolicy {
|
|
configPath: string;
|
|
totalBudgetSeconds: number;
|
|
heartbeatSeconds: number;
|
|
terminationGraceSeconds: number;
|
|
probes: Record<NodeRuntimeStatusProbe, boolean>;
|
|
}
|
|
|
|
type ScopedStatus = ReturnType<typeof parseNodeScopedDelegatedOptions>;
|
|
|
|
export function readNodeRuntimeStatusPolicy(): NodeRuntimeStatusPolicy {
|
|
const parsed = requiredRecord(Bun.YAML.parse(readFileSync(rootPath(STATUS_CONFIG_PATH), "utf8")) as unknown, STATUS_CONFIG_PATH);
|
|
const status = requiredRecord(parsed.status, `${STATUS_CONFIG_PATH}.status`);
|
|
const probesRaw = requiredRecord(status.probes, `${STATUS_CONFIG_PATH}.status.probes`);
|
|
const probes = Object.fromEntries(NODE_RUNTIME_STATUS_PROBES.map((probe) => [probe, requiredBoolean(probesRaw[probe], `${STATUS_CONFIG_PATH}.status.probes.${probe}`)])) as Record<NodeRuntimeStatusProbe, boolean>;
|
|
if (probes["pipeline-run"] && !probes.source) throw new Error(`${STATUS_CONFIG_PATH}.status.probes.source must be true when pipeline-run is enabled`);
|
|
return {
|
|
configPath: STATUS_CONFIG_PATH,
|
|
totalBudgetSeconds: requiredPositiveInteger(status.totalBudgetSeconds, `${STATUS_CONFIG_PATH}.status.totalBudgetSeconds`),
|
|
heartbeatSeconds: requiredPositiveInteger(status.heartbeatSeconds, `${STATUS_CONFIG_PATH}.status.heartbeatSeconds`),
|
|
terminationGraceSeconds: requiredPositiveInteger(status.terminationGraceSeconds, `${STATUS_CONFIG_PATH}.status.terminationGraceSeconds`),
|
|
probes,
|
|
};
|
|
}
|
|
|
|
export function isNodeRuntimeStatusWorker(): boolean {
|
|
return process.env[NODE_RUNTIME_STATUS_WORKER_ENV] === "1";
|
|
}
|
|
|
|
export function emitNodeRuntimeStatusWorkerEvent(event: NodeRuntimeStatusWorkerEvent): void {
|
|
process.stderr.write(`${WORKER_PROGRESS_PREFIX}${JSON.stringify(event)}\n`);
|
|
}
|
|
|
|
export async function runObservedNodeRuntimeStatus(scoped: ScopedStatus): Promise<{ full: Record<string, unknown>; summary: Record<string, unknown> }> {
|
|
const policy = readNodeRuntimeStatusPolicy();
|
|
const explicitBudget = scoped.originalArgs.includes("--timeout-seconds");
|
|
const budgetSeconds = explicitBudget ? scoped.timeoutSeconds : policy.totalBudgetSeconds;
|
|
const startedAtMs = Date.now();
|
|
const startedAt = new Date(startedAtMs).toISOString();
|
|
const tracker = createProbeTracker(policy);
|
|
const abortController = new AbortController();
|
|
let interruptSignal: "SIGINT" | "SIGTERM" | null = null;
|
|
const interrupt = (signal: "SIGINT" | "SIGTERM") => {
|
|
if (interruptSignal !== null) return;
|
|
interruptSignal = signal;
|
|
abortController.abort();
|
|
};
|
|
const onSigint = () => interrupt("SIGINT");
|
|
const onSigterm = () => interrupt("SIGTERM");
|
|
process.on("SIGINT", onSigint);
|
|
process.on("SIGTERM", onSigterm);
|
|
|
|
emitStatusProgress(scoped, {
|
|
state: "started",
|
|
phase: tracker.currentProbe,
|
|
elapsedMs: 0,
|
|
budgetSeconds,
|
|
heartbeatSeconds: policy.heartbeatSeconds,
|
|
configPath: policy.configPath,
|
|
});
|
|
|
|
let stderrBuffer = "";
|
|
let commandResult: CommandResult;
|
|
try {
|
|
commandResult = await runCommandObserved(workerCommand(scoped, budgetSeconds), repoRoot, {
|
|
timeoutMs: budgetSeconds * 1000,
|
|
heartbeatMs: policy.heartbeatSeconds * 1000,
|
|
killAfterMs: policy.terminationGraceSeconds * 1000,
|
|
maxCaptureChars: WORKER_CAPTURE_CHARS,
|
|
signal: abortController.signal,
|
|
env: {
|
|
...process.env,
|
|
[NODE_RUNTIME_STATUS_WORKER_ENV]: "1",
|
|
UNIDESK_CLI_OUTPUT_DUMP_DISABLED: "1",
|
|
},
|
|
onStderr: (text) => {
|
|
stderrBuffer += text;
|
|
stderrBuffer = consumeWorkerProgress(stderrBuffer, (event) => {
|
|
tracker.accept(event);
|
|
emitStatusProgress(scoped, tracker.progress(event.state, budgetSeconds));
|
|
});
|
|
},
|
|
onProgress: (progress) => emitStatusProgress(scoped, tracker.heartbeat(progress, budgetSeconds)),
|
|
});
|
|
} finally {
|
|
process.off("SIGINT", onSigint);
|
|
process.off("SIGTERM", onSigterm);
|
|
}
|
|
|
|
const workerPayload = commandResult.timedOut || commandResult.aborted === true || commandResult.stdoutTruncated === true
|
|
? null
|
|
: parseWorkerPayload(commandResult.stdout);
|
|
const closeout = statusCloseout(commandResult, workerPayload);
|
|
const observation = tracker.observation({
|
|
state: closeout.state,
|
|
reason: closeout.reason,
|
|
signal: interruptSignal,
|
|
startedAt,
|
|
startedAtMs,
|
|
budgetSeconds,
|
|
budgetSource: explicitBudget ? "cli-override" : "yaml",
|
|
policy,
|
|
commandResult,
|
|
});
|
|
emitStatusProgress(scoped, {
|
|
state: "closeout",
|
|
phase: observation.currentPhase,
|
|
elapsedMs: observation.durationMs,
|
|
reason: observation.reason,
|
|
completedProbes: observation.completedProbes,
|
|
unfinishedProbes: observation.unfinishedProbes,
|
|
processGroupCleaned: observation.processGroup.cleaned,
|
|
});
|
|
|
|
if (workerPayload !== null) {
|
|
const summary = { ...record(workerPayload.summary), observation };
|
|
return { full: { ...workerPayload, observation, summary }, summary };
|
|
}
|
|
const summary = partialStatusSummary(scoped, closeout.reason, observation);
|
|
return { full: { ...summary, mode: "node-scoped-runtime-status-closeout", summary }, summary };
|
|
}
|
|
|
|
function workerCommand(scoped: ScopedStatus, budgetSeconds: number): string[] {
|
|
const args: string[] = [];
|
|
for (let index = 0; index < scoped.originalArgs.length; index += 1) {
|
|
const arg = scoped.originalArgs[index] ?? "";
|
|
if (arg === "--raw" || arg === "--full" || arg === "--json") continue;
|
|
if (arg === "--timeout-seconds") {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
args.push(arg);
|
|
}
|
|
return [process.execPath, "scripts/cli.ts", "hwlab", "nodes", "control-plane", ...args, "--timeout-seconds", String(budgetSeconds), "--raw"];
|
|
}
|
|
|
|
function consumeWorkerProgress(buffer: string, onEvent: (event: NodeRuntimeStatusWorkerEvent) => void): string {
|
|
let remaining = buffer;
|
|
while (true) {
|
|
const newline = remaining.indexOf("\n");
|
|
if (newline === -1) return remaining.slice(-8192);
|
|
const line = remaining.slice(0, newline).trimEnd();
|
|
remaining = remaining.slice(newline + 1);
|
|
if (!line.startsWith(WORKER_PROGRESS_PREFIX)) continue;
|
|
try {
|
|
const parsed = JSON.parse(line.slice(WORKER_PROGRESS_PREFIX.length)) as NodeRuntimeStatusWorkerEvent;
|
|
if (NODE_RUNTIME_STATUS_PROBES.includes(parsed.probe) && ["started", "completed", "failed", "skipped"].includes(parsed.state)) onEvent(parsed);
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
function parseWorkerPayload(stdout: string): Record<string, unknown> | null {
|
|
try {
|
|
const envelope = record(JSON.parse(stdout) as unknown);
|
|
const data = record(envelope.data);
|
|
return Object.keys(data).length === 0 ? null : data;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function statusCloseout(result: CommandResult, payload: Record<string, unknown> | null): { state: string; reason: string } {
|
|
if (result.processGroupAliveAfterCleanup === true) return { state: "failed", reason: "status-process-group-cleanup-incomplete" };
|
|
if (result.timedOut) return { state: "timed-out", reason: "status-total-budget-exceeded" };
|
|
if (result.aborted === true) return { state: "interrupted", reason: "status-interrupted" };
|
|
if (payload === null) return { state: "failed", reason: "status-worker-failed" };
|
|
return { state: "completed", reason: "status-collected" };
|
|
}
|
|
|
|
function partialStatusSummary(scoped: ScopedStatus, reason: string, observation: Record<string, unknown>): Record<string, unknown> {
|
|
const status = `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`;
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
mode: "node-scoped-runtime-status-summary",
|
|
mutation: false,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
sourceCommit: null,
|
|
degradedReason: reason,
|
|
pipelineRun: {},
|
|
argo: {},
|
|
runtime: {},
|
|
publicProbe: {},
|
|
gitMirror: {},
|
|
observation,
|
|
nextAction: `${status} --json`,
|
|
next: { status, json: `${status} --json`, full: `${status} --full`, raw: `${status} --raw` },
|
|
};
|
|
}
|
|
|
|
function createProbeTracker(policy: NodeRuntimeStatusPolicy) {
|
|
const started = new Map<NodeRuntimeStatusProbe, number>();
|
|
const completed: NodeRuntimeStatusProbe[] = [];
|
|
const failed: NodeRuntimeStatusProbe[] = [];
|
|
const skipped: NodeRuntimeStatusProbe[] = [];
|
|
const timings: Record<string, unknown>[] = [];
|
|
let currentProbe: string = "starting";
|
|
const enabled = NODE_RUNTIME_STATUS_PROBES.filter((probe) => policy.probes[probe]);
|
|
const accept = (event: NodeRuntimeStatusWorkerEvent) => {
|
|
currentProbe = event.probe;
|
|
if (event.state === "started") started.set(event.probe, Date.now());
|
|
if (event.state === "completed" || event.state === "failed" || event.state === "skipped") {
|
|
const destination = event.state === "completed" ? completed : event.state === "failed" ? failed : skipped;
|
|
if (!destination.includes(event.probe)) destination.push(event.probe);
|
|
timings.push({ probe: event.probe, state: event.state, durationMs: event.durationMs ?? Math.max(0, Date.now() - (started.get(event.probe) ?? Date.now())), reason: event.reason ?? null });
|
|
}
|
|
};
|
|
const unfinished = () => enabled.filter((probe) => !completed.includes(probe) && !failed.includes(probe) && !skipped.includes(probe));
|
|
return {
|
|
get currentProbe() { return currentProbe; },
|
|
accept,
|
|
progress: (state: string, budgetSeconds: number) => ({ state: `probe-${state}`, phase: currentProbe, elapsedMs: null, budgetSeconds, completedProbes: [...completed], unfinishedProbes: unfinished() }),
|
|
heartbeat: (progress: CommandProgress, budgetSeconds: number) => ({ state: "heartbeat", phase: currentProbe, elapsedMs: progress.elapsedMs, budgetSeconds, completedProbes: [...completed], unfinishedProbes: unfinished(), stdoutBytes: progress.stdoutBytes, stderrBytes: progress.stderrBytes }),
|
|
observation: (input: { state: string; reason: string; signal: string | null; startedAt: string; startedAtMs: number; budgetSeconds: number; budgetSource: string; policy: NodeRuntimeStatusPolicy; commandResult: CommandResult }) => ({
|
|
state: input.state,
|
|
reason: input.reason,
|
|
signal: input.signal,
|
|
startedAt: input.startedAt,
|
|
finishedAt: new Date().toISOString(),
|
|
durationMs: Math.max(0, Date.now() - input.startedAtMs),
|
|
currentPhase: currentProbe,
|
|
totalBudgetSeconds: input.budgetSeconds,
|
|
budgetSource: input.budgetSource,
|
|
heartbeatSeconds: input.policy.heartbeatSeconds,
|
|
configPath: input.policy.configPath,
|
|
completedProbes: [...completed],
|
|
failedProbes: [...failed],
|
|
skippedProbes: [...skipped],
|
|
unfinishedProbes: unfinished(),
|
|
phaseTimings: timings.slice(0, NODE_RUNTIME_STATUS_PROBES.length),
|
|
worker: {
|
|
exitCode: input.commandResult.exitCode,
|
|
signal: input.commandResult.signal,
|
|
timedOut: input.commandResult.timedOut,
|
|
aborted: input.commandResult.aborted === true,
|
|
stdoutBytes: input.commandResult.stdoutBytes ?? 0,
|
|
stderrBytes: input.commandResult.stderrBytes ?? 0,
|
|
stdoutTruncated: input.commandResult.stdoutTruncated === true,
|
|
stderrTruncated: input.commandResult.stderrTruncated === true,
|
|
...nodeRuntimeStatusWorkerDiagnostics(input.commandResult, input.reason === "status-worker-failed"),
|
|
},
|
|
processGroup: {
|
|
id: input.commandResult.processGroupId ?? null,
|
|
isolated: process.platform !== "win32",
|
|
terminationReason: input.commandResult.terminationReason ?? null,
|
|
cleaned: input.commandResult.terminationReason === null || input.commandResult.processGroupAliveAfterCleanup === null
|
|
? null
|
|
: input.commandResult.processGroupAliveAfterCleanup === false,
|
|
},
|
|
}),
|
|
};
|
|
}
|
|
|
|
export function runNodeRuntimeStatusProbe<T>(
|
|
probe: NodeRuntimeStatusProbe,
|
|
policy: NodeRuntimeStatusPolicy,
|
|
onProgress: ((event: NodeRuntimeStatusWorkerEvent) => void) | undefined,
|
|
run: () => T,
|
|
skipped: () => T,
|
|
): T {
|
|
const startedAt = Date.now();
|
|
if (!policy.probes[probe]) {
|
|
onProgress?.({ probe, state: "skipped", at: new Date(startedAt).toISOString(), durationMs: 0, reason: "disabled-by-yaml" });
|
|
return skipped();
|
|
}
|
|
onProgress?.({ probe, state: "started", at: new Date(startedAt).toISOString() });
|
|
try {
|
|
const value = run();
|
|
onProgress?.({ probe, state: "completed", at: new Date().toISOString(), durationMs: Date.now() - startedAt });
|
|
return value;
|
|
} catch (error) {
|
|
onProgress?.({ probe, state: "failed", at: new Date().toISOString(), durationMs: Date.now() - startedAt, reason: error instanceof Error ? error.message.slice(0, 300) : String(error).slice(0, 300) });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export function skippedNodeRuntimeStatusCommand(name: string): CommandResult {
|
|
return {
|
|
command: ["<skipped>", name],
|
|
cwd: repoRoot,
|
|
exitCode: 0,
|
|
stdout: "",
|
|
stderr: "",
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeStatusWorkerDiagnostics(result: CommandResult, includeStdoutTail: boolean): Record<string, unknown> {
|
|
return {
|
|
stderrTail: statusDiagnosticTail(result.stderr, WORKER_STDERR_TAIL_CHARS, true),
|
|
stdoutTail: includeStdoutTail ? statusDiagnosticTail(result.stdout, WORKER_STDOUT_TAIL_CHARS, false) : null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function statusDiagnosticTail(text: string, maxChars: number, filterProgress: boolean): string | null {
|
|
const filtered = filterProgress
|
|
? text.split(/\r?\n/u).filter((line) => !line.trimStart().startsWith(WORKER_PROGRESS_PREFIX)).join("\n")
|
|
: text;
|
|
const redacted = redactStatusDiagnostic(filtered).trim();
|
|
return redacted.length === 0 ? null : redacted.slice(-maxChars);
|
|
}
|
|
|
|
function redactStatusDiagnostic(text: string): string {
|
|
return text
|
|
.replace(/-----BEGIN [^-\r\n]*PRIVATE KEY-----[\s\S]*?-----END [^-\r\n]*PRIVATE KEY-----/giu, "<private-key:redacted>")
|
|
.replace(/((?:postgres(?:ql)?|https?|ssh):\/\/)[^@\s"']+@/giu, "$1<redacted>@")
|
|
.replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+\/=:-]+/giu, "$1 <redacted>")
|
|
.replace(/\b(?:sk-[A-Za-z0-9_-]{16,}|(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9_]{16,})\b/gu, "<credential:redacted>")
|
|
.replace(/([?&](?:access[_-]?token|api[_-]?key|apikey|authorization|cookie|password|secret|token)=)[^&\s"']+/giu, "$1<redacted>")
|
|
.replace(/(["']?(?:access[_-]?token|api[_-]?key|apikey|authorization|cookie|database[_-]?url|jwt[_-]?secret|n8n_encryption_key|passwd|password|private[_-]?key|secret|token)["']?\s*[:=]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,}\r\n]+)/giu, "$1<redacted>");
|
|
}
|
|
|
|
function emitStatusProgress(scoped: ScopedStatus, detail: Record<string, unknown>): void {
|
|
process.stderr.write(`${JSON.stringify({ event: "hwlab.control-plane.status.progress", at: new Date().toISOString(), node: scoped.node, lane: scoped.lane, ...detail })}\n`);
|
|
}
|
|
|
|
function requiredRecord(value: unknown, path: string): Record<string, unknown> {
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function requiredPositiveInteger(value: unknown, path: string): number {
|
|
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
function requiredBoolean(value: unknown, path: string): boolean {
|
|
if (typeof value !== "boolean") throw new Error(`${path} must be a boolean`);
|
|
return value;
|
|
}
|