fix: 修复 control-plane status 长时无输出
This commit is contained in:
Codex
+54
-10
@@ -14,6 +14,10 @@ export interface CommandResult {
|
||||
stderrBytes?: number;
|
||||
stdoutTruncated?: boolean;
|
||||
stderrTruncated?: boolean;
|
||||
aborted?: boolean;
|
||||
terminationReason?: "timeout" | "abort" | null;
|
||||
processGroupId?: number | null;
|
||||
processGroupAliveAfterCleanup?: boolean | null;
|
||||
}
|
||||
|
||||
export interface CommandProgress {
|
||||
@@ -25,13 +29,15 @@ export interface CommandProgress {
|
||||
lastOutputAgeMs: number | null;
|
||||
}
|
||||
|
||||
interface ObservedCommandOptions {
|
||||
export interface ObservedCommandOptions {
|
||||
timeoutMs?: number;
|
||||
heartbeatMs?: number;
|
||||
killAfterMs?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
input?: string;
|
||||
maxCaptureChars?: number;
|
||||
signal?: AbortSignal;
|
||||
onStderr?: (text: string) => void;
|
||||
onProgress?: (progress: CommandProgress) => void;
|
||||
}
|
||||
|
||||
@@ -70,9 +76,12 @@ export async function runCommandObserved(command: string[], cwd: string, options
|
||||
let stderrTruncated = false;
|
||||
let lastOutputAt: number | null = null;
|
||||
let timedOut = false;
|
||||
let aborted = false;
|
||||
let terminationReason: "timeout" | "abort" | null = null;
|
||||
let timeoutTimer: NodeJS.Timeout | undefined;
|
||||
let killTimer: NodeJS.Timeout | undefined;
|
||||
let heartbeatTimer: NodeJS.Timeout | undefined;
|
||||
let abortListener: (() => void) | undefined;
|
||||
|
||||
const child = spawn(command[0], command.slice(1), {
|
||||
cwd,
|
||||
@@ -91,6 +100,7 @@ export async function runCommandObserved(command: string[], cwd: string, options
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
const text = chunk.toString();
|
||||
options.onStderr?.(text);
|
||||
stderrBytes += Buffer.byteLength(text);
|
||||
lastOutputAt = Date.now();
|
||||
const captured = appendBounded(stderr, text, maxCaptureChars);
|
||||
@@ -113,14 +123,23 @@ export async function runCommandObserved(command: string[], cwd: string, options
|
||||
}, heartbeatMs);
|
||||
}
|
||||
|
||||
if (timeoutMs !== undefined) {
|
||||
timeoutTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
killProcessTree(child, "SIGTERM");
|
||||
killTimer = setTimeout(() => {
|
||||
killProcessTree(child, "SIGKILL");
|
||||
}, Math.max(100, options.killAfterMs ?? 5_000));
|
||||
}, timeoutMs);
|
||||
const terminate = (reason: "timeout" | "abort") => {
|
||||
if (terminationReason !== null) return;
|
||||
terminationReason = reason;
|
||||
timedOut = reason === "timeout";
|
||||
aborted = reason === "abort";
|
||||
killProcessTree(child, "SIGTERM");
|
||||
killTimer = setTimeout(() => {
|
||||
killProcessTree(child, "SIGKILL");
|
||||
killTimer = undefined;
|
||||
}, Math.max(100, options.killAfterMs ?? 5_000));
|
||||
};
|
||||
|
||||
if (timeoutMs !== undefined) timeoutTimer = setTimeout(() => terminate("timeout"), timeoutMs);
|
||||
if (options.signal !== undefined) {
|
||||
abortListener = () => terminate("abort");
|
||||
options.signal.addEventListener("abort", abortListener, { once: true });
|
||||
if (options.signal.aborted) abortListener();
|
||||
}
|
||||
|
||||
const completion = await new Promise<{ exitCode: number | null; signal: NodeJS.Signals | null; error?: Error }>((resolve) => {
|
||||
@@ -135,8 +154,20 @@ export async function runCommandObserved(command: string[], cwd: string, options
|
||||
});
|
||||
|
||||
if (timeoutTimer !== undefined) clearTimeout(timeoutTimer);
|
||||
if (killTimer !== undefined) clearTimeout(killTimer);
|
||||
if (killTimer !== undefined) {
|
||||
clearTimeout(killTimer);
|
||||
killTimer = undefined;
|
||||
}
|
||||
if (heartbeatTimer !== undefined) clearInterval(heartbeatTimer);
|
||||
if (abortListener !== undefined) options.signal?.removeEventListener("abort", abortListener);
|
||||
|
||||
const processGroupId = process.platform === "win32" ? null : child.pid ?? null;
|
||||
let processGroupAliveAfterCleanup: boolean | null = null;
|
||||
if (terminationReason !== null && processGroupId !== null) {
|
||||
killProcessTree(child, "SIGKILL");
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
processGroupAliveAfterCleanup = processGroupExists(processGroupId);
|
||||
}
|
||||
|
||||
if (completion.error !== undefined && stderr.length === 0) {
|
||||
stderr = completion.error.message;
|
||||
@@ -156,9 +187,22 @@ export async function runCommandObserved(command: string[], cwd: string, options
|
||||
stderrBytes,
|
||||
stdoutTruncated,
|
||||
stderrTruncated,
|
||||
aborted,
|
||||
terminationReason,
|
||||
processGroupId,
|
||||
processGroupAliveAfterCleanup,
|
||||
};
|
||||
}
|
||||
|
||||
function processGroupExists(processGroupId: number): boolean {
|
||||
try {
|
||||
process.kill(-processGroupId, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function appendBounded(current: string, next: string, maxChars: number): { text: string; truncated: boolean } {
|
||||
const combined = `${current}${next}`;
|
||||
if (combined.length <= maxChars) return { text: combined, truncated: false };
|
||||
|
||||
@@ -12,6 +12,7 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
examples: [
|
||||
"bun scripts/cli.ts hwlab nodes control-plane infra plan --node <node> --lane <lane>",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane status --node <node> --lane <lane>",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane status --node <node> --lane <lane> --json",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane source-workspace sync --node JD01 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node JD01 --lane v03 --min-age-minutes 30 --limit 200 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane cleanup-released-pvs --node JD01 --lane v03 --limit 200 --dry-run",
|
||||
@@ -43,6 +44,7 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
"cleanup-legacy-docker-images is a transitional legacy-cache GC for Docker images matching YAML allowlisted repositories; it protects container-referenced images, does not run prune, and does not touch Docker volumes.",
|
||||
"cleanup-legacy-docker-registry-volume removes only an exited legacy Docker registry container and its unique /var/lib/registry volume after the YAML-declared k8s node-local-registry is ready.",
|
||||
"`trigger-current --confirm --wait` is the one-command CICD path for current node/lane runtime publish.",
|
||||
"control-plane status reads its total budget, heartbeat cadence, termination grace period, and probe switches from config/hwlab-node-control-plane.yaml; --json keeps stdout as one compact JSON document while progress remains on stderr.",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { runCommandObserved, type CommandProgress, type CommandResult } from "../command";
|
||||
import { repoRoot, rootPath } from "../config";
|
||||
import type { parseNodeScopedDelegatedOptions } from "./plan";
|
||||
import { record } from "./utils";
|
||||
|
||||
export const NODE_RUNTIME_STATUS_WORKER_ENV = "UNIDESK_HWLAB_CONTROL_PLANE_STATUS_WORKER";
|
||||
const WORKER_PROGRESS_PREFIX = "UNIDESK_HWLAB_CONTROL_PLANE_STATUS_PROGRESS ";
|
||||
const STATUS_CONFIG_PATH = "config/hwlab-node-control-plane.yaml";
|
||||
const WORKER_CAPTURE_CHARS = 2 * 1024 * 1024;
|
||||
const WORKER_STDERR_TAIL_CHARS = 1600;
|
||||
const WORKER_STDOUT_TAIL_CHARS = 1200;
|
||||
|
||||
export const NODE_RUNTIME_STATUS_PROBES = [
|
||||
"source",
|
||||
"control-plane",
|
||||
"pipeline-run",
|
||||
"argo",
|
||||
"runtime",
|
||||
"public",
|
||||
"git-mirror",
|
||||
] as const;
|
||||
|
||||
export type NodeRuntimeStatusProbe = typeof NODE_RUNTIME_STATUS_PROBES[number];
|
||||
export type NodeRuntimeStatusProbeState = "started" | "completed" | "failed" | "skipped";
|
||||
|
||||
export interface NodeRuntimeStatusWorkerEvent {
|
||||
probe: NodeRuntimeStatusProbe;
|
||||
state: NodeRuntimeStatusProbeState;
|
||||
at: string;
|
||||
durationMs?: number;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface NodeRuntimeStatusPolicy {
|
||||
configPath: string;
|
||||
totalBudgetSeconds: number;
|
||||
heartbeatSeconds: number;
|
||||
terminationGraceSeconds: number;
|
||||
probes: Record<NodeRuntimeStatusProbe, boolean>;
|
||||
}
|
||||
|
||||
type ScopedStatus = ReturnType<typeof parseNodeScopedDelegatedOptions>;
|
||||
|
||||
export function readNodeRuntimeStatusPolicy(): NodeRuntimeStatusPolicy {
|
||||
const parsed = requiredRecord(Bun.YAML.parse(readFileSync(rootPath(STATUS_CONFIG_PATH), "utf8")) as unknown, STATUS_CONFIG_PATH);
|
||||
const status = requiredRecord(parsed.status, `${STATUS_CONFIG_PATH}.status`);
|
||||
const probesRaw = requiredRecord(status.probes, `${STATUS_CONFIG_PATH}.status.probes`);
|
||||
const probes = Object.fromEntries(NODE_RUNTIME_STATUS_PROBES.map((probe) => [probe, requiredBoolean(probesRaw[probe], `${STATUS_CONFIG_PATH}.status.probes.${probe}`)])) as Record<NodeRuntimeStatusProbe, boolean>;
|
||||
if (probes["pipeline-run"] && !probes.source) throw new Error(`${STATUS_CONFIG_PATH}.status.probes.source must be true when pipeline-run is enabled`);
|
||||
return {
|
||||
configPath: STATUS_CONFIG_PATH,
|
||||
totalBudgetSeconds: requiredPositiveInteger(status.totalBudgetSeconds, `${STATUS_CONFIG_PATH}.status.totalBudgetSeconds`),
|
||||
heartbeatSeconds: requiredPositiveInteger(status.heartbeatSeconds, `${STATUS_CONFIG_PATH}.status.heartbeatSeconds`),
|
||||
terminationGraceSeconds: requiredPositiveInteger(status.terminationGraceSeconds, `${STATUS_CONFIG_PATH}.status.terminationGraceSeconds`),
|
||||
probes,
|
||||
};
|
||||
}
|
||||
|
||||
export function isNodeRuntimeStatusWorker(): boolean {
|
||||
return process.env[NODE_RUNTIME_STATUS_WORKER_ENV] === "1";
|
||||
}
|
||||
|
||||
export function emitNodeRuntimeStatusWorkerEvent(event: NodeRuntimeStatusWorkerEvent): void {
|
||||
process.stderr.write(`${WORKER_PROGRESS_PREFIX}${JSON.stringify(event)}\n`);
|
||||
}
|
||||
|
||||
export async function runObservedNodeRuntimeStatus(scoped: ScopedStatus): Promise<{ full: Record<string, unknown>; summary: Record<string, unknown> }> {
|
||||
const policy = readNodeRuntimeStatusPolicy();
|
||||
const explicitBudget = scoped.originalArgs.includes("--timeout-seconds");
|
||||
const budgetSeconds = explicitBudget ? scoped.timeoutSeconds : policy.totalBudgetSeconds;
|
||||
const startedAtMs = Date.now();
|
||||
const startedAt = new Date(startedAtMs).toISOString();
|
||||
const tracker = createProbeTracker(policy);
|
||||
const abortController = new AbortController();
|
||||
let interruptSignal: "SIGINT" | "SIGTERM" | null = null;
|
||||
const interrupt = (signal: "SIGINT" | "SIGTERM") => {
|
||||
if (interruptSignal !== null) return;
|
||||
interruptSignal = signal;
|
||||
abortController.abort();
|
||||
};
|
||||
const onSigint = () => interrupt("SIGINT");
|
||||
const onSigterm = () => interrupt("SIGTERM");
|
||||
process.on("SIGINT", onSigint);
|
||||
process.on("SIGTERM", onSigterm);
|
||||
|
||||
emitStatusProgress(scoped, {
|
||||
state: "started",
|
||||
phase: tracker.currentProbe,
|
||||
elapsedMs: 0,
|
||||
budgetSeconds,
|
||||
heartbeatSeconds: policy.heartbeatSeconds,
|
||||
configPath: policy.configPath,
|
||||
});
|
||||
|
||||
let stderrBuffer = "";
|
||||
let commandResult: CommandResult;
|
||||
try {
|
||||
commandResult = await runCommandObserved(workerCommand(scoped, budgetSeconds), repoRoot, {
|
||||
timeoutMs: budgetSeconds * 1000,
|
||||
heartbeatMs: policy.heartbeatSeconds * 1000,
|
||||
killAfterMs: policy.terminationGraceSeconds * 1000,
|
||||
maxCaptureChars: WORKER_CAPTURE_CHARS,
|
||||
signal: abortController.signal,
|
||||
env: {
|
||||
...process.env,
|
||||
[NODE_RUNTIME_STATUS_WORKER_ENV]: "1",
|
||||
UNIDESK_CLI_OUTPUT_DUMP_DISABLED: "1",
|
||||
},
|
||||
onStderr: (text) => {
|
||||
stderrBuffer += text;
|
||||
stderrBuffer = consumeWorkerProgress(stderrBuffer, (event) => {
|
||||
tracker.accept(event);
|
||||
emitStatusProgress(scoped, tracker.progress(event.state, budgetSeconds));
|
||||
});
|
||||
},
|
||||
onProgress: (progress) => emitStatusProgress(scoped, tracker.heartbeat(progress, budgetSeconds)),
|
||||
});
|
||||
} finally {
|
||||
process.off("SIGINT", onSigint);
|
||||
process.off("SIGTERM", onSigterm);
|
||||
}
|
||||
|
||||
const workerPayload = commandResult.timedOut || commandResult.aborted === true || commandResult.stdoutTruncated === true
|
||||
? null
|
||||
: parseWorkerPayload(commandResult.stdout);
|
||||
const closeout = statusCloseout(commandResult, workerPayload);
|
||||
const observation = tracker.observation({
|
||||
state: closeout.state,
|
||||
reason: closeout.reason,
|
||||
signal: interruptSignal,
|
||||
startedAt,
|
||||
startedAtMs,
|
||||
budgetSeconds,
|
||||
budgetSource: explicitBudget ? "cli-override" : "yaml",
|
||||
policy,
|
||||
commandResult,
|
||||
});
|
||||
emitStatusProgress(scoped, {
|
||||
state: "closeout",
|
||||
phase: observation.currentPhase,
|
||||
elapsedMs: observation.durationMs,
|
||||
reason: observation.reason,
|
||||
completedProbes: observation.completedProbes,
|
||||
unfinishedProbes: observation.unfinishedProbes,
|
||||
processGroupCleaned: observation.processGroup.cleaned,
|
||||
});
|
||||
|
||||
if (workerPayload !== null) {
|
||||
const summary = { ...record(workerPayload.summary), observation };
|
||||
return { full: { ...workerPayload, observation, summary }, summary };
|
||||
}
|
||||
const summary = partialStatusSummary(scoped, closeout.reason, observation);
|
||||
return { full: { ...summary, mode: "node-scoped-runtime-status-closeout", summary }, summary };
|
||||
}
|
||||
|
||||
function workerCommand(scoped: ScopedStatus, budgetSeconds: number): string[] {
|
||||
const args: string[] = [];
|
||||
for (let index = 0; index < scoped.originalArgs.length; index += 1) {
|
||||
const arg = scoped.originalArgs[index] ?? "";
|
||||
if (arg === "--raw" || arg === "--full" || arg === "--json") continue;
|
||||
if (arg === "--timeout-seconds") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
args.push(arg);
|
||||
}
|
||||
return [process.execPath, "scripts/cli.ts", "hwlab", "nodes", "control-plane", ...args, "--timeout-seconds", String(budgetSeconds), "--raw"];
|
||||
}
|
||||
|
||||
function consumeWorkerProgress(buffer: string, onEvent: (event: NodeRuntimeStatusWorkerEvent) => void): string {
|
||||
let remaining = buffer;
|
||||
while (true) {
|
||||
const newline = remaining.indexOf("\n");
|
||||
if (newline === -1) return remaining.slice(-8192);
|
||||
const line = remaining.slice(0, newline).trimEnd();
|
||||
remaining = remaining.slice(newline + 1);
|
||||
if (!line.startsWith(WORKER_PROGRESS_PREFIX)) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line.slice(WORKER_PROGRESS_PREFIX.length)) as NodeRuntimeStatusWorkerEvent;
|
||||
if (NODE_RUNTIME_STATUS_PROBES.includes(parsed.probe) && ["started", "completed", "failed", "skipped"].includes(parsed.state)) onEvent(parsed);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseWorkerPayload(stdout: string): Record<string, unknown> | null {
|
||||
try {
|
||||
const envelope = record(JSON.parse(stdout) as unknown);
|
||||
const data = record(envelope.data);
|
||||
return Object.keys(data).length === 0 ? null : data;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function statusCloseout(result: CommandResult, payload: Record<string, unknown> | null): { state: string; reason: string } {
|
||||
if (result.processGroupAliveAfterCleanup === true) return { state: "failed", reason: "status-process-group-cleanup-incomplete" };
|
||||
if (result.timedOut) return { state: "timed-out", reason: "status-total-budget-exceeded" };
|
||||
if (result.aborted === true) return { state: "interrupted", reason: "status-interrupted" };
|
||||
if (payload === null) return { state: "failed", reason: "status-worker-failed" };
|
||||
return { state: "completed", reason: "status-collected" };
|
||||
}
|
||||
|
||||
function partialStatusSummary(scoped: ScopedStatus, reason: string, observation: Record<string, unknown>): Record<string, unknown> {
|
||||
const status = `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`;
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
mode: "node-scoped-runtime-status-summary",
|
||||
mutation: false,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
sourceCommit: null,
|
||||
degradedReason: reason,
|
||||
pipelineRun: {},
|
||||
argo: {},
|
||||
runtime: {},
|
||||
publicProbe: {},
|
||||
gitMirror: {},
|
||||
observation,
|
||||
nextAction: `${status} --json`,
|
||||
next: { status, json: `${status} --json`, full: `${status} --full`, raw: `${status} --raw` },
|
||||
};
|
||||
}
|
||||
|
||||
function createProbeTracker(policy: NodeRuntimeStatusPolicy) {
|
||||
const started = new Map<NodeRuntimeStatusProbe, number>();
|
||||
const completed: NodeRuntimeStatusProbe[] = [];
|
||||
const failed: NodeRuntimeStatusProbe[] = [];
|
||||
const skipped: NodeRuntimeStatusProbe[] = [];
|
||||
const timings: Record<string, unknown>[] = [];
|
||||
let currentProbe: string = "starting";
|
||||
const enabled = NODE_RUNTIME_STATUS_PROBES.filter((probe) => policy.probes[probe]);
|
||||
const accept = (event: NodeRuntimeStatusWorkerEvent) => {
|
||||
currentProbe = event.probe;
|
||||
if (event.state === "started") started.set(event.probe, Date.now());
|
||||
if (event.state === "completed" || event.state === "failed" || event.state === "skipped") {
|
||||
const destination = event.state === "completed" ? completed : event.state === "failed" ? failed : skipped;
|
||||
if (!destination.includes(event.probe)) destination.push(event.probe);
|
||||
timings.push({ probe: event.probe, state: event.state, durationMs: event.durationMs ?? Math.max(0, Date.now() - (started.get(event.probe) ?? Date.now())), reason: event.reason ?? null });
|
||||
}
|
||||
};
|
||||
const unfinished = () => enabled.filter((probe) => !completed.includes(probe) && !failed.includes(probe) && !skipped.includes(probe));
|
||||
return {
|
||||
get currentProbe() { return currentProbe; },
|
||||
accept,
|
||||
progress: (state: string, budgetSeconds: number) => ({ state: `probe-${state}`, phase: currentProbe, elapsedMs: null, budgetSeconds, completedProbes: [...completed], unfinishedProbes: unfinished() }),
|
||||
heartbeat: (progress: CommandProgress, budgetSeconds: number) => ({ state: "heartbeat", phase: currentProbe, elapsedMs: progress.elapsedMs, budgetSeconds, completedProbes: [...completed], unfinishedProbes: unfinished(), stdoutBytes: progress.stdoutBytes, stderrBytes: progress.stderrBytes }),
|
||||
observation: (input: { state: string; reason: string; signal: string | null; startedAt: string; startedAtMs: number; budgetSeconds: number; budgetSource: string; policy: NodeRuntimeStatusPolicy; commandResult: CommandResult }) => ({
|
||||
state: input.state,
|
||||
reason: input.reason,
|
||||
signal: input.signal,
|
||||
startedAt: input.startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
durationMs: Math.max(0, Date.now() - input.startedAtMs),
|
||||
currentPhase: currentProbe,
|
||||
totalBudgetSeconds: input.budgetSeconds,
|
||||
budgetSource: input.budgetSource,
|
||||
heartbeatSeconds: input.policy.heartbeatSeconds,
|
||||
configPath: input.policy.configPath,
|
||||
completedProbes: [...completed],
|
||||
failedProbes: [...failed],
|
||||
skippedProbes: [...skipped],
|
||||
unfinishedProbes: unfinished(),
|
||||
phaseTimings: timings.slice(0, NODE_RUNTIME_STATUS_PROBES.length),
|
||||
worker: {
|
||||
exitCode: input.commandResult.exitCode,
|
||||
signal: input.commandResult.signal,
|
||||
timedOut: input.commandResult.timedOut,
|
||||
aborted: input.commandResult.aborted === true,
|
||||
stdoutBytes: input.commandResult.stdoutBytes ?? 0,
|
||||
stderrBytes: input.commandResult.stderrBytes ?? 0,
|
||||
stdoutTruncated: input.commandResult.stdoutTruncated === true,
|
||||
stderrTruncated: input.commandResult.stderrTruncated === true,
|
||||
...nodeRuntimeStatusWorkerDiagnostics(input.commandResult, input.reason === "status-worker-failed"),
|
||||
},
|
||||
processGroup: {
|
||||
id: input.commandResult.processGroupId ?? null,
|
||||
isolated: process.platform !== "win32",
|
||||
terminationReason: input.commandResult.terminationReason ?? null,
|
||||
cleaned: input.commandResult.terminationReason === null || input.commandResult.processGroupAliveAfterCleanup === null
|
||||
? null
|
||||
: input.commandResult.processGroupAliveAfterCleanup === false,
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function runNodeRuntimeStatusProbe<T>(
|
||||
probe: NodeRuntimeStatusProbe,
|
||||
policy: NodeRuntimeStatusPolicy,
|
||||
onProgress: ((event: NodeRuntimeStatusWorkerEvent) => void) | undefined,
|
||||
run: () => T,
|
||||
skipped: () => T,
|
||||
): T {
|
||||
const startedAt = Date.now();
|
||||
if (!policy.probes[probe]) {
|
||||
onProgress?.({ probe, state: "skipped", at: new Date(startedAt).toISOString(), durationMs: 0, reason: "disabled-by-yaml" });
|
||||
return skipped();
|
||||
}
|
||||
onProgress?.({ probe, state: "started", at: new Date(startedAt).toISOString() });
|
||||
try {
|
||||
const value = run();
|
||||
onProgress?.({ probe, state: "completed", at: new Date().toISOString(), durationMs: Date.now() - startedAt });
|
||||
return value;
|
||||
} catch (error) {
|
||||
onProgress?.({ probe, state: "failed", at: new Date().toISOString(), durationMs: Date.now() - startedAt, reason: error instanceof Error ? error.message.slice(0, 300) : String(error).slice(0, 300) });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function skippedNodeRuntimeStatusCommand(name: string): CommandResult {
|
||||
return {
|
||||
command: ["<skipped>", name],
|
||||
cwd: repoRoot,
|
||||
exitCode: 0,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeRuntimeStatusWorkerDiagnostics(result: CommandResult, includeStdoutTail: boolean): Record<string, unknown> {
|
||||
return {
|
||||
stderrTail: statusDiagnosticTail(result.stderr, WORKER_STDERR_TAIL_CHARS, true),
|
||||
stdoutTail: includeStdoutTail ? statusDiagnosticTail(result.stdout, WORKER_STDOUT_TAIL_CHARS, false) : null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function statusDiagnosticTail(text: string, maxChars: number, filterProgress: boolean): string | null {
|
||||
const filtered = filterProgress
|
||||
? text.split(/\r?\n/u).filter((line) => !line.trimStart().startsWith(WORKER_PROGRESS_PREFIX)).join("\n")
|
||||
: text;
|
||||
const redacted = redactStatusDiagnostic(filtered).trim();
|
||||
return redacted.length === 0 ? null : redacted.slice(-maxChars);
|
||||
}
|
||||
|
||||
function redactStatusDiagnostic(text: string): string {
|
||||
return text
|
||||
.replace(/-----BEGIN [^-\r\n]*PRIVATE KEY-----[\s\S]*?-----END [^-\r\n]*PRIVATE KEY-----/giu, "<private-key:redacted>")
|
||||
.replace(/((?:postgres(?:ql)?|https?|ssh):\/\/)[^@\s"']+@/giu, "$1<redacted>@")
|
||||
.replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+\/=:-]+/giu, "$1 <redacted>")
|
||||
.replace(/\b(?:sk-[A-Za-z0-9_-]{16,}|(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9_]{16,})\b/gu, "<credential:redacted>")
|
||||
.replace(/([?&](?:access[_-]?token|api[_-]?key|apikey|authorization|cookie|password|secret|token)=)[^&\s"']+/giu, "$1<redacted>")
|
||||
.replace(/(["']?(?:access[_-]?token|api[_-]?key|apikey|authorization|cookie|database[_-]?url|jwt[_-]?secret|n8n_encryption_key|passwd|password|private[_-]?key|secret|token)["']?\s*[:=]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,}\r\n]+)/giu, "$1<redacted>");
|
||||
}
|
||||
|
||||
function emitStatusProgress(scoped: ScopedStatus, detail: Record<string, unknown>): void {
|
||||
process.stderr.write(`${JSON.stringify({ event: "hwlab.control-plane.status.progress", at: new Date().toISOString(), node: scoped.node, lane: scoped.lane, ...detail })}\n`);
|
||||
}
|
||||
|
||||
function requiredRecord(value: unknown, path: string): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function requiredPositiveInteger(value: unknown, path: string): number {
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredBoolean(value: unknown, path: string): boolean {
|
||||
if (typeof value !== "boolean") throw new Error(`${path} must be a boolean`);
|
||||
return value;
|
||||
}
|
||||
@@ -42,6 +42,7 @@ import { runNodeEndpointBridge } from "./secret-scripts";
|
||||
import { nodeRuntimeSourceWorkspaceCommand } from "./source-workspace";
|
||||
import { nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeScopedFullOutput, withNodeRuntimeGitMirrorRendered } from "./status";
|
||||
import { assertNodeId, positiveIntegerOption, requiredOption } from "./utils";
|
||||
import { emitNodeRuntimeStatusWorkerEvent, isNodeRuntimeStatusWorker, readNodeRuntimeStatusPolicy, runObservedNodeRuntimeStatus } from "./control-plane-status-observed";
|
||||
import { legacyHwlabNodeWebProbeUnsupported } from "../web-probe";
|
||||
import { startNodeDelegatedJob } from "./web-probe";
|
||||
import { assertKnownOptions } from "./web-probe-observe";
|
||||
@@ -639,10 +640,17 @@ export async function runNodeDelegatedDomain(config: Config, domain: DelegatedNo
|
||||
}
|
||||
if (domain === "control-plane") {
|
||||
if (scoped.action === "status") {
|
||||
const result = nodeRuntimeControlPlaneStatus(scoped);
|
||||
if (scoped.originalArgs.includes("--raw")) return result;
|
||||
if (scoped.originalArgs.includes("--full")) return withNodeRuntimeControlPlaneStatusFullRendered(result, scoped);
|
||||
return withNodeRuntimeControlPlaneStatusRendered(result, scoped);
|
||||
if (isNodeRuntimeStatusWorker()) {
|
||||
return nodeRuntimeControlPlaneStatus(scoped, {
|
||||
policy: readNodeRuntimeStatusPolicy(),
|
||||
onProgress: emitNodeRuntimeStatusWorkerEvent,
|
||||
});
|
||||
}
|
||||
const observed = await runObservedNodeRuntimeStatus(scoped);
|
||||
if (scoped.originalArgs.includes("--raw")) return observed.full;
|
||||
if (scoped.originalArgs.includes("--json")) return observed.summary;
|
||||
if (scoped.originalArgs.includes("--full")) return withNodeRuntimeControlPlaneStatusFullRendered(observed.full, scoped);
|
||||
return withNodeRuntimeControlPlaneStatusRendered(observed.summary, scoped);
|
||||
}
|
||||
if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration" || scoped.action === "cleanup-runs" || scoped.action === "cleanup-released-pvs" || scoped.action === "cleanup-legacy-docker-images" || scoped.action === "cleanup-legacy-docker-registry-volume") {
|
||||
if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped);
|
||||
|
||||
@@ -774,15 +774,16 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string,
|
||||
webObserveShort(webObserveText(pipelineRun.name), 36),
|
||||
]],
|
||||
),
|
||||
...nodeRuntimeStatusObservationLines(result),
|
||||
"",
|
||||
webObserveTable(
|
||||
["STAGE", "STATUS", "DETAIL"],
|
||||
[
|
||||
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", pipelineDetail],
|
||||
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)} target=${shortValue(argo.targetGitopsRevision)} op=${webObserveText(argoOperation.phase)}`],
|
||||
["runtime", runtime.ready === true ? "ok" : "failed", runtimeDetail],
|
||||
["public", publicProbe.ready === true ? "ok" : "failed", publicProbeDetail],
|
||||
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `snapshot=${webObserveText(gitMirror.sourceSnapshotReady)} pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
||||
["pipeline", pipelineRun.skipped === true ? "skipped" : pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", pipelineDetail],
|
||||
["argo", argo.skipped === true ? "skipped" : argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)} target=${shortValue(argo.targetGitopsRevision)} op=${webObserveText(argoOperation.phase)}`],
|
||||
["runtime", runtime.skipped === true ? "skipped" : runtime.ready === true ? "ok" : "failed", runtimeDetail],
|
||||
["public", publicProbe.skipped === true ? "skipped" : publicProbe.ready === true ? "ok" : "failed", publicProbeDetail],
|
||||
["git-mirror", gitMirror.skipped === true ? "skipped" : gitMirror.ready === true ? "ok" : "failed", `snapshot=${webObserveText(gitMirror.sourceSnapshotReady)} pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
||||
],
|
||||
),
|
||||
...(pendingTaskRunCommand === null && pendingPodCommand === null ? [] : [
|
||||
@@ -807,7 +808,7 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string,
|
||||
` ${result.nextAction ?? next.full ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --full`}`,
|
||||
"",
|
||||
"Disclosure:",
|
||||
" default view is a bounded control-plane status summary; use --full or --raw for the complete JSON payload.",
|
||||
" default view is a bounded control-plane status summary; use --json for compact JSON, --full for bounded drill-down, or --raw for complete JSON.",
|
||||
].join("\n");
|
||||
return { ...result, renderedText, contentType: "text/plain" };
|
||||
}
|
||||
@@ -869,15 +870,16 @@ export function withNodeRuntimeControlPlaneStatusFullRendered(result: Record<str
|
||||
webObserveShort(webObserveText(pipelineRun.name ?? result.pipelineRun), 36),
|
||||
]],
|
||||
),
|
||||
...nodeRuntimeStatusObservationLines(result),
|
||||
"",
|
||||
webObserveTable(
|
||||
["STAGE", "STATUS", "DETAIL"],
|
||||
[
|
||||
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 80)],
|
||||
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)} target=${shortValue(argo.targetGitopsRevision)}`],
|
||||
["runtime", runtimeSummary.ready === true ? "ok" : "failed", `namespace=${webObserveText(runtimeSummary.namespace)} workloads=${webObserveText(runtimeSummary.workloadReady)} pg=${webObserveText(runtimeSummary.externalPostgresReady ?? runtimeSummary.localPostgresReady)}`],
|
||||
["public", publicProbe.ready === true ? "ok" : "failed", publicProbeDetail],
|
||||
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
||||
["pipeline", pipelineRun.skipped === true ? "skipped" : pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 80)],
|
||||
["argo", argo.skipped === true ? "skipped" : argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)} target=${shortValue(argo.targetGitopsRevision)}`],
|
||||
["runtime", runtimeSummary.skipped === true ? "skipped" : runtimeSummary.ready === true ? "ok" : "failed", `namespace=${webObserveText(runtimeSummary.namespace)} workloads=${webObserveText(runtimeSummary.workloadReady)} pg=${webObserveText(runtimeSummary.externalPostgresReady ?? runtimeSummary.localPostgresReady)}`],
|
||||
["public", publicProbe.skipped === true ? "skipped" : publicProbe.ready === true ? "ok" : "failed", publicProbeDetail],
|
||||
["git-mirror", gitMirror.skipped === true ? "skipped" : gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
||||
],
|
||||
),
|
||||
...(pendingTaskRunCommand === null && pendingPodCommand === null ? [] : [
|
||||
@@ -971,6 +973,37 @@ export function withNodeRuntimeControlPlaneStatusFullRendered(result: Record<str
|
||||
return { ...result, renderedText, contentType: "text/plain" };
|
||||
}
|
||||
|
||||
function nodeRuntimeStatusObservationLines(result: Record<string, unknown>): string[] {
|
||||
const observation = record(result.observation);
|
||||
if (Object.keys(observation).length === 0) return [];
|
||||
const processGroup = record(observation.processGroup);
|
||||
const worker = record(observation.worker);
|
||||
const completed = statusProbeNames(observation.completedProbes);
|
||||
const unfinished = statusProbeNames(observation.unfinishedProbes);
|
||||
const workerDiagnostic = webObserveText(worker.stderrTail ?? worker.stdoutTail ?? "").replace(/\s+/gu, " ").trim();
|
||||
return [
|
||||
"",
|
||||
"OBSERVATION",
|
||||
webObserveTable(
|
||||
["STATE", "PHASE", "ELAPSED", "BUDGET", "REASON", "PROCESS_GROUP"],
|
||||
[[
|
||||
webObserveText(observation.state),
|
||||
webObserveText(observation.currentPhase),
|
||||
formatElapsedMs(observation.durationMs),
|
||||
`${webObserveText(observation.totalBudgetSeconds)}s`,
|
||||
webObserveShort(webObserveText(observation.reason), 48),
|
||||
processGroup.cleaned === true ? "cleaned" : processGroup.cleaned === false ? "residual" : "not-requested",
|
||||
]],
|
||||
),
|
||||
` completed=${completed.length === 0 ? "-" : completed.join(",")} unfinished=${unfinished.length === 0 ? "-" : unfinished.join(",")}`,
|
||||
...(workerDiagnostic === "-" ? [] : [` worker-error=${webObserveShort(workerDiagnostic, 160)}`]),
|
||||
];
|
||||
}
|
||||
|
||||
function statusProbeNames(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string").slice(0, 12) : [];
|
||||
}
|
||||
|
||||
function nodeRuntimePublicProbeDetail(publicProbe: Record<string, unknown>): string {
|
||||
const web = record(publicProbe.web);
|
||||
const apiHealth = record(publicProbe.apiHealth);
|
||||
|
||||
@@ -38,6 +38,7 @@ import { compactNodeRuntimeGitMirrorStatus, nodeRuntimeGitMirrorStatus } from ".
|
||||
import { keyValueLinesFromText, numericField, optionValue, record, shellQuote } from "./utils";
|
||||
import { externalPostgresBridgeStatus, externalPostgresSecretStatus, getNodeRuntimePipelineRun, isLocalPostgresObject, nodeRuntimeCodeAgentRuntimeStatus, nodeRuntimeRenderOverlay } from "./web-probe";
|
||||
import { webObserveShort, webObserveText } from "./web-probe-observe";
|
||||
import { readNodeRuntimeStatusPolicy, runNodeRuntimeStatusProbe, skippedNodeRuntimeStatusCommand, type NodeRuntimeStatusPolicy, type NodeRuntimeStatusWorkerEvent } from "./control-plane-status-observed";
|
||||
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
|
||||
|
||||
const runtimeGitopsObservabilityNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-observability.mjs"), "utf8").trimEnd();
|
||||
@@ -172,69 +173,120 @@ export function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGi
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
export interface NodeRuntimeControlPlaneStatusOptions {
|
||||
policy?: NodeRuntimeStatusPolicy;
|
||||
onProgress?: (event: NodeRuntimeStatusWorkerEvent) => void;
|
||||
}
|
||||
|
||||
export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, options: NodeRuntimeControlPlaneStatusOptions = {}): Record<string, unknown> {
|
||||
const statusPolicy = options.policy ?? readNodeRuntimeStatusPolicy();
|
||||
const spec = scoped.spec;
|
||||
const probeTimeoutSeconds = Math.max(1, Math.min(60, scoped.timeoutSeconds));
|
||||
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
|
||||
const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run");
|
||||
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
|
||||
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
|
||||
const expectedPipelineRun = sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit);
|
||||
const sourceState = runNodeRuntimeStatusProbe("source", statusPolicy, options.onProgress, () => {
|
||||
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
|
||||
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
|
||||
const expectedPipelineRun = sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit);
|
||||
return { head, sourceCommit, expectedPipelineRun };
|
||||
}, () => ({ head: null, sourceCommit: sourceCommitOverride ?? null, expectedPipelineRun: null }));
|
||||
const { head, sourceCommit, expectedPipelineRun } = sourceState;
|
||||
let pipelineRun = pipelineRunOverride ?? expectedPipelineRun;
|
||||
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds);
|
||||
const namespaceExists = namespace.exitCode === 0;
|
||||
const postgresObjects = namespaceExists
|
||||
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds)
|
||||
: null;
|
||||
const localPostgresObjects = postgresObjects === null
|
||||
? []
|
||||
: postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
|
||||
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds);
|
||||
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds);
|
||||
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds);
|
||||
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
|
||||
let pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
|
||||
const pipelineRunResolution = pipelineRunOverride !== undefined
|
||||
|
||||
const controlPlane = runNodeRuntimeStatusProbe("control-plane", statusPolicy, options.onProgress, () => ({
|
||||
serviceAccount: runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds),
|
||||
pipeline: runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds),
|
||||
}), () => ({ serviceAccount: skippedNodeRuntimeStatusCommand("service-account"), pipeline: skippedNodeRuntimeStatusCommand("pipeline") }));
|
||||
const { serviceAccount, pipeline } = controlPlane;
|
||||
|
||||
let pipelineRunProbe: Record<string, unknown> | null = null;
|
||||
let pipelineRunDiagnostics: Record<string, unknown> | null = null;
|
||||
const pipelineRunResolution: Record<string, unknown> = pipelineRunOverride !== undefined
|
||||
? { mode: "explicit-pipeline-run", expectedName: pipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null }
|
||||
: sourceCommit === null || expectedPipelineRun === null
|
||||
? { mode: "source-unresolved", expectedName: null, resolvedName: null, fallbackUsed: false, result: null }
|
||||
: { mode: "source-commit", expectedName: expectedPipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null as Record<string, unknown> | null };
|
||||
if (pipelineRunOverride === undefined && sourceCommit !== null && expectedPipelineRun !== null && pipelineRunProbe !== null && pipelineRunProbe.exists !== true) {
|
||||
const resolved = resolveNodeRuntimePipelineRunBySourceCommit(spec, sourceCommit);
|
||||
pipelineRunResolution.result = compactRuntimeCommand(resolved.result);
|
||||
if (resolved.name !== null) {
|
||||
pipelineRun = resolved.name;
|
||||
pipelineRunResolution.resolvedName = resolved.name;
|
||||
pipelineRunResolution.fallbackUsed = true;
|
||||
pipelineRunProbe = getNodeRuntimePipelineRun(spec, resolved.name);
|
||||
: { mode: "source-commit", expectedName: expectedPipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null };
|
||||
runNodeRuntimeStatusProbe("pipeline-run", statusPolicy, options.onProgress, () => {
|
||||
pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
|
||||
if (pipelineRunOverride === undefined && sourceCommit !== null && expectedPipelineRun !== null && pipelineRunProbe !== null && pipelineRunProbe.exists !== true) {
|
||||
const resolved = resolveNodeRuntimePipelineRunBySourceCommit(spec, sourceCommit);
|
||||
pipelineRunResolution.result = compactRuntimeCommand(resolved.result);
|
||||
if (resolved.name !== null) {
|
||||
pipelineRun = resolved.name;
|
||||
pipelineRunResolution.resolvedName = resolved.name;
|
||||
pipelineRunResolution.fallbackUsed = true;
|
||||
pipelineRunProbe = getNodeRuntimePipelineRun(spec, resolved.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
|
||||
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
|
||||
: null;
|
||||
const workloads = namespaceExists
|
||||
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds)
|
||||
: null;
|
||||
const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
const workloadReadinessProbe = namespaceExists
|
||||
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds)
|
||||
: null;
|
||||
const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
|
||||
const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
|
||||
const secrets = externalPostgresSecretStatus(spec, namespaceExists);
|
||||
const codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists);
|
||||
const publicProbes = nodeRuntimePublicProbeStatus(spec);
|
||||
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
|
||||
const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror);
|
||||
pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
|
||||
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
|
||||
: null;
|
||||
return true;
|
||||
}, () => {
|
||||
pipelineRunProbe = { exists: false, ready: true, skipped: true, reason: "probe-disabled" };
|
||||
return false;
|
||||
});
|
||||
|
||||
let namespace = skippedNodeRuntimeStatusCommand("runtime-namespace");
|
||||
let namespaceExists = false;
|
||||
let postgresObjects: CommandResult | null = null;
|
||||
let localPostgresObjects: string[] = [];
|
||||
let workloads: CommandResult | null = null;
|
||||
let workloadNames: string[] = [];
|
||||
let workloadReadinessProbe: CommandResult | null = null;
|
||||
let workloadReadiness: Array<Record<string, unknown>> = [];
|
||||
let bridge: Record<string, unknown> = { required: false, ready: true, skipped: true };
|
||||
let secrets: Record<string, unknown> = { required: false, ready: true, skipped: true, valuesPrinted: false };
|
||||
let codeAgentRuntime: Record<string, unknown> = { required: false, ready: true, skipped: true };
|
||||
runNodeRuntimeStatusProbe("runtime", statusPolicy, options.onProgress, () => {
|
||||
namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds);
|
||||
namespaceExists = namespace.exitCode === 0;
|
||||
postgresObjects = namespaceExists
|
||||
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds)
|
||||
: null;
|
||||
localPostgresObjects = postgresObjects === null ? [] : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
|
||||
workloads = namespaceExists
|
||||
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds)
|
||||
: null;
|
||||
workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
workloadReadinessProbe = namespaceExists
|
||||
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds)
|
||||
: null;
|
||||
workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
|
||||
bridge = externalPostgresBridgeStatus(spec, namespaceExists);
|
||||
secrets = externalPostgresSecretStatus(spec, namespaceExists);
|
||||
codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists);
|
||||
return true;
|
||||
}, () => false);
|
||||
|
||||
const publicProbes = runNodeRuntimeStatusProbe("public", statusPolicy, options.onProgress, () => nodeRuntimePublicProbeStatus(spec), () => ({
|
||||
ready: true,
|
||||
skipped: true,
|
||||
reason: "probe-disabled",
|
||||
web: { url: spec.publicWebUrl, ok: true, skipped: true, httpStatus: null },
|
||||
apiHealth: { url: spec.publicApiUrl, ok: true, skipped: true, httpStatus: null },
|
||||
}));
|
||||
const gitMirror = runNodeRuntimeStatusProbe("git-mirror", statusPolicy, options.onProgress, () => nodeRuntimeGitMirrorStatus(scoped), () => ({ ok: true, ready: true, skipped: true, compact: { skipped: true, sourceSnapshotReady: true, pendingFlush: false, githubInSync: true } }));
|
||||
const gitMirrorCompact = statusPolicy.probes["git-mirror"] ? compactNodeRuntimeGitMirrorStatus(gitMirror) : record(gitMirror.compact);
|
||||
|
||||
const argoState = runNodeRuntimeStatusProbe("argo", statusPolicy, options.onProgress, () => {
|
||||
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds);
|
||||
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
|
||||
const ready = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
|
||||
const diagnostics = argo.exitCode === 0 && !ready ? nodeRuntimeArgoDiagnostics(spec, probeTimeoutSeconds) : null;
|
||||
return { argo, repoURL, targetRevision, path, syncRevision, syncStatus, health, ready, diagnostics };
|
||||
}, () => ({ argo: skippedNodeRuntimeStatusCommand("argo"), repoURL: "", targetRevision: "", path: "", syncRevision: "", syncStatus: "", health: "", ready: true, diagnostics: null }));
|
||||
const { argo, repoURL, targetRevision, path, syncRevision, syncStatus, health, ready: argoReady, diagnostics: argoDiagnostics } = argoState;
|
||||
|
||||
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
|
||||
const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0;
|
||||
const controlPlaneReady = !statusPolicy.probes["control-plane"] || serviceAccount.exitCode === 0 && pipeline.exitCode === 0;
|
||||
const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready);
|
||||
const localPostgresExpectedAbsent = nodeRuntimeLocalPostgresExpectedAbsent(spec);
|
||||
const localPostgresReady = localPostgresExpectedAbsent ? localPostgresObjects.length === 0 : localPostgresObjects.length > 0;
|
||||
const postgresReady = activeExternalPostgres === undefined
|
||||
? localPostgresReady
|
||||
: bridge.ready === true && secrets.ready === true;
|
||||
const runtimeReady = namespaceExists && postgresReady && workloadsReady && codeAgentRuntime.ready === true;
|
||||
const runtimeReady = !statusPolicy.probes.runtime || namespaceExists && postgresReady && workloadsReady && codeAgentRuntime.ready === true;
|
||||
const codeAgentRuntimeDegradedReason = typeof codeAgentRuntime.degradedReason === "string" ? codeAgentRuntime.degradedReason : null;
|
||||
let runtimeDegradedReason = "runtime-not-ready";
|
||||
if (!namespaceExists) {
|
||||
@@ -250,18 +302,14 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
|
||||
} else if (codeAgentRuntime.ready !== true) {
|
||||
runtimeDegradedReason = codeAgentRuntimeDegradedReason ?? "code-agent-runtime-not-ready";
|
||||
}
|
||||
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
|
||||
const argoDiagnostics = argo.exitCode === 0 && !argoReady
|
||||
? nodeRuntimeArgoDiagnostics(spec, probeTimeoutSeconds)
|
||||
: null;
|
||||
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
|
||||
const pipelineRunReady = !statusPolicy.probes["pipeline-run"] || pipelineRunProbe !== null && pipelineRunProbe.status === "True";
|
||||
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
|
||||
? pipelineRunDiagnostics.degradedReason
|
||||
: pipelineRunProbe === null || pipelineRunProbe.exists !== true
|
||||
? "pipelinerun-not-found"
|
||||
: "pipelinerun-not-succeeded";
|
||||
const publicReady = publicProbes.ready === true;
|
||||
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
|
||||
const publicReady = !statusPolicy.probes.public || publicProbes.ready === true;
|
||||
const gitMirrorReady = !statusPolicy.probes["git-mirror"] || gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
|
||||
const gitMirrorDegradedReason = gitMirrorCompact.sourceSnapshotReady === false
|
||||
? "source-snapshot-missing"
|
||||
: gitMirrorCompact.pendingFlush === true
|
||||
@@ -307,16 +355,24 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
|
||||
expectedPipelineRun,
|
||||
pipelineRunResolution,
|
||||
expected: nodeRuntimeExpected(spec),
|
||||
sourceHead: head === null
|
||||
statusPolicy: {
|
||||
configPath: statusPolicy.configPath,
|
||||
probes: statusPolicy.probes,
|
||||
},
|
||||
sourceHead: !statusPolicy.probes.source
|
||||
? { ok: true, skipped: true, reason: "probe-disabled" }
|
||||
: head === null
|
||||
? { ok: sourceCommit !== null, value: sourceCommit, source: "option" }
|
||||
: { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) },
|
||||
controlPlane: {
|
||||
ready: controlPlaneReady,
|
||||
skipped: !statusPolicy.probes["control-plane"],
|
||||
serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) },
|
||||
pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) },
|
||||
},
|
||||
argo: {
|
||||
ready: argoReady,
|
||||
skipped: !statusPolicy.probes.argo,
|
||||
application: spec.app,
|
||||
repoURL,
|
||||
expectedRepoURL: spec.argoRepoUrl,
|
||||
@@ -333,6 +389,7 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
|
||||
pipelineRunDiagnostics,
|
||||
runtime: {
|
||||
ready: runtimeReady,
|
||||
skipped: !statusPolicy.probes.runtime,
|
||||
namespace: spec.runtimeNamespace,
|
||||
namespaceExists,
|
||||
localPostgresObjects,
|
||||
@@ -694,11 +751,12 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
|
||||
pipelineRun: {
|
||||
name: pipelineRun.name ?? null,
|
||||
exists: pipelineRun.exists === true,
|
||||
skipped: pipelineRun.skipped === true,
|
||||
status: pipelineRun.status ?? null,
|
||||
reason: pipelineRun.reason ?? null,
|
||||
message: pipelineRun.message ?? null,
|
||||
createdAt: pipelineRun.createdAt ?? null,
|
||||
ready: pipelineRun.status === "True",
|
||||
ready: pipelineRun.skipped === true || pipelineRun.status === "True",
|
||||
diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : {
|
||||
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
|
||||
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
|
||||
@@ -715,6 +773,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
|
||||
argo: {
|
||||
application: argo.application ?? null,
|
||||
ready: argo.ready === true,
|
||||
skipped: argo.skipped === true,
|
||||
syncRevision: argo.syncRevision ?? null,
|
||||
targetGitopsRevision: argo.targetGitopsRevision ?? null,
|
||||
revisionObserved: typeof argo.targetGitopsRevision === "string" && argo.syncRevision === argo.targetGitopsRevision,
|
||||
@@ -726,6 +785,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
|
||||
namespace: runtime.namespace ?? null,
|
||||
namespaceExists: runtime.namespaceExists === true,
|
||||
ready: runtime.ready === true,
|
||||
skipped: runtime.skipped === true,
|
||||
workloadCount,
|
||||
workloadReady: `${readyWorkloads}/${workloadReadiness.length}`,
|
||||
notReadyWorkloads,
|
||||
@@ -740,6 +800,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
|
||||
},
|
||||
publicProbe: {
|
||||
ready: publicProbes.ready === true,
|
||||
skipped: publicProbes.skipped === true,
|
||||
web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
|
||||
apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
|
||||
targetHost: Object.keys(targetHostProbe).length === 0 ? null : {
|
||||
@@ -765,6 +826,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
|
||||
},
|
||||
gitMirror: {
|
||||
ready: gitMirror.ready === true,
|
||||
skipped: gitMirror.skipped === true,
|
||||
localSource: gitMirrorCompact.localSource ?? null,
|
||||
githubSource: gitMirrorCompact.githubSource ?? null,
|
||||
sourceAuthority: gitMirrorCompact.sourceAuthority ?? null,
|
||||
|
||||
Reference in New Issue
Block a user