diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index d44c3561..0f6baf49 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -14,6 +14,7 @@ bun scripts/cli.ts cicd status --node NC01 bun scripts/cli.ts cicd status --node bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --once --dry-run bun scripts/cli.ts hwlab g14 control-plane status --lane v02 +bun scripts/cli.ts hwlab nodes control-plane status --node NC01 --lane v03 --json bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait bun scripts/cli.ts hwlab g14 git-mirror status --lane v02 bun scripts/cli.ts agentrun control-plane status @@ -56,6 +57,11 @@ bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 --id < - CI/CD validation 阶段只能验证部署对象的 `/health` 端点和必要 provenance；禁止在 CI/CD gate 中运行 web-probe、Playwright、远程浏览器截图、用户路径 E2E 或等价重型业务探针。业务/用户入口验证只能作为发布后的独立 post-deploy validation 证据，不得阻塞 CI/CD 一键交付。 - 任一 CI/CD 阶段或总耗时超过 2 分钟时，不要继续死等或把超长等待视为正常；先输出阶段耗时分解，并优先从 env reuse、git mirror、BuildKit/cache、GitOps/Argo watch 和 runtime readiness 探测方向优化后再继续交付。 - node-scoped `trigger-current --wait` 必须把 source sync、pre/post flush、PipelineRun、GitOps/Argo、runtime readiness 和 `/health` closeout 放进同一 120s 端到端预算；超预算时由 CLI 输出阶段分解、Argo target revision、runtime/public 状态和 TaskRun/Pod drill-down，不继续死等，也不要求操作者手动串联多个状态/flush 命令才能完成一次交付。 +- node-scoped `control-plane status` 可见性规则： + - 必须立即输出 started/阶段，并按 owning YAML 输出 heartbeat； + - 正常、总预算超时、中断和 worker 失败都要返回有界 closeout； + - closeout 必须披露探针完成情况、进程组清理结果和脱敏的 worker 错误摘要； + - 显式 `--json` 的 stdout 只允许一个合法 JSON 文档，progress 写入 stderr。 - 触发或验收 rollout 时必须绑定 lane、source commit、PipelineRun/GitOps revision、runtime ready 和 `/health` 端点验证结果；web-probe/Playwright 结果只能作为单独的 post-deploy 证据。 - CI/CD 状态、日志和事件查询必须减少 trans/SSH 传输：能在目标 NODE/k8s 内解析、聚合、裁剪的内容，必须在目标侧计算成短 JSON/table 摘要后再回传；禁止为了本地解析而把完整 ConfigMap、大对象、长日志或原始 API payload 透传回来。 - branch-follower 排障必须优先使用 `debug-step` 做单步调试：`state-read`、`status-read`、`decide`、`state-write` 分别定位状态读取、K8s native status、决策和 ConfigMap patch，不要通过反复推小提交触发整条自动跟随大回环来定位同一问题。 diff --git a/.agents/skills/unidesk-cicd/references/control-plane.md b/.agents/skills/unidesk-cicd/references/control-plane.md index e5a4b744..a45972c2 100644 --- a/.agents/skills/unidesk-cicd/references/control-plane.md +++ b/.agents/skills/unidesk-cicd/references/control-plane.md @@ -38,6 +38,18 @@ bun scripts/cli.ts hwlab nodes control-plane refresh --node --lane v03 -- `status` 默认返回 compact summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe、git mirror 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。 +- `status` 可见性契约： + - 启动后立即在 stderr 输出 `started` 和当前阶段； + - 等待期间按 `config/hwlab-node-control-plane.yaml#status` 输出有界 heartbeat； + - 正常完成、总预算超时和中断都返回阶段耗时、已完成探针、未完成探针和 typed reason； + - 超时或中断由 observed worker 终止整组子进程，并在 closeout 中披露进程组清理结果； + - worker 启动或解析失败时，closeout 返回过滤 progress 后的有界脱敏错误摘要； + - 显式 `--json` 的 stdout 只包含一个 compact JSON 文档，progress 保持在 stderr； + - `--full` 保留有界文本 drill-down，`--raw` 保留完整 JSON 的渐进披露入口。 +- `status` 配置真相： + - 总预算、heartbeat 周期、终止宽限期和探针开关只由 `config/hwlab-node-control-plane.yaml#status` 维护； + - `--timeout-seconds` 只作为本次调用的显式总预算覆盖，不能变成代码默认值或第二份长期配置。 + `control-plane apply --confirm` 是 node/lane 控制面更新入口：先从 `config/hwlab-node-control-plane.yaml` 收敛 node-local infra、git-mirror ConfigMap、RBAC、Tekton/Argo 前置对象，再应用 source-render 的 runtime namespace、Pipeline 和 Argo Application。node-scoped git-mirror 脚本只有 infra manifest 是权威来源；不要把 HWLAB source render 生成的 `devops-infra/git-mirror.yaml` 当作正式 apply 来源。 `trigger-current --confirm --wait` 是 node/lane CI/CD 一键入口：按 YAML `sourceAuthority` 解析并同步 source snapshot，legacy lane 执行 k8s git-mirror source snapshot sync，Gitea/PaC 迁移 lane 执行受控 Gitea mirror sync 并读取 immutable snapshot ref；随后执行必要 pre-flush，刷新 control-plane，创建或复用 commit-pinned PipelineRun，等待 PipelineRun 终态，并在终态成功后继续执行 post-flush、GitOps/Argo、runtime readiness 和 public `/health` closeout。 diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 7fd4bbec..c19bc26f 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -9,6 +9,7 @@ metadata: - 1119 - 1148 - 1234 + - 1721 imagePolicy: requireReproducibleBuildSource: true forbidPrivateOrNodeLocalImagesAsInputs: true @@ -16,6 +17,18 @@ imagePolicy: requiredSourceKinds: - dockerfile - docker-compose +status: + totalBudgetSeconds: 120 + heartbeatSeconds: 5 + terminationGraceSeconds: 2 + probes: + source: true + control-plane: true + pipeline-run: true + argo: true + runtime: true + public: true + git-mirror: true nodes: D601: route: D601 diff --git a/scripts/src/command.ts b/scripts/src/command.ts index 2a3cd72f..c9f8554a 100644 --- a/scripts/src/command.ts +++ b/scripts/src/command.ts @@ -14,6 +14,10 @@ export interface CommandResult { stderrBytes?: number; stdoutTruncated?: boolean; stderrTruncated?: boolean; + aborted?: boolean; + terminationReason?: "timeout" | "abort" | null; + processGroupId?: number | null; + processGroupAliveAfterCleanup?: boolean | null; } export interface CommandProgress { @@ -25,13 +29,15 @@ export interface CommandProgress { lastOutputAgeMs: number | null; } -interface ObservedCommandOptions { +export interface ObservedCommandOptions { timeoutMs?: number; heartbeatMs?: number; killAfterMs?: number; env?: NodeJS.ProcessEnv; input?: string; maxCaptureChars?: number; + signal?: AbortSignal; + onStderr?: (text: string) => void; onProgress?: (progress: CommandProgress) => void; } @@ -70,9 +76,12 @@ export async function runCommandObserved(command: string[], cwd: string, options let stderrTruncated = false; let lastOutputAt: number | null = null; let timedOut = false; + let aborted = false; + let terminationReason: "timeout" | "abort" | null = null; let timeoutTimer: NodeJS.Timeout | undefined; let killTimer: NodeJS.Timeout | undefined; let heartbeatTimer: NodeJS.Timeout | undefined; + let abortListener: (() => void) | undefined; const child = spawn(command[0], command.slice(1), { cwd, @@ -91,6 +100,7 @@ export async function runCommandObserved(command: string[], cwd: string, options }); child.stderr.on("data", (chunk: Buffer | string) => { const text = chunk.toString(); + options.onStderr?.(text); stderrBytes += Buffer.byteLength(text); lastOutputAt = Date.now(); const captured = appendBounded(stderr, text, maxCaptureChars); @@ -113,14 +123,23 @@ export async function runCommandObserved(command: string[], cwd: string, options }, heartbeatMs); } - if (timeoutMs !== undefined) { - timeoutTimer = setTimeout(() => { - timedOut = true; - killProcessTree(child, "SIGTERM"); - killTimer = setTimeout(() => { - killProcessTree(child, "SIGKILL"); - }, Math.max(100, options.killAfterMs ?? 5_000)); - }, timeoutMs); + const terminate = (reason: "timeout" | "abort") => { + if (terminationReason !== null) return; + terminationReason = reason; + timedOut = reason === "timeout"; + aborted = reason === "abort"; + killProcessTree(child, "SIGTERM"); + killTimer = setTimeout(() => { + killProcessTree(child, "SIGKILL"); + killTimer = undefined; + }, Math.max(100, options.killAfterMs ?? 5_000)); + }; + + if (timeoutMs !== undefined) timeoutTimer = setTimeout(() => terminate("timeout"), timeoutMs); + if (options.signal !== undefined) { + abortListener = () => terminate("abort"); + options.signal.addEventListener("abort", abortListener, { once: true }); + if (options.signal.aborted) abortListener(); } const completion = await new Promise<{ exitCode: number | null; signal: NodeJS.Signals | null; error?: Error }>((resolve) => { @@ -135,8 +154,20 @@ export async function runCommandObserved(command: string[], cwd: string, options }); if (timeoutTimer !== undefined) clearTimeout(timeoutTimer); - if (killTimer !== undefined) clearTimeout(killTimer); + if (killTimer !== undefined) { + clearTimeout(killTimer); + killTimer = undefined; + } if (heartbeatTimer !== undefined) clearInterval(heartbeatTimer); + if (abortListener !== undefined) options.signal?.removeEventListener("abort", abortListener); + + const processGroupId = process.platform === "win32" ? null : child.pid ?? null; + let processGroupAliveAfterCleanup: boolean | null = null; + if (terminationReason !== null && processGroupId !== null) { + killProcessTree(child, "SIGKILL"); + await new Promise((resolve) => setTimeout(resolve, 25)); + processGroupAliveAfterCleanup = processGroupExists(processGroupId); + } if (completion.error !== undefined && stderr.length === 0) { stderr = completion.error.message; @@ -156,9 +187,22 @@ export async function runCommandObserved(command: string[], cwd: string, options stderrBytes, stdoutTruncated, stderrTruncated, + aborted, + terminationReason, + processGroupId, + processGroupAliveAfterCleanup, }; } +function processGroupExists(processGroupId: number): boolean { + try { + process.kill(-processGroupId, 0); + return true; + } catch { + return false; + } +} + function appendBounded(current: string, next: string, maxChars: number): { text: string; truncated: boolean } { const combined = `${current}${next}`; if (combined.length <= maxChars) return { text: combined, truncated: false }; diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index f9e604be..853e6103 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -12,6 +12,7 @@ export function hwlabNodeHelp(): Record { examples: [ "bun scripts/cli.ts hwlab nodes control-plane infra plan --node --lane ", "bun scripts/cli.ts hwlab nodes control-plane status --node --lane ", + "bun scripts/cli.ts hwlab nodes control-plane status --node --lane --json", "bun scripts/cli.ts hwlab nodes control-plane source-workspace sync --node JD01 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node JD01 --lane v03 --min-age-minutes 30 --limit 200 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane cleanup-released-pvs --node JD01 --lane v03 --limit 200 --dry-run", @@ -43,6 +44,7 @@ export function hwlabNodeHelp(): Record { "cleanup-legacy-docker-images is a transitional legacy-cache GC for Docker images matching YAML allowlisted repositories; it protects container-referenced images, does not run prune, and does not touch Docker volumes.", "cleanup-legacy-docker-registry-volume removes only an exited legacy Docker registry container and its unique /var/lib/registry volume after the YAML-declared k8s node-local-registry is ready.", "`trigger-current --confirm --wait` is the one-command CICD path for current node/lane runtime publish.", + "control-plane status reads its total budget, heartbeat cadence, termination grace period, and probe switches from config/hwlab-node-control-plane.yaml; --json keeps stdout as one compact JSON document while progress remains on stderr.", ], }; } diff --git a/scripts/src/hwlab-node/control-plane-status-observed.ts b/scripts/src/hwlab-node/control-plane-status-observed.ts new file mode 100644 index 00000000..bfefb3e9 --- /dev/null +++ b/scripts/src/hwlab-node/control-plane-status-observed.ts @@ -0,0 +1,369 @@ +import { readFileSync } from "node:fs"; +import { runCommandObserved, type CommandProgress, type CommandResult } from "../command"; +import { repoRoot, rootPath } from "../config"; +import type { parseNodeScopedDelegatedOptions } from "./plan"; +import { record } from "./utils"; + +export const NODE_RUNTIME_STATUS_WORKER_ENV = "UNIDESK_HWLAB_CONTROL_PLANE_STATUS_WORKER"; +const WORKER_PROGRESS_PREFIX = "UNIDESK_HWLAB_CONTROL_PLANE_STATUS_PROGRESS "; +const STATUS_CONFIG_PATH = "config/hwlab-node-control-plane.yaml"; +const WORKER_CAPTURE_CHARS = 2 * 1024 * 1024; +const WORKER_STDERR_TAIL_CHARS = 1600; +const WORKER_STDOUT_TAIL_CHARS = 1200; + +export const NODE_RUNTIME_STATUS_PROBES = [ + "source", + "control-plane", + "pipeline-run", + "argo", + "runtime", + "public", + "git-mirror", +] as const; + +export type NodeRuntimeStatusProbe = typeof NODE_RUNTIME_STATUS_PROBES[number]; +export type NodeRuntimeStatusProbeState = "started" | "completed" | "failed" | "skipped"; + +export interface NodeRuntimeStatusWorkerEvent { + probe: NodeRuntimeStatusProbe; + state: NodeRuntimeStatusProbeState; + at: string; + durationMs?: number; + reason?: string; +} + +export interface NodeRuntimeStatusPolicy { + configPath: string; + totalBudgetSeconds: number; + heartbeatSeconds: number; + terminationGraceSeconds: number; + probes: Record; +} + +type ScopedStatus = ReturnType; + +export function readNodeRuntimeStatusPolicy(): NodeRuntimeStatusPolicy { + const parsed = requiredRecord(Bun.YAML.parse(readFileSync(rootPath(STATUS_CONFIG_PATH), "utf8")) as unknown, STATUS_CONFIG_PATH); + const status = requiredRecord(parsed.status, `${STATUS_CONFIG_PATH}.status`); + const probesRaw = requiredRecord(status.probes, `${STATUS_CONFIG_PATH}.status.probes`); + const probes = Object.fromEntries(NODE_RUNTIME_STATUS_PROBES.map((probe) => [probe, requiredBoolean(probesRaw[probe], `${STATUS_CONFIG_PATH}.status.probes.${probe}`)])) as Record; + if (probes["pipeline-run"] && !probes.source) throw new Error(`${STATUS_CONFIG_PATH}.status.probes.source must be true when pipeline-run is enabled`); + return { + configPath: STATUS_CONFIG_PATH, + totalBudgetSeconds: requiredPositiveInteger(status.totalBudgetSeconds, `${STATUS_CONFIG_PATH}.status.totalBudgetSeconds`), + heartbeatSeconds: requiredPositiveInteger(status.heartbeatSeconds, `${STATUS_CONFIG_PATH}.status.heartbeatSeconds`), + terminationGraceSeconds: requiredPositiveInteger(status.terminationGraceSeconds, `${STATUS_CONFIG_PATH}.status.terminationGraceSeconds`), + probes, + }; +} + +export function isNodeRuntimeStatusWorker(): boolean { + return process.env[NODE_RUNTIME_STATUS_WORKER_ENV] === "1"; +} + +export function emitNodeRuntimeStatusWorkerEvent(event: NodeRuntimeStatusWorkerEvent): void { + process.stderr.write(`${WORKER_PROGRESS_PREFIX}${JSON.stringify(event)}\n`); +} + +export async function runObservedNodeRuntimeStatus(scoped: ScopedStatus): Promise<{ full: Record; summary: Record }> { + const policy = readNodeRuntimeStatusPolicy(); + const explicitBudget = scoped.originalArgs.includes("--timeout-seconds"); + const budgetSeconds = explicitBudget ? scoped.timeoutSeconds : policy.totalBudgetSeconds; + const startedAtMs = Date.now(); + const startedAt = new Date(startedAtMs).toISOString(); + const tracker = createProbeTracker(policy); + const abortController = new AbortController(); + let interruptSignal: "SIGINT" | "SIGTERM" | null = null; + const interrupt = (signal: "SIGINT" | "SIGTERM") => { + if (interruptSignal !== null) return; + interruptSignal = signal; + abortController.abort(); + }; + const onSigint = () => interrupt("SIGINT"); + const onSigterm = () => interrupt("SIGTERM"); + process.on("SIGINT", onSigint); + process.on("SIGTERM", onSigterm); + + emitStatusProgress(scoped, { + state: "started", + phase: tracker.currentProbe, + elapsedMs: 0, + budgetSeconds, + heartbeatSeconds: policy.heartbeatSeconds, + configPath: policy.configPath, + }); + + let stderrBuffer = ""; + let commandResult: CommandResult; + try { + commandResult = await runCommandObserved(workerCommand(scoped, budgetSeconds), repoRoot, { + timeoutMs: budgetSeconds * 1000, + heartbeatMs: policy.heartbeatSeconds * 1000, + killAfterMs: policy.terminationGraceSeconds * 1000, + maxCaptureChars: WORKER_CAPTURE_CHARS, + signal: abortController.signal, + env: { + ...process.env, + [NODE_RUNTIME_STATUS_WORKER_ENV]: "1", + UNIDESK_CLI_OUTPUT_DUMP_DISABLED: "1", + }, + onStderr: (text) => { + stderrBuffer += text; + stderrBuffer = consumeWorkerProgress(stderrBuffer, (event) => { + tracker.accept(event); + emitStatusProgress(scoped, tracker.progress(event.state, budgetSeconds)); + }); + }, + onProgress: (progress) => emitStatusProgress(scoped, tracker.heartbeat(progress, budgetSeconds)), + }); + } finally { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + } + + const workerPayload = commandResult.timedOut || commandResult.aborted === true || commandResult.stdoutTruncated === true + ? null + : parseWorkerPayload(commandResult.stdout); + const closeout = statusCloseout(commandResult, workerPayload); + const observation = tracker.observation({ + state: closeout.state, + reason: closeout.reason, + signal: interruptSignal, + startedAt, + startedAtMs, + budgetSeconds, + budgetSource: explicitBudget ? "cli-override" : "yaml", + policy, + commandResult, + }); + emitStatusProgress(scoped, { + state: "closeout", + phase: observation.currentPhase, + elapsedMs: observation.durationMs, + reason: observation.reason, + completedProbes: observation.completedProbes, + unfinishedProbes: observation.unfinishedProbes, + processGroupCleaned: observation.processGroup.cleaned, + }); + + if (workerPayload !== null) { + const summary = { ...record(workerPayload.summary), observation }; + return { full: { ...workerPayload, observation, summary }, summary }; + } + const summary = partialStatusSummary(scoped, closeout.reason, observation); + return { full: { ...summary, mode: "node-scoped-runtime-status-closeout", summary }, summary }; +} + +function workerCommand(scoped: ScopedStatus, budgetSeconds: number): string[] { + const args: string[] = []; + for (let index = 0; index < scoped.originalArgs.length; index += 1) { + const arg = scoped.originalArgs[index] ?? ""; + if (arg === "--raw" || arg === "--full" || arg === "--json") continue; + if (arg === "--timeout-seconds") { + index += 1; + continue; + } + args.push(arg); + } + return [process.execPath, "scripts/cli.ts", "hwlab", "nodes", "control-plane", ...args, "--timeout-seconds", String(budgetSeconds), "--raw"]; +} + +function consumeWorkerProgress(buffer: string, onEvent: (event: NodeRuntimeStatusWorkerEvent) => void): string { + let remaining = buffer; + while (true) { + const newline = remaining.indexOf("\n"); + if (newline === -1) return remaining.slice(-8192); + const line = remaining.slice(0, newline).trimEnd(); + remaining = remaining.slice(newline + 1); + if (!line.startsWith(WORKER_PROGRESS_PREFIX)) continue; + try { + const parsed = JSON.parse(line.slice(WORKER_PROGRESS_PREFIX.length)) as NodeRuntimeStatusWorkerEvent; + if (NODE_RUNTIME_STATUS_PROBES.includes(parsed.probe) && ["started", "completed", "failed", "skipped"].includes(parsed.state)) onEvent(parsed); + } catch { + continue; + } + } +} + +function parseWorkerPayload(stdout: string): Record | null { + try { + const envelope = record(JSON.parse(stdout) as unknown); + const data = record(envelope.data); + return Object.keys(data).length === 0 ? null : data; + } catch { + return null; + } +} + +function statusCloseout(result: CommandResult, payload: Record | null): { state: string; reason: string } { + if (result.processGroupAliveAfterCleanup === true) return { state: "failed", reason: "status-process-group-cleanup-incomplete" }; + if (result.timedOut) return { state: "timed-out", reason: "status-total-budget-exceeded" }; + if (result.aborted === true) return { state: "interrupted", reason: "status-interrupted" }; + if (payload === null) return { state: "failed", reason: "status-worker-failed" }; + return { state: "completed", reason: "status-collected" }; +} + +function partialStatusSummary(scoped: ScopedStatus, reason: string, observation: Record): Record { + const status = `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`; + return { + ok: false, + command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, + mode: "node-scoped-runtime-status-summary", + mutation: false, + node: scoped.node, + lane: scoped.lane, + sourceCommit: null, + degradedReason: reason, + pipelineRun: {}, + argo: {}, + runtime: {}, + publicProbe: {}, + gitMirror: {}, + observation, + nextAction: `${status} --json`, + next: { status, json: `${status} --json`, full: `${status} --full`, raw: `${status} --raw` }, + }; +} + +function createProbeTracker(policy: NodeRuntimeStatusPolicy) { + const started = new Map(); + const completed: NodeRuntimeStatusProbe[] = []; + const failed: NodeRuntimeStatusProbe[] = []; + const skipped: NodeRuntimeStatusProbe[] = []; + const timings: Record[] = []; + let currentProbe: string = "starting"; + const enabled = NODE_RUNTIME_STATUS_PROBES.filter((probe) => policy.probes[probe]); + const accept = (event: NodeRuntimeStatusWorkerEvent) => { + currentProbe = event.probe; + if (event.state === "started") started.set(event.probe, Date.now()); + if (event.state === "completed" || event.state === "failed" || event.state === "skipped") { + const destination = event.state === "completed" ? completed : event.state === "failed" ? failed : skipped; + if (!destination.includes(event.probe)) destination.push(event.probe); + timings.push({ probe: event.probe, state: event.state, durationMs: event.durationMs ?? Math.max(0, Date.now() - (started.get(event.probe) ?? Date.now())), reason: event.reason ?? null }); + } + }; + const unfinished = () => enabled.filter((probe) => !completed.includes(probe) && !failed.includes(probe) && !skipped.includes(probe)); + return { + get currentProbe() { return currentProbe; }, + accept, + progress: (state: string, budgetSeconds: number) => ({ state: `probe-${state}`, phase: currentProbe, elapsedMs: null, budgetSeconds, completedProbes: [...completed], unfinishedProbes: unfinished() }), + heartbeat: (progress: CommandProgress, budgetSeconds: number) => ({ state: "heartbeat", phase: currentProbe, elapsedMs: progress.elapsedMs, budgetSeconds, completedProbes: [...completed], unfinishedProbes: unfinished(), stdoutBytes: progress.stdoutBytes, stderrBytes: progress.stderrBytes }), + observation: (input: { state: string; reason: string; signal: string | null; startedAt: string; startedAtMs: number; budgetSeconds: number; budgetSource: string; policy: NodeRuntimeStatusPolicy; commandResult: CommandResult }) => ({ + state: input.state, + reason: input.reason, + signal: input.signal, + startedAt: input.startedAt, + finishedAt: new Date().toISOString(), + durationMs: Math.max(0, Date.now() - input.startedAtMs), + currentPhase: currentProbe, + totalBudgetSeconds: input.budgetSeconds, + budgetSource: input.budgetSource, + heartbeatSeconds: input.policy.heartbeatSeconds, + configPath: input.policy.configPath, + completedProbes: [...completed], + failedProbes: [...failed], + skippedProbes: [...skipped], + unfinishedProbes: unfinished(), + phaseTimings: timings.slice(0, NODE_RUNTIME_STATUS_PROBES.length), + worker: { + exitCode: input.commandResult.exitCode, + signal: input.commandResult.signal, + timedOut: input.commandResult.timedOut, + aborted: input.commandResult.aborted === true, + stdoutBytes: input.commandResult.stdoutBytes ?? 0, + stderrBytes: input.commandResult.stderrBytes ?? 0, + stdoutTruncated: input.commandResult.stdoutTruncated === true, + stderrTruncated: input.commandResult.stderrTruncated === true, + ...nodeRuntimeStatusWorkerDiagnostics(input.commandResult, input.reason === "status-worker-failed"), + }, + processGroup: { + id: input.commandResult.processGroupId ?? null, + isolated: process.platform !== "win32", + terminationReason: input.commandResult.terminationReason ?? null, + cleaned: input.commandResult.terminationReason === null || input.commandResult.processGroupAliveAfterCleanup === null + ? null + : input.commandResult.processGroupAliveAfterCleanup === false, + }, + }), + }; +} + +export function runNodeRuntimeStatusProbe( + probe: NodeRuntimeStatusProbe, + policy: NodeRuntimeStatusPolicy, + onProgress: ((event: NodeRuntimeStatusWorkerEvent) => void) | undefined, + run: () => T, + skipped: () => T, +): T { + const startedAt = Date.now(); + if (!policy.probes[probe]) { + onProgress?.({ probe, state: "skipped", at: new Date(startedAt).toISOString(), durationMs: 0, reason: "disabled-by-yaml" }); + return skipped(); + } + onProgress?.({ probe, state: "started", at: new Date(startedAt).toISOString() }); + try { + const value = run(); + onProgress?.({ probe, state: "completed", at: new Date().toISOString(), durationMs: Date.now() - startedAt }); + return value; + } catch (error) { + onProgress?.({ probe, state: "failed", at: new Date().toISOString(), durationMs: Date.now() - startedAt, reason: error instanceof Error ? error.message.slice(0, 300) : String(error).slice(0, 300) }); + throw error; + } +} + +export function skippedNodeRuntimeStatusCommand(name: string): CommandResult { + return { + command: ["", name], + cwd: repoRoot, + exitCode: 0, + stdout: "", + stderr: "", + signal: null, + timedOut: false, + }; +} + +export function nodeRuntimeStatusWorkerDiagnostics(result: CommandResult, includeStdoutTail: boolean): Record { + return { + stderrTail: statusDiagnosticTail(result.stderr, WORKER_STDERR_TAIL_CHARS, true), + stdoutTail: includeStdoutTail ? statusDiagnosticTail(result.stdout, WORKER_STDOUT_TAIL_CHARS, false) : null, + valuesRedacted: true, + }; +} + +function statusDiagnosticTail(text: string, maxChars: number, filterProgress: boolean): string | null { + const filtered = filterProgress + ? text.split(/\r?\n/u).filter((line) => !line.trimStart().startsWith(WORKER_PROGRESS_PREFIX)).join("\n") + : text; + const redacted = redactStatusDiagnostic(filtered).trim(); + return redacted.length === 0 ? null : redacted.slice(-maxChars); +} + +function redactStatusDiagnostic(text: string): string { + return text + .replace(/-----BEGIN [^-\r\n]*PRIVATE KEY-----[\s\S]*?-----END [^-\r\n]*PRIVATE KEY-----/giu, "") + .replace(/((?:postgres(?:ql)?|https?|ssh):\/\/)[^@\s"']+@/giu, "$1@") + .replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+\/=:-]+/giu, "$1 ") + .replace(/\b(?:sk-[A-Za-z0-9_-]{16,}|(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9_]{16,})\b/gu, "") + .replace(/([?&](?:access[_-]?token|api[_-]?key|apikey|authorization|cookie|password|secret|token)=)[^&\s"']+/giu, "$1") + .replace(/(["']?(?:access[_-]?token|api[_-]?key|apikey|authorization|cookie|database[_-]?url|jwt[_-]?secret|n8n_encryption_key|passwd|password|private[_-]?key|secret|token)["']?\s*[:=]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,}\r\n]+)/giu, "$1"); +} + +function emitStatusProgress(scoped: ScopedStatus, detail: Record): void { + process.stderr.write(`${JSON.stringify({ event: "hwlab.control-plane.status.progress", at: new Date().toISOString(), node: scoped.node, lane: scoped.lane, ...detail })}\n`); +} + +function requiredRecord(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`); + return value as Record; +} + +function requiredPositiveInteger(value: unknown, path: string): number { + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`); + return value; +} + +function requiredBoolean(value: unknown, path: string): boolean { + if (typeof value !== "boolean") throw new Error(`${path} must be a boolean`); + return value; +} diff --git a/scripts/src/hwlab-node/entry.ts b/scripts/src/hwlab-node/entry.ts index 7e89d82a..b8190c26 100644 --- a/scripts/src/hwlab-node/entry.ts +++ b/scripts/src/hwlab-node/entry.ts @@ -42,6 +42,7 @@ import { runNodeEndpointBridge } from "./secret-scripts"; import { nodeRuntimeSourceWorkspaceCommand } from "./source-workspace"; import { nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeScopedFullOutput, withNodeRuntimeGitMirrorRendered } from "./status"; import { assertNodeId, positiveIntegerOption, requiredOption } from "./utils"; +import { emitNodeRuntimeStatusWorkerEvent, isNodeRuntimeStatusWorker, readNodeRuntimeStatusPolicy, runObservedNodeRuntimeStatus } from "./control-plane-status-observed"; import { legacyHwlabNodeWebProbeUnsupported } from "../web-probe"; import { startNodeDelegatedJob } from "./web-probe"; import { assertKnownOptions } from "./web-probe-observe"; @@ -639,10 +640,17 @@ export async function runNodeDelegatedDomain(config: Config, domain: DelegatedNo } if (domain === "control-plane") { if (scoped.action === "status") { - const result = nodeRuntimeControlPlaneStatus(scoped); - if (scoped.originalArgs.includes("--raw")) return result; - if (scoped.originalArgs.includes("--full")) return withNodeRuntimeControlPlaneStatusFullRendered(result, scoped); - return withNodeRuntimeControlPlaneStatusRendered(result, scoped); + if (isNodeRuntimeStatusWorker()) { + return nodeRuntimeControlPlaneStatus(scoped, { + policy: readNodeRuntimeStatusPolicy(), + onProgress: emitNodeRuntimeStatusWorkerEvent, + }); + } + const observed = await runObservedNodeRuntimeStatus(scoped); + if (scoped.originalArgs.includes("--raw")) return observed.full; + if (scoped.originalArgs.includes("--json")) return observed.summary; + if (scoped.originalArgs.includes("--full")) return withNodeRuntimeControlPlaneStatusFullRendered(observed.full, scoped); + return withNodeRuntimeControlPlaneStatusRendered(observed.summary, scoped); } if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration" || scoped.action === "cleanup-runs" || scoped.action === "cleanup-released-pvs" || scoped.action === "cleanup-legacy-docker-images" || scoped.action === "cleanup-legacy-docker-registry-volume") { if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped); diff --git a/scripts/src/hwlab-node/git-mirror.ts b/scripts/src/hwlab-node/git-mirror.ts index a7a917b9..869bd515 100644 --- a/scripts/src/hwlab-node/git-mirror.ts +++ b/scripts/src/hwlab-node/git-mirror.ts @@ -774,15 +774,16 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record): string[] { + const observation = record(result.observation); + if (Object.keys(observation).length === 0) return []; + const processGroup = record(observation.processGroup); + const worker = record(observation.worker); + const completed = statusProbeNames(observation.completedProbes); + const unfinished = statusProbeNames(observation.unfinishedProbes); + const workerDiagnostic = webObserveText(worker.stderrTail ?? worker.stdoutTail ?? "").replace(/\s+/gu, " ").trim(); + return [ + "", + "OBSERVATION", + webObserveTable( + ["STATE", "PHASE", "ELAPSED", "BUDGET", "REASON", "PROCESS_GROUP"], + [[ + webObserveText(observation.state), + webObserveText(observation.currentPhase), + formatElapsedMs(observation.durationMs), + `${webObserveText(observation.totalBudgetSeconds)}s`, + webObserveShort(webObserveText(observation.reason), 48), + processGroup.cleaned === true ? "cleaned" : processGroup.cleaned === false ? "residual" : "not-requested", + ]], + ), + ` completed=${completed.length === 0 ? "-" : completed.join(",")} unfinished=${unfinished.length === 0 ? "-" : unfinished.join(",")}`, + ...(workerDiagnostic === "-" ? [] : [` worker-error=${webObserveShort(workerDiagnostic, 160)}`]), + ]; +} + +function statusProbeNames(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string").slice(0, 12) : []; +} + function nodeRuntimePublicProbeDetail(publicProbe: Record): string { const web = record(publicProbe.web); const apiHealth = record(publicProbe.apiHealth); diff --git a/scripts/src/hwlab-node/render.ts b/scripts/src/hwlab-node/render.ts index 18707b70..3d0ac356 100644 --- a/scripts/src/hwlab-node/render.ts +++ b/scripts/src/hwlab-node/render.ts @@ -38,6 +38,7 @@ import { compactNodeRuntimeGitMirrorStatus, nodeRuntimeGitMirrorStatus } from ". import { keyValueLinesFromText, numericField, optionValue, record, shellQuote } from "./utils"; import { externalPostgresBridgeStatus, externalPostgresSecretStatus, getNodeRuntimePipelineRun, isLocalPostgresObject, nodeRuntimeCodeAgentRuntimeStatus, nodeRuntimeRenderOverlay } from "./web-probe"; import { webObserveShort, webObserveText } from "./web-probe-observe"; +import { readNodeRuntimeStatusPolicy, runNodeRuntimeStatusProbe, skippedNodeRuntimeStatusCommand, type NodeRuntimeStatusPolicy, type NodeRuntimeStatusWorkerEvent } from "./control-plane-status-observed"; import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes"; const runtimeGitopsObservabilityNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-observability.mjs"), "utf8").trimEnd(); @@ -172,69 +173,120 @@ export function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGi }; } -export function nodeRuntimeControlPlaneStatus(scoped: ReturnType): Record { +export interface NodeRuntimeControlPlaneStatusOptions { + policy?: NodeRuntimeStatusPolicy; + onProgress?: (event: NodeRuntimeStatusWorkerEvent) => void; +} + +export function nodeRuntimeControlPlaneStatus(scoped: ReturnType, options: NodeRuntimeControlPlaneStatusOptions = {}): Record { + const statusPolicy = options.policy ?? readNodeRuntimeStatusPolicy(); const spec = scoped.spec; const probeTimeoutSeconds = Math.max(1, Math.min(60, scoped.timeoutSeconds)); const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit"); const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run"); - const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null; - const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null; - const expectedPipelineRun = sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit); + const sourceState = runNodeRuntimeStatusProbe("source", statusPolicy, options.onProgress, () => { + const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null; + const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null; + const expectedPipelineRun = sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit); + return { head, sourceCommit, expectedPipelineRun }; + }, () => ({ head: null, sourceCommit: sourceCommitOverride ?? null, expectedPipelineRun: null })); + const { head, sourceCommit, expectedPipelineRun } = sourceState; let pipelineRun = pipelineRunOverride ?? expectedPipelineRun; - const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds); - const namespaceExists = namespace.exitCode === 0; - const postgresObjects = namespaceExists - ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds) - : null; - const localPostgresObjects = postgresObjects === null - ? [] - : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec)); - const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds); - const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds); - const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds); - const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u); - let pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun); - const pipelineRunResolution = pipelineRunOverride !== undefined + + const controlPlane = runNodeRuntimeStatusProbe("control-plane", statusPolicy, options.onProgress, () => ({ + serviceAccount: runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds), + pipeline: runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds), + }), () => ({ serviceAccount: skippedNodeRuntimeStatusCommand("service-account"), pipeline: skippedNodeRuntimeStatusCommand("pipeline") })); + const { serviceAccount, pipeline } = controlPlane; + + let pipelineRunProbe: Record | null = null; + let pipelineRunDiagnostics: Record | null = null; + const pipelineRunResolution: Record = pipelineRunOverride !== undefined ? { mode: "explicit-pipeline-run", expectedName: pipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null } : sourceCommit === null || expectedPipelineRun === null ? { mode: "source-unresolved", expectedName: null, resolvedName: null, fallbackUsed: false, result: null } - : { mode: "source-commit", expectedName: expectedPipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null as Record | null }; - if (pipelineRunOverride === undefined && sourceCommit !== null && expectedPipelineRun !== null && pipelineRunProbe !== null && pipelineRunProbe.exists !== true) { - const resolved = resolveNodeRuntimePipelineRunBySourceCommit(spec, sourceCommit); - pipelineRunResolution.result = compactRuntimeCommand(resolved.result); - if (resolved.name !== null) { - pipelineRun = resolved.name; - pipelineRunResolution.resolvedName = resolved.name; - pipelineRunResolution.fallbackUsed = true; - pipelineRunProbe = getNodeRuntimePipelineRun(spec, resolved.name); + : { mode: "source-commit", expectedName: expectedPipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null }; + runNodeRuntimeStatusProbe("pipeline-run", statusPolicy, options.onProgress, () => { + pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun); + if (pipelineRunOverride === undefined && sourceCommit !== null && expectedPipelineRun !== null && pipelineRunProbe !== null && pipelineRunProbe.exists !== true) { + const resolved = resolveNodeRuntimePipelineRunBySourceCommit(spec, sourceCommit); + pipelineRunResolution.result = compactRuntimeCommand(resolved.result); + if (resolved.name !== null) { + pipelineRun = resolved.name; + pipelineRunResolution.resolvedName = resolved.name; + pipelineRunResolution.fallbackUsed = true; + pipelineRunProbe = getNodeRuntimePipelineRun(spec, resolved.name); + } } - } - const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True" - ? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun) - : null; - const workloads = namespaceExists - ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds) - : null; - const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); - const workloadReadinessProbe = namespaceExists - ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds) - : null; - const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? ""); - const bridge = externalPostgresBridgeStatus(spec, namespaceExists); - const secrets = externalPostgresSecretStatus(spec, namespaceExists); - const codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists); - const publicProbes = nodeRuntimePublicProbeStatus(spec); - const gitMirror = nodeRuntimeGitMirrorStatus(scoped); - const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror); + pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True" + ? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun) + : null; + return true; + }, () => { + pipelineRunProbe = { exists: false, ready: true, skipped: true, reason: "probe-disabled" }; + return false; + }); + + let namespace = skippedNodeRuntimeStatusCommand("runtime-namespace"); + let namespaceExists = false; + let postgresObjects: CommandResult | null = null; + let localPostgresObjects: string[] = []; + let workloads: CommandResult | null = null; + let workloadNames: string[] = []; + let workloadReadinessProbe: CommandResult | null = null; + let workloadReadiness: Array> = []; + let bridge: Record = { required: false, ready: true, skipped: true }; + let secrets: Record = { required: false, ready: true, skipped: true, valuesPrinted: false }; + let codeAgentRuntime: Record = { required: false, ready: true, skipped: true }; + runNodeRuntimeStatusProbe("runtime", statusPolicy, options.onProgress, () => { + namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds); + namespaceExists = namespace.exitCode === 0; + postgresObjects = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds) + : null; + localPostgresObjects = postgresObjects === null ? [] : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec)); + workloads = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds) + : null; + workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + workloadReadinessProbe = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds) + : null; + workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? ""); + bridge = externalPostgresBridgeStatus(spec, namespaceExists); + secrets = externalPostgresSecretStatus(spec, namespaceExists); + codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists); + return true; + }, () => false); + + const publicProbes = runNodeRuntimeStatusProbe("public", statusPolicy, options.onProgress, () => nodeRuntimePublicProbeStatus(spec), () => ({ + ready: true, + skipped: true, + reason: "probe-disabled", + web: { url: spec.publicWebUrl, ok: true, skipped: true, httpStatus: null }, + apiHealth: { url: spec.publicApiUrl, ok: true, skipped: true, httpStatus: null }, + })); + const gitMirror = runNodeRuntimeStatusProbe("git-mirror", statusPolicy, options.onProgress, () => nodeRuntimeGitMirrorStatus(scoped), () => ({ ok: true, ready: true, skipped: true, compact: { skipped: true, sourceSnapshotReady: true, pendingFlush: false, githubInSync: true } })); + const gitMirrorCompact = statusPolicy.probes["git-mirror"] ? compactNodeRuntimeGitMirrorStatus(gitMirror) : record(gitMirror.compact); + + const argoState = runNodeRuntimeStatusProbe("argo", statusPolicy, options.onProgress, () => { + const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds); + const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u); + const ready = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy"; + const diagnostics = argo.exitCode === 0 && !ready ? nodeRuntimeArgoDiagnostics(spec, probeTimeoutSeconds) : null; + return { argo, repoURL, targetRevision, path, syncRevision, syncStatus, health, ready, diagnostics }; + }, () => ({ argo: skippedNodeRuntimeStatusCommand("argo"), repoURL: "", targetRevision: "", path: "", syncRevision: "", syncStatus: "", health: "", ready: true, diagnostics: null })); + const { argo, repoURL, targetRevision, path, syncRevision, syncStatus, health, ready: argoReady, diagnostics: argoDiagnostics } = argoState; + const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec); - const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0; + const controlPlaneReady = !statusPolicy.probes["control-plane"] || serviceAccount.exitCode === 0 && pipeline.exitCode === 0; const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready); const localPostgresExpectedAbsent = nodeRuntimeLocalPostgresExpectedAbsent(spec); const localPostgresReady = localPostgresExpectedAbsent ? localPostgresObjects.length === 0 : localPostgresObjects.length > 0; const postgresReady = activeExternalPostgres === undefined ? localPostgresReady : bridge.ready === true && secrets.ready === true; - const runtimeReady = namespaceExists && postgresReady && workloadsReady && codeAgentRuntime.ready === true; + const runtimeReady = !statusPolicy.probes.runtime || namespaceExists && postgresReady && workloadsReady && codeAgentRuntime.ready === true; const codeAgentRuntimeDegradedReason = typeof codeAgentRuntime.degradedReason === "string" ? codeAgentRuntime.degradedReason : null; let runtimeDegradedReason = "runtime-not-ready"; if (!namespaceExists) { @@ -250,18 +302,14 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType