fix: 修复 control-plane status 长时无输出

This commit is contained in:
Codex
2026-07-10 22:00:56 +02:00
parent 16579dbc44
commit d0c2c3f250
9 changed files with 630 additions and 81 deletions
+118 -56
View File
@@ -38,6 +38,7 @@ import { compactNodeRuntimeGitMirrorStatus, nodeRuntimeGitMirrorStatus } from ".
import { keyValueLinesFromText, numericField, optionValue, record, shellQuote } from "./utils";
import { externalPostgresBridgeStatus, externalPostgresSecretStatus, getNodeRuntimePipelineRun, isLocalPostgresObject, nodeRuntimeCodeAgentRuntimeStatus, nodeRuntimeRenderOverlay } from "./web-probe";
import { webObserveShort, webObserveText } from "./web-probe-observe";
import { readNodeRuntimeStatusPolicy, runNodeRuntimeStatusProbe, skippedNodeRuntimeStatusCommand, type NodeRuntimeStatusPolicy, type NodeRuntimeStatusWorkerEvent } from "./control-plane-status-observed";
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
const runtimeGitopsObservabilityNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-observability.mjs"), "utf8").trimEnd();
@@ -172,69 +173,120 @@ export function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGi
};
}
export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
export interface NodeRuntimeControlPlaneStatusOptions {
policy?: NodeRuntimeStatusPolicy;
onProgress?: (event: NodeRuntimeStatusWorkerEvent) => void;
}
export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, options: NodeRuntimeControlPlaneStatusOptions = {}): Record<string, unknown> {
const statusPolicy = options.policy ?? readNodeRuntimeStatusPolicy();
const spec = scoped.spec;
const probeTimeoutSeconds = Math.max(1, Math.min(60, scoped.timeoutSeconds));
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run");
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
const expectedPipelineRun = sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit);
const sourceState = runNodeRuntimeStatusProbe("source", statusPolicy, options.onProgress, () => {
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
const expectedPipelineRun = sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit);
return { head, sourceCommit, expectedPipelineRun };
}, () => ({ head: null, sourceCommit: sourceCommitOverride ?? null, expectedPipelineRun: null }));
const { head, sourceCommit, expectedPipelineRun } = sourceState;
let pipelineRun = pipelineRunOverride ?? expectedPipelineRun;
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds);
const namespaceExists = namespace.exitCode === 0;
const postgresObjects = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds)
: null;
const localPostgresObjects = postgresObjects === null
? []
: postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds);
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds);
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds);
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
let pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
const pipelineRunResolution = pipelineRunOverride !== undefined
const controlPlane = runNodeRuntimeStatusProbe("control-plane", statusPolicy, options.onProgress, () => ({
serviceAccount: runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds),
pipeline: runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds),
}), () => ({ serviceAccount: skippedNodeRuntimeStatusCommand("service-account"), pipeline: skippedNodeRuntimeStatusCommand("pipeline") }));
const { serviceAccount, pipeline } = controlPlane;
let pipelineRunProbe: Record<string, unknown> | null = null;
let pipelineRunDiagnostics: Record<string, unknown> | null = null;
const pipelineRunResolution: Record<string, unknown> = pipelineRunOverride !== undefined
? { mode: "explicit-pipeline-run", expectedName: pipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null }
: sourceCommit === null || expectedPipelineRun === null
? { mode: "source-unresolved", expectedName: null, resolvedName: null, fallbackUsed: false, result: null }
: { mode: "source-commit", expectedName: expectedPipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null as Record<string, unknown> | null };
if (pipelineRunOverride === undefined && sourceCommit !== null && expectedPipelineRun !== null && pipelineRunProbe !== null && pipelineRunProbe.exists !== true) {
const resolved = resolveNodeRuntimePipelineRunBySourceCommit(spec, sourceCommit);
pipelineRunResolution.result = compactRuntimeCommand(resolved.result);
if (resolved.name !== null) {
pipelineRun = resolved.name;
pipelineRunResolution.resolvedName = resolved.name;
pipelineRunResolution.fallbackUsed = true;
pipelineRunProbe = getNodeRuntimePipelineRun(spec, resolved.name);
: { mode: "source-commit", expectedName: expectedPipelineRun, resolvedName: pipelineRun, fallbackUsed: false, result: null };
runNodeRuntimeStatusProbe("pipeline-run", statusPolicy, options.onProgress, () => {
pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
if (pipelineRunOverride === undefined && sourceCommit !== null && expectedPipelineRun !== null && pipelineRunProbe !== null && pipelineRunProbe.exists !== true) {
const resolved = resolveNodeRuntimePipelineRunBySourceCommit(spec, sourceCommit);
pipelineRunResolution.result = compactRuntimeCommand(resolved.result);
if (resolved.name !== null) {
pipelineRun = resolved.name;
pipelineRunResolution.resolvedName = resolved.name;
pipelineRunResolution.fallbackUsed = true;
pipelineRunProbe = getNodeRuntimePipelineRun(spec, resolved.name);
}
}
}
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
: null;
const workloads = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds)
: null;
const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
const workloadReadinessProbe = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds)
: null;
const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
const secrets = externalPostgresSecretStatus(spec, namespaceExists);
const codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists);
const publicProbes = nodeRuntimePublicProbeStatus(spec);
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror);
pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
: null;
return true;
}, () => {
pipelineRunProbe = { exists: false, ready: true, skipped: true, reason: "probe-disabled" };
return false;
});
let namespace = skippedNodeRuntimeStatusCommand("runtime-namespace");
let namespaceExists = false;
let postgresObjects: CommandResult | null = null;
let localPostgresObjects: string[] = [];
let workloads: CommandResult | null = null;
let workloadNames: string[] = [];
let workloadReadinessProbe: CommandResult | null = null;
let workloadReadiness: Array<Record<string, unknown>> = [];
let bridge: Record<string, unknown> = { required: false, ready: true, skipped: true };
let secrets: Record<string, unknown> = { required: false, ready: true, skipped: true, valuesPrinted: false };
let codeAgentRuntime: Record<string, unknown> = { required: false, ready: true, skipped: true };
runNodeRuntimeStatusProbe("runtime", statusPolicy, options.onProgress, () => {
namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds);
namespaceExists = namespace.exitCode === 0;
postgresObjects = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds)
: null;
localPostgresObjects = postgresObjects === null ? [] : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
workloads = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds)
: null;
workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
workloadReadinessProbe = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds)
: null;
workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
bridge = externalPostgresBridgeStatus(spec, namespaceExists);
secrets = externalPostgresSecretStatus(spec, namespaceExists);
codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists);
return true;
}, () => false);
const publicProbes = runNodeRuntimeStatusProbe("public", statusPolicy, options.onProgress, () => nodeRuntimePublicProbeStatus(spec), () => ({
ready: true,
skipped: true,
reason: "probe-disabled",
web: { url: spec.publicWebUrl, ok: true, skipped: true, httpStatus: null },
apiHealth: { url: spec.publicApiUrl, ok: true, skipped: true, httpStatus: null },
}));
const gitMirror = runNodeRuntimeStatusProbe("git-mirror", statusPolicy, options.onProgress, () => nodeRuntimeGitMirrorStatus(scoped), () => ({ ok: true, ready: true, skipped: true, compact: { skipped: true, sourceSnapshotReady: true, pendingFlush: false, githubInSync: true } }));
const gitMirrorCompact = statusPolicy.probes["git-mirror"] ? compactNodeRuntimeGitMirrorStatus(gitMirror) : record(gitMirror.compact);
const argoState = runNodeRuntimeStatusProbe("argo", statusPolicy, options.onProgress, () => {
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds);
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
const ready = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
const diagnostics = argo.exitCode === 0 && !ready ? nodeRuntimeArgoDiagnostics(spec, probeTimeoutSeconds) : null;
return { argo, repoURL, targetRevision, path, syncRevision, syncStatus, health, ready, diagnostics };
}, () => ({ argo: skippedNodeRuntimeStatusCommand("argo"), repoURL: "", targetRevision: "", path: "", syncRevision: "", syncStatus: "", health: "", ready: true, diagnostics: null }));
const { argo, repoURL, targetRevision, path, syncRevision, syncStatus, health, ready: argoReady, diagnostics: argoDiagnostics } = argoState;
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0;
const controlPlaneReady = !statusPolicy.probes["control-plane"] || serviceAccount.exitCode === 0 && pipeline.exitCode === 0;
const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready);
const localPostgresExpectedAbsent = nodeRuntimeLocalPostgresExpectedAbsent(spec);
const localPostgresReady = localPostgresExpectedAbsent ? localPostgresObjects.length === 0 : localPostgresObjects.length > 0;
const postgresReady = activeExternalPostgres === undefined
? localPostgresReady
: bridge.ready === true && secrets.ready === true;
const runtimeReady = namespaceExists && postgresReady && workloadsReady && codeAgentRuntime.ready === true;
const runtimeReady = !statusPolicy.probes.runtime || namespaceExists && postgresReady && workloadsReady && codeAgentRuntime.ready === true;
const codeAgentRuntimeDegradedReason = typeof codeAgentRuntime.degradedReason === "string" ? codeAgentRuntime.degradedReason : null;
let runtimeDegradedReason = "runtime-not-ready";
if (!namespaceExists) {
@@ -250,18 +302,14 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
} else if (codeAgentRuntime.ready !== true) {
runtimeDegradedReason = codeAgentRuntimeDegradedReason ?? "code-agent-runtime-not-ready";
}
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
const argoDiagnostics = argo.exitCode === 0 && !argoReady
? nodeRuntimeArgoDiagnostics(spec, probeTimeoutSeconds)
: null;
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
const pipelineRunReady = !statusPolicy.probes["pipeline-run"] || pipelineRunProbe !== null && pipelineRunProbe.status === "True";
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
? pipelineRunDiagnostics.degradedReason
: pipelineRunProbe === null || pipelineRunProbe.exists !== true
? "pipelinerun-not-found"
: "pipelinerun-not-succeeded";
const publicReady = publicProbes.ready === true;
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
const publicReady = !statusPolicy.probes.public || publicProbes.ready === true;
const gitMirrorReady = !statusPolicy.probes["git-mirror"] || gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
const gitMirrorDegradedReason = gitMirrorCompact.sourceSnapshotReady === false
? "source-snapshot-missing"
: gitMirrorCompact.pendingFlush === true
@@ -307,16 +355,24 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
expectedPipelineRun,
pipelineRunResolution,
expected: nodeRuntimeExpected(spec),
sourceHead: head === null
statusPolicy: {
configPath: statusPolicy.configPath,
probes: statusPolicy.probes,
},
sourceHead: !statusPolicy.probes.source
? { ok: true, skipped: true, reason: "probe-disabled" }
: head === null
? { ok: sourceCommit !== null, value: sourceCommit, source: "option" }
: { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) },
controlPlane: {
ready: controlPlaneReady,
skipped: !statusPolicy.probes["control-plane"],
serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) },
pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) },
},
argo: {
ready: argoReady,
skipped: !statusPolicy.probes.argo,
application: spec.app,
repoURL,
expectedRepoURL: spec.argoRepoUrl,
@@ -333,6 +389,7 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
pipelineRunDiagnostics,
runtime: {
ready: runtimeReady,
skipped: !statusPolicy.probes.runtime,
namespace: spec.runtimeNamespace,
namespaceExists,
localPostgresObjects,
@@ -694,11 +751,12 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
pipelineRun: {
name: pipelineRun.name ?? null,
exists: pipelineRun.exists === true,
skipped: pipelineRun.skipped === true,
status: pipelineRun.status ?? null,
reason: pipelineRun.reason ?? null,
message: pipelineRun.message ?? null,
createdAt: pipelineRun.createdAt ?? null,
ready: pipelineRun.status === "True",
ready: pipelineRun.skipped === true || pipelineRun.status === "True",
diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : {
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
@@ -715,6 +773,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
argo: {
application: argo.application ?? null,
ready: argo.ready === true,
skipped: argo.skipped === true,
syncRevision: argo.syncRevision ?? null,
targetGitopsRevision: argo.targetGitopsRevision ?? null,
revisionObserved: typeof argo.targetGitopsRevision === "string" && argo.syncRevision === argo.targetGitopsRevision,
@@ -726,6 +785,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
namespace: runtime.namespace ?? null,
namespaceExists: runtime.namespaceExists === true,
ready: runtime.ready === true,
skipped: runtime.skipped === true,
workloadCount,
workloadReady: `${readyWorkloads}/${workloadReadiness.length}`,
notReadyWorkloads,
@@ -740,6 +800,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
},
publicProbe: {
ready: publicProbes.ready === true,
skipped: publicProbes.skipped === true,
web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
targetHost: Object.keys(targetHostProbe).length === 0 ? null : {
@@ -765,6 +826,7 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
},
gitMirror: {
ready: gitMirror.ready === true,
skipped: gitMirror.skipped === true,
localSource: gitMirrorCompact.localSource ?? null,
githubSource: gitMirrorCompact.githubSource ?? null,
sourceAuthority: gitMirrorCompact.sourceAuthority ?? null,