diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 5c2003b5..7cd6927f 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -82,10 +82,16 @@ interface SentinelObservedStatus { readonly sourceMirror: Record; readonly registry: Record; readonly gitMirror: Record; + readonly gitops: Record; readonly argo: Record; readonly runtime: Record; } +interface SentinelObservedExpectation { + readonly gitopsRevision: string | null; + readonly runtimeImage: string | null; +} + interface SentinelRemoteJobResult { readonly ok: boolean; readonly phase: string; @@ -540,23 +546,30 @@ function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "con return rendered(true, command, renderAsyncJobResult(result)); } -function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number): SentinelObservedStatus { +function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus { + const registry = probeImageRegistry(state, timeoutSeconds); + const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds); + const effectiveExpectation = { + gitopsRevision: expectation?.gitopsRevision ?? nonEmptyString(gitops.revision), + runtimeImage: expectation?.runtimeImage ?? nonEmptyString(gitops.image) ?? expectedRuntimeImageFromRegistry(state, registry), + }; return { sourceMirror: probeSourceMirror(state, timeoutSeconds), - registry: probeImageRegistry(state, timeoutSeconds), + registry, gitMirror: runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds), - argo: probeArgoApplication(state, timeoutSeconds), - runtime: probeRuntimeObjects(state, timeoutSeconds), + gitops, + argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision), + runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage), }; } -function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number): SentinelObservedStatus { +function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus { const startedAt = Date.now(); const timeoutMs = Math.max(30_000, Math.min(timeoutSeconds * 1000, 900_000)); - let observed = collectSentinelObservedStatus(state, timeoutSeconds); + let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation); while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) { runCommand(["sleep", "5"], repoRoot, { timeoutMs: 6_000 }); - observed = collectSentinelObservedStatus(state, timeoutSeconds); + observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation); } return observed; } @@ -566,6 +579,7 @@ function sentinelObservedReady(value: Record | SentinelObserved return record(observed.sourceMirror).ok === true && record(record(observed.registry).probe).present === true && record(observed.gitMirror).ok === true + && record(observed.gitops).ok === true && record(observed.argo).ok === true && record(observed.runtime).ok === true; } @@ -607,26 +621,60 @@ function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Re return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) }; } -function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record { +function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record { const namespace = stringAt(state.cicd, "argo.namespace"); const applicationName = stringAt(state.cicd, "argo.applicationName"); - const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const app = parseJsonObject(result.stdout); - const status = record(app?.status); - const sync = record(status.sync); - const health = record(status.health); - const ok = result.exitCode === 0 && sync.status === "Synced" && health.status === "Healthy"; + const jsonpath = "{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.sync.revision}{\"\\n\"}"; + const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", `jsonpath=${jsonpath}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const [syncStatusRaw, healthStatusRaw, revisionRaw] = result.stdout.trim().split(/\r?\n/u); + const syncStatus = nonEmptyString(syncStatusRaw); + const healthStatus = nonEmptyString(healthStatusRaw); + const revision = nonEmptyString(revisionRaw); + const revisionMatches = expectedRevision === null || revision === expectedRevision; + const ok = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy" && revisionMatches; return { ok, present: result.exitCode === 0, - syncStatus: sync.status ?? null, - healthStatus: health.status ?? null, - revision: sync.revision ?? null, + syncStatus, + healthStatus, + revision, + expectedRevision, + revisionMatches, result: compactCommand(result), }; } -function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number): Record { +function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record { + const namespace = stringAt(state.cicd, "builder.namespace"); + const repository = stringAt(state.controlPlaneTarget, "source.repository"); + const branch = stringAt(state.cicd, "argo.targetRevision"); + const manifestPath = `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`; + const repoPath = `/cache/${repository}.git`; + const inner = [ + "set -eu", + `git --git-dir=${shellQuote(repoPath)} rev-parse ${shellQuote(`refs/heads/${branch}`)}`, + `git --git-dir=${shellQuote(repoPath)} show ${shellQuote(`refs/heads/${branch}:${manifestPath}`)}`, + ].join("\n"); + const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "exec", "deploy/git-mirror-http", "--", "sh", "-lc", inner], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const [revisionLine, ...manifestLines] = result.stdout.split(/\r?\n/u); + const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null; + const manifest = manifestLines.join("\n"); + const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]); + const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`); + const compact = compactCommand(result); + return { + ok: result.exitCode === 0 && revision !== null && imageMatchesRepository, + revision, + branch, + manifestPath, + image, + imageMatchesRepository, + result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` }, + valuesRedacted: true, + }; +} + +function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, expectedImage: string | null): Record { const namespace = stringAt(state.runtime, "namespace"); const deploymentName = stringAt(state.runtime, "deploymentName"); const serviceName = stringAt(state.runtime, "serviceName"); @@ -647,9 +695,11 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number): "kubectl -n \"$namespace\" get pvc \"$pvc\" -o json >\"$tmp/pvc.json\" 2>/dev/null; echo $? >\"$tmp/pvc.rc\"", "kubectl -n \"$namespace\" get cm \"$configmap\" -o json >\"$tmp/cm.json\" 2>/dev/null; echo $? >\"$tmp/cm.rc\"", "kubectl -n \"$namespace\" get sa \"$serviceaccount\" -o json >\"$tmp/sa.json\" 2>/dev/null; echo $? >\"$tmp/sa.rc\"", - "node - \"$tmp\" <<'NODE'", + `expected_image=${shellQuote(expectedImage ?? "")}`, + "node - \"$tmp\" \"$expected_image\" <<'NODE'", "const fs = require('node:fs');", "const dir = process.argv[2];", + "const expectedImage = process.argv[3] || null;", "function rc(name){ return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); }", "function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }", "const dep = json('deploy');", @@ -658,15 +708,16 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number): "const ready = Number(dep?.status?.readyReplicas ?? 0);", "const updated = Number(dep?.status?.updatedReplicas ?? 0);", "const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;", + "const imageMatches = expectedImage === null || image === expectedImage;", "const payload = {", - " deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image },", + " deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },", " service: { present: rc('svc') === 0 },", " pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },", " configMap: { present: rc('cm') === 0 },", " serviceAccount: { present: rc('sa') === 0 },", " valuesRedacted: true", "};", - "payload.ok = payload.deployment.present && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;", + "payload.ok = payload.deployment.present && imageMatches && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;", "console.log(JSON.stringify(payload));", "NODE", ].join("\n"); @@ -675,6 +726,12 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number): return { ok: result.exitCode === 0 && probe?.ok === true, probe, result: compactCommand(result) }; } +function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record): string | null { + const digest = nonEmptyString(record(record(registry).probe).digest); + if (digest === null) return null; + return `${state.image.repository}@${digest}`; +} + function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult { const jobName = `${stringAt(state.cicd, "builder.jobPrefix")}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); const manifest = sentinelPublishJobManifest(state, jobName, publishGitops); @@ -1071,6 +1128,7 @@ function renderObservedStatus(observed: Record): string { observedStatusRow("source", observed.sourceMirror), observedStatusRow("registry", observed.registry), observedStatusRow("git-mirror", observed.gitMirror), + observedStatusRow("gitops", observed.gitops), observedStatusRow("argo", observed.argo), observedStatusRow("runtime", observed.runtime), ].filter((row) => row !== null); @@ -1088,11 +1146,12 @@ function observedStatusRow(name: string, value: unknown): unknown[] | null { function observedDetail(name: string, item: Record): string { if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`; if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`; - if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}`; + if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`; + if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}`; if (name === "runtime") { const probe = record(item.probe); const deployment = record(probe.deployment); - return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}`; + return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`; } return "-"; } @@ -1168,6 +1227,10 @@ function stringAt(value: unknown, path: string): string { return found; } +function nonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + function stringField(value: Record, path: string): string { return stringAt(value, path); }