mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: require sentinel gitops revision convergence (#904)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-25 23:52:08 +08:00
committed by GitHub
parent 2963986251
commit 9a30427b2b
1 changed files with 86 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-web-sentinel-cicd.ts
+86 -23
View File
@@ -82,10 +82,16 @@ interface SentinelObservedStatus {
readonly sourceMirror: Record<string, unknown>;
readonly registry: Record<string, unknown>;
readonly gitMirror: Record<string, unknown>;
readonly gitops: Record<string, unknown>;
readonly argo: Record<string, unknown>;
readonly runtime: Record<string, unknown>;
}
interface SentinelObservedExpectation {
readonly gitopsRevision: string | null;
readonly runtimeImage: string | null;
}
interface SentinelRemoteJobResult {
readonly ok: boolean;
readonly phase: string;
@@ -540,23 +546,30 @@ function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "con
return rendered(true, command, renderAsyncJobResult(result));
}
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number): SentinelObservedStatus {
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus {
const registry = probeImageRegistry(state, timeoutSeconds);
const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds);
const effectiveExpectation = {
gitopsRevision: expectation?.gitopsRevision ?? nonEmptyString(gitops.revision),
runtimeImage: expectation?.runtimeImage ?? nonEmptyString(gitops.image) ?? expectedRuntimeImageFromRegistry(state, registry),
};
return {
sourceMirror: probeSourceMirror(state, timeoutSeconds),
registry: probeImageRegistry(state, timeoutSeconds),
registry,
gitMirror: runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds),
argo: probeArgoApplication(state, timeoutSeconds),
runtime: probeRuntimeObjects(state, timeoutSeconds),
gitops,
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
};
}
function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number): SentinelObservedStatus {
function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus {
const startedAt = Date.now();
const timeoutMs = Math.max(30_000, Math.min(timeoutSeconds * 1000, 900_000));
let observed = collectSentinelObservedStatus(state, timeoutSeconds);
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation);
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
runCommand(["sleep", "5"], repoRoot, { timeoutMs: 6_000 });
observed = collectSentinelObservedStatus(state, timeoutSeconds);
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation);
}
return observed;
}
@@ -566,6 +579,7 @@ function sentinelObservedReady(value: Record<string, unknown> | SentinelObserved
return record(observed.sourceMirror).ok === true
&& record(record(observed.registry).probe).present === true
&& record(observed.gitMirror).ok === true
&& record(observed.gitops).ok === true
&& record(observed.argo).ok === true
&& record(observed.runtime).ok === true;
}
@@ -607,26 +621,60 @@ function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Re
return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) };
}
function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record<string, unknown> {
const namespace = stringAt(state.cicd, "argo.namespace");
const applicationName = stringAt(state.cicd, "argo.applicationName");
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
const app = parseJsonObject(result.stdout);
const status = record(app?.status);
const sync = record(status.sync);
const health = record(status.health);
const ok = result.exitCode === 0 && sync.status === "Synced" && health.status === "Healthy";
const jsonpath = "{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.sync.revision}{\"\\n\"}";
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", `jsonpath=${jsonpath}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
const [syncStatusRaw, healthStatusRaw, revisionRaw] = result.stdout.trim().split(/\r?\n/u);
const syncStatus = nonEmptyString(syncStatusRaw);
const healthStatus = nonEmptyString(healthStatusRaw);
const revision = nonEmptyString(revisionRaw);
const revisionMatches = expectedRevision === null || revision === expectedRevision;
const ok = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy" && revisionMatches;
return {
ok,
present: result.exitCode === 0,
syncStatus: sync.status ?? null,
healthStatus: health.status ?? null,
revision: sync.revision ?? null,
syncStatus,
healthStatus,
revision,
expectedRevision,
revisionMatches,
result: compactCommand(result),
};
}
function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
const namespace = stringAt(state.cicd, "builder.namespace");
const repository = stringAt(state.controlPlaneTarget, "source.repository");
const branch = stringAt(state.cicd, "argo.targetRevision");
const manifestPath = `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`;
const repoPath = `/cache/${repository}.git`;
const inner = [
"set -eu",
`git --git-dir=${shellQuote(repoPath)} rev-parse ${shellQuote(`refs/heads/${branch}`)}`,
`git --git-dir=${shellQuote(repoPath)} show ${shellQuote(`refs/heads/${branch}:${manifestPath}`)}`,
].join("\n");
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "exec", "deploy/git-mirror-http", "--", "sh", "-lc", inner], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
const [revisionLine, ...manifestLines] = result.stdout.split(/\r?\n/u);
const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null;
const manifest = manifestLines.join("\n");
const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]);
const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`);
const compact = compactCommand(result);
return {
ok: result.exitCode === 0 && revision !== null && imageMatchesRepository,
revision,
branch,
manifestPath,
image,
imageMatchesRepository,
result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` },
valuesRedacted: true,
};
}
function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, expectedImage: string | null): Record<string, unknown> {
const namespace = stringAt(state.runtime, "namespace");
const deploymentName = stringAt(state.runtime, "deploymentName");
const serviceName = stringAt(state.runtime, "serviceName");
@@ -647,9 +695,11 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number):
"kubectl -n \"$namespace\" get pvc \"$pvc\" -o json >\"$tmp/pvc.json\" 2>/dev/null; echo $? >\"$tmp/pvc.rc\"",
"kubectl -n \"$namespace\" get cm \"$configmap\" -o json >\"$tmp/cm.json\" 2>/dev/null; echo $? >\"$tmp/cm.rc\"",
"kubectl -n \"$namespace\" get sa \"$serviceaccount\" -o json >\"$tmp/sa.json\" 2>/dev/null; echo $? >\"$tmp/sa.rc\"",
"node - \"$tmp\" <<'NODE'",
`expected_image=${shellQuote(expectedImage ?? "")}`,
"node - \"$tmp\" \"$expected_image\" <<'NODE'",
"const fs = require('node:fs');",
"const dir = process.argv[2];",
"const expectedImage = process.argv[3] || null;",
"function rc(name){ return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); }",
"function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }",
"const dep = json('deploy');",
@@ -658,15 +708,16 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number):
"const ready = Number(dep?.status?.readyReplicas ?? 0);",
"const updated = Number(dep?.status?.updatedReplicas ?? 0);",
"const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;",
"const imageMatches = expectedImage === null || image === expectedImage;",
"const payload = {",
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image },",
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },",
" service: { present: rc('svc') === 0 },",
" pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },",
" configMap: { present: rc('cm') === 0 },",
" serviceAccount: { present: rc('sa') === 0 },",
" valuesRedacted: true",
"};",
"payload.ok = payload.deployment.present && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;",
"payload.ok = payload.deployment.present && imageMatches && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;",
"console.log(JSON.stringify(payload));",
"NODE",
].join("\n");
@@ -675,6 +726,12 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number):
return { ok: result.exitCode === 0 && probe?.ok === true, probe, result: compactCommand(result) };
}
function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record<string, unknown>): string | null {
const digest = nonEmptyString(record(record(registry).probe).digest);
if (digest === null) return null;
return `${state.image.repository}@${digest}`;
}
function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult {
const jobName = `${stringAt(state.cicd, "builder.jobPrefix")}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
const manifest = sentinelPublishJobManifest(state, jobName, publishGitops);
@@ -1071,6 +1128,7 @@ function renderObservedStatus(observed: Record<string, unknown>): string {
observedStatusRow("source", observed.sourceMirror),
observedStatusRow("registry", observed.registry),
observedStatusRow("git-mirror", observed.gitMirror),
observedStatusRow("gitops", observed.gitops),
observedStatusRow("argo", observed.argo),
observedStatusRow("runtime", observed.runtime),
].filter((row) => row !== null);
@@ -1088,11 +1146,12 @@ function observedStatusRow(name: string, value: unknown): unknown[] | null {
function observedDetail(name: string, item: Record<string, unknown>): string {
if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`;
if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`;
if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}`;
if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`;
if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}`;
if (name === "runtime") {
const probe = record(item.probe);
const deployment = record(probe.deployment);
return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}`;
return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`;
}
return "-";
}
@@ -1168,6 +1227,10 @@ function stringAt(value: unknown, path: string): string {
return found;
}
function nonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.length > 0 ? value : null;
}
function stringField(value: Record<string, unknown>, path: string): string {
return stringAt(value, path);
}