fix(sentinel): add YAML shortest-path validation
This commit is contained in:
@@ -3,6 +3,7 @@ import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { resolveSentinelChildJson } from "./hwlab-node-web-sentinel-cicd-shared";
|
||||
import { sentinelDeliveryStatus, type SentinelCicdState } from "./hwlab-node-web-sentinel-cicd";
|
||||
|
||||
describe("sentinel CI/CD child JSON recovery", () => {
|
||||
test("recovers remote probe JSON from trans truncation dump", () => {
|
||||
@@ -37,3 +38,22 @@ describe("sentinel CI/CD child JSON recovery", () => {
|
||||
expect(resolved.diagnostics.source).toBe("dump");
|
||||
});
|
||||
});
|
||||
|
||||
test("sentinel delivery status distinguishes cadence convergence without blocking", () => {
|
||||
const commit = "f87cd3ac8fd2494f9dfed51e7d8ececd3fc42939";
|
||||
const state = { sourceHead: { commit } } as unknown as SentinelCicdState;
|
||||
const base = {
|
||||
sourceMirror: { ok: true },
|
||||
registry: { probe: { present: true } },
|
||||
gitMirror: { ok: true },
|
||||
gitops: { ok: true, sourceCommit: commit, schedule: "0 */2 * * *", revision: "gitops-revision" },
|
||||
argo: { ok: true, syncStatus: "Synced", healthStatus: "Healthy" },
|
||||
runtime: { ok: true, probe: { deployment: { sourceCommit: commit } } },
|
||||
};
|
||||
const converging = sentinelDeliveryStatus(state, { ...base, cadence: { ok: false, probe: { schedule: "0 * * * *" } } }, "pipeline-run");
|
||||
expect(converging.outcome).toBe("converging");
|
||||
expect(converging.blocking).toBe(false);
|
||||
const converged = sentinelDeliveryStatus(state, { ...base, cadence: { ok: true, probe: { schedule: "0 */2 * * *" } } }, "pipeline-run");
|
||||
expect(converged.outcome).toBe("converged");
|
||||
expect(converged.desiredSourceCommit).toBe(commit);
|
||||
});
|
||||
|
||||
@@ -102,6 +102,7 @@ export type WebProbeSentinelOptions =
|
||||
readonly wait: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
readonly quickVerify: boolean;
|
||||
readonly localOnly: boolean;
|
||||
}
|
||||
| {
|
||||
readonly kind: "report";
|
||||
@@ -659,6 +660,7 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
|
||||
const gitops = record(result.gitops);
|
||||
const argo = record(result.argo);
|
||||
const validation = record(result.validation);
|
||||
const deliveryStatus = record(result.deliveryStatus);
|
||||
const observability = record(result.observability);
|
||||
const observed = record(result.observed);
|
||||
const sourceMirrorSync = record(result.sourceMirrorSync);
|
||||
@@ -751,6 +753,11 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
|
||||
"",
|
||||
renderObservedStatus(observed),
|
||||
"",
|
||||
Object.keys(deliveryStatus).length === 0 ? "DELIVERY_STATUS\n-" : table(
|
||||
["OUTCOME", "DESIRED_COMMIT", "RUNTIME_COMMIT", "DESIRED_SCHEDULE", "RUNTIME_SCHEDULE", "ARGO", "READY", "CONVERGING", "BLOCKING"],
|
||||
[[deliveryStatus.outcome, short(deliveryStatus.desiredSourceCommit), short(deliveryStatus.runtimeSourceCommit), deliveryStatus.desiredSchedule ?? "-", deliveryStatus.runtimeSchedule ?? "-", `${deliveryStatus.argoSyncStatus ?? "-"}/${deliveryStatus.argoHealthStatus ?? "-"}`, deliveryStatus.runtimeReady, deliveryStatus.converging, deliveryStatus.blocking]],
|
||||
),
|
||||
"",
|
||||
Object.keys(statusDiagnosis).length === 0 ? "STATUS_DIAGNOSIS\n-" : [
|
||||
"STATUS_DIAGNOSIS",
|
||||
table(["CODE", "PHASE", "PIPELINERUN", "SOURCE", "REGISTRY", "GIT_MIRROR", "GITOPS", "ARGO", "RUNTIME"], [[
|
||||
@@ -965,8 +972,8 @@ export function renderAsyncJobResult(result: Record<string, unknown>): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function rendered(ok: boolean, command: string, text: string): RenderedCliResult {
|
||||
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" };
|
||||
export function rendered(ok: boolean, command: string, text: string, projection?: Record<string, unknown>): RenderedCliResult {
|
||||
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain", ...(projection === undefined ? {} : { projection }) };
|
||||
}
|
||||
|
||||
export function sentinelProgressEvent(event: string, payload: Record<string, unknown>): void {
|
||||
|
||||
@@ -168,7 +168,16 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
|
||||
if (blocked !== null) return blocked;
|
||||
}
|
||||
requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
|
||||
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options), sentinelSourceOverrideFromOptions(options));
|
||||
const localCommit = options.kind === "validate" && options.localOnly
|
||||
? runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 5_000 }).stdout.trim()
|
||||
: null;
|
||||
const localSourceOverride = localCommit === null ? null : {
|
||||
commit: localCommit,
|
||||
stageRef: null,
|
||||
mirrorCommit: localCommit,
|
||||
sourceAuthority: "gitea-snapshot" as const,
|
||||
};
|
||||
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options), localSourceOverride ?? sentinelSourceOverrideFromOptions(options));
|
||||
if (options.kind === "image") return runSentinelImage(state, options);
|
||||
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
|
||||
if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
|
||||
@@ -229,13 +238,17 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
|
||||
if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds);
|
||||
return runSentinelControlPlaneConfirmed(state, options);
|
||||
}
|
||||
const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null;
|
||||
const observed = options.action === "status"
|
||||
? options.wait
|
||||
? waitForSentinelObservedStatus(state, options.timeoutSeconds)
|
||||
: collectSentinelObservedStatus(state, options.timeoutSeconds)
|
||||
: null;
|
||||
const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
|
||||
const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
|
||||
const pipelineRun = sentinelPipelineRunName(state, options.rerun);
|
||||
const statusDiagnosis = options.action === "status" && !observedReady ? sentinelObservedStatusDiagnosis(state, observed, pipelineRun) : null;
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && observedReady,
|
||||
ok: state.configReady && state.sourceHead.ok && (options.action === "status" || observedReady),
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
@@ -283,17 +296,18 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
|
||||
cicd: state.cicd,
|
||||
}),
|
||||
observed,
|
||||
deliveryStatus: options.action === "status" ? sentinelDeliveryStatus(state, record(observed), pipelineRun) : null,
|
||||
statusDiagnosis,
|
||||
drillDown: controlPlaneDrillDown(state, pipelineRun, null),
|
||||
warnings: mergeWarnings(observedWarnings, record(statusDiagnosis).warning),
|
||||
blocker: observedReady
|
||||
blocker: options.action === "status" || observedReady
|
||||
? null
|
||||
: record(statusDiagnosis).blocker ?? { code: "sentinel-control-plane-observed-not-ready", reason: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass", valuesRedacted: true },
|
||||
recoveryNext: record(statusDiagnosis).recoveryNext ?? null,
|
||||
next: controlPlaneNext(state, options.action),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(result.ok, command, renderControlPlaneResult(result));
|
||||
return rendered(result.ok, command, renderControlPlaneResult(result), options.action === "status" ? record(result.deliveryStatus) : undefined);
|
||||
}
|
||||
|
||||
function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
|
||||
@@ -1924,7 +1938,7 @@ function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
|
||||
gitops,
|
||||
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
|
||||
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
|
||||
cadence: probeCadenceCronJob(state, timeoutSeconds),
|
||||
cadence: probeCadenceCronJob(state, timeoutSeconds, nonEmptyString(gitops.schedule)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1934,7 +1948,8 @@ function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
|
||||
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
||||
let polls = 1;
|
||||
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
|
||||
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
||||
const pollSeconds = numberAtNullable(state.cicd, "confirmWait.pollSeconds") ?? 2;
|
||||
runCommand(["sleep", String(pollSeconds)], repoRoot, { timeoutMs: (pollSeconds + 1) * 1_000 });
|
||||
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
||||
polls += 1;
|
||||
}
|
||||
@@ -1964,6 +1979,35 @@ function sentinelObservedReady(value: Record<string, unknown> | SentinelObserved
|
||||
&& record(observed.cadence).ok === true;
|
||||
}
|
||||
|
||||
export function sentinelDeliveryStatus(state: SentinelCicdState, observed: Record<string, unknown>, pipelineRun: string): Record<string, unknown> {
|
||||
const gitops = record(observed.gitops);
|
||||
const runtimeDeployment = record(record(observed.runtime).probe).deployment;
|
||||
const cadenceProbe = record(record(observed.cadence).probe);
|
||||
const argo = record(observed.argo);
|
||||
const desiredSourceCommit = nonEmptyString(gitops.sourceCommit) ?? state.sourceHead.commit;
|
||||
const runtimeSourceCommit = nonEmptyString(record(runtimeDeployment).sourceCommit);
|
||||
const desiredSchedule = nonEmptyString(gitops.schedule);
|
||||
const runtimeSchedule = nonEmptyString(cadenceProbe.schedule);
|
||||
const converging = desiredSourceCommit !== runtimeSourceCommit || desiredSchedule !== runtimeSchedule || !sentinelObservedReady(observed);
|
||||
return {
|
||||
desiredSourceCommit,
|
||||
runtimeSourceCommit,
|
||||
pipelineRun,
|
||||
gitopsRevision: gitops.revision ?? null,
|
||||
argoSyncStatus: argo.syncStatus ?? null,
|
||||
argoHealthStatus: argo.healthStatus ?? null,
|
||||
runtimeReady: record(observed.runtime).ok === true,
|
||||
desiredSchedule,
|
||||
runtimeSchedule,
|
||||
converging,
|
||||
warning: converging ? "GitOps 期望状态尚未完全收敛;这是只读、非阻塞 warning。" : null,
|
||||
blocking: false,
|
||||
outcome: converging ? "converging" : "converged",
|
||||
mutation: false,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObservedStatus | null): string[] {
|
||||
const observed = record(value);
|
||||
const argo = record(observed.argo);
|
||||
@@ -2253,6 +2297,8 @@ function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: nu
|
||||
const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null;
|
||||
const manifest = manifestLines.join("\n");
|
||||
const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]);
|
||||
const sourceCommit = nonEmptyString(manifest.match(/unidesk\.ai\/source-commit:\s*["']?([0-9a-f]{40})/iu)?.[1]);
|
||||
const schedule = nonEmptyString(manifest.match(/\bschedule:\s*["']?([^,"'\r\n#]+)/u)?.[1]?.trim());
|
||||
const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`);
|
||||
const compact = compactCommand(result);
|
||||
return {
|
||||
@@ -2261,6 +2307,8 @@ function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: nu
|
||||
branch,
|
||||
manifestPath,
|
||||
image,
|
||||
sourceCommit,
|
||||
schedule,
|
||||
imageMatchesRepository,
|
||||
result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` },
|
||||
valuesRedacted: true,
|
||||
@@ -2301,9 +2349,10 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e
|
||||
"const ready = Number(dep?.status?.readyReplicas ?? 0);",
|
||||
"const updated = Number(dep?.status?.updatedReplicas ?? 0);",
|
||||
"const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;",
|
||||
"const sourceCommit = dep?.spec?.template?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? dep?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? null;",
|
||||
"const imageMatches = expectedImage === null || image === expectedImage;",
|
||||
"const payload = {",
|
||||
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },",
|
||||
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches, sourceCommit },",
|
||||
" service: { present: rc('svc') === 0 },",
|
||||
" pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },",
|
||||
" configMap: { present: rc('cm') === 0 },",
|
||||
@@ -2320,7 +2369,7 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e
|
||||
return { ok: result.exitCode === 0 && probe?.ok === true, probe, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result) };
|
||||
}
|
||||
|
||||
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number, desiredSchedule: string | null = null): Record<string, unknown> {
|
||||
const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null;
|
||||
if (expected === null) {
|
||||
return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true };
|
||||
@@ -2329,7 +2378,7 @@ function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number):
|
||||
const spec = record(expected.spec);
|
||||
const namespace = stringAt(metadata, "namespace");
|
||||
const name = stringAt(metadata, "name");
|
||||
const expectedSchedule = stringAt(spec, "schedule");
|
||||
const expectedSchedule = desiredSchedule ?? stringAt(spec, "schedule");
|
||||
const script = [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
|
||||
@@ -128,6 +128,48 @@ export function runSentinelMaintenance(state: SentinelCicdState, options: Extrac
|
||||
|
||||
export function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
|
||||
const command = "web-probe sentinel validate";
|
||||
if (options.localOnly) {
|
||||
const cronJob = state.manifests.find((item) => item.kind === "CronJob") ?? null;
|
||||
const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
|
||||
const scenarioRows = Array.isArray(state.scenarios)
|
||||
? state.scenarios
|
||||
: Array.isArray(record(state.scenarios).scenarios)
|
||||
? record(state.scenarios).scenarios as unknown[]
|
||||
: [];
|
||||
const scenario = scenarioRows.map(record).find((item) => item.id === scenarioId) ?? null;
|
||||
const cadence = scenario === null ? null : scenario.cadence ?? null;
|
||||
const renderedCron = cronJob === null ? null : record(cronJob.spec).schedule ?? null;
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && cadence !== null && renderedCron !== null,
|
||||
command,
|
||||
mode: "local",
|
||||
mutation: false,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
configRef: state.configRefs.scenarios,
|
||||
scenarioId,
|
||||
cadence,
|
||||
renderedCron,
|
||||
sourceCommit: state.sourceHead.commit,
|
||||
manifestSha256: state.manifestSha256,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(result.ok, command, [
|
||||
command,
|
||||
`status: ${result.ok ? "ok" : "blocked"}`,
|
||||
`node: ${result.node}`,
|
||||
`lane: ${result.lane}`,
|
||||
`sentinelId: ${result.sentinelId}`,
|
||||
`configRef: ${result.configRef}`,
|
||||
`scenarioId: ${result.scenarioId}`,
|
||||
`cadence: ${result.cadence ?? "-"}`,
|
||||
`renderedCron: ${result.renderedCron ?? "-"}`,
|
||||
`sourceCommit: ${result.sourceCommit ?? "-"}`,
|
||||
`manifestSha256: ${result.manifestSha256}`,
|
||||
"mutation: false",
|
||||
].join("\n"), result);
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
|
||||
let quickVerify: Record<string, unknown> | null = null;
|
||||
|
||||
@@ -87,7 +87,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
"--source-stage-ref",
|
||||
"--source-mirror-commit",
|
||||
"--source-authority",
|
||||
]), new Set(["--dry-run", "--confirm", "--wait", "--rerun", "--manual-recovery", "--recovery", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page", "--diagnose-monitor-web"]));
|
||||
]), new Set(["--dry-run", "--confirm", "--wait", "--local", "--rerun", "--manual-recovery", "--recovery", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page", "--diagnose-monitor-web"]));
|
||||
const nodeOption = optionValue(args, "--node") ?? null;
|
||||
const laneOption = optionValue(args, "--lane") ?? null;
|
||||
const inspectMode = sentinelActionRaw === "inspect-url" || sentinelActionRaw === "inspect-id";
|
||||
@@ -183,7 +183,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
quickVerify: maintenanceAction === "stop" || args.includes("--quick-verify"),
|
||||
};
|
||||
} else if (sentinelActionRaw === "validate") {
|
||||
sentinel = { kind: "validate", action: "validate", node, lane, sentinelId, dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, quickVerify: args.includes("--quick-verify") };
|
||||
sentinel = { kind: "validate", action: "validate", node, lane, sentinelId, dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, quickVerify: args.includes("--quick-verify"), localOnly: args.includes("--local") };
|
||||
} else if (sentinelActionRaw === "dashboard") {
|
||||
const dashboardAction = parseWebProbeSentinelDashboardAction(args[1]);
|
||||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000);
|
||||
|
||||
@@ -16,6 +16,7 @@ export interface RenderedCliResult {
|
||||
command: string;
|
||||
renderedText: string;
|
||||
contentType: "text/plain" | "application/json" | "application/yaml";
|
||||
projection?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface CliInputErrorOptions {
|
||||
|
||||
@@ -12,7 +12,12 @@ export function webProbeHelp(): Record<string, unknown> {
|
||||
|
||||
export async function runWebProbeCommand(_config: Config, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (args.length === 0 || args.includes("--help") || args.includes("-h") || args[0] === "help") return webProbeHelp();
|
||||
return runNodeWebProbe(parseNodeWebProbeOptions(args));
|
||||
const format = args.includes("--json") ? "json" : args.includes("--yaml") ? "yaml" : "text";
|
||||
const parsedArgs = args.filter((item) => item !== "--json" && item !== "--yaml");
|
||||
const result = runNodeWebProbe(parseNodeWebProbeOptions(parsedArgs));
|
||||
if (format === "text" || !("projection" in result) || result.projection === undefined) return result;
|
||||
if (format === "json") return result.projection;
|
||||
return { ...result, renderedText: Bun.YAML.stringify(result.projection), contentType: "application/yaml" };
|
||||
}
|
||||
|
||||
export function legacyHwlabNodeWebProbeUnsupported(args: string[]): Record<string, unknown> {
|
||||
|
||||
Reference in New Issue
Block a user