fix(sentinel): add YAML shortest-path validation

This commit is contained in:
Codex
2026-07-12 17:28:17 +02:00
parent e5c763f5a7
commit 5a99f4efd7
11 changed files with 153 additions and 15 deletions
+1
View File
@@ -113,6 +113,7 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help
- PaC migrated consumer 的旧手动 publish/debug 命令不得作为正式 closeout 引导;调查命令默认只能指向只读的 `platform-infra pipelines-as-code status|history``closeout` 仅保留只读兼容能力,不得作为主代理、子代理或操作者的 PR 合并后步骤。需要保留的旧 mutation 命令只能属于 YAML 明确的 legacy authority 或独立平台维护,不得标记或引导 manual recovery;单步确认的自动链缺陷必须回到 YAML/controller/源码修复。
- Secret 只通过 YAML sourceRef/targetKey 和受控 CLI 下发;输出只披露 presence/fingerprint。
- 长命令用异步 job 或短轮询;不要长时间挂住 trans/ssh。
- Web 哨兵 YAML cadence 变更不新增发布 orchestrator:提交前用 `web-probe sentinel validate --local`;合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单,不能用操作者旧 worktree 的本地渲染值做阻塞门禁;收敛差异统一为非阻塞 warning。
## 何时读取 reference
+2
View File
@@ -75,6 +75,8 @@ bun scripts/cli.ts web-probe observe analyze <observerId>
## Architecture Preference
Web 哨兵 YAML 小改动使用两条现有入口完成闭环:提交前运行 `web-probe sentinel validate --local`,提交合并后运行 `web-probe sentinel control-plane status --wait`。前者只验证当前 worktree 的 owning YAML 和渲染结果;后者只读观察 PaC/GitOps/Argo/runtime/cadence 收敛,不得触发 publish、sync、refresh 或 patch。收敛中的版本或 cadence 差异只能显示为 `warning`,不得阻塞用户业务。
Prefer Kubernetes-native discovery and isolation before inventing a custom control plane:
- Labels/selectors identify sentinel runners.
+3
View File
@@ -87,6 +87,9 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/
## Validation
- Web 哨兵 cadence 例外采用现有 CLI 的轻量投影:`web-probe sentinel validate --local` 必须证明当前 worktree owning YAML 的 `configRef`、场景 cadence 和渲染 Cron,且 `mutation=false`
- 合并后只允许 `web-probe sentinel control-plane status --wait` 观察 GitOps 期望与 runtime 事实;旧 worktree 值不得参与线上阻塞判定。
- CLI 轻量变更:默认最多做 `bun --check`/TypeScript 语法、相关 `--help`、目标 `plan`/`validate`/`sync --dry-run` 或等价命令形态确认。
- 配置数值变更:只验证能被目标 CLI 读取和渲染,数值本身以 YAML 为准。
- 运行面变更:用原受控 CLI 入口验证实际对象、fingerprint、presence、health 或 public URL;不要绕到原生 `kubectl`/`curl` 作为正式控制面。
+8
View File
@@ -192,3 +192,11 @@ Avoid these patterns:
Long-term references should point to this architecture for common YAML-first ops rules, then document only domain-specific ownership and entrypoints. They should not repeat common Secret, exposure, target, redaction or no-hardcoding rules unless a domain adds a stricter constraint.
When a recurring operation becomes stable, update the owning reference document and the relevant skill with the domain entrypoint and decision boundary. Do not document one-off manual recovery as the standard path; manual repair remains recovery evidence until the YAML and CLI path exists.
## Web 哨兵最短变更路径
- 修改 cadence 前,使用 `web-probe sentinel validate --local` 直接读取当前 worktree 的 owning YAML。
- 本地验证必须输出 `configRef``scenarioId``cadence`、渲染后的 Cron 表达式和 `mutation=false`,不得读取远端 authority 来替代未提交 YAML。
- 合并后的观察使用 `web-probe sentinel control-plane status --wait`;该入口只读,等待和轮询预算来自 owning YAML。
- 状态必须同时区分 GitOps 期望和运行时事实,并输出 `desiredSourceCommit``runtimeSourceCommit``desiredSchedule``runtimeSchedule``converging``warning``blocking=false`
- 旧 worktree 的本地渲染值不得作为线上 cadence 对齐门禁;线上期望值来自 GitOps 清单。
@@ -3,6 +3,7 @@ import { join } from "node:path";
import { tmpdir } from "node:os";
import { describe, expect, test } from "bun:test";
import { resolveSentinelChildJson } from "./hwlab-node-web-sentinel-cicd-shared";
import { sentinelDeliveryStatus, type SentinelCicdState } from "./hwlab-node-web-sentinel-cicd";
describe("sentinel CI/CD child JSON recovery", () => {
test("recovers remote probe JSON from trans truncation dump", () => {
@@ -37,3 +38,22 @@ describe("sentinel CI/CD child JSON recovery", () => {
expect(resolved.diagnostics.source).toBe("dump");
});
});
test("sentinel delivery status distinguishes cadence convergence without blocking", () => {
const commit = "f87cd3ac8fd2494f9dfed51e7d8ececd3fc42939";
const state = { sourceHead: { commit } } as unknown as SentinelCicdState;
const base = {
sourceMirror: { ok: true },
registry: { probe: { present: true } },
gitMirror: { ok: true },
gitops: { ok: true, sourceCommit: commit, schedule: "0 */2 * * *", revision: "gitops-revision" },
argo: { ok: true, syncStatus: "Synced", healthStatus: "Healthy" },
runtime: { ok: true, probe: { deployment: { sourceCommit: commit } } },
};
const converging = sentinelDeliveryStatus(state, { ...base, cadence: { ok: false, probe: { schedule: "0 * * * *" } } }, "pipeline-run");
expect(converging.outcome).toBe("converging");
expect(converging.blocking).toBe(false);
const converged = sentinelDeliveryStatus(state, { ...base, cadence: { ok: true, probe: { schedule: "0 */2 * * *" } } }, "pipeline-run");
expect(converged.outcome).toBe("converged");
expect(converged.desiredSourceCommit).toBe(commit);
});
@@ -102,6 +102,7 @@ export type WebProbeSentinelOptions =
readonly wait: boolean;
readonly timeoutSeconds: number;
readonly quickVerify: boolean;
readonly localOnly: boolean;
}
| {
readonly kind: "report";
@@ -659,6 +660,7 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
const gitops = record(result.gitops);
const argo = record(result.argo);
const validation = record(result.validation);
const deliveryStatus = record(result.deliveryStatus);
const observability = record(result.observability);
const observed = record(result.observed);
const sourceMirrorSync = record(result.sourceMirrorSync);
@@ -751,6 +753,11 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
"",
renderObservedStatus(observed),
"",
Object.keys(deliveryStatus).length === 0 ? "DELIVERY_STATUS\n-" : table(
["OUTCOME", "DESIRED_COMMIT", "RUNTIME_COMMIT", "DESIRED_SCHEDULE", "RUNTIME_SCHEDULE", "ARGO", "READY", "CONVERGING", "BLOCKING"],
[[deliveryStatus.outcome, short(deliveryStatus.desiredSourceCommit), short(deliveryStatus.runtimeSourceCommit), deliveryStatus.desiredSchedule ?? "-", deliveryStatus.runtimeSchedule ?? "-", `${deliveryStatus.argoSyncStatus ?? "-"}/${deliveryStatus.argoHealthStatus ?? "-"}`, deliveryStatus.runtimeReady, deliveryStatus.converging, deliveryStatus.blocking]],
),
"",
Object.keys(statusDiagnosis).length === 0 ? "STATUS_DIAGNOSIS\n-" : [
"STATUS_DIAGNOSIS",
table(["CODE", "PHASE", "PIPELINERUN", "SOURCE", "REGISTRY", "GIT_MIRROR", "GITOPS", "ARGO", "RUNTIME"], [[
@@ -965,8 +972,8 @@ export function renderAsyncJobResult(result: Record<string, unknown>): string {
].join("\n");
}
export function rendered(ok: boolean, command: string, text: string): RenderedCliResult {
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" };
export function rendered(ok: boolean, command: string, text: string, projection?: Record<string, unknown>): RenderedCliResult {
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain", ...(projection === undefined ? {} : { projection }) };
}
export function sentinelProgressEvent(event: string, payload: Record<string, unknown>): void {
+59 -10
View File
@@ -168,7 +168,16 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
if (blocked !== null) return blocked;
}
requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options), sentinelSourceOverrideFromOptions(options));
const localCommit = options.kind === "validate" && options.localOnly
? runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 5_000 }).stdout.trim()
: null;
const localSourceOverride = localCommit === null ? null : {
commit: localCommit,
stageRef: null,
mirrorCommit: localCommit,
sourceAuthority: "gitea-snapshot" as const,
};
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options), localSourceOverride ?? sentinelSourceOverrideFromOptions(options));
if (options.kind === "image") return runSentinelImage(state, options);
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
@@ -229,13 +238,17 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds);
return runSentinelControlPlaneConfirmed(state, options);
}
const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null;
const observed = options.action === "status"
? options.wait
? waitForSentinelObservedStatus(state, options.timeoutSeconds)
: collectSentinelObservedStatus(state, options.timeoutSeconds)
: null;
const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
const pipelineRun = sentinelPipelineRunName(state, options.rerun);
const statusDiagnosis = options.action === "status" && !observedReady ? sentinelObservedStatusDiagnosis(state, observed, pipelineRun) : null;
const result = {
ok: state.configReady && state.sourceHead.ok && observedReady,
ok: state.configReady && state.sourceHead.ok && (options.action === "status" || observedReady),
command,
node: state.spec.nodeId,
lane: state.spec.lane,
@@ -283,17 +296,18 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
cicd: state.cicd,
}),
observed,
deliveryStatus: options.action === "status" ? sentinelDeliveryStatus(state, record(observed), pipelineRun) : null,
statusDiagnosis,
drillDown: controlPlaneDrillDown(state, pipelineRun, null),
warnings: mergeWarnings(observedWarnings, record(statusDiagnosis).warning),
blocker: observedReady
blocker: options.action === "status" || observedReady
? null
: record(statusDiagnosis).blocker ?? { code: "sentinel-control-plane-observed-not-ready", reason: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass", valuesRedacted: true },
recoveryNext: record(statusDiagnosis).recoveryNext ?? null,
next: controlPlaneNext(state, options.action),
valuesRedacted: true,
};
return rendered(result.ok, command, renderControlPlaneResult(result));
return rendered(result.ok, command, renderControlPlaneResult(result), options.action === "status" ? record(result.deliveryStatus) : undefined);
}
function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
@@ -1924,7 +1938,7 @@ function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
gitops,
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
cadence: probeCadenceCronJob(state, timeoutSeconds),
cadence: probeCadenceCronJob(state, timeoutSeconds, nonEmptyString(gitops.schedule)),
};
}
@@ -1934,7 +1948,8 @@ function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
let polls = 1;
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
const pollSeconds = numberAtNullable(state.cicd, "confirmWait.pollSeconds") ?? 2;
runCommand(["sleep", String(pollSeconds)], repoRoot, { timeoutMs: (pollSeconds + 1) * 1_000 });
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
polls += 1;
}
@@ -1964,6 +1979,35 @@ function sentinelObservedReady(value: Record<string, unknown> | SentinelObserved
&& record(observed.cadence).ok === true;
}
export function sentinelDeliveryStatus(state: SentinelCicdState, observed: Record<string, unknown>, pipelineRun: string): Record<string, unknown> {
const gitops = record(observed.gitops);
const runtimeDeployment = record(record(observed.runtime).probe).deployment;
const cadenceProbe = record(record(observed.cadence).probe);
const argo = record(observed.argo);
const desiredSourceCommit = nonEmptyString(gitops.sourceCommit) ?? state.sourceHead.commit;
const runtimeSourceCommit = nonEmptyString(record(runtimeDeployment).sourceCommit);
const desiredSchedule = nonEmptyString(gitops.schedule);
const runtimeSchedule = nonEmptyString(cadenceProbe.schedule);
const converging = desiredSourceCommit !== runtimeSourceCommit || desiredSchedule !== runtimeSchedule || !sentinelObservedReady(observed);
return {
desiredSourceCommit,
runtimeSourceCommit,
pipelineRun,
gitopsRevision: gitops.revision ?? null,
argoSyncStatus: argo.syncStatus ?? null,
argoHealthStatus: argo.healthStatus ?? null,
runtimeReady: record(observed.runtime).ok === true,
desiredSchedule,
runtimeSchedule,
converging,
warning: converging ? "GitOps 期望状态尚未完全收敛;这是只读、非阻塞 warning。" : null,
blocking: false,
outcome: converging ? "converging" : "converged",
mutation: false,
valuesRedacted: true,
};
}
function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObservedStatus | null): string[] {
const observed = record(value);
const argo = record(observed.argo);
@@ -2253,6 +2297,8 @@ function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: nu
const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null;
const manifest = manifestLines.join("\n");
const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]);
const sourceCommit = nonEmptyString(manifest.match(/unidesk\.ai\/source-commit:\s*["']?([0-9a-f]{40})/iu)?.[1]);
const schedule = nonEmptyString(manifest.match(/\bschedule:\s*["']?([^,"'\r\n#]+)/u)?.[1]?.trim());
const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`);
const compact = compactCommand(result);
return {
@@ -2261,6 +2307,8 @@ function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: nu
branch,
manifestPath,
image,
sourceCommit,
schedule,
imageMatchesRepository,
result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` },
valuesRedacted: true,
@@ -2301,9 +2349,10 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e
"const ready = Number(dep?.status?.readyReplicas ?? 0);",
"const updated = Number(dep?.status?.updatedReplicas ?? 0);",
"const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;",
"const sourceCommit = dep?.spec?.template?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? dep?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? null;",
"const imageMatches = expectedImage === null || image === expectedImage;",
"const payload = {",
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },",
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches, sourceCommit },",
" service: { present: rc('svc') === 0 },",
" pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },",
" configMap: { present: rc('cm') === 0 },",
@@ -2320,7 +2369,7 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e
return { ok: result.exitCode === 0 && probe?.ok === true, probe, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result) };
}
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number, desiredSchedule: string | null = null): Record<string, unknown> {
const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null;
if (expected === null) {
return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true };
@@ -2329,7 +2378,7 @@ function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number):
const spec = record(expected.spec);
const namespace = stringAt(metadata, "namespace");
const name = stringAt(metadata, "name");
const expectedSchedule = stringAt(spec, "schedule");
const expectedSchedule = desiredSchedule ?? stringAt(spec, "schedule");
const script = [
"set +e",
`namespace=${shellQuote(namespace)}`,
+42
View File
@@ -128,6 +128,48 @@ export function runSentinelMaintenance(state: SentinelCicdState, options: Extrac
export function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
const command = "web-probe sentinel validate";
if (options.localOnly) {
const cronJob = state.manifests.find((item) => item.kind === "CronJob") ?? null;
const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
const scenarioRows = Array.isArray(state.scenarios)
? state.scenarios
: Array.isArray(record(state.scenarios).scenarios)
? record(state.scenarios).scenarios as unknown[]
: [];
const scenario = scenarioRows.map(record).find((item) => item.id === scenarioId) ?? null;
const cadence = scenario === null ? null : scenario.cadence ?? null;
const renderedCron = cronJob === null ? null : record(cronJob.spec).schedule ?? null;
const result = {
ok: state.configReady && state.sourceHead.ok && cadence !== null && renderedCron !== null,
command,
mode: "local",
mutation: false,
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
configRef: state.configRefs.scenarios,
scenarioId,
cadence,
renderedCron,
sourceCommit: state.sourceHead.commit,
manifestSha256: state.manifestSha256,
valuesRedacted: true,
};
return rendered(result.ok, command, [
command,
`status: ${result.ok ? "ok" : "blocked"}`,
`node: ${result.node}`,
`lane: ${result.lane}`,
`sentinelId: ${result.sentinelId}`,
`configRef: ${result.configRef}`,
`scenarioId: ${result.scenarioId}`,
`cadence: ${result.cadence ?? "-"}`,
`renderedCron: ${result.renderedCron ?? "-"}`,
`sourceCommit: ${result.sourceCommit ?? "-"}`,
`manifestSha256: ${result.manifestSha256}`,
"mutation: false",
].join("\n"), result);
}
const startedAt = Date.now();
const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
let quickVerify: Record<string, unknown> | null = null;
+2 -2
View File
@@ -87,7 +87,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
"--source-stage-ref",
"--source-mirror-commit",
"--source-authority",
]), new Set(["--dry-run", "--confirm", "--wait", "--rerun", "--manual-recovery", "--recovery", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page", "--diagnose-monitor-web"]));
]), new Set(["--dry-run", "--confirm", "--wait", "--local", "--rerun", "--manual-recovery", "--recovery", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page", "--diagnose-monitor-web"]));
const nodeOption = optionValue(args, "--node") ?? null;
const laneOption = optionValue(args, "--lane") ?? null;
const inspectMode = sentinelActionRaw === "inspect-url" || sentinelActionRaw === "inspect-id";
@@ -183,7 +183,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
quickVerify: maintenanceAction === "stop" || args.includes("--quick-verify"),
};
} else if (sentinelActionRaw === "validate") {
sentinel = { kind: "validate", action: "validate", node, lane, sentinelId, dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, quickVerify: args.includes("--quick-verify") };
sentinel = { kind: "validate", action: "validate", node, lane, sentinelId, dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, quickVerify: args.includes("--quick-verify"), localOnly: args.includes("--local") };
} else if (sentinelActionRaw === "dashboard") {
const dashboardAction = parseWebProbeSentinelDashboardAction(args[1]);
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000);
+1
View File
@@ -16,6 +16,7 @@ export interface RenderedCliResult {
command: string;
renderedText: string;
contentType: "text/plain" | "application/json" | "application/yaml";
projection?: Record<string, unknown>;
}
export interface CliInputErrorOptions {
+6 -1
View File
@@ -12,7 +12,12 @@ export function webProbeHelp(): Record<string, unknown> {
export async function runWebProbeCommand(_config: Config, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
if (args.length === 0 || args.includes("--help") || args.includes("-h") || args[0] === "help") return webProbeHelp();
return runNodeWebProbe(parseNodeWebProbeOptions(args));
const format = args.includes("--json") ? "json" : args.includes("--yaml") ? "yaml" : "text";
const parsedArgs = args.filter((item) => item !== "--json" && item !== "--yaml");
const result = runNodeWebProbe(parseNodeWebProbeOptions(parsedArgs));
if (format === "text" || !("projection" in result) || result.projection === undefined) return result;
if (format === "json") return result.projection;
return { ...result, renderedText: Bun.YAML.stringify(result.projection), contentType: "application/yaml" };
}
export function legacyHwlabNodeWebProbeUnsupported(args: string[]): Record<string, unknown> {