fix(pac): 延长 watcher 启动 backlog 探针窗口

This commit is contained in:
Codex
2026-07-15 09:35:54 +02:00
parent 1db64cd322
commit ba934f2298
4 changed files with 171 additions and 2 deletions
@@ -42,6 +42,37 @@ release:
controllerServiceName: pipelines-as-code-controller
controllerServicePort: 8080
waitTimeoutSeconds: 55
workloads:
watcher:
deploymentName: pipelines-as-code-watcher
containerName: pac-watcher
startupProbe:
httpGet:
path: /live
port: probes
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 60
successThreshold: 1
readinessProbe:
httpGet:
path: /live
port: probes
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
successThreshold: 1
livenessProbe:
httpGet:
path: /live
port: probes
initialDelaySeconds: 180
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
successThreshold: 1
deliveryProvenance:
version: admission-pac-v2
markerAnnotation: unidesk.ai/pac-admission-provenance
@@ -78,3 +78,28 @@
- 无源码或定向验证阻塞。
- 运行面验收仍等待 PR 合并及主代理执行受控 bootstrap,因此任务保持 `in_progress`
## 启动 backlog 后续
- warning-only admission 收敛后,新 PaC run 已可创建,但 NC01 官方 watcher 在重放历史 PipelineRun 时暴露第二断点:
- Pod `pipelines-as-code-watcher-7f6c87497c-7lhfh``/live` 在集中 reconcile 期间超过 `5s` timeout
- 默认 liveness 在 `initialDelaySeconds: 5``failureThreshold: 3` 下连续失败并重启容器;
- 同期 readiness 失败只影响流量就绪,liveness 重启会反复打断历史 backlog 初始同步。
- owning YAML 新增 `release.workloads.watcher`,明确声明:
- watcher Deployment 与 container identity
- startup、readiness 与 liveness 三类 HTTP probe
- startup backlog 窗口和 liveness 延迟全部由 YAML 持有,代码不提供隐藏数值默认值。
- PaC bootstrap 下载官方 v0.48.0 release manifest 后,在 apply 前按 YAML 精确匹配 watcher Deployment/container 并渲染探针:
- 未匹配 Deployment 或 container 时直接失败;
- 不使用 runtime patch、第二份 manifest 或手工 `kubectl patch`
- 后续官方 release reapply 仍由同一 bootstrap 渲染相同 desired state。
- 启动窗口采用:
- startup:延迟 `30s`,每 `10s` 检查,单次 timeout `5s`,允许 `60` 次失败;
- liveness:延迟 `180s`,每 `10s` 检查,允许 `6` 次失败;
- readiness:保留 `5s` 延迟、`10s` 周期和 `3` 次失败,只控制就绪状态。
- 定向验证:
- `bun test scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts``13 pass, 0 fail, 66 expect()`
- 使用真实 v0.48.0 上游 manifest 验证:共解析 `35` 个对象,精确命中 `pipelines-as-code-watcher/pac-watcher` 并渲染三类 YAML-owned probe
- `bun scripts/cli.ts platform-infra pipelines-as-code plan --target NC01 --consumer hwlab-nc01-v03 --json`:成功披露 watcher probes`warnings=[]`
- `git diff --check`:通过。
- 合并后只允许由 PaC migrated consumer 的正常 PR merge 自动链触发交付;不得人工执行 PipelineRun、mirror、trigger、sync、flush、Argo sync、runtime patch 或删除历史业务对象。
@@ -4,7 +4,7 @@ import { resolve } from "node:path";
import type { GiteaConfig, GiteaMirrorRepository } from "./platform-infra-gitea-config";
import { renderMirrorBootstrap } from "./platform-infra-gitea-render";
import { pacBootstrapYamlMatchFailure, projectPacBootstrapResult, renderPacBootstrap, resolvePacBootstrapMirrorRepository } from "./platform-infra-pipelines-as-code-bootstrap";
import { parsePacConfigDocument, readPacConfig, runPlatformInfraPipelinesAsCodeCommand, validatePacConfig, validPacConsumers } from "./platform-infra-pipelines-as-code";
import { parsePacConfigDocument, readPacConfig, renderPacReleaseManifest, runPlatformInfraPipelinesAsCodeCommand, validatePacConfig, validPacConsumers } from "./platform-infra-pipelines-as-code";
import { materializeYamlComposition } from "./yaml-composition";
const mirror: GiteaMirrorRepository = {
@@ -185,6 +185,38 @@ test("PaC apply checks the Gitea repository before dry-run return and cluster mu
expect(statusAction).toContain("blocking: false");
});
test("PaC release manifest renders YAML-owned watcher startup backlog probes", () => {
const pac = readPacConfig({ consumerId: "hwlab-nc01-v03", selectDefault: true });
const manifest = renderPacReleaseManifest(pac, `apiVersion: apps/v1
kind: Deployment
metadata:
name: pipelines-as-code-watcher
namespace: pipelines-as-code
spec:
template:
spec:
containers:
- name: pac-watcher
image: example.invalid/pac-watcher:v0
readinessProbe:
httpGet:
path: /live
port: probes
---
apiVersion: v1
kind: Service
metadata:
name: pipelines-as-code-watcher
namespace: pipelines-as-code
`);
const documents = manifest.split(/^---\s*$/mu).map((item) => item.trim()).filter(Boolean).map((item) => Bun.YAML.parse(item) as any);
const watcher = documents[0].spec.template.spec.containers[0];
expect(watcher.startupProbe).toEqual(pac.release.workloads.watcher.startupProbe);
expect(watcher.readinessProbe).toEqual(pac.release.workloads.watcher.readinessProbe);
expect(watcher.livenessProbe).toEqual(pac.release.workloads.watcher.livenessProbe);
expect(documents[1].kind).toBe("Service");
});
test("platform bootstrap help advertises the composite entry before low-level apply", async () => {
const help = await runPlatformInfraPipelinesAsCodeCommand({} as never, ["help", "platform-bootstrap"]);
const usage = (help as Record<string, unknown>).usage as string[];
@@ -71,6 +71,26 @@ interface PacTarget {
enabled: boolean;
}
interface PacHttpProbe {
httpGet: {
path: string;
port: string;
};
initialDelaySeconds: number;
periodSeconds: number;
timeoutSeconds: number;
failureThreshold: number;
successThreshold: number;
}
interface PacWorkloadProbes {
deploymentName: string;
containerName: string;
startupProbe: PacHttpProbe;
readinessProbe: PacHttpProbe;
livenessProbe: PacHttpProbe;
}
export interface PacConfig {
version: number;
kind: "platform-infra-pipelines-as-code";
@@ -104,6 +124,9 @@ export interface PacConfig {
controllerServiceName: string;
controllerServicePort: number;
waitTimeoutSeconds: number;
workloads: {
watcher: PacWorkloadProbes;
};
};
deliveryProvenance: {
version: string;
@@ -474,6 +497,8 @@ export function parsePacConfigDocument(
selection: { readonly consumerId?: string | null; readonly selectDefault?: boolean } = {},
): PacConfig {
const release = y.objectField(root, "release", "");
const releaseWorkloads = y.objectField(release, "workloads", "release");
const watcherWorkload = y.objectField(releaseWorkloads, "watcher", "release.workloads");
const deliveryProvenance = y.objectField(root, "deliveryProvenance", "");
const provenanceController = y.objectField(deliveryProvenance, "controller", "deliveryProvenance");
const provenanceAdmission = y.objectField(deliveryProvenance, "admission", "deliveryProvenance");
@@ -542,6 +567,9 @@ export function parsePacConfigDocument(
controllerServiceName: y.kubernetesNameField(release, "controllerServiceName", "release"),
controllerServicePort: y.portField(release, "controllerServicePort", "release"),
waitTimeoutSeconds: positiveInteger(release, "waitTimeoutSeconds", "release"),
workloads: {
watcher: parsePacWorkloadProbes(watcherWorkload, "release.workloads.watcher"),
},
},
deliveryProvenance: {
version: y.stringField(deliveryProvenance, "version", "deliveryProvenance"),
@@ -1714,7 +1742,35 @@ async function fetchReleaseManifest(pac: PacConfig): Promise<string> {
if (!response.ok) throw new Error(`failed to fetch ${pac.release.manifestUrl}: HTTP ${response.status}`);
const text = await response.text();
if (!text.includes("repositories.pipelinesascode.tekton.dev")) throw new Error(`${pac.release.manifestUrl} did not contain the Repository CRD`);
return text;
return renderPacReleaseManifest(pac, text);
}
export function renderPacReleaseManifest(pac: PacConfig, source: string): string {
const workload = pac.release.workloads.watcher;
let matched = false;
const documents = source
.split(/^---\s*$/mu)
.map((item) => item.trim())
.filter(Boolean)
.map((item) => Bun.YAML.parse(item) as unknown)
.map((item) => {
const document = record(item);
const metadata = record(document.metadata);
if (document.kind !== "Deployment" || metadata.name !== workload.deploymentName || metadata.namespace !== pac.release.namespace) return document;
const spec = record(document.spec);
const template = record(spec.template);
const podSpec = record(template.spec);
const containers = arrayRecords(podSpec.containers);
const container = containers.find((candidate) => candidate.name === workload.containerName);
if (container === undefined) throw new Error(`${configLabel}.release.workloads.watcher.containerName did not match ${workload.deploymentName}`);
container.startupProbe = structuredClone(workload.startupProbe);
container.readinessProbe = structuredClone(workload.readinessProbe);
container.livenessProbe = structuredClone(workload.livenessProbe);
matched = true;
return document;
});
if (!matched) throw new Error(`${configLabel}.release.workloads.watcher.deploymentName did not match the release manifest`);
return `${documents.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
}
function remoteScript(action: "apply" | "status" | "history" | "debug-step", pac: PacConfig, target: PacTarget, repository: PacRepository, consumer: PacConsumer, options: ApplyOptions | HistoryOptions, secrets: SecretMaterial, releaseManifest: string, historyConsumers: PacConsumer[] = [consumer]): string {
@@ -3187,6 +3243,31 @@ function positiveInteger(obj: Record<string, unknown>, key: string, path: string
return value;
}
function parsePacHttpProbe(obj: Record<string, unknown>, path: string): PacHttpProbe {
const httpGet = y.objectField(obj, "httpGet", path);
return {
httpGet: {
path: y.stringField(httpGet, "path", `${path}.httpGet`),
port: y.stringField(httpGet, "port", `${path}.httpGet`),
},
initialDelaySeconds: positiveInteger(obj, "initialDelaySeconds", path),
periodSeconds: positiveInteger(obj, "periodSeconds", path),
timeoutSeconds: positiveInteger(obj, "timeoutSeconds", path),
failureThreshold: positiveInteger(obj, "failureThreshold", path),
successThreshold: positiveInteger(obj, "successThreshold", path),
};
}
function parsePacWorkloadProbes(obj: Record<string, unknown>, path: string): PacWorkloadProbes {
return {
deploymentName: y.kubernetesNameField(obj, "deploymentName", path),
containerName: y.kubernetesNameField(obj, "containerName", path),
startupProbe: parsePacHttpProbe(y.objectField(obj, "startupProbe", path), `${path}.startupProbe`),
readinessProbe: parsePacHttpProbe(y.objectField(obj, "readinessProbe", path), `${path}.readinessProbe`),
livenessProbe: parsePacHttpProbe(y.objectField(obj, "livenessProbe", path), `${path}.livenessProbe`),
};
}
function urlField(obj: Record<string, unknown>, key: string, path: string): string {
const value = y.stringField(obj, key, path);
const parsed = new URL(value);