fix(pac): 延长 watcher 启动 backlog 探针窗口
This commit is contained in:
@@ -42,6 +42,37 @@ release:
|
||||
controllerServiceName: pipelines-as-code-controller
|
||||
controllerServicePort: 8080
|
||||
waitTimeoutSeconds: 55
|
||||
workloads:
|
||||
watcher:
|
||||
deploymentName: pipelines-as-code-watcher
|
||||
containerName: pac-watcher
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /live
|
||||
port: probes
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 60
|
||||
successThreshold: 1
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /live
|
||||
port: probes
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
successThreshold: 1
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /live
|
||||
port: probes
|
||||
initialDelaySeconds: 180
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
successThreshold: 1
|
||||
deliveryProvenance:
|
||||
version: admission-pac-v2
|
||||
markerAnnotation: unidesk.ai/pac-admission-provenance
|
||||
|
||||
@@ -78,3 +78,28 @@
|
||||
|
||||
- 无源码或定向验证阻塞。
|
||||
- 运行面验收仍等待 PR 合并及主代理执行受控 bootstrap,因此任务保持 `in_progress`。
|
||||
|
||||
## 启动 backlog 后续
|
||||
|
||||
- warning-only admission 收敛后,新 PaC run 已可创建,但 NC01 官方 watcher 在重放历史 PipelineRun 时暴露第二断点:
|
||||
- Pod `pipelines-as-code-watcher-7f6c87497c-7lhfh` 的 `/live` 在集中 reconcile 期间超过 `5s` timeout;
|
||||
- 默认 liveness 在 `initialDelaySeconds: 5`、`failureThreshold: 3` 下连续失败并重启容器;
|
||||
- 同期 readiness 失败只影响流量就绪,liveness 重启会反复打断历史 backlog 初始同步。
|
||||
- owning YAML 新增 `release.workloads.watcher`,明确声明:
|
||||
- watcher Deployment 与 container identity;
|
||||
- startup、readiness 与 liveness 三类 HTTP probe;
|
||||
- startup backlog 窗口和 liveness 延迟全部由 YAML 持有,代码不提供隐藏数值默认值。
|
||||
- PaC bootstrap 下载官方 v0.48.0 release manifest 后,在 apply 前按 YAML 精确匹配 watcher Deployment/container 并渲染探针:
|
||||
- 未匹配 Deployment 或 container 时直接失败;
|
||||
- 不使用 runtime patch、第二份 manifest 或手工 `kubectl patch`;
|
||||
- 后续官方 release reapply 仍由同一 bootstrap 渲染相同 desired state。
|
||||
- 启动窗口采用:
|
||||
- startup:延迟 `30s`,每 `10s` 检查,单次 timeout `5s`,允许 `60` 次失败;
|
||||
- liveness:延迟 `180s`,每 `10s` 检查,允许 `6` 次失败;
|
||||
- readiness:保留 `5s` 延迟、`10s` 周期和 `3` 次失败,只控制就绪状态。
|
||||
- 定向验证:
|
||||
- `bun test scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts`:`13 pass, 0 fail, 66 expect()`;
|
||||
- 使用真实 v0.48.0 上游 manifest 验证:共解析 `35` 个对象,精确命中 `pipelines-as-code-watcher/pac-watcher` 并渲染三类 YAML-owned probe;
|
||||
- `bun scripts/cli.ts platform-infra pipelines-as-code plan --target NC01 --consumer hwlab-nc01-v03 --json`:成功披露 watcher probes,`warnings=[]`;
|
||||
- `git diff --check`:通过。
|
||||
- 合并后只允许由 PaC migrated consumer 的正常 PR merge 自动链触发交付;不得人工执行 PipelineRun、mirror、trigger、sync、flush、Argo sync、runtime patch 或删除历史业务对象。
|
||||
|
||||
@@ -4,7 +4,7 @@ import { resolve } from "node:path";
|
||||
import type { GiteaConfig, GiteaMirrorRepository } from "./platform-infra-gitea-config";
|
||||
import { renderMirrorBootstrap } from "./platform-infra-gitea-render";
|
||||
import { pacBootstrapYamlMatchFailure, projectPacBootstrapResult, renderPacBootstrap, resolvePacBootstrapMirrorRepository } from "./platform-infra-pipelines-as-code-bootstrap";
|
||||
import { parsePacConfigDocument, readPacConfig, runPlatformInfraPipelinesAsCodeCommand, validatePacConfig, validPacConsumers } from "./platform-infra-pipelines-as-code";
|
||||
import { parsePacConfigDocument, readPacConfig, renderPacReleaseManifest, runPlatformInfraPipelinesAsCodeCommand, validatePacConfig, validPacConsumers } from "./platform-infra-pipelines-as-code";
|
||||
import { materializeYamlComposition } from "./yaml-composition";
|
||||
|
||||
const mirror: GiteaMirrorRepository = {
|
||||
@@ -185,6 +185,38 @@ test("PaC apply checks the Gitea repository before dry-run return and cluster mu
|
||||
expect(statusAction).toContain("blocking: false");
|
||||
});
|
||||
|
||||
test("PaC release manifest renders YAML-owned watcher startup backlog probes", () => {
|
||||
const pac = readPacConfig({ consumerId: "hwlab-nc01-v03", selectDefault: true });
|
||||
const manifest = renderPacReleaseManifest(pac, `apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pipelines-as-code-watcher
|
||||
namespace: pipelines-as-code
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: pac-watcher
|
||||
image: example.invalid/pac-watcher:v0
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /live
|
||||
port: probes
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: pipelines-as-code-watcher
|
||||
namespace: pipelines-as-code
|
||||
`);
|
||||
const documents = manifest.split(/^---\s*$/mu).map((item) => item.trim()).filter(Boolean).map((item) => Bun.YAML.parse(item) as any);
|
||||
const watcher = documents[0].spec.template.spec.containers[0];
|
||||
expect(watcher.startupProbe).toEqual(pac.release.workloads.watcher.startupProbe);
|
||||
expect(watcher.readinessProbe).toEqual(pac.release.workloads.watcher.readinessProbe);
|
||||
expect(watcher.livenessProbe).toEqual(pac.release.workloads.watcher.livenessProbe);
|
||||
expect(documents[1].kind).toBe("Service");
|
||||
});
|
||||
|
||||
test("platform bootstrap help advertises the composite entry before low-level apply", async () => {
|
||||
const help = await runPlatformInfraPipelinesAsCodeCommand({} as never, ["help", "platform-bootstrap"]);
|
||||
const usage = (help as Record<string, unknown>).usage as string[];
|
||||
|
||||
@@ -71,6 +71,26 @@ interface PacTarget {
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
interface PacHttpProbe {
|
||||
httpGet: {
|
||||
path: string;
|
||||
port: string;
|
||||
};
|
||||
initialDelaySeconds: number;
|
||||
periodSeconds: number;
|
||||
timeoutSeconds: number;
|
||||
failureThreshold: number;
|
||||
successThreshold: number;
|
||||
}
|
||||
|
||||
interface PacWorkloadProbes {
|
||||
deploymentName: string;
|
||||
containerName: string;
|
||||
startupProbe: PacHttpProbe;
|
||||
readinessProbe: PacHttpProbe;
|
||||
livenessProbe: PacHttpProbe;
|
||||
}
|
||||
|
||||
export interface PacConfig {
|
||||
version: number;
|
||||
kind: "platform-infra-pipelines-as-code";
|
||||
@@ -104,6 +124,9 @@ export interface PacConfig {
|
||||
controllerServiceName: string;
|
||||
controllerServicePort: number;
|
||||
waitTimeoutSeconds: number;
|
||||
workloads: {
|
||||
watcher: PacWorkloadProbes;
|
||||
};
|
||||
};
|
||||
deliveryProvenance: {
|
||||
version: string;
|
||||
@@ -474,6 +497,8 @@ export function parsePacConfigDocument(
|
||||
selection: { readonly consumerId?: string | null; readonly selectDefault?: boolean } = {},
|
||||
): PacConfig {
|
||||
const release = y.objectField(root, "release", "");
|
||||
const releaseWorkloads = y.objectField(release, "workloads", "release");
|
||||
const watcherWorkload = y.objectField(releaseWorkloads, "watcher", "release.workloads");
|
||||
const deliveryProvenance = y.objectField(root, "deliveryProvenance", "");
|
||||
const provenanceController = y.objectField(deliveryProvenance, "controller", "deliveryProvenance");
|
||||
const provenanceAdmission = y.objectField(deliveryProvenance, "admission", "deliveryProvenance");
|
||||
@@ -542,6 +567,9 @@ export function parsePacConfigDocument(
|
||||
controllerServiceName: y.kubernetesNameField(release, "controllerServiceName", "release"),
|
||||
controllerServicePort: y.portField(release, "controllerServicePort", "release"),
|
||||
waitTimeoutSeconds: positiveInteger(release, "waitTimeoutSeconds", "release"),
|
||||
workloads: {
|
||||
watcher: parsePacWorkloadProbes(watcherWorkload, "release.workloads.watcher"),
|
||||
},
|
||||
},
|
||||
deliveryProvenance: {
|
||||
version: y.stringField(deliveryProvenance, "version", "deliveryProvenance"),
|
||||
@@ -1714,7 +1742,35 @@ async function fetchReleaseManifest(pac: PacConfig): Promise<string> {
|
||||
if (!response.ok) throw new Error(`failed to fetch ${pac.release.manifestUrl}: HTTP ${response.status}`);
|
||||
const text = await response.text();
|
||||
if (!text.includes("repositories.pipelinesascode.tekton.dev")) throw new Error(`${pac.release.manifestUrl} did not contain the Repository CRD`);
|
||||
return text;
|
||||
return renderPacReleaseManifest(pac, text);
|
||||
}
|
||||
|
||||
export function renderPacReleaseManifest(pac: PacConfig, source: string): string {
|
||||
const workload = pac.release.workloads.watcher;
|
||||
let matched = false;
|
||||
const documents = source
|
||||
.split(/^---\s*$/mu)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean)
|
||||
.map((item) => Bun.YAML.parse(item) as unknown)
|
||||
.map((item) => {
|
||||
const document = record(item);
|
||||
const metadata = record(document.metadata);
|
||||
if (document.kind !== "Deployment" || metadata.name !== workload.deploymentName || metadata.namespace !== pac.release.namespace) return document;
|
||||
const spec = record(document.spec);
|
||||
const template = record(spec.template);
|
||||
const podSpec = record(template.spec);
|
||||
const containers = arrayRecords(podSpec.containers);
|
||||
const container = containers.find((candidate) => candidate.name === workload.containerName);
|
||||
if (container === undefined) throw new Error(`${configLabel}.release.workloads.watcher.containerName did not match ${workload.deploymentName}`);
|
||||
container.startupProbe = structuredClone(workload.startupProbe);
|
||||
container.readinessProbe = structuredClone(workload.readinessProbe);
|
||||
container.livenessProbe = structuredClone(workload.livenessProbe);
|
||||
matched = true;
|
||||
return document;
|
||||
});
|
||||
if (!matched) throw new Error(`${configLabel}.release.workloads.watcher.deploymentName did not match the release manifest`);
|
||||
return `${documents.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
||||
}
|
||||
|
||||
function remoteScript(action: "apply" | "status" | "history" | "debug-step", pac: PacConfig, target: PacTarget, repository: PacRepository, consumer: PacConsumer, options: ApplyOptions | HistoryOptions, secrets: SecretMaterial, releaseManifest: string, historyConsumers: PacConsumer[] = [consumer]): string {
|
||||
@@ -3187,6 +3243,31 @@ function positiveInteger(obj: Record<string, unknown>, key: string, path: string
|
||||
return value;
|
||||
}
|
||||
|
||||
function parsePacHttpProbe(obj: Record<string, unknown>, path: string): PacHttpProbe {
|
||||
const httpGet = y.objectField(obj, "httpGet", path);
|
||||
return {
|
||||
httpGet: {
|
||||
path: y.stringField(httpGet, "path", `${path}.httpGet`),
|
||||
port: y.stringField(httpGet, "port", `${path}.httpGet`),
|
||||
},
|
||||
initialDelaySeconds: positiveInteger(obj, "initialDelaySeconds", path),
|
||||
periodSeconds: positiveInteger(obj, "periodSeconds", path),
|
||||
timeoutSeconds: positiveInteger(obj, "timeoutSeconds", path),
|
||||
failureThreshold: positiveInteger(obj, "failureThreshold", path),
|
||||
successThreshold: positiveInteger(obj, "successThreshold", path),
|
||||
};
|
||||
}
|
||||
|
||||
function parsePacWorkloadProbes(obj: Record<string, unknown>, path: string): PacWorkloadProbes {
|
||||
return {
|
||||
deploymentName: y.kubernetesNameField(obj, "deploymentName", path),
|
||||
containerName: y.kubernetesNameField(obj, "containerName", path),
|
||||
startupProbe: parsePacHttpProbe(y.objectField(obj, "startupProbe", path), `${path}.startupProbe`),
|
||||
readinessProbe: parsePacHttpProbe(y.objectField(obj, "readinessProbe", path), `${path}.readinessProbe`),
|
||||
livenessProbe: parsePacHttpProbe(y.objectField(obj, "livenessProbe", path), `${path}.livenessProbe`),
|
||||
};
|
||||
}
|
||||
|
||||
function urlField(obj: Record<string, unknown>, key: string, path: string): string {
|
||||
const value = y.stringField(obj, key, path);
|
||||
const parsed = new URL(value);
|
||||
|
||||
Reference in New Issue
Block a user