mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make hwlab v02 cd latest-only

Browse Source
This commit is contained in:
Codex
2026-06-04 14:44:02 +00:00
parent e19c77fd86
commit 08da0805bb
4 changed files with 103 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-g14.ts
+72 -113
View File
@@ -1433,6 +1433,36 @@ function v02TargetValidation(input: {
};
}
export function v02LatestOnlyTargetValidation(input: {
targetMode: string;
sourceCommit: string | null;
pipelineRun: Record<string, unknown> | null;
commitAlignment: Record<string, unknown>;
targetValidation: Record<string, unknown>;
}): Record<string, unknown> {
if (input.targetMode === "latest-source-head") return input.targetValidation;
if (input.sourceCommit === null) return input.targetValidation;
if (input.pipelineRun === null || input.pipelineRun.status !== "True") return input.targetValidation;
const validation = record(input.targetValidation);
if (validation.ok === true || validation.state === "passed") return input.targetValidation;
const staleReasons = stringArray(input.commitAlignment.staleReasons);
const sourceHeadAdvancedReasons = staleReasons.filter((reason) => reason === "origin-head-mismatch" || reason === "cicd-source-repo-stale");
if (sourceHeadAdvancedReasons.length === 0) return input.targetValidation;
const failures = Array.isArray(validation.failures) ? validation.failures : [];
return {
...validation,
ok: true,
state: "superseded",
superseded: true,
latestOnlySuperseded: true,
latestOnlyReasons: sourceHeadAdvancedReasons,
originalState: validation.state ?? null,
summary: `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`,
supersededFailures: failures.slice(0, 10),
failures: [],
};
}
export function v02CommitAlignment(input: {
expectedSourceHead: string | null;
sourceHeads: Record<string, unknown>;
@@ -2038,16 +2068,21 @@ function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): Com
`manifest_b64=${shellQuote(manifestB64)}`,
`manifest_path=/tmp/${pipelineRun}.json`,
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
"kubectl create -f \"$manifest_path\"",
"if kubectl create -f \"$manifest_path\"; then",
" :",
"else",
" code=$?",
` if kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} >/dev/null 2>&1; then`,
` printf 'PipelineRun %s already exists; reusing existing object\\n' ${shellQuote(pipelineRun)} >&2`,
" else",
" exit \"$code\"",
" fi",
"fi",
`kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`,
].join("\n");
return g14K3s(["script", "--", script], timeoutSeconds * 1000);
}
function deleteV02PipelineRun(pipelineRun: string): CommandJsonResult {
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, pipelineRun, "--ignore-not-found=true"], 60_000);
}
function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record<string, unknown> {
const targetMode: V02StatusTargetMode = target.mode
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
@@ -2141,7 +2176,7 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record
runtimeWorkloads,
webAssets,
});
const targetValidation = v02TargetValidation({
const targetValidationBase = v02TargetValidation({
targetMode,
sourceCommit,
pipelineRun: pipelineRunInfo,
@@ -2155,6 +2190,13 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record
gitMirror,
recentPipelineRuns,
});
const targetValidation = v02LatestOnlyTargetValidation({
targetMode,
sourceCommit,
pipelineRun: pipelineRunInfo,
commitAlignment,
targetValidation: targetValidationBase,
});
const falseGreenGuard = targetValidation.state === "superseded"
? {
ok: null,
@@ -2310,16 +2352,20 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
};
}
if (before.status === "True" || before.status === "Unknown") {
if (before.exists === true && before.status !== null) {
const alreadyUsable = before.status === "True" || before.status === "Unknown";
return {
ok: false,
ok: alreadyUsable,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
degradedReason: "refuse-active-or-successful-pipelinerun",
skipped: true,
reason: alreadyUsable ? "existing-pipelinerun-reused" : "existing-pipelinerun-terminal-failed",
degradedReason: alreadyUsable ? undefined : "existing-pipelinerun-terminal-failed",
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default",
};
}
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
@@ -2375,16 +2421,11 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
degradedReason: "git-mirror-pre-sync-failed",
};
}
printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
const deletePipelineRun = deleteV02PipelineRun(v02PipelineRunName(sourceCommit));
printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: isCommandSuccess(deletePipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: deletePipelineRun.exitCode });
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
const createPipelineRun = isCommandSuccess(deletePipelineRun)
? createV02PipelineRun(sourceCommit, options.timeoutSeconds)
: null;
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: createPipelineRun !== null && isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: createPipelineRun?.exitCode ?? null });
const createPipelineRun = createV02PipelineRun(sourceCommit, options.timeoutSeconds);
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: createPipelineRun.exitCode });
return {
ok: isCommandSuccess(deletePipelineRun) && createPipelineRun !== null && isCommandSuccess(createPipelineRun),
ok: isCommandSuccess(createPipelineRun),
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
@@ -2393,9 +2434,9 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
before,
controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh),
gitMirrorPreSync,
deletePipelineRun: compactTriggerCommandResult(compactCommandResult(deletePipelineRun)),
createPipelineRun: createPipelineRun === null ? null : compactTriggerCommandResult(compactCommandResult(createPipelineRun)),
createPipelineRun: compactTriggerCommandResult(compactCommandResult(createPipelineRun)),
after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)),
latestOnlyPolicy: "no old PipelineRun is canceled; stale commits self-supersede before GitOps writeback",
disclosure: {
fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.",
},
@@ -4252,35 +4293,6 @@ async function waitForV02Cd(sourceCommit: string, timeoutSeconds: number): Promi
};
}
async function waitForV02LaneIdle(timeoutSeconds: number): Promise<Record<string, unknown>> {
const started = Date.now();
let lastStatus: Record<string, unknown> = {};
let activeRuns: Record<string, unknown>[] = [];
while (Date.now() - started < timeoutSeconds * 1000) {
lastStatus = v02ControlPlaneStatus();
activeRuns = activeV02PipelineRuns(lastStatus);
printEvent("v02.cd.lane-idle", { activeCount: activeRuns.length, activeRuns: activeRuns.slice(0, 5) });
printV02PrMonitorProgress({ stage: "lane-idle", status: activeRuns.length === 0 ? "succeeded" : "running", activeCount: activeRuns.length });
if (activeRuns.length === 0) {
return {
ok: true,
waitedSeconds: Math.round((Date.now() - started) / 1000),
status: summarizeV02CdStatus(lastStatus),
activeRuns,
};
}
await sleep(30_000);
}
return {
ok: false,
phase: "active-run-timeout",
timeoutSeconds,
waitedSeconds: Math.round((Date.now() - started) / 1000),
status: summarizeV02CdStatus(lastStatus),
activeRuns,
};
}
async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
@@ -4340,7 +4352,7 @@ async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unk
? "dry-run：PR 已达到自动合并条件；本轮不会写 GitHub merge 或 CD。"
: mergeRaceState === "merged"
? "PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 v0.2 CD。后续成功、失败或超时会继续在本 PR 下回复。"
: "PR 已自动合并，v0.2 CD 准备开始。后续成功、失败或超时会继续在本 PR 下回复。",
: "PR 已自动合并，v0.2 CD 准备开始；其他 commit 的运行中 PipelineRun 不会阻塞本轮 CI，旧 run 若发现 source head 已推进会以 superseded/no-op 收口。后续成功、superseded、失败或超时会继续在本 PR 下回复。",
});
if (!options.dryRun && record(startedComment).ok !== true) {
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
@@ -4367,48 +4379,13 @@ async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unk
const before = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
const beforeSummary = summarizeV02CdStatus(before);
const activeRuns = activeV02PipelineRuns(before);
if (activeRuns.length > 0 && !v02CdPassed(before)) {
printV02PrMonitorProgress({ stage: "lane-idle", status: "running", pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length });
const comment = commentV02PullRequest({
pr,
phase: "cd-active-run",
state: "cd-blocked",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
cd: beforeSummary,
dryRun: false,
message: "PR 已合并，但 v0.2 当前已有运行中的 PipelineRun；worker 会等待 lane 空闲后继续触发当前 merge commit 的 CD。",
});
if (record(comment).ok !== true) return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, activeRuns, before: beforeSummary, comment };
const idle = await waitForV02LaneIdle(options.timeoutSeconds);
if (record(idle).ok !== true) {
const timeoutComment = commentV02PullRequest({
pr,
phase: "cd-active-run-timeout",
state: "cd-timeout",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
cd: record(idle.status),
dryRun: false,
message: "PR 已合并，但 v0.2 lane 长时间被已有 PipelineRun 占用，当前 merge commit 尚未触发 CD；本评论保留 active run 状态用于接续排障。",
});
return { ok: false, phase: "cd-active-run-timeout", pr, sourceCommit, pipelineRun, activeRuns, before: beforeSummary, idle, comment, timeoutComment };
}
if (activeRuns.length > 0) {
printEvent("v02.cd.active-runs-observed", { pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, activeRuns: activeRuns.slice(0, 5), latestOnlyPolicy: "do-not-wait-or-cancel-old-runs" });
}
const trigger = v02CdPassed(before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerV02Current(Math.min(options.timeoutSeconds, 600));
printEvent("v02.cd.trigger", { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
printV02PrMonitorProgress({ stage: "cd-trigger", status: record(trigger).ok === true || record(trigger).degradedReason === "refuse-active-or-successful-pipelinerun" ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
if (record(trigger).ok !== true && record(trigger).degradedReason !== "refuse-active-or-successful-pipelinerun") {
printV02PrMonitorProgress({ stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
if (record(trigger).ok !== true) {
const comment = commentV02PullRequest({
pr,
phase: "cd-trigger",
@@ -4431,10 +4408,11 @@ async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unk
const flush = record(cd.flush);
const cdOk = cd.ok === true;
const cdPhase = String(cd.phase ?? "");
const cdSuperseded = cdOk && cdStatus.targetValidationState === "superseded";
const finalComment = commentV02PullRequest({
pr,
phase: cdPhase,
state: cdOk ? "cd-succeeded" : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
state: cdOk ? (cdSuperseded ? "cd-superseded" : "cd-succeeded") : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
@@ -4446,14 +4424,16 @@ async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unk
flush,
dryRun: false,
message: cdOk
? "v0.2 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。"
? cdSuperseded
? "v0.2 自动 CD 已收口：本 merge head 的 PipelineRun 已完成，但 source head 已被后续提交推进，本轮按 latest-only 语义 superseded/no-op，未回写旧 GitOps revision。"
: "v0.2 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。"
: cdPhase === "timeout"
? "v0.2 自动 CD 超时未收口；本评论保留最后一次 targetValidation / PipelineRun / Git mirror 状态用于接续排障。"
: "v0.2 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。",
});
return {
ok: cdOk && record(finalComment).ok === true,
action: cdOk ? "merged-and-rolled-v02" : "v02-cd-failed",
action: cdOk ? (cdSuperseded ? "merged-and-superseded-v02" : "merged-and-rolled-v02") : "v02-cd-failed",
phase: cdOk ? "cd-passed" : cdPhase,
pr,
sourceCommit,
@@ -4558,27 +4538,6 @@ async function monitorV02Cycle(options: G14MonitorOptions, cycle: number): Promi
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
continue;
}
const currentStatus = v02ControlPlaneStatus();
const activeRuns = activeV02PipelineRuns(currentStatus);
if (activeRuns.length > 0) {
const cd = summarizeV02CdStatus(currentStatus);
const comment = commentV02PullRequest({
pr,
phase: "cd-active-before-merge",
state: "cd-blocked",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
cd,
dryRun: options.dryRun,
message: "PR 已通过 CI / mergeability，但 v0.2 lane 当前已有运行中的 PipelineRun；为保持 PR merge commit 与 CD 目标一一对应，本轮暂不合并，待 lane 空闲后自动继续。",
});
observations.push({ pullRequest: pr, preflight, activeRuns, cd, comment });
printV02PrMonitorProgress({ stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, activeCount: activeRuns.length });
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
continue;
}
const merge = mergePullRequest(pr.number, options.dryRun);
printEvent("v02.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
printV02PrMonitorProgress({ stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
@@ -4678,7 +4637,7 @@ export function hwlabG14Help(): Record<string, unknown> {
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
],
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs, triggers v0.2 CD, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.",
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.",
defaults: {
repo: HWLAB_REPO,
base: G14_SOURCE_BRANCH,