Codex
674 lines
31 KiB
TypeScript
674 lines
31 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. cd-trigger module for scripts/src/hwlab-g14.ts.
|
||
|
||
// Moved mechanically from scripts/src/hwlab-g14.ts:9508-10162 for #903.
|
||
|
||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||
import { dirname, join } from "node:path";
|
||
import { createHash, randomBytes } from "node:crypto";
|
||
import { repoRoot, rootPath, type Config } from "../config";
|
||
import { runCommand } from "../command";
|
||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||
|
||
import type { CommandJsonResult, G14MonitorOptions, OpenPullRequest } from "./types";
|
||
import { runRuntimeLaneControlPlane, runV02ControlPlane, runtimeLaneControlPlaneStatus, v02ControlPlaneStatus } from "./control-plane";
|
||
import { runG14GitMirror, runGitMirrorStatus } from "./git-mirror";
|
||
import { activeV02PipelineRuns, commentRuntimeLanePullRequest, commentV02PullRequest, durationSeconds, reportRuntimeLaneAutomationIssue, runtimeLaneCdFailed, runtimeLaneCdPassed, sourceCommitFromMergeResult, summarizeRuntimeLaneCdStatus, summarizeV02CdStatus, v02CdFailed, v02CdPassed, v02PipelineSucceeded, waitForV02Head } from "./pr-monitor";
|
||
import { commandData, compactCommandResult, isCommandSuccess, nested, printEvent, printRuntimeLanePrMonitorProgress, printV02PrMonitorProgress, record, sleep, stringOrNull } from "./remote";
|
||
import { resolveRuntimeLaneHead, runtimeLanePipelineRunName, v02PipelineRunName } from "./source";
|
||
|
||
export function triggerV02Current(timeoutSeconds: number): Record<string, unknown> {
|
||
return runV02ControlPlane({
|
||
action: "trigger-current",
|
||
lane: "v02",
|
||
dryRun: false,
|
||
confirm: true,
|
||
wait: true,
|
||
allowLiveDbRead: false,
|
||
timeoutSeconds,
|
||
minAgeMinutes: 30,
|
||
limit: 20,
|
||
});
|
||
}
|
||
|
||
export function flushV02GitMirrorIfNeeded(status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
|
||
const pendingFlush = nested(status, ["gitMirror", "summary", "pendingFlush"]);
|
||
if (pendingFlush !== true) return { ok: true, skipped: true, reason: "git-mirror-already-flushed" };
|
||
return runG14GitMirror({
|
||
action: "flush",
|
||
lane: "v02",
|
||
confirm: true,
|
||
dryRun: false,
|
||
wait: true,
|
||
timeoutSeconds,
|
||
});
|
||
}
|
||
|
||
export function triggerRuntimeLaneCurrent(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
|
||
return runRuntimeLaneControlPlane(spec, {
|
||
action: "trigger-current",
|
||
lane: spec.lane,
|
||
dryRun: false,
|
||
confirm: true,
|
||
wait: true,
|
||
allowLiveDbRead: false,
|
||
timeoutSeconds,
|
||
minAgeMinutes: 30,
|
||
limit: 20,
|
||
history: false,
|
||
});
|
||
}
|
||
|
||
export function runtimeLaneStatusWithGitMirror(spec: HwlabRuntimeLaneSpec, sourceCommit: string): Record<string, unknown> {
|
||
const status = runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "source-commit" });
|
||
const gitMirrorStatus = runGitMirrorStatus(spec.lane);
|
||
return {
|
||
...status,
|
||
gitMirror: record(gitMirrorStatus.cache),
|
||
gitMirrorStatus,
|
||
};
|
||
}
|
||
|
||
export function runtimeLaneGitopsSynced(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>): boolean {
|
||
const summary = record(nested(status, ["gitMirror", "summary"]));
|
||
const inSync = spec.lane === "v03" ? summary.v03GitopsInSync : summary.githubInSync;
|
||
return summary.pendingFlush !== true && inSync === true;
|
||
}
|
||
|
||
export function runtimeLaneCdPassedWithMirror(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>): boolean {
|
||
return runtimeLaneCdPassed(status) && runtimeLaneGitopsSynced(spec, status);
|
||
}
|
||
|
||
export function flushRuntimeLaneGitMirrorIfNeeded(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
|
||
if (runtimeLaneGitopsSynced(spec, status)) return { ok: true, skipped: true, reason: `${spec.lane}-git-mirror-already-flushed` };
|
||
return runG14GitMirror({
|
||
action: "flush",
|
||
lane: spec.lane,
|
||
confirm: true,
|
||
dryRun: false,
|
||
wait: true,
|
||
timeoutSeconds,
|
||
});
|
||
}
|
||
|
||
export function pullRequestFromMergeCommand(merge: CommandJsonResult): Record<string, unknown> {
|
||
const data = commandData(merge);
|
||
const direct = record(data.pullRequest);
|
||
if (Object.keys(direct).length > 0) return direct;
|
||
return record(nested(data, ["details", "details", "pullRequest"]));
|
||
}
|
||
|
||
export function mergeCommandRaceState(merge: CommandJsonResult): "merged" | "closed" | null {
|
||
const pullRequest = pullRequestFromMergeCommand(merge);
|
||
if (pullRequest.merged === true || pullRequest.stateDetail === "merged") return "merged";
|
||
if (pullRequest.closed === true || pullRequest.state === "closed") return "closed";
|
||
return null;
|
||
}
|
||
|
||
export async function waitForRuntimeLaneHead(spec: HwlabRuntimeLaneSpec, expectedCommit: string | null, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
let head: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
head = resolveRuntimeLaneHead(spec).sourceCommit;
|
||
if (expectedCommit === null) {
|
||
if (head !== null) return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||
} else if (head === expectedCommit) {
|
||
return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||
}
|
||
printEvent(`${spec.lane}.source.wait`, { lane: spec.lane, expectedCommit, head, waitedSeconds: Math.round((Date.now() - started) / 1000) });
|
||
await sleep(5_000);
|
||
}
|
||
return { ok: false, sourceCommit: head, expectedCommit, matched: false, waitedSeconds: Math.round((Date.now() - started) / 1000), degradedReason: `${spec.lane}-source-head-not-aligned-after-merge` };
|
||
}
|
||
|
||
export async function waitForV02Cd(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
const startedAt = new Date(started).toISOString();
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
let lastStatus: Record<string, unknown> = {};
|
||
let firstPassedAt: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
lastStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
const summary = summarizeV02CdStatus(lastStatus);
|
||
printEvent("v02.cd.status", { sourceCommit, pipelineRun, summary });
|
||
printV02PrMonitorProgress({ stage: "cd-status", status: "running", sourceCommit, pipelineRun, targetValidationState: summary.targetValidationState ?? null, pipelineStatus: summary.pipelineStatus ?? null, pendingFlush: summary.pendingFlush ?? null });
|
||
if (v02PipelineSucceeded(lastStatus) && summary.pendingFlush === true) {
|
||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||
if (record(flush).ok !== true) {
|
||
return {
|
||
ok: false,
|
||
phase: "git-mirror-flush",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summary,
|
||
rawStatus: lastStatus,
|
||
flush,
|
||
};
|
||
}
|
||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
const finalSummary = summarizeV02CdStatus(finalStatus);
|
||
if (v02CdPassed(finalStatus)) {
|
||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: "succeeded", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||
return {
|
||
ok: true,
|
||
phase: "cd-passed",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: finalSummary,
|
||
rawStatus: finalStatus,
|
||
flush,
|
||
};
|
||
}
|
||
lastStatus = finalStatus;
|
||
printEvent("v02.cd.after-flush", { sourceCommit, pipelineRun, flushOk: record(flush).ok, summary: finalSummary });
|
||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: record(flush).ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||
}
|
||
if (v02CdPassed(lastStatus)) {
|
||
firstPassedAt ??= new Date().toISOString();
|
||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
printV02PrMonitorProgress({ stage: "cd-status", status: v02CdPassed(finalStatus) ? "succeeded" : "failed", sourceCommit, pipelineRun, targetValidationState: nested(finalStatus, ["targetValidation", "state"]) ?? null });
|
||
return {
|
||
ok: v02CdPassed(finalStatus),
|
||
phase: v02CdPassed(finalStatus) ? "cd-passed" : "git-mirror-flush",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summarizeV02CdStatus(finalStatus),
|
||
rawStatus: finalStatus,
|
||
flush,
|
||
};
|
||
}
|
||
if (v02CdFailed(lastStatus)) {
|
||
printV02PrMonitorProgress({ stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
|
||
return {
|
||
ok: false,
|
||
phase: "cd-failed",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summary,
|
||
rawStatus: lastStatus,
|
||
};
|
||
}
|
||
await sleep(30_000);
|
||
}
|
||
return {
|
||
ok: false,
|
||
phase: "timeout",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
timeoutSeconds,
|
||
firstPassedAt,
|
||
status: summarizeV02CdStatus(lastStatus),
|
||
rawStatus: lastStatus,
|
||
};
|
||
}
|
||
|
||
export async function waitForRuntimeLaneCd(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
const startedAt = new Date(started).toISOString();
|
||
let pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit);
|
||
let lastStatus: Record<string, unknown> = {};
|
||
let firstPassedAt: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
lastStatus = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
|
||
const summary = summarizeRuntimeLaneCdStatus(spec, lastStatus);
|
||
pipelineRun = stringOrNull(summary.pipelineRun) ?? pipelineRun;
|
||
printEvent(`${spec.lane}.cd.status`, { lane: spec.lane, sourceCommit, pipelineRun, summary });
|
||
printRuntimeLanePrMonitorProgress(spec, {
|
||
stage: "cd-status",
|
||
status: "running",
|
||
sourceCommit,
|
||
pipelineRun,
|
||
pipelineStatus: summary.pipelineStatus ?? null,
|
||
pipelineReason: summary.pipelineReason ?? null,
|
||
argoSync: summary.argoSync ?? null,
|
||
argoHealth: summary.argoHealth ?? null,
|
||
publicProbesOk: summary.publicProbesOk ?? null,
|
||
pendingFlush: summary.pendingFlush ?? null,
|
||
});
|
||
if (runtimeLaneCdPassed(lastStatus)) {
|
||
firstPassedAt ??= new Date().toISOString();
|
||
const flush = flushRuntimeLaneGitMirrorIfNeeded(spec, lastStatus, Math.min(timeoutSeconds, 600));
|
||
if (record(flush).ok !== true) {
|
||
return {
|
||
ok: false,
|
||
phase: "git-mirror-flush",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summary,
|
||
rawStatus: lastStatus,
|
||
flush,
|
||
};
|
||
}
|
||
const finalStatus = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
|
||
const finalSummary = summarizeRuntimeLaneCdStatus(spec, finalStatus);
|
||
printRuntimeLanePrMonitorProgress(spec, {
|
||
stage: "git-mirror-flush",
|
||
status: runtimeLaneCdPassedWithMirror(spec, finalStatus) ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
pipelineRun,
|
||
pendingFlush: finalSummary.pendingFlush ?? null,
|
||
});
|
||
return {
|
||
ok: runtimeLaneCdPassedWithMirror(spec, finalStatus),
|
||
phase: runtimeLaneCdPassedWithMirror(spec, finalStatus) ? "cd-passed" : "git-mirror-flush",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: finalSummary,
|
||
rawStatus: finalStatus,
|
||
flush,
|
||
};
|
||
}
|
||
if (runtimeLaneCdFailed(lastStatus)) {
|
||
printRuntimeLanePrMonitorProgress(spec, { stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
|
||
return {
|
||
ok: false,
|
||
phase: "cd-failed",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summary,
|
||
rawStatus: lastStatus,
|
||
};
|
||
}
|
||
await sleep(30_000);
|
||
}
|
||
return {
|
||
ok: false,
|
||
phase: "timeout",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
timeoutSeconds,
|
||
firstPassedAt,
|
||
status: summarizeRuntimeLaneCdStatus(spec, lastStatus),
|
||
rawStatus: lastStatus,
|
||
};
|
||
}
|
||
|
||
export async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
|
||
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
|
||
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
|
||
printV02PrMonitorProgress({ stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "merge",
|
||
state: "merge-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: compactCommandResult(merge),
|
||
dryRun: options.dryRun,
|
||
message: mergeRaceState === "closed"
|
||
? "PR preflight 曾通过，但自动合并前 PR 已被关闭且未确认 merged；本轮不会触发 CD。"
|
||
: "PR preflight 已通过，但自动合并失败；需要处理 GitHub 返回的 merge blocker 后重试。",
|
||
});
|
||
return { ok: mergeRaceState === "closed", phase: "merge", action: "merge-race-closed", pr, merge, comment };
|
||
}
|
||
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
|
||
if (!options.dryRun && expectedCommit === null) {
|
||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "merge-commit",
|
||
state: "cd-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit: null,
|
||
pipelineRun: null,
|
||
dryRun: false,
|
||
message: "PR 已合并，但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit；为避免触发错误 source head，本轮未启动 CD。",
|
||
});
|
||
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, degradedReason: "merge-commit-unresolved" };
|
||
}
|
||
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForV02Head(expectedCommit, 120);
|
||
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
|
||
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
|
||
printV02PrMonitorProgress({ stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
|
||
const startedComment = commentV02PullRequest({
|
||
pr,
|
||
phase: options.dryRun ? "dry-run" : "cd-trigger",
|
||
state: "cd-started",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
dryRun: options.dryRun,
|
||
message: options.dryRun
|
||
? "dry-run：PR 已达到自动合并条件；本轮不会写 GitHub merge 或 CD。"
|
||
: mergeRaceState === "merged"
|
||
? "PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 v0.2 CD。后续成功、失败或超时会继续在本 PR 下回复。"
|
||
: "PR 已自动合并，v0.2 CD 准备开始；其他 commit 的运行中 PipelineRun 不会阻塞本轮 CI，旧 run 若发现 source head 已推进会以 superseded/no-op 收口。后续成功、superseded、失败或超时会继续在本 PR 下回复。",
|
||
});
|
||
if (!options.dryRun && record(startedComment).ok !== true) {
|
||
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
|
||
}
|
||
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
|
||
if (sourceCommit === null || record(headWait).ok !== true) {
|
||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "v02-source-head",
|
||
state: "cd-timeout",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
dryRun: false,
|
||
message: "PR 已合并，但 G14 v0.2 CI/CD source repo 没有在等待窗口内对齐 merge commit；为避免触发旧 head，本轮未启动 CD。",
|
||
});
|
||
return { ok: false, phase: "v02-head", pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, degradedReason: "v02-head-unresolved-after-merge" };
|
||
}
|
||
const before = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
const beforeSummary = summarizeV02CdStatus(before);
|
||
const activeRuns = activeV02PipelineRuns(before);
|
||
if (activeRuns.length > 0) {
|
||
printEvent("v02.cd.active-runs-observed", { pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, activeRuns: activeRuns.slice(0, 5), latestOnlyPolicy: "do-not-wait-or-cancel-old-runs" });
|
||
}
|
||
const trigger = v02CdPassed(before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerV02Current(Math.min(options.timeoutSeconds, 600));
|
||
printEvent("v02.cd.trigger", { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||
printV02PrMonitorProgress({ stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||
if (record(trigger).ok !== true) {
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "cd-trigger",
|
||
state: "cd-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
cd: beforeSummary,
|
||
dryRun: false,
|
||
message: "PR 已合并，但 v0.2 CD 触发失败；需要查看 trigger 阶段的 degradedReason。",
|
||
});
|
||
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment };
|
||
}
|
||
const cd = await waitForV02Cd(sourceCommit, options.timeoutSeconds);
|
||
const cdStatus = record(cd.status);
|
||
const flush = record(cd.flush);
|
||
const cdOk = cd.ok === true;
|
||
const cdPhase = String(cd.phase ?? "");
|
||
const cdSuperseded = cdOk && cdStatus.targetValidationState === "superseded";
|
||
const finalComment = commentV02PullRequest({
|
||
pr,
|
||
phase: cdPhase,
|
||
state: cdOk ? (cdSuperseded ? "cd-superseded" : "cd-succeeded") : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
cd: cdStatus,
|
||
flush,
|
||
dryRun: false,
|
||
message: cdOk
|
||
? cdSuperseded
|
||
? "v0.2 自动 CD 已收口：本 merge head 的 PipelineRun 已完成，但 source head 已被后续提交推进，本轮按 latest-only 语义 superseded/no-op，未回写旧 GitOps revision。"
|
||
: "v0.2 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。"
|
||
: cdPhase === "timeout"
|
||
? "v0.2 自动 CD 超时未收口；本评论保留最后一次 targetValidation / PipelineRun / Git mirror 状态用于接续排障。"
|
||
: "v0.2 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。",
|
||
});
|
||
return {
|
||
ok: cdOk && record(finalComment).ok === true,
|
||
action: cdOk ? (cdSuperseded ? "merged-and-superseded-v02" : "merged-and-rolled-v02") : "v02-cd-failed",
|
||
phase: cdOk ? "cd-passed" : cdPhase,
|
||
pr,
|
||
sourceCommit,
|
||
pipelineRun,
|
||
trigger,
|
||
cd,
|
||
comment: finalComment,
|
||
};
|
||
}
|
||
|
||
export async function runRuntimeLanePrAutoCd(spec: HwlabRuntimeLaneSpec, pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
|
||
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
|
||
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
|
||
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
|
||
const observedAt = new Date().toISOString();
|
||
const comment = commentRuntimeLanePullRequest(spec, {
|
||
pr,
|
||
phase: "merge",
|
||
state: "merge-failed",
|
||
startedAt,
|
||
observedAt,
|
||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||
preflight,
|
||
merge: compactCommandResult(merge),
|
||
dryRun: options.dryRun,
|
||
message: mergeRaceState === "closed"
|
||
? `PR preflight 曾通过，但自动合并前 PR 已被关闭且未确认 merged；本轮不会触发 ${spec.version} CD。`
|
||
: "PR preflight 已通过，但自动合并失败；需要处理 GitHub 返回的 merge blocker 后重试。",
|
||
});
|
||
const issue = reportRuntimeLaneAutomationIssue({
|
||
spec,
|
||
pr,
|
||
state: "merge-failed",
|
||
phase: "merge",
|
||
startedAt,
|
||
observedAt,
|
||
preflight,
|
||
details: { merge: compactCommandResult(merge), mergeRaceState },
|
||
message: "自动合并失败，v0.3 monitor 已暂停 CD 触发。",
|
||
dryRun: options.dryRun,
|
||
});
|
||
return { ok: mergeRaceState === "closed" && record(comment).ok === true && record(issue).ok === true, phase: "merge", action: "merge-race-closed", pr, merge, comment, issue };
|
||
}
|
||
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
|
||
if (!options.dryRun && expectedCommit === null) {
|
||
const observedAt = new Date().toISOString();
|
||
printRuntimeLanePrMonitorProgress(spec, { stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
|
||
const comment = commentRuntimeLanePullRequest(spec, {
|
||
pr,
|
||
phase: "merge-commit",
|
||
state: "cd-failed",
|
||
startedAt,
|
||
observedAt,
|
||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit: null,
|
||
pipelineRun: null,
|
||
dryRun: false,
|
||
message: "PR 已合并，但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit；为避免触发错误 source head，本轮未启动 CD。",
|
||
});
|
||
const issue = reportRuntimeLaneAutomationIssue({
|
||
spec,
|
||
pr,
|
||
state: "cd-failed",
|
||
phase: "merge-commit",
|
||
startedAt,
|
||
observedAt,
|
||
preflight,
|
||
details: { merge: commandData(merge) },
|
||
message: "PR 已合并但 merge commit 不可解析，v0.3 CD 未启动。",
|
||
dryRun: false,
|
||
});
|
||
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, issue, degradedReason: "merge-commit-unresolved" };
|
||
}
|
||
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForRuntimeLaneHead(spec, expectedCommit, 120);
|
||
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
|
||
const pipelineRun = sourceCommit === null ? null : runtimeLanePipelineRunName(spec, sourceCommit);
|
||
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
|
||
const startedComment = commentRuntimeLanePullRequest(spec, {
|
||
pr,
|
||
phase: options.dryRun ? "dry-run" : "cd-trigger",
|
||
state: "cd-started",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
dryRun: options.dryRun,
|
||
message: options.dryRun
|
||
? `dry-run：PR 已达到 ${spec.version} 自动合并条件；本轮不会写 GitHub merge 或 CD。`
|
||
: mergeRaceState === "merged"
|
||
? `PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 ${spec.version} CD。后续成功、失败或超时会继续在本 PR 下回复。`
|
||
: `PR 已自动合并，${spec.version} CD 准备开始；后续成功、失败或超时会继续在本 PR 下回复。`,
|
||
});
|
||
if (!options.dryRun && record(startedComment).ok !== true) {
|
||
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
|
||
}
|
||
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
|
||
if (sourceCommit === null || record(headWait).ok !== true) {
|
||
const observedAt = new Date().toISOString();
|
||
printRuntimeLanePrMonitorProgress(spec, { stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
|
||
const comment = commentRuntimeLanePullRequest(spec, {
|
||
pr,
|
||
phase: `${spec.lane}-source-head`,
|
||
state: "cd-timeout",
|
||
startedAt,
|
||
observedAt,
|
||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
dryRun: false,
|
||
message: `PR 已合并，但 G14 ${spec.version} CI/CD source repo 没有在等待窗口内对齐 merge commit；为避免触发旧 head，本轮未启动 CD。`,
|
||
});
|
||
const issue = reportRuntimeLaneAutomationIssue({
|
||
spec,
|
||
pr,
|
||
state: "cd-timeout",
|
||
phase: `${spec.lane}-source-head`,
|
||
startedAt,
|
||
observedAt,
|
||
preflight,
|
||
sourceCommit,
|
||
pipelineRun,
|
||
details: { expectedCommit, headWait },
|
||
message: `${spec.version} source head 未在等待窗口内对齐 merge commit，CD 未启动。`,
|
||
dryRun: false,
|
||
});
|
||
return { ok: false, phase: `${spec.lane}-head`, pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, issue, degradedReason: `${spec.lane}-head-unresolved-after-merge` };
|
||
}
|
||
const before = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
|
||
const beforeSummary = summarizeRuntimeLaneCdStatus(spec, before);
|
||
const trigger = runtimeLaneCdPassedWithMirror(spec, before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerRuntimeLaneCurrent(spec, Math.min(options.timeoutSeconds, 600));
|
||
printEvent(`${spec.lane}.cd.trigger`, { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||
printRuntimeLanePrMonitorProgress(spec, { stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||
if (record(trigger).ok !== true) {
|
||
const observedAt = new Date().toISOString();
|
||
const comment = commentRuntimeLanePullRequest(spec, {
|
||
pr,
|
||
phase: "cd-trigger",
|
||
state: "cd-failed",
|
||
startedAt,
|
||
observedAt,
|
||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
cd: beforeSummary,
|
||
dryRun: false,
|
||
message: `PR 已合并，但 ${spec.version} CD 触发失败；需要查看 trigger 阶段的 degradedReason。`,
|
||
});
|
||
const issue = reportRuntimeLaneAutomationIssue({
|
||
spec,
|
||
pr,
|
||
state: "cd-failed",
|
||
phase: "cd-trigger",
|
||
startedAt,
|
||
observedAt,
|
||
preflight,
|
||
sourceCommit,
|
||
pipelineRun,
|
||
cd: beforeSummary,
|
||
details: { trigger },
|
||
message: `${spec.version} CD 触发失败，PipelineRun 未进入可等待状态。`,
|
||
dryRun: false,
|
||
});
|
||
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment, issue };
|
||
}
|
||
const cd = await waitForRuntimeLaneCd(spec, sourceCommit, options.timeoutSeconds);
|
||
const cdStatus = record(cd.status);
|
||
const flush = record(cd.flush);
|
||
const cdOk = cd.ok === true;
|
||
const cdPhase = String(cd.phase ?? "");
|
||
const cdPipelineRun = stringOrNull(cdStatus.pipelineRun) ?? pipelineRun;
|
||
const observedAt = new Date().toISOString();
|
||
const finalComment = commentRuntimeLanePullRequest(spec, {
|
||
pr,
|
||
phase: cdPhase,
|
||
state: cdOk ? "cd-succeeded" : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||
startedAt,
|
||
observedAt,
|
||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun: cdPipelineRun,
|
||
cd: cdStatus,
|
||
flush,
|
||
dryRun: false,
|
||
message: cdOk
|
||
? `${spec.version} 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。`
|
||
: cdPhase === "timeout"
|
||
? `${spec.version} 自动 CD 超时未收口；本评论保留最后一次 PipelineRun / Argo / public probe / Git mirror 状态用于接续排障。`
|
||
: `${spec.version} 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。`,
|
||
});
|
||
const issue = cdOk ? { ok: true, skipped: true, reason: "cd-succeeded" } : reportRuntimeLaneAutomationIssue({
|
||
spec,
|
||
pr,
|
||
state: cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||
phase: cdPhase || "cd",
|
||
startedAt,
|
||
observedAt,
|
||
preflight,
|
||
sourceCommit,
|
||
pipelineRun: cdPipelineRun,
|
||
cd: cdStatus,
|
||
details: { cd },
|
||
message: cdPhase === "timeout" ? `${spec.version} 自动 CD 超时。` : `${spec.version} 自动 CD 失败。`,
|
||
dryRun: false,
|
||
});
|
||
return {
|
||
ok: cdOk && record(finalComment).ok === true && record(issue).ok === true,
|
||
action: cdOk ? `merged-and-rolled-${spec.lane}` : `${spec.lane}-cd-failed`,
|
||
phase: cdOk ? "cd-passed" : cdPhase,
|
||
pr,
|
||
sourceCommit,
|
||
pipelineRun: cdPipelineRun,
|
||
trigger,
|
||
cd,
|
||
comment: finalComment,
|
||
issue,
|
||
};
|
||
}
|