feat: automate HWLAB v02 PR CD monitor
This commit is contained in:
+714
-23
@@ -1,5 +1,5 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "./config";
|
||||
import { runCommand } from "./command";
|
||||
@@ -65,7 +65,10 @@ const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||||
const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
|
||||
const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
|
||||
|
||||
export type G14MonitorLane = "g14" | "v02";
|
||||
|
||||
interface G14MonitorOptions {
|
||||
lane: G14MonitorLane;
|
||||
intervalSeconds: number;
|
||||
maxCycles: number;
|
||||
once: boolean;
|
||||
@@ -149,7 +152,7 @@ interface ShellSection {
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
interface OpenPullRequest {
|
||||
export interface OpenPullRequest {
|
||||
number: number;
|
||||
title?: string;
|
||||
url?: string;
|
||||
@@ -184,22 +187,49 @@ interface CiPipelineMetrics {
|
||||
rawText?: string;
|
||||
}
|
||||
|
||||
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun">): string {
|
||||
if (options.once && options.dryRun) return "latest-once-dry-run-job.json";
|
||||
if (options.once) return "latest-once-job.json";
|
||||
if (options.dryRun) return "latest-dry-run-job.json";
|
||||
return "latest-monitor-job.json";
|
||||
export interface V02PrCommentInput {
|
||||
pr: OpenPullRequest;
|
||||
phase: string;
|
||||
state: "waiting-ci" | "blocked" | "merge-failed" | "cd-started" | "cd-succeeded" | "cd-failed" | "cd-timeout" | "cd-blocked";
|
||||
startedAt: string;
|
||||
observedAt: string;
|
||||
elapsedSeconds: number | null;
|
||||
preflight?: Record<string, unknown>;
|
||||
merge?: Record<string, unknown>;
|
||||
sourceCommit?: string | null;
|
||||
pipelineRun?: string | null;
|
||||
cd?: Record<string, unknown>;
|
||||
flush?: Record<string, unknown>;
|
||||
dryRun?: boolean;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun">): string {
|
||||
if (options.once && options.dryRun) return "once-dry-run";
|
||||
if (options.once) return "once";
|
||||
if (options.dryRun) return "dry-run-monitor";
|
||||
return "monitor";
|
||||
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||||
const prefix = options.lane === "v02" ? "latest-v02-" : "latest-";
|
||||
if (options.once && options.dryRun) return options.lane === "v02" ? `${prefix}once-dry-run-job.json` : "latest-once-dry-run-job.json";
|
||||
if (options.once && options.lane === "v02") return `${prefix}once-job.json`;
|
||||
if (options.once) return "latest-once-job.json";
|
||||
if (options.dryRun) return `${prefix}dry-run-job.json`;
|
||||
return `${prefix}monitor-job.json`;
|
||||
}
|
||||
|
||||
function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||||
const lanePrefix = options.lane === "v02" ? "v02-" : "";
|
||||
if (options.once && options.dryRun) return `${lanePrefix}once-dry-run`;
|
||||
if (options.once) return `${lanePrefix}once`;
|
||||
if (options.dryRun) return `${lanePrefix}dry-run-monitor`;
|
||||
return `${lanePrefix}monitor`;
|
||||
}
|
||||
|
||||
function parseMonitorLane(args: string[]): G14MonitorLane {
|
||||
const lane = optionValue(args, "--lane") ?? "g14";
|
||||
if (lane !== "g14" && lane !== "v02") throw new Error("monitor-prs --lane must be g14 or v02");
|
||||
return lane;
|
||||
}
|
||||
|
||||
function parseOptions(args: string[]): G14MonitorOptions {
|
||||
return {
|
||||
lane: parseMonitorLane(args),
|
||||
intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400),
|
||||
maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000),
|
||||
once: args.includes("--once"),
|
||||
@@ -519,12 +549,16 @@ function isCommandSuccess(result: CommandJsonResult): boolean {
|
||||
return dataOk !== false;
|
||||
}
|
||||
|
||||
function extractPullRequests(result: CommandJsonResult): OpenPullRequest[] {
|
||||
function monitorBaseBranch(lane: G14MonitorLane): string {
|
||||
return lane === "v02" ? V02_SOURCE_BRANCH : G14_SOURCE_BRANCH;
|
||||
}
|
||||
|
||||
function extractPullRequests(result: CommandJsonResult, baseBranch = G14_SOURCE_BRANCH): OpenPullRequest[] {
|
||||
const prs = nested(result.parsed, ["data", "pullRequests"]);
|
||||
if (!Array.isArray(prs)) return [];
|
||||
return prs
|
||||
.map((item) => record(item))
|
||||
.filter((item) => item.baseRefName === G14_SOURCE_BRANCH || record(item.base).ref === G14_SOURCE_BRANCH)
|
||||
.filter((item) => item.baseRefName === baseBranch || record(item.base).ref === baseBranch)
|
||||
.map((item) => ({
|
||||
number: Number(item.number),
|
||||
title: typeof item.title === "string" ? item.title : undefined,
|
||||
@@ -550,6 +584,10 @@ function printProgressEvent(event: string, data: Record<string, unknown> = {}):
|
||||
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||||
}
|
||||
|
||||
function printV02PrMonitorProgress(data: Record<string, unknown> = {}): void {
|
||||
printProgressEvent("hwlab.v02.pr-monitor.progress", data);
|
||||
}
|
||||
|
||||
function shortSha(sha: string): string {
|
||||
return sha.slice(0, 12);
|
||||
}
|
||||
@@ -3753,12 +3791,36 @@ function summarizePrFiles(filesResult: CommandJsonResult): Record<string, unknow
|
||||
|
||||
function mergeCommitFromPr(prData: Record<string, unknown>): string | null {
|
||||
const mergeCommit = record(record(prData.json).mergeCommit);
|
||||
const fromJson = mergeCommit.oid;
|
||||
const fromJson = mergeCommit.oid ?? mergeCommit.sha;
|
||||
if (typeof fromJson === "string") return fromJson;
|
||||
const fromSummary = record(record(prData.pullRequest).mergeCommit).oid;
|
||||
const fromSummary = record(record(prData.pullRequest).mergeCommit).oid ?? record(record(prData.pullRequest).mergeCommit).sha;
|
||||
return typeof fromSummary === "string" ? fromSummary : null;
|
||||
}
|
||||
|
||||
function sourceCommitFromMergeResult(merge: CommandJsonResult, prNumber: number): string | null {
|
||||
const fromMerge = mergeCommitFromPr(commandData(merge));
|
||||
if (fromMerge !== null) return fromMerge;
|
||||
const prRead = readPullRequest(prNumber);
|
||||
if (!isCommandSuccess(prRead)) return null;
|
||||
return mergeCommitFromPr(commandData(prRead));
|
||||
}
|
||||
|
||||
async function waitForV02Head(expectedCommit: string | null, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
let head: string | null = null;
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
head = getV02Head();
|
||||
if (expectedCommit === null) {
|
||||
if (head !== null) return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||||
} else if (head === expectedCommit) {
|
||||
return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||||
}
|
||||
printEvent("v02.source.wait", { expectedCommit, head, waitedSeconds: Math.round((Date.now() - started) / 1000) });
|
||||
await sleep(5_000);
|
||||
}
|
||||
return { ok: false, sourceCommit: head, expectedCommit, matched: false, waitedSeconds: Math.round((Date.now() - started) / 1000), degradedReason: "v02-source-head-not-aligned-after-merge" };
|
||||
}
|
||||
|
||||
function currentGitopsRevision(): string | null {
|
||||
const argo = getArgoStatus();
|
||||
if (!isCommandSuccess(argo)) return null;
|
||||
@@ -3868,6 +3930,540 @@ function appendRolloutBrief(options: G14RecordRolloutOptions, rollout: Record<st
|
||||
return { ok: true, date: now.date, brief, sourceCommit, pipelineRun, gitopsRevision, ciMetrics, bodyPath, update: commandData(update), ensured };
|
||||
}
|
||||
|
||||
function v02PrCommentStatePath(): string {
|
||||
return rootPath(".state", "hwlab-g14", "v02-pr-comment-signatures.json");
|
||||
}
|
||||
|
||||
function readV02PrCommentState(): Record<string, string> {
|
||||
const path = v02PrCommentStatePath();
|
||||
if (!existsSync(path)) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
||||
const state: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(record(parsed))) {
|
||||
if (typeof value === "string") state[key] = value;
|
||||
}
|
||||
return state;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function writeV02PrCommentState(state: Record<string, string>): string {
|
||||
const path = v02PrCommentStatePath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
||||
return path;
|
||||
}
|
||||
|
||||
function preflightSummary(preflight: CommandJsonResult): Record<string, unknown> {
|
||||
const data = commandData(preflight);
|
||||
const mergeability = record(data.mergeability);
|
||||
return {
|
||||
ok: isCommandSuccess(preflight),
|
||||
conclusion: stringOrNull(mergeability.conclusion) ?? (isCommandSuccess(preflight) ? "unknown" : "preflight-command-failed"),
|
||||
readyForCommanderMerge: mergeability.readyForCommanderMerge === true,
|
||||
blockers: Array.isArray(mergeability.blockers) ? mergeability.blockers.map(String).slice(0, 20) : [],
|
||||
pending: Array.isArray(mergeability.pending) ? mergeability.pending.map(String).slice(0, 20) : [],
|
||||
mergeable: mergeability.mergeable ?? nested(data, ["pullRequest", "mergeable"]) ?? null,
|
||||
mergeStateStatus: mergeability.mergeStateStatus ?? nested(data, ["pullRequest", "mergeStateStatus"]) ?? null,
|
||||
headRefName: nested(data, ["pullRequest", "headRefName"]) ?? null,
|
||||
baseRefName: nested(data, ["pullRequest", "baseRefName"]) ?? null,
|
||||
statusChecks: record(data.statusChecks).summary ?? data.statusChecks ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function v02PrConflictState(summary: Record<string, unknown>): string {
|
||||
const mergeable = String(summary.mergeable ?? "").toUpperCase();
|
||||
const mergeStateStatus = String(summary.mergeStateStatus ?? "").toUpperCase();
|
||||
const blockers = Array.isArray(summary.blockers) ? summary.blockers.map(String).join("\n") : "";
|
||||
if (mergeable.includes("CONFLICT") || mergeStateStatus.includes("DIRTY") || /conflict/i.test(blockers)) return "conflict";
|
||||
return "clear-or-unknown";
|
||||
}
|
||||
|
||||
function summarizeV02CdStatus(status: Record<string, unknown>): Record<string, unknown> {
|
||||
const pipelineRun = record(status.pipelineRun);
|
||||
const targetValidation = record(status.targetValidation);
|
||||
const gitMirror = record(status.gitMirror);
|
||||
const gitMirrorSummary = record(gitMirror.summary);
|
||||
const planArtifacts = record(status.planArtifacts);
|
||||
const planSummary = record(planArtifacts.summary);
|
||||
const argo = record(status.argo);
|
||||
const webAssets = record(status.webAssets);
|
||||
return {
|
||||
ok: status.ok === true,
|
||||
sourceCommit: status.sourceCommit ?? nested(status, ["statusTarget", "sourceCommit"]) ?? null,
|
||||
pipelineRun: pipelineRun.pipelineRun ?? pipelineRun.name ?? nested(status, ["statusTarget", "pipelineRun"]) ?? null,
|
||||
pipelineStatus: pipelineRun.status ?? null,
|
||||
pipelineReason: pipelineRun.reason ?? null,
|
||||
targetValidationState: targetValidation.state ?? null,
|
||||
targetValidationOk: targetValidation.ok ?? null,
|
||||
targetValidationFailures: Array.isArray(targetValidation.failures) ? targetValidation.failures.slice(0, 10) : [],
|
||||
targetValidationWarnings: Array.isArray(targetValidation.warnings) ? targetValidation.warnings.slice(0, 10) : [],
|
||||
argoSync: argo.syncStatus ?? nested(argo, ["summary", "syncStatus"]) ?? null,
|
||||
argoHealth: argo.healthStatus ?? nested(argo, ["summary", "healthStatus"]) ?? null,
|
||||
webAssetsOk: webAssets.ok ?? null,
|
||||
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
|
||||
githubInSync: gitMirrorSummary.githubInSync ?? null,
|
||||
buildServices: planSummary.buildServices ?? planArtifacts.buildServices ?? null,
|
||||
reusedServices: planSummary.reusedServices ?? planArtifacts.reusedServices ?? null,
|
||||
rolloutServices: planSummary.rolloutServices ?? planArtifacts.rolloutServices ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function v02CdPassed(status: Record<string, unknown>): boolean {
|
||||
const state = String(nested(status, ["targetValidation", "state"]) ?? "");
|
||||
return state === "passed" || state === "superseded";
|
||||
}
|
||||
|
||||
function v02PipelineSucceeded(status: Record<string, unknown>): boolean {
|
||||
return String(nested(status, ["pipelineRun", "status"]) ?? "") === "True";
|
||||
}
|
||||
|
||||
function v02CdFailed(status: Record<string, unknown>): boolean {
|
||||
const pipelineStatus = String(nested(status, ["pipelineRun", "status"]) ?? "");
|
||||
return pipelineStatus === "False";
|
||||
}
|
||||
|
||||
function activeV02PipelineRuns(status: Record<string, unknown>): Record<string, unknown>[] {
|
||||
const active = record(status.activePipelineRuns);
|
||||
const items = Array.isArray(active.items) ? active.items : [];
|
||||
return items.map((item) => record(item)).filter((item) => String(item.status ?? "") === "Unknown");
|
||||
}
|
||||
|
||||
function v02PrCommentSignature(input: V02PrCommentInput): string {
|
||||
const summary = input.preflight ?? {};
|
||||
const cd = input.cd ?? {};
|
||||
const flush = input.flush ?? {};
|
||||
return textHash(JSON.stringify({
|
||||
pr: input.pr.number,
|
||||
state: input.state,
|
||||
phase: input.phase,
|
||||
conclusion: summary.conclusion ?? null,
|
||||
readyForCommanderMerge: summary.readyForCommanderMerge ?? null,
|
||||
conflict: v02PrConflictState(summary),
|
||||
blockers: summary.blockers ?? [],
|
||||
pending: summary.pending ?? [],
|
||||
sourceCommit: input.sourceCommit ?? null,
|
||||
pipelineRun: input.pipelineRun ?? null,
|
||||
pipelineStatus: cd.pipelineStatus ?? null,
|
||||
pipelineReason: cd.pipelineReason ?? null,
|
||||
targetValidationState: cd.targetValidationState ?? null,
|
||||
pendingFlush: cd.pendingFlush ?? null,
|
||||
flushOk: flush.ok ?? null,
|
||||
dryRun: input.dryRun === true,
|
||||
}));
|
||||
}
|
||||
|
||||
function listValue(value: unknown): string {
|
||||
if (!Array.isArray(value) || value.length === 0) return "none";
|
||||
return value.map(String).slice(0, 8).join("; ");
|
||||
}
|
||||
|
||||
function scalarValue(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "n/a";
|
||||
if (typeof value === "object") return JSON.stringify(value).slice(0, 240);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
export function v02PrAutomationCommentBody(input: V02PrCommentInput): string {
|
||||
const preflight = input.preflight ?? {};
|
||||
const cd = input.cd ?? {};
|
||||
const flush = input.flush ?? {};
|
||||
const title = input.pr.title ?? `PR #${input.pr.number}`;
|
||||
const url = input.pr.url ?? `https://github.com/${HWLAB_REPO}/pull/${input.pr.number}`;
|
||||
const elapsed = formatDuration(input.elapsedSeconds);
|
||||
const dryRunPrefix = input.dryRun === true ? "[dry-run] " : "";
|
||||
return [
|
||||
`### ${dryRunPrefix}v0.2 自动 CI/CD 状态`,
|
||||
"",
|
||||
`${input.message ?? "UniDesk monitor 已记录本轮 PR 自动化状态。"}`,
|
||||
"",
|
||||
"- PR:",
|
||||
` - [#${input.pr.number} ${title}](${url})`,
|
||||
` - base: \`${input.pr.baseRefName ?? V02_SOURCE_BRANCH}\`; head: \`${input.pr.headRefName ?? "unknown"}\``,
|
||||
"- 自动化状态:",
|
||||
` - state: \`${input.state}\`; phase: \`${input.phase}\``,
|
||||
` - startedAt: \`${input.startedAt}\`; observedAt: \`${input.observedAt}\`; elapsed: ${elapsed}`,
|
||||
"- CI / mergeability:",
|
||||
` - conclusion: \`${scalarValue(preflight.conclusion)}\`; readyForCommanderMerge: \`${String(preflight.readyForCommanderMerge ?? "n/a")}\`; conflict: \`${v02PrConflictState(preflight)}\``,
|
||||
` - blockers: ${listValue(preflight.blockers)}`,
|
||||
` - pending: ${listValue(preflight.pending)}`,
|
||||
"- CD / runtime:",
|
||||
` - sourceCommit: \`${input.sourceCommit ?? "n/a"}\``,
|
||||
` - PipelineRun: \`${input.pipelineRun ?? scalarValue(cd.pipelineRun)}\``,
|
||||
` - pipeline: \`${scalarValue(cd.pipelineStatus)} / ${scalarValue(cd.pipelineReason)}\`; targetValidation: \`${scalarValue(cd.targetValidationState)}\``,
|
||||
` - Argo: \`${scalarValue(cd.argoSync)} / ${scalarValue(cd.argoHealth)}\`; webAssets.ok: \`${String(cd.webAssetsOk ?? "n/a")}\``,
|
||||
` - Git mirror: pendingFlush=\`${String(cd.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(cd.githubInSync ?? "n/a")}\`, flush.ok=\`${String(flush.ok ?? "n/a")}\``,
|
||||
` - rolloutServices: ${listValue(cd.rolloutServices)}; buildServices: ${listValue(cd.buildServices)}; reusedServices: ${listValue(cd.reusedServices)}`,
|
||||
"",
|
||||
"后续动作由 `bun scripts/cli.ts hwlab g14 monitor-prs --lane v02` 继续驱动;同一状态签名不会重复刷评论。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function commentV02PullRequest(input: V02PrCommentInput): Record<string, unknown> {
|
||||
const body = v02PrAutomationCommentBody(input);
|
||||
const signature = v02PrCommentSignature(input);
|
||||
const cacheKey = `pr-${input.pr.number}`;
|
||||
if (input.dryRun === true) {
|
||||
return { ok: true, dryRun: true, skipped: true, signature, wouldComment: { bodyPreview: body.slice(0, 1200), bodyChars: body.length } };
|
||||
}
|
||||
const state = readV02PrCommentState();
|
||||
if (state[cacheKey] === signature) {
|
||||
return { ok: true, skipped: true, reason: "same-pr-comment-signature", signature, stateFile: v02PrCommentStatePath() };
|
||||
}
|
||||
const bodyPath = writeStateFile(`v02-pr-${input.pr.number}-${input.state}-${signature}.md`, `${body}\n`);
|
||||
const create = cliJson(["gh", "pr", "comment", "create", String(input.pr.number), "--repo", HWLAB_REPO, "--body-file", bodyPath], 100_000);
|
||||
if (!isCommandSuccess(create)) return { ok: false, phase: "pr-comment", signature, bodyPath, create };
|
||||
state[cacheKey] = signature;
|
||||
const stateFile = writeV02PrCommentState(state);
|
||||
return { ok: true, signature, bodyPath, stateFile, create: commandData(create) };
|
||||
}
|
||||
|
||||
function triggerV02Current(timeoutSeconds: number): Record<string, unknown> {
|
||||
return runV02ControlPlane({
|
||||
action: "trigger-current",
|
||||
lane: "v02",
|
||||
dryRun: false,
|
||||
confirm: true,
|
||||
wait: true,
|
||||
allowLiveDbRead: false,
|
||||
timeoutSeconds,
|
||||
minAgeMinutes: 30,
|
||||
limit: 20,
|
||||
});
|
||||
}
|
||||
|
||||
function flushV02GitMirrorIfNeeded(status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
|
||||
const pendingFlush = nested(status, ["gitMirror", "summary", "pendingFlush"]);
|
||||
if (pendingFlush !== true) return { ok: true, skipped: true, reason: "git-mirror-already-flushed" };
|
||||
return runG14GitMirror({
|
||||
action: "flush",
|
||||
confirm: true,
|
||||
dryRun: false,
|
||||
wait: true,
|
||||
timeoutSeconds,
|
||||
});
|
||||
}
|
||||
|
||||
function pullRequestFromMergeCommand(merge: CommandJsonResult): Record<string, unknown> {
|
||||
const data = commandData(merge);
|
||||
const direct = record(data.pullRequest);
|
||||
if (Object.keys(direct).length > 0) return direct;
|
||||
return record(nested(data, ["details", "details", "pullRequest"]));
|
||||
}
|
||||
|
||||
function mergeCommandRaceState(merge: CommandJsonResult): "merged" | "closed" | null {
|
||||
const pullRequest = pullRequestFromMergeCommand(merge);
|
||||
if (pullRequest.merged === true || pullRequest.stateDetail === "merged") return "merged";
|
||||
if (pullRequest.closed === true || pullRequest.state === "closed") return "closed";
|
||||
return null;
|
||||
}
|
||||
|
||||
async function waitForV02Cd(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
const startedAt = new Date(started).toISOString();
|
||||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||||
let lastStatus: Record<string, unknown> = {};
|
||||
let firstPassedAt: string | null = null;
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
lastStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
const summary = summarizeV02CdStatus(lastStatus);
|
||||
printEvent("v02.cd.status", { sourceCommit, pipelineRun, summary });
|
||||
printV02PrMonitorProgress({ stage: "cd-status", status: "running", sourceCommit, pipelineRun, targetValidationState: summary.targetValidationState ?? null, pipelineStatus: summary.pipelineStatus ?? null, pendingFlush: summary.pendingFlush ?? null });
|
||||
if (v02PipelineSucceeded(lastStatus) && summary.pendingFlush === true) {
|
||||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||||
if (record(flush).ok !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "git-mirror-flush",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summary,
|
||||
rawStatus: lastStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
const finalSummary = summarizeV02CdStatus(finalStatus);
|
||||
if (v02CdPassed(finalStatus)) {
|
||||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: "succeeded", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||||
return {
|
||||
ok: true,
|
||||
phase: "cd-passed",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: finalSummary,
|
||||
rawStatus: finalStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
lastStatus = finalStatus;
|
||||
printEvent("v02.cd.after-flush", { sourceCommit, pipelineRun, flushOk: record(flush).ok, summary: finalSummary });
|
||||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: record(flush).ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||||
}
|
||||
if (v02CdPassed(lastStatus)) {
|
||||
firstPassedAt ??= new Date().toISOString();
|
||||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
printV02PrMonitorProgress({ stage: "cd-status", status: v02CdPassed(finalStatus) ? "succeeded" : "failed", sourceCommit, pipelineRun, targetValidationState: nested(finalStatus, ["targetValidation", "state"]) ?? null });
|
||||
return {
|
||||
ok: v02CdPassed(finalStatus),
|
||||
phase: v02CdPassed(finalStatus) ? "cd-passed" : "git-mirror-flush",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summarizeV02CdStatus(finalStatus),
|
||||
rawStatus: finalStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
if (v02CdFailed(lastStatus)) {
|
||||
printV02PrMonitorProgress({ stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
|
||||
return {
|
||||
ok: false,
|
||||
phase: "cd-failed",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summary,
|
||||
rawStatus: lastStatus,
|
||||
};
|
||||
}
|
||||
await sleep(30_000);
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
phase: "timeout",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
timeoutSeconds,
|
||||
firstPassedAt,
|
||||
status: summarizeV02CdStatus(lastStatus),
|
||||
rawStatus: lastStatus,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForV02LaneIdle(timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
let lastStatus: Record<string, unknown> = {};
|
||||
let activeRuns: Record<string, unknown>[] = [];
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
lastStatus = v02ControlPlaneStatus();
|
||||
activeRuns = activeV02PipelineRuns(lastStatus);
|
||||
printEvent("v02.cd.lane-idle", { activeCount: activeRuns.length, activeRuns: activeRuns.slice(0, 5) });
|
||||
printV02PrMonitorProgress({ stage: "lane-idle", status: activeRuns.length === 0 ? "succeeded" : "running", activeCount: activeRuns.length });
|
||||
if (activeRuns.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
waitedSeconds: Math.round((Date.now() - started) / 1000),
|
||||
status: summarizeV02CdStatus(lastStatus),
|
||||
activeRuns,
|
||||
};
|
||||
}
|
||||
await sleep(30_000);
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
phase: "active-run-timeout",
|
||||
timeoutSeconds,
|
||||
waitedSeconds: Math.round((Date.now() - started) / 1000),
|
||||
status: summarizeV02CdStatus(lastStatus),
|
||||
activeRuns,
|
||||
};
|
||||
}
|
||||
|
||||
async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
|
||||
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
|
||||
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
|
||||
printV02PrMonitorProgress({ stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "merge",
|
||||
state: "merge-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: compactCommandResult(merge),
|
||||
dryRun: options.dryRun,
|
||||
message: mergeRaceState === "closed"
|
||||
? "PR preflight 曾通过,但自动合并前 PR 已被关闭且未确认 merged;本轮不会触发 CD。"
|
||||
: "PR preflight 已通过,但自动合并失败;需要处理 GitHub 返回的 merge blocker 后重试。",
|
||||
});
|
||||
return { ok: mergeRaceState === "closed", phase: "merge", action: "merge-race-closed", pr, merge, comment };
|
||||
}
|
||||
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
|
||||
if (!options.dryRun && expectedCommit === null) {
|
||||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "merge-commit",
|
||||
state: "cd-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit: null,
|
||||
pipelineRun: null,
|
||||
dryRun: false,
|
||||
message: "PR 已合并,但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit;为避免触发错误 source head,本轮未启动 CD。",
|
||||
});
|
||||
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, degradedReason: "merge-commit-unresolved" };
|
||||
}
|
||||
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForV02Head(expectedCommit, 120);
|
||||
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
|
||||
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
|
||||
printV02PrMonitorProgress({ stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
|
||||
const startedComment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: options.dryRun ? "dry-run" : "cd-trigger",
|
||||
state: "cd-started",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
dryRun: options.dryRun,
|
||||
message: options.dryRun
|
||||
? "dry-run:PR 已达到自动合并条件;本轮不会写 GitHub merge 或 CD。"
|
||||
: mergeRaceState === "merged"
|
||||
? "PR 在本轮 merge 前已被合并;worker 继续对齐 merge commit 并驱动 v0.2 CD。后续成功、失败或超时会继续在本 PR 下回复。"
|
||||
: "PR 已自动合并,v0.2 CD 准备开始。后续成功、失败或超时会继续在本 PR 下回复。",
|
||||
});
|
||||
if (!options.dryRun && record(startedComment).ok !== true) {
|
||||
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
|
||||
}
|
||||
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
|
||||
if (sourceCommit === null || record(headWait).ok !== true) {
|
||||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "v02-source-head",
|
||||
state: "cd-timeout",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
dryRun: false,
|
||||
message: "PR 已合并,但 G14 v0.2 CI/CD source repo 没有在等待窗口内对齐 merge commit;为避免触发旧 head,本轮未启动 CD。",
|
||||
});
|
||||
return { ok: false, phase: "v02-head", pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, degradedReason: "v02-head-unresolved-after-merge" };
|
||||
}
|
||||
const before = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
const beforeSummary = summarizeV02CdStatus(before);
|
||||
const activeRuns = activeV02PipelineRuns(before);
|
||||
if (activeRuns.length > 0 && !v02CdPassed(before)) {
|
||||
printV02PrMonitorProgress({ stage: "lane-idle", status: "running", pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length });
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "cd-active-run",
|
||||
state: "cd-blocked",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: beforeSummary,
|
||||
dryRun: false,
|
||||
message: "PR 已合并,但 v0.2 当前已有运行中的 PipelineRun;worker 会等待 lane 空闲后继续触发当前 merge commit 的 CD。",
|
||||
});
|
||||
if (record(comment).ok !== true) return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, activeRuns, before: beforeSummary, comment };
|
||||
const idle = await waitForV02LaneIdle(options.timeoutSeconds);
|
||||
if (record(idle).ok !== true) {
|
||||
const timeoutComment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "cd-active-run-timeout",
|
||||
state: "cd-timeout",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: record(idle.status),
|
||||
dryRun: false,
|
||||
message: "PR 已合并,但 v0.2 lane 长时间被已有 PipelineRun 占用,当前 merge commit 尚未触发 CD;本评论保留 active run 状态用于接续排障。",
|
||||
});
|
||||
return { ok: false, phase: "cd-active-run-timeout", pr, sourceCommit, pipelineRun, activeRuns, before: beforeSummary, idle, comment, timeoutComment };
|
||||
}
|
||||
}
|
||||
const trigger = v02CdPassed(before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerV02Current(Math.min(options.timeoutSeconds, 600));
|
||||
printEvent("v02.cd.trigger", { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||||
printV02PrMonitorProgress({ stage: "cd-trigger", status: record(trigger).ok === true || record(trigger).degradedReason === "refuse-active-or-successful-pipelinerun" ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||||
if (record(trigger).ok !== true && record(trigger).degradedReason !== "refuse-active-or-successful-pipelinerun") {
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "cd-trigger",
|
||||
state: "cd-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: beforeSummary,
|
||||
dryRun: false,
|
||||
message: "PR 已合并,但 v0.2 CD 触发失败;需要查看 trigger 阶段的 degradedReason。",
|
||||
});
|
||||
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment };
|
||||
}
|
||||
const cd = await waitForV02Cd(sourceCommit, options.timeoutSeconds);
|
||||
const cdStatus = record(cd.status);
|
||||
const flush = record(cd.flush);
|
||||
const cdOk = cd.ok === true;
|
||||
const cdPhase = String(cd.phase ?? "");
|
||||
const finalComment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: cdPhase,
|
||||
state: cdOk ? "cd-succeeded" : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: cdStatus,
|
||||
flush,
|
||||
dryRun: false,
|
||||
message: cdOk
|
||||
? "v0.2 自动 CD 已完成:PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。"
|
||||
: cdPhase === "timeout"
|
||||
? "v0.2 自动 CD 超时未收口;本评论保留最后一次 targetValidation / PipelineRun / Git mirror 状态用于接续排障。"
|
||||
: "v0.2 自动 CD 失败;本评论保留失败阶段和最后一次状态用于接续排障。",
|
||||
});
|
||||
return {
|
||||
ok: cdOk && record(finalComment).ok === true,
|
||||
action: cdOk ? "merged-and-rolled-v02" : "v02-cd-failed",
|
||||
phase: cdOk ? "cd-passed" : cdPhase,
|
||||
pr,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
trigger,
|
||||
cd,
|
||||
comment: finalComment,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
const startedAt = new Date(started).toISOString();
|
||||
@@ -3910,7 +4506,88 @@ async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Prom
|
||||
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
|
||||
}
|
||||
|
||||
async function monitorV02Cycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||||
printEvent("v02.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||||
const listed = listOpenG14PullRequests();
|
||||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||||
const prs = extractPullRequests(listed, V02_SOURCE_BRANCH);
|
||||
printEvent("v02.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||||
if (prs.length === 0) return { ok: true, cycle, action: "none", lane: "v02", pullRequests: [] };
|
||||
const observations: unknown[] = [];
|
||||
for (const pr of prs) {
|
||||
const startedAt = new Date().toISOString();
|
||||
const preflightResult = preflightPullRequest(pr.number);
|
||||
const preflight = preflightSummary(preflightResult);
|
||||
printEvent("v02.pr.preflight", {
|
||||
cycle,
|
||||
number: pr.number,
|
||||
ok: isCommandSuccess(preflightResult),
|
||||
conclusion: preflight.conclusion,
|
||||
readyForCommanderMerge: preflight.readyForCommanderMerge,
|
||||
conflict: v02PrConflictState(preflight),
|
||||
});
|
||||
printV02PrMonitorProgress({
|
||||
stage: "preflight",
|
||||
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
|
||||
pr: pr.number,
|
||||
conclusion: preflight.conclusion,
|
||||
conflict: v02PrConflictState(preflight),
|
||||
});
|
||||
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
|
||||
const conclusion = String(preflight.conclusion ?? "unknown");
|
||||
const blocked = conclusion === "blocked" || v02PrConflictState(preflight) === "conflict" || !isCommandSuccess(preflightResult);
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "preflight",
|
||||
state: blocked ? "blocked" : "waiting-ci",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
dryRun: options.dryRun,
|
||||
message: blocked
|
||||
? "v0.2 自动化已暂停在 PR preflight:存在 blocker 或冲突,需要先修复后再继续。"
|
||||
: "v0.2 自动化正在等待 GitHub CI / mergeability 收敛;CI 通过且无冲突后会自动合并并触发 CD。",
|
||||
});
|
||||
observations.push({ pullRequest: pr, preflight, comment });
|
||||
printV02PrMonitorProgress({ stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
|
||||
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
|
||||
continue;
|
||||
}
|
||||
const currentStatus = v02ControlPlaneStatus();
|
||||
const activeRuns = activeV02PipelineRuns(currentStatus);
|
||||
if (activeRuns.length > 0) {
|
||||
const cd = summarizeV02CdStatus(currentStatus);
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "cd-active-before-merge",
|
||||
state: "cd-blocked",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
cd,
|
||||
dryRun: options.dryRun,
|
||||
message: "PR 已通过 CI / mergeability,但 v0.2 lane 当前已有运行中的 PipelineRun;为保持 PR merge commit 与 CD 目标一一对应,本轮暂不合并,待 lane 空闲后自动继续。",
|
||||
});
|
||||
observations.push({ pullRequest: pr, preflight, activeRuns, cd, comment });
|
||||
printV02PrMonitorProgress({ stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, activeCount: activeRuns.length });
|
||||
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
|
||||
continue;
|
||||
}
|
||||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||||
printEvent("v02.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||||
printV02PrMonitorProgress({ stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
|
||||
const result = await runV02PrAutoCd(pr, preflight, merge, options, startedAt);
|
||||
observations.push(result);
|
||||
if (record(result).ok !== true) return { ok: false, cycle, lane: "v02", phase: record(result).phase ?? "v02-auto-cd", pullRequest: pr, result, observations };
|
||||
return { ok: true, cycle, lane: "v02", action: record(result).action ?? (options.dryRun ? "dry-run-merge" : "merged-and-rolled-v02"), result, observations };
|
||||
}
|
||||
return { ok: true, cycle, lane: "v02", action: "none", observations };
|
||||
}
|
||||
|
||||
async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||||
if (options.lane === "v02") return monitorV02Cycle(options, cycle);
|
||||
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||||
const precheck = precheckWorkspace();
|
||||
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
|
||||
@@ -3950,10 +4627,10 @@ async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<stri
|
||||
cycle += 1;
|
||||
const result = await monitorCycle(options, cycle);
|
||||
results.push(result);
|
||||
printEvent("g14.monitor.cycle.done", { cycle, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
||||
printEvent(`${options.lane}.monitor.cycle.done`, { cycle, lane: options.lane, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
||||
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
|
||||
if (options.once || record(result).action !== "none") return { ok: true, cycles: cycle, lastResult: result, results };
|
||||
printEvent("g14.monitor.sleep", { cycle, intervalSeconds: options.intervalSeconds });
|
||||
if (options.once || (options.lane === "g14" && record(result).action !== "none")) return { ok: true, cycles: cycle, lastResult: result, results };
|
||||
printEvent(`${options.lane}.monitor.sleep`, { cycle, lane: options.lane, intervalSeconds: options.intervalSeconds });
|
||||
await sleep(options.intervalSeconds * 1000);
|
||||
}
|
||||
return { ok: true, cycles: cycle, results };
|
||||
@@ -3965,7 +4642,9 @@ export function hwlabG14Help(): Record<string, unknown> {
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --once --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --once --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 record-rollout --pr <number> [--source-commit sha]",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
|
||||
@@ -3994,10 +4673,11 @@ export function hwlabG14Help(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||||
],
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.",
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs, triggers v0.2 CD, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.",
|
||||
defaults: {
|
||||
repo: HWLAB_REPO,
|
||||
base: G14_SOURCE_BRANCH,
|
||||
v02Base: V02_SOURCE_BRANCH,
|
||||
provider: G14_PROVIDER,
|
||||
workspace: G14_WORKSPACE,
|
||||
v02Workspace: V02_WORKSPACE,
|
||||
@@ -4012,6 +4692,11 @@ export function hwlabG14Help(): Record<string, unknown> {
|
||||
once: ".state/hwlab-g14/latest-once-job.json",
|
||||
dryRun: ".state/hwlab-g14/latest-dry-run-job.json",
|
||||
onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json",
|
||||
v02Monitor: ".state/hwlab-g14/latest-v02-monitor-job.json",
|
||||
v02Once: ".state/hwlab-g14/latest-v02-once-job.json",
|
||||
v02DryRun: ".state/hwlab-g14/latest-v02-dry-run-job.json",
|
||||
v02OnceDryRun: ".state/hwlab-g14/latest-v02-once-dry-run-job.json",
|
||||
v02PrCommentSignatures: ".state/hwlab-g14/v02-pr-comment-signatures.json",
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -4050,8 +4735,12 @@ export async function runHwlabG14Command(_config: Config, args: string[]): Promi
|
||||
}
|
||||
const options = parseOptions(args.slice(1));
|
||||
if (options.worker) return runMonitorWorker(options);
|
||||
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
|
||||
const job = startJob("hwlab_g14_pr_monitor", command, `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`);
|
||||
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--lane", options.lane, "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
|
||||
const jobName = options.lane === "v02" ? "hwlab_g14_v02_pr_monitor" : "hwlab_g14_pr_monitor";
|
||||
const jobNote = options.lane === "v02"
|
||||
? `Monitor ${HWLAB_REPO} PRs targeting ${V02_SOURCE_BRANCH}, merge ready PRs, trigger v0.2 CD, and comment PR progress`
|
||||
: `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`;
|
||||
const job = startJob(jobName, command, jobNote);
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
|
||||
const stateDir = rootPath(".state", "hwlab-g14");
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
@@ -4059,10 +4748,12 @@ export async function runHwlabG14Command(_config: Config, args: string[]): Promi
|
||||
const stateFileRole = hwlabG14MonitorStateRole(options);
|
||||
const latestPath = join(stateDir, stateFileName);
|
||||
const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null;
|
||||
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole }, null, 2)}\n`, "utf8");
|
||||
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole, lane: options.lane, baseBranch: monitorBaseBranch(options.lane) }, null, 2)}\n`, "utf8");
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 monitor-prs",
|
||||
lane: options.lane,
|
||||
baseBranch: monitorBaseBranch(options.lane),
|
||||
mode: "async-job",
|
||||
job,
|
||||
statusCommand,
|
||||
|
||||
+9
-3
@@ -224,14 +224,20 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
|
||||
|
||||
function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
|
||||
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
|
||||
const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
|
||||
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
|
||||
if (!knownWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
|
||||
if (!knownWorkflow && !v02PrMonitorWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
|
||||
const nowMs = Date.now();
|
||||
const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
|
||||
const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
|
||||
const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
|
||||
if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs);
|
||||
const events = parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress");
|
||||
const events = v02PrMonitorWorkflow
|
||||
? [
|
||||
...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"),
|
||||
...parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress"),
|
||||
].sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")))
|
||||
: parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress");
|
||||
const lastEvent = events.at(-1) ?? {};
|
||||
const stage = stringField(lastEvent.stage);
|
||||
const stageStatus = stringField(lastEvent.status);
|
||||
@@ -243,7 +249,7 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
|
||||
? false
|
||||
: null;
|
||||
const lastEventAt = stringField(lastEvent.at);
|
||||
const kind = events.length > 0 || knownWorkflow ? "hwlab-v02-trigger" : "generic";
|
||||
const kind = events.length > 0 || knownWorkflow || v02PrMonitorWorkflow ? "hwlab-v02-trigger" : "generic";
|
||||
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
||||
const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs);
|
||||
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
|
||||
|
||||
Reference in New Issue
Block a user