Codex
277 lines
17 KiB
TypeScript
277 lines
17 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. monitor module for scripts/src/hwlab-g14.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-g14.ts:10163-10420 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult, G14MonitorOptions } from "./types";
|
|
import { runRuntimeLanePrAutoCd, runV02PrAutoCd } from "./cd-trigger";
|
|
import { appendRolloutBrief, collectPipelineMetrics, commentRuntimeLanePullRequest, commentV02PullRequest, durationSeconds, getArgoStatus, getDevWorkloads, getG14Head, getLiveHealth, getPipelineStatus, listOpenG14PullRequests, mergePullRequest, preflightPullRequest, preflightSummary, refreshArgoDev, reportRuntimeLaneAutomationIssue, statusText, v02PrConflictState, workloadReadiness } from "./pr-monitor";
|
|
import { commandData, compactCommandResult, extractPullRequests, isCommandSuccess, nested, precheckWorkspace, printEvent, printRuntimeLanePrMonitorProgress, printV02PrMonitorProgress, record, shortSha, sleep } from "./remote";
|
|
import { legacyG14RetirementBlocksMonitor } from "./retirement";
|
|
import { V02_SOURCE_BRANCH } from "./types";
|
|
|
|
export async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
|
const started = Date.now();
|
|
const startedAt = new Date(started).toISOString();
|
|
let pipelineText = "";
|
|
let argoText = "";
|
|
let healthOk = false;
|
|
let workloadsReady: Record<string, unknown> = { ready: false };
|
|
let pipelineSucceededAt: string | null = null;
|
|
while (Date.now() - started < timeoutSeconds * 1000) {
|
|
const pipeline = getPipelineStatus(sourceCommit);
|
|
pipelineText = statusText(pipeline);
|
|
printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) });
|
|
if (!pipelineText.startsWith("True\nSucceeded")) {
|
|
if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText };
|
|
await sleep(30_000);
|
|
continue;
|
|
}
|
|
pipelineSucceededAt ??= new Date().toISOString();
|
|
refreshArgoDev();
|
|
const argo = getArgoStatus();
|
|
argoText = statusText(argo);
|
|
printEvent("g14.argo.status", { text: argoText.slice(0, 500) });
|
|
const argoLines = argoText.split(/\r?\n/u);
|
|
const synced = argoLines[1] === "Synced";
|
|
const healthy = argoLines[2] === "Healthy";
|
|
const gitopsRevision = argoLines[0] ?? null;
|
|
const workloads = getDevWorkloads();
|
|
const workloadsText = statusText(workloads);
|
|
workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads));
|
|
const health = getLiveHealth();
|
|
healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout);
|
|
if (synced && healthy && record(workloadsReady).ready === true && healthOk) {
|
|
const finishedAt = new Date().toISOString();
|
|
const pipelineRun = `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
|
const ciMetrics = collectPipelineMetrics(pipelineRun);
|
|
return { ok: true, sourceCommit, gitopsRevision, pipelineRun, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk, ciMetrics };
|
|
}
|
|
await sleep(30_000);
|
|
}
|
|
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
|
|
}
|
|
|
|
export async function monitorV02Cycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
|
printEvent("v02.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
|
printV02PrMonitorProgress({ stage: "cycle", status: "started", cycle, dryRun: options.dryRun });
|
|
const listed = listOpenG14PullRequests();
|
|
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
|
const prs = extractPullRequests(listed, V02_SOURCE_BRANCH);
|
|
printEvent("v02.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
|
if (prs.length === 0) {
|
|
printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, pullRequests: 0, intervalSeconds: options.intervalSeconds });
|
|
return { ok: true, cycle, action: "none", lane: "v02", pullRequests: [] };
|
|
}
|
|
const observations: unknown[] = [];
|
|
for (const pr of prs) {
|
|
const startedAt = new Date().toISOString();
|
|
const preflightResult = preflightPullRequest(pr.number);
|
|
const preflight = preflightSummary(preflightResult);
|
|
printEvent("v02.pr.preflight", {
|
|
cycle,
|
|
number: pr.number,
|
|
ok: isCommandSuccess(preflightResult),
|
|
conclusion: preflight.conclusion,
|
|
readyForCommanderMerge: preflight.readyForCommanderMerge,
|
|
conflict: v02PrConflictState(preflight),
|
|
});
|
|
printV02PrMonitorProgress({
|
|
stage: "preflight",
|
|
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
|
|
pr: pr.number,
|
|
conclusion: preflight.conclusion,
|
|
conflict: v02PrConflictState(preflight),
|
|
});
|
|
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
|
|
const conclusion = String(preflight.conclusion ?? "unknown");
|
|
const blocked = conclusion === "blocked" || v02PrConflictState(preflight) === "conflict" || !isCommandSuccess(preflightResult);
|
|
const comment = commentV02PullRequest({
|
|
pr,
|
|
phase: "preflight",
|
|
state: blocked ? "blocked" : "waiting-ci",
|
|
startedAt,
|
|
observedAt: new Date().toISOString(),
|
|
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
|
preflight,
|
|
dryRun: options.dryRun,
|
|
message: blocked
|
|
? "v0.2 自动化已暂停在 PR preflight：存在 blocker 或冲突，需要先修复后再继续。"
|
|
: "v0.2 自动化正在等待 GitHub CI / mergeability 收敛；CI 通过且无冲突后会自动合并并触发 CD。",
|
|
});
|
|
observations.push({ pullRequest: pr, preflight, comment });
|
|
printV02PrMonitorProgress({ stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
|
|
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
|
|
continue;
|
|
}
|
|
const merge = mergePullRequest(pr.number, options.dryRun);
|
|
printEvent("v02.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
|
printV02PrMonitorProgress({ stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
|
|
const result = await runV02PrAutoCd(pr, preflight, merge, options, startedAt);
|
|
observations.push(result);
|
|
if (record(result).ok !== true) return { ok: false, cycle, lane: "v02", phase: record(result).phase ?? "v02-auto-cd", pullRequest: pr, result, observations };
|
|
return { ok: true, cycle, lane: "v02", action: record(result).action ?? (options.dryRun ? "dry-run-merge" : "merged-and-rolled-v02"), result, observations };
|
|
}
|
|
return { ok: true, cycle, lane: "v02", action: "none", observations };
|
|
}
|
|
|
|
export function runtimeLanePreflightBlocked(preflightResult: CommandJsonResult, preflight: Record<string, unknown>): boolean {
|
|
const blockers = Array.isArray(preflight.blockers) ? preflight.blockers : [];
|
|
const pending = Array.isArray(preflight.pending) ? preflight.pending : [];
|
|
const conclusion = String(preflight.conclusion ?? "unknown").toLowerCase();
|
|
return !isCommandSuccess(preflightResult)
|
|
|| v02PrConflictState(preflight) === "conflict"
|
|
|| conclusion === "blocked"
|
|
|| conclusion === "failed"
|
|
|| conclusion === "failure"
|
|
|| conclusion === "error"
|
|
|| (blockers.length > 0 && pending.length === 0);
|
|
}
|
|
|
|
export async function monitorRuntimeLaneCycle(spec: HwlabRuntimeLaneSpec, options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
|
printEvent(`${spec.lane}.monitor.cycle.start`, { cycle, dryRun: options.dryRun });
|
|
printRuntimeLanePrMonitorProgress(spec, { stage: "cycle", status: "started", cycle, dryRun: options.dryRun });
|
|
const listed = listOpenG14PullRequests();
|
|
if (!isCommandSuccess(listed)) return { ok: false, cycle, lane: spec.lane, phase: "list-prs", listed };
|
|
const prs = extractPullRequests(listed, spec.sourceBranch);
|
|
printEvent(`${spec.lane}.monitor.prs`, { cycle, count: prs.length, pullRequests: prs });
|
|
if (prs.length === 0) {
|
|
printRuntimeLanePrMonitorProgress(spec, { stage: "idle", status: "waiting", cycle, pullRequests: 0, intervalSeconds: options.intervalSeconds });
|
|
return { ok: true, cycle, action: "none", lane: spec.lane, pullRequests: [] };
|
|
}
|
|
const observations: unknown[] = [];
|
|
for (const pr of prs) {
|
|
const startedAt = new Date().toISOString();
|
|
const preflightResult = preflightPullRequest(pr.number);
|
|
const preflight = preflightSummary(preflightResult);
|
|
printEvent(`${spec.lane}.pr.preflight`, {
|
|
cycle,
|
|
number: pr.number,
|
|
ok: isCommandSuccess(preflightResult),
|
|
conclusion: preflight.conclusion,
|
|
readyForCommanderMerge: preflight.readyForCommanderMerge,
|
|
conflict: v02PrConflictState(preflight),
|
|
});
|
|
printRuntimeLanePrMonitorProgress(spec, {
|
|
stage: "preflight",
|
|
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
|
|
pr: pr.number,
|
|
conclusion: preflight.conclusion,
|
|
conflict: v02PrConflictState(preflight),
|
|
});
|
|
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
|
|
const blocked = runtimeLanePreflightBlocked(preflightResult, preflight);
|
|
const observedAt = new Date().toISOString();
|
|
const comment = commentRuntimeLanePullRequest(spec, {
|
|
pr,
|
|
phase: "preflight",
|
|
state: blocked ? "blocked" : "waiting-ci",
|
|
startedAt,
|
|
observedAt,
|
|
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
|
preflight,
|
|
dryRun: options.dryRun,
|
|
message: blocked
|
|
? `${spec.version} 自动化已暂停在 PR preflight：存在 blocker、失败 check 或冲突，需要先修复后再继续。`
|
|
: `${spec.version} 自动化正在等待 GitHub CI / mergeability 收敛；CI 通过且无冲突后会自动合并并触发 CD。`,
|
|
});
|
|
const issue = blocked ? reportRuntimeLaneAutomationIssue({
|
|
spec,
|
|
pr,
|
|
state: "blocked",
|
|
phase: "preflight",
|
|
startedAt,
|
|
observedAt,
|
|
preflight,
|
|
details: { preflightResult: compactCommandResult(preflightResult) },
|
|
message: `${spec.version} PR preflight 未通过或存在冲突，自动合并/CD 已暂停。`,
|
|
dryRun: options.dryRun,
|
|
}) : { ok: true, skipped: true, reason: "waiting-ci-without-blocker" };
|
|
observations.push({ pullRequest: pr, preflight, comment, issue });
|
|
printRuntimeLanePrMonitorProgress(spec, { stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
|
|
if (record(comment).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: "pr-comment", pullRequest: pr, preflight, comment, issue, observations };
|
|
if (record(issue).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: "failure-issue", pullRequest: pr, preflight, comment, issue, observations };
|
|
continue;
|
|
}
|
|
const merge = mergePullRequest(pr.number, options.dryRun);
|
|
printEvent(`${spec.lane}.pr.merge`, { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
|
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
|
|
const result = await runRuntimeLanePrAutoCd(spec, pr, preflight, merge, options, startedAt);
|
|
observations.push(result);
|
|
if (record(result).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: record(result).phase ?? `${spec.lane}-auto-cd`, pullRequest: pr, result, observations };
|
|
return { ok: true, cycle, lane: spec.lane, action: record(result).action ?? (options.dryRun ? "dry-run-merge" : `merged-and-rolled-${spec.lane}`), result, observations };
|
|
}
|
|
return { ok: true, cycle, lane: spec.lane, action: "none", observations };
|
|
}
|
|
|
|
export async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
|
if (options.lane === "v02") return monitorV02Cycle(options, cycle);
|
|
if (options.lane !== "g14") return monitorRuntimeLaneCycle(hwlabRuntimeLaneSpec(options.lane), options, cycle);
|
|
const retirement = legacyG14RetirementBlocksMonitor();
|
|
if (retirement !== null) {
|
|
return {
|
|
ok: false,
|
|
cycle,
|
|
phase: "retired",
|
|
degradedReason: "legacy-g14-dev-prod-retired",
|
|
retirement,
|
|
next: { status: "bun scripts/cli.ts hwlab g14 retirement status", v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02" },
|
|
};
|
|
}
|
|
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
|
const precheck = precheckWorkspace();
|
|
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
|
|
const listed = listOpenG14PullRequests();
|
|
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
|
const prs = extractPullRequests(listed);
|
|
printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
|
if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] };
|
|
const merged: unknown[] = [];
|
|
for (const pr of prs) {
|
|
const preflight = preflightPullRequest(pr.number);
|
|
printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) });
|
|
if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) {
|
|
return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight };
|
|
}
|
|
const merge = mergePullRequest(pr.number, options.dryRun);
|
|
printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
|
if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge };
|
|
const sourceCommit = getG14Head();
|
|
const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds);
|
|
const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true
|
|
? { skipped: true, dryRun: options.dryRun }
|
|
: appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout);
|
|
printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null });
|
|
merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief });
|
|
if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged };
|
|
if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged };
|
|
}
|
|
return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged };
|
|
}
|
|
|
|
export async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<string, unknown>> {
|
|
const maxCycles = options.once ? 1 : options.maxCycles;
|
|
let cycle = 0;
|
|
const results: unknown[] = [];
|
|
while (maxCycles === 0 || cycle < maxCycles) {
|
|
cycle += 1;
|
|
const result = await monitorCycle(options, cycle);
|
|
results.push(result);
|
|
printEvent(`${options.lane}.monitor.cycle.done`, { cycle, lane: options.lane, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
|
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
|
|
if (options.once || (options.lane === "g14" && record(result).action !== "none")) return { ok: true, cycles: cycle, lastResult: result, results };
|
|
printEvent(`${options.lane}.monitor.sleep`, { cycle, lane: options.lane, intervalSeconds: options.intervalSeconds });
|
|
if (options.lane === "v02") printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, intervalSeconds: options.intervalSeconds });
|
|
else if (options.lane !== "g14") printRuntimeLanePrMonitorProgress(hwlabRuntimeLaneSpec(options.lane), { stage: "idle", status: "waiting", cycle, intervalSeconds: options.intervalSeconds });
|
|
await sleep(options.intervalSeconds * 1000);
|
|
}
|
|
return { ok: true, cycles: cycle, results };
|
|
}
|