feat: automate HWLAB G14 PR rollout monitoring
This commit is contained in:
Codex
@@ -0,0 +1,687 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { repoRoot, rootPath, type Config } from "./config";
|
||||
import { runCommand } from "./command";
|
||||
import { startJob } from "./jobs";
|
||||
|
||||
const HWLAB_REPO = "pikasTech/HWLAB";
|
||||
const G14_SOURCE_BRANCH = "G14";
|
||||
const G14_PROVIDER = "G14";
|
||||
const G14_WORKSPACE = "/root/hwlab";
|
||||
const DEV_NAMESPACE = "hwlab-dev";
|
||||
const CI_NAMESPACE = "hwlab-ci";
|
||||
const ARGO_NAMESPACE = "argocd";
|
||||
const DEV_APP = "hwlab-g14-dev";
|
||||
const DEFAULT_INTERVAL_SECONDS = 600;
|
||||
const DEFAULT_MAX_CYCLES = 0;
|
||||
const DEFAULT_TIMEOUT_SECONDS = 1800;
|
||||
const G14_BRIEF_INDEX_ISSUE = 7;
|
||||
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||||
|
||||
interface G14MonitorOptions {
|
||||
intervalSeconds: number;
|
||||
maxCycles: number;
|
||||
once: boolean;
|
||||
dryRun: boolean;
|
||||
worker: boolean;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
interface G14RecordRolloutOptions {
|
||||
prNumber: number;
|
||||
sourceCommit?: string;
|
||||
pipelineRun?: string;
|
||||
gitopsRevision?: string;
|
||||
mergedAt?: string;
|
||||
pipelineSucceededAt?: string;
|
||||
finishedAt?: string;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface CommandJsonResult {
|
||||
ok: boolean;
|
||||
command: string[];
|
||||
exitCode: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
parsed: unknown | null;
|
||||
}
|
||||
|
||||
interface OpenPullRequest {
|
||||
number: number;
|
||||
title?: string;
|
||||
url?: string;
|
||||
baseRefName?: string;
|
||||
headRefName?: string;
|
||||
mergeable?: string | null;
|
||||
mergeStateStatus?: string | null;
|
||||
draft?: boolean;
|
||||
}
|
||||
|
||||
function parseOptions(args: string[]): G14MonitorOptions {
|
||||
return {
|
||||
intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400),
|
||||
maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000),
|
||||
once: args.includes("--once"),
|
||||
dryRun: args.includes("--dry-run"),
|
||||
worker: args.includes("--worker"),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", DEFAULT_TIMEOUT_SECONDS, 86400),
|
||||
};
|
||||
}
|
||||
|
||||
function optionValue(args: string[], name: string): string | undefined {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return undefined;
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||||
const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number");
|
||||
const prNumber = Number(prRaw);
|
||||
if (!Number.isInteger(prNumber) || prNumber <= 0) throw new Error("record-rollout requires --pr <number>");
|
||||
return {
|
||||
prNumber,
|
||||
sourceCommit: optionValue(args, "--source-commit"),
|
||||
pipelineRun: optionValue(args, "--pipeline-run"),
|
||||
gitopsRevision: optionValue(args, "--gitops-revision"),
|
||||
mergedAt: optionValue(args, "--merged-at"),
|
||||
pipelineSucceededAt: optionValue(args, "--pipeline-succeeded-at"),
|
||||
finishedAt: optionValue(args, "--finished-at"),
|
||||
dryRun: args.includes("--dry-run"),
|
||||
};
|
||||
}
|
||||
|
||||
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return defaultValue;
|
||||
const raw = args[index + 1];
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||||
return Math.min(value, maxValue);
|
||||
}
|
||||
|
||||
function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
|
||||
const result = runCommand(command, repoRoot, { timeoutMs });
|
||||
let parsed: unknown | null = null;
|
||||
if (result.stdout.trim().length > 0) {
|
||||
try {
|
||||
parsed = JSON.parse(result.stdout) as unknown;
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: result.exitCode === 0,
|
||||
command,
|
||||
exitCode: result.exitCode,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
parsed,
|
||||
};
|
||||
}
|
||||
|
||||
function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||||
return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs);
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function nested(value: unknown, keys: string[]): unknown {
|
||||
let current = value;
|
||||
for (const key of keys) current = record(current)[key];
|
||||
return current;
|
||||
}
|
||||
|
||||
function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown> {
|
||||
const dumpPath = nested(result.parsed, ["data", "dump", "path"]);
|
||||
if (typeof dumpPath === "string" && existsSync(dumpPath)) {
|
||||
try {
|
||||
return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown);
|
||||
} catch {
|
||||
return record(result.parsed);
|
||||
}
|
||||
}
|
||||
return record(result.parsed);
|
||||
}
|
||||
|
||||
function commandData(result: CommandJsonResult): Record<string, unknown> {
|
||||
return record(expandedParsedRoot(result).data);
|
||||
}
|
||||
|
||||
function isCommandSuccess(result: CommandJsonResult): boolean {
|
||||
if (!result.ok) return false;
|
||||
const topOk = record(result.parsed).ok;
|
||||
if (topOk === false) return false;
|
||||
const dataOk = nested(result.parsed, ["data", "ok"]);
|
||||
return dataOk !== false;
|
||||
}
|
||||
|
||||
function extractPullRequests(result: CommandJsonResult): OpenPullRequest[] {
|
||||
const prs = nested(result.parsed, ["data", "pullRequests"]);
|
||||
if (!Array.isArray(prs)) return [];
|
||||
return prs
|
||||
.map((item) => record(item))
|
||||
.filter((item) => item.baseRefName === G14_SOURCE_BRANCH || record(item.base).ref === G14_SOURCE_BRANCH)
|
||||
.map((item) => ({
|
||||
number: Number(item.number),
|
||||
title: typeof item.title === "string" ? item.title : undefined,
|
||||
url: typeof item.url === "string" ? item.url : undefined,
|
||||
baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined,
|
||||
headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined,
|
||||
mergeable: typeof item.mergeable === "string" ? item.mergeable : null,
|
||||
mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null,
|
||||
draft: item.draft === true,
|
||||
}))
|
||||
.filter((item) => Number.isInteger(item.number) && item.number > 0);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function printEvent(event: string, data: Record<string, unknown> = {}): void {
|
||||
process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||||
}
|
||||
|
||||
function shortSha(sha: string): string {
|
||||
return sha.slice(0, 12);
|
||||
}
|
||||
|
||||
function precheckWorkspace(): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
|
||||
}
|
||||
|
||||
function listOpenG14PullRequests(): CommandJsonResult {
|
||||
return cliJson(["gh", "pr", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "30", "--json", "number,title,state,url,head,base,draft,headRefName,baseRefName"], 60_000);
|
||||
}
|
||||
|
||||
function preflightPullRequest(number: number): CommandJsonResult {
|
||||
return cliJson(["gh", "pr", "preflight", String(number), "--repo", HWLAB_REPO], 80_000);
|
||||
}
|
||||
|
||||
function mergePullRequest(number: number, dryRun: boolean): CommandJsonResult {
|
||||
return cliJson(["gh", "pr", "merge", String(number), "--repo", HWLAB_REPO, "--merge", "--delete-branch", ...(dryRun ? ["--dry-run"] : [])], 100_000);
|
||||
}
|
||||
|
||||
function getG14Head(): string | null {
|
||||
const result = cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "git fetch origin G14 --prune >/dev/null 2>&1; git rev-parse origin/G14"], 120_000);
|
||||
if (!isCommandSuccess(result)) return null;
|
||||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||||
return match?.[0] ?? null;
|
||||
}
|
||||
|
||||
function refreshArgoDev(): void {
|
||||
cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "annotate", "application", "-n", ARGO_NAMESPACE, DEV_APP, "argocd.argoproj.io/refresh=hard", "--overwrite"], 60_000);
|
||||
}
|
||||
|
||||
function getPipelineStatus(sourceCommit: string): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}"], 60_000);
|
||||
}
|
||||
|
||||
function getArgoStatus(): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "application", "-n", ARGO_NAMESPACE, DEV_APP, "-o", "jsonpath={.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.operationState.phase}{\"\\n\"}{.status.operationState.syncResult.revision}{\"\\n\"}"], 60_000);
|
||||
}
|
||||
|
||||
function getDevWorkloads(): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "deploy,statefulset,pod", "-n", DEV_NAMESPACE, "-o", "wide"], 60_000);
|
||||
}
|
||||
|
||||
function getLiveHealth(): CommandJsonResult {
|
||||
return commandJson(["curl", "-fsS", "--max-time", "20", "http://74.48.78.17:17667/health/live"], 30_000);
|
||||
}
|
||||
|
||||
function statusText(result: CommandJsonResult): string {
|
||||
return String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||||
}
|
||||
|
||||
function workloadReadiness(workloadsText: string, commandOk: boolean): Record<string, unknown> {
|
||||
const blockers: string[] = [];
|
||||
const ignoredPods: string[] = [];
|
||||
const readyWorkloads: string[] = [];
|
||||
const allowedZeroDeployments = new Set(["deployment.apps/hwlab-gateway", "deployment.apps/hwlab-tunnel-client"]);
|
||||
for (const line of workloadsText.split(/\r?\n/u)) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0 || trimmed.startsWith("NAME ")) continue;
|
||||
const fields = trimmed.split(/\s+/u);
|
||||
const name = fields[0] ?? "";
|
||||
const ready = fields[1] ?? "";
|
||||
if (name.startsWith("deployment.apps/") || name.startsWith("statefulset.apps/")) {
|
||||
if (ready === "0/0" && allowedZeroDeployments.has(name)) {
|
||||
readyWorkloads.push(`${name}:${ready}:scaled-zero`);
|
||||
continue;
|
||||
}
|
||||
const match = /^(\d+)\/(\d+)$/u.exec(ready);
|
||||
if (match === null) {
|
||||
blockers.push(`${name}:unparseable-ready:${ready}`);
|
||||
continue;
|
||||
}
|
||||
const readyCount = Number(match[1]);
|
||||
const desiredCount = Number(match[2]);
|
||||
if (desiredCount > 0 && readyCount < desiredCount) blockers.push(`${name}:not-ready:${ready}`);
|
||||
else readyWorkloads.push(`${name}:${ready}`);
|
||||
continue;
|
||||
}
|
||||
if (name.startsWith("pod/")) {
|
||||
const status = fields[2] ?? "";
|
||||
if (status === "Completed" || status === "Succeeded") ignoredPods.push(`${name}:${status}`);
|
||||
if (/^(CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError|RunContainerError)$/u.test(status)) {
|
||||
blockers.push(`${name}:pod-status:${status}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
ready: commandOk && blockers.length === 0,
|
||||
blockers,
|
||||
readyWorkloads: readyWorkloads.slice(0, 20),
|
||||
ignoredPods: ignoredPods.slice(0, 20),
|
||||
};
|
||||
}
|
||||
|
||||
function beijingParts(date = new Date()): { date: string; time: string; iso: string } {
|
||||
const shifted = new Date(date.getTime() + BEIJING_OFFSET_MS).toISOString();
|
||||
return { date: shifted.slice(0, 10), time: shifted.slice(11, 19), iso: shifted };
|
||||
}
|
||||
|
||||
function durationSeconds(start?: string | null, end?: string | null): number | null {
|
||||
if (start === undefined || start === null || end === undefined || end === null) return null;
|
||||
const startMs = Date.parse(start);
|
||||
const endMs = Date.parse(end);
|
||||
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) return null;
|
||||
return Math.round((endMs - startMs) / 1000);
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number | null): string {
|
||||
if (seconds === null) return "n/a";
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const rest = seconds % 60;
|
||||
if (minutes === 0) return `${rest}s`;
|
||||
return `${minutes}m${String(rest).padStart(2, "0")}s`;
|
||||
}
|
||||
|
||||
function readIssue(issueNumber: number): CommandJsonResult {
|
||||
return cliJson(["gh", "issue", "read", String(issueNumber), "--repo", HWLAB_REPO, "--json", "title,body,state,updatedAt", "--raw"], 80_000);
|
||||
}
|
||||
|
||||
function issueFromRead(result: CommandJsonResult): Record<string, unknown> {
|
||||
return record(commandData(result).issue);
|
||||
}
|
||||
|
||||
function parseBriefIssueFromIndex(indexBody: string, date: string): { number: number; url: string } | null {
|
||||
const escapedDate = date.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||||
const linePattern = new RegExp(`^\\|\\s*${escapedDate}\\s*\\|([^\\n]+)$`, "mu");
|
||||
const line = linePattern.exec(indexBody)?.[0];
|
||||
if (line === undefined) return null;
|
||||
const match = /\[#(\d+)\]\((https:\/\/github\.com\/pikasTech\/HWLAB\/issues\/\d+)\)/u.exec(line);
|
||||
if (match === null) return null;
|
||||
return { number: Number(match[1]), url: match[2] };
|
||||
}
|
||||
|
||||
function dailyBriefTitle(date: string): string {
|
||||
return `${date} 指挥简报（北京时间）`;
|
||||
}
|
||||
|
||||
function writeStateFile(name: string, content: string): string {
|
||||
const dir = rootPath(".state", "hwlab-g14");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const path = join(dir, name);
|
||||
writeFileSync(path, content, "utf8");
|
||||
return path;
|
||||
}
|
||||
|
||||
function listIssues(): CommandJsonResult {
|
||||
return cliJson(["gh", "issue", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "100", "--json", "number,title,state,url"], 80_000);
|
||||
}
|
||||
|
||||
function findBriefIssueInList(date: string): { number: number; url: string } | null {
|
||||
const listed = listIssues();
|
||||
if (!isCommandSuccess(listed)) return null;
|
||||
const issues = commandData(listed).issues;
|
||||
if (!Array.isArray(issues)) return null;
|
||||
const title = dailyBriefTitle(date);
|
||||
const match = issues.map((item) => record(item)).find((issue) => issue.title === title);
|
||||
if (match === undefined) return null;
|
||||
return { number: Number(match.number), url: String(match.url) };
|
||||
}
|
||||
|
||||
function newDailyBriefBody(date: string): string {
|
||||
return [
|
||||
`# ${dailyBriefTitle(date)}`,
|
||||
"",
|
||||
"## 当日更新",
|
||||
"",
|
||||
"## 常驻观察与长期建议",
|
||||
"",
|
||||
"- G14 HWLAB source truth 固定为 G14 `/root/hwlab` 与 `origin/G14`；DEV rollout 只接受 G14 Tekton/GitOps/Argo 和公网 health 证据。",
|
||||
"- GitHub issue/PR 写操作必须走 UniDesk `gh` 子命令；G14 k3s 操作必须走 UniDesk `ssh G14:k3s`。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function insertBriefIndexRow(indexBody: string, date: string, brief: { number: number; url: string }): string {
|
||||
if (parseBriefIssueFromIndex(indexBody, date) !== null) return indexBody;
|
||||
const row = `| ${date} | [#${brief.number}](${brief.url}) | 已创建当日 G14 DEV rollout 自动简报入口；后续每次 DEV 上线记录 CI/CD 耗时和上线 changelog。 | 继续监控 HWLAB base=G14 PR，ready 后自动合并并滚动到 G14 DEV。 |`;
|
||||
const marker = "### 指挥简报索引";
|
||||
const markerIndex = indexBody.indexOf(marker);
|
||||
if (markerIndex === -1) return `${indexBody.trimEnd()}\n\n${marker}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n`;
|
||||
const afterMarker = indexBody.slice(markerIndex);
|
||||
const separator = "| --- | --- | --- | --- |";
|
||||
const separatorInBody = afterMarker.indexOf(separator);
|
||||
if (separatorInBody === -1) return `${indexBody.slice(0, markerIndex + marker.length)}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n${indexBody.slice(markerIndex + marker.length)}`;
|
||||
const insertAt = markerIndex + separatorInBody + separator.length;
|
||||
return `${indexBody.slice(0, insertAt)}\n${row}${indexBody.slice(insertAt)}`;
|
||||
}
|
||||
|
||||
function ensureDailyBriefIssue(date: string, dryRun: boolean): Record<string, unknown> {
|
||||
const indexRead = readIssue(G14_BRIEF_INDEX_ISSUE);
|
||||
if (!isCommandSuccess(indexRead)) return { ok: false, phase: "read-index", indexRead };
|
||||
const indexIssue = issueFromRead(indexRead);
|
||||
const indexBody = String(indexIssue.body ?? "");
|
||||
let brief = parseBriefIssueFromIndex(indexBody, date) ?? findBriefIssueInList(date);
|
||||
const actions: Record<string, unknown>[] = [];
|
||||
if (brief === null) {
|
||||
const bodyPath = writeStateFile(`daily-brief-${date}.md`, `${newDailyBriefBody(date)}\n`);
|
||||
if (dryRun) {
|
||||
brief = { number: 0, url: `https://github.com/pikasTech/HWLAB/issues/new?title=${encodeURIComponent(dailyBriefTitle(date))}` };
|
||||
actions.push({ action: "would-create-daily-brief", title: dailyBriefTitle(date), bodyPath });
|
||||
} else {
|
||||
const created = cliJson(["gh", "issue", "create", "--repo", HWLAB_REPO, "--title", dailyBriefTitle(date), "--body-file", bodyPath], 80_000);
|
||||
if (!isCommandSuccess(created)) return { ok: false, phase: "create-daily-brief", created };
|
||||
const createdIssue = record(commandData(created).issue);
|
||||
brief = { number: Number(createdIssue.number), url: String(createdIssue.url) };
|
||||
actions.push({ action: "created-daily-brief", issue: brief, bodyPath });
|
||||
}
|
||||
}
|
||||
if (brief.number > 0 && parseBriefIssueFromIndex(indexBody, date) === null) {
|
||||
const nextBody = insertBriefIndexRow(indexBody, date, brief);
|
||||
const bodyPath = writeStateFile(`issue-${G14_BRIEF_INDEX_ISSUE}-brief-index-${date}.md`, `${nextBody.trimEnd()}\n`);
|
||||
if (dryRun) {
|
||||
actions.push({ action: "would-update-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||||
} else {
|
||||
const bodySha = String(indexIssue.bodySha ?? "");
|
||||
const update = cliJson(["gh", "issue", "update", String(G14_BRIEF_INDEX_ISSUE), "--repo", HWLAB_REPO, "--mode", "replace", "--body-file", bodyPath, ...(bodySha.length > 0 ? ["--expect-body-sha", bodySha] : [])], 100_000);
|
||||
if (!isCommandSuccess(update)) return { ok: false, phase: "update-brief-index", update, bodyPath };
|
||||
actions.push({ action: "updated-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||||
}
|
||||
}
|
||||
return { ok: true, date, brief, actions };
|
||||
}
|
||||
|
||||
function readPullRequest(number: number): CommandJsonResult {
|
||||
return cliJson(["gh", "pr", "read", String(number), "--repo", HWLAB_REPO, "--json", "title,url,stateDetail,merged,mergedAt,mergeCommit,headRefName,baseRefName"], 80_000);
|
||||
}
|
||||
|
||||
function readPullRequestFiles(number: number): CommandJsonResult {
|
||||
return cliJson(["gh", "pr", "files", String(number), "--repo", HWLAB_REPO, "--limit", "30"], 80_000);
|
||||
}
|
||||
|
||||
function summarizePrFiles(filesResult: CommandJsonResult): Record<string, unknown> {
|
||||
if (!isCommandSuccess(filesResult)) return { ok: false, filesResult };
|
||||
const data = commandData(filesResult);
|
||||
const files = Array.isArray(data.files) ? data.files.map((item) => record(item)) : [];
|
||||
return {
|
||||
ok: true,
|
||||
summary: data.summary ?? null,
|
||||
keyFiles: files.slice(0, 8).map((file) => `${file.status ?? "changed"} ${file.filename ?? ""}`),
|
||||
};
|
||||
}
|
||||
|
||||
function mergeCommitFromPr(prData: Record<string, unknown>): string | null {
|
||||
const mergeCommit = record(record(prData.json).mergeCommit);
|
||||
const fromJson = mergeCommit.oid;
|
||||
if (typeof fromJson === "string") return fromJson;
|
||||
const fromSummary = record(record(prData.pullRequest).mergeCommit).oid;
|
||||
return typeof fromSummary === "string" ? fromSummary : null;
|
||||
}
|
||||
|
||||
function currentGitopsRevision(): string | null {
|
||||
const argo = getArgoStatus();
|
||||
if (!isCommandSuccess(argo)) return null;
|
||||
return statusText(argo).split(/\r?\n/u)[0] ?? null;
|
||||
}
|
||||
|
||||
function rolloutRecordBody(input: {
|
||||
pr: OpenPullRequest;
|
||||
prData: Record<string, unknown>;
|
||||
fileSummary: Record<string, unknown>;
|
||||
sourceCommit: string;
|
||||
pipelineRun: string;
|
||||
gitopsRevision: string | null;
|
||||
mergedAt: string | null;
|
||||
pipelineSucceededAt: string | null;
|
||||
finishedAt: string | null;
|
||||
rollout: Record<string, unknown>;
|
||||
}): string {
|
||||
const now = beijingParts();
|
||||
const title = String(record(input.prData.json).title ?? input.pr.title ?? `PR #${input.pr.number}`);
|
||||
const url = String(record(input.prData.json).url ?? input.pr.url ?? `https://github.com/pikasTech/HWLAB/pull/${input.pr.number}`);
|
||||
const summary = record(input.fileSummary.summary);
|
||||
const keyFiles = Array.isArray(input.fileSummary.keyFiles) ? input.fileSummary.keyFiles.map(String) : [];
|
||||
const mergeToPipeline = durationSeconds(input.mergedAt, input.pipelineSucceededAt);
|
||||
const pipelineToDev = durationSeconds(input.pipelineSucceededAt, input.finishedAt);
|
||||
const mergeToDev = durationSeconds(input.mergedAt, input.finishedAt);
|
||||
return [
|
||||
"",
|
||||
"",
|
||||
`## 更新 ${now.date} ${now.time.slice(0, 5)} 北京时间`,
|
||||
"",
|
||||
`### G14 DEV rollout：PR #${input.pr.number}`,
|
||||
"",
|
||||
`- PR: [#${input.pr.number} ${title}](${url})`,
|
||||
`- 合并 commit: \`${input.sourceCommit}\``,
|
||||
`- GitOps revision: \`${input.gitopsRevision ?? "unknown"}\``,
|
||||
`- PipelineRun: \`${input.pipelineRun}\``,
|
||||
"- CI/CD 耗时:",
|
||||
` - merge -> pipeline succeeded: ${formatDuration(mergeToPipeline)}`,
|
||||
` - pipeline succeeded -> DEV Healthy: ${formatDuration(pipelineToDev)}`,
|
||||
` - merge -> DEV Healthy: ${formatDuration(mergeToDev)}`,
|
||||
"- 上线 changelog:",
|
||||
` - ${title}`,
|
||||
` - changed files: ${String(summary.files ?? "n/a")}; +${String(summary.additions ?? "n/a")} / -${String(summary.deletions ?? "n/a")}; commits: ${String(summary.commits ?? "n/a")}`,
|
||||
...keyFiles.map((file) => ` - ${file}`),
|
||||
"- DEV 验证:",
|
||||
` - Tekton: ${String(input.rollout.pipelineText ?? "Succeeded").split(/\r?\n/u).join(" / ")}`,
|
||||
` - Argo: ${String(input.rollout.argoText ?? "Synced / Healthy").split(/\r?\n/u).slice(0, 5).join(" / ")}`,
|
||||
` - health/live: ${input.rollout.healthOk === false ? "not-ok" : "ok"}`,
|
||||
" - workload readiness: Deployment/StatefulSet ready；历史 Completed smoke/debug pod 不作为 DEV rollout blocker。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function appendRolloutBrief(options: G14RecordRolloutOptions, rollout: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
const now = beijingParts();
|
||||
const ensured = ensureDailyBriefIssue(now.date, options.dryRun);
|
||||
if (record(ensured).ok !== true) return ensured;
|
||||
const brief = record(record(ensured).brief);
|
||||
const prRead = readPullRequest(options.prNumber);
|
||||
if (!isCommandSuccess(prRead)) return { ok: false, phase: "read-pr", prRead };
|
||||
const prData = commandData(prRead);
|
||||
const sourceCommit = options.sourceCommit ?? mergeCommitFromPr(prData);
|
||||
if (sourceCommit === null || sourceCommit === undefined) return { ok: false, phase: "source-commit", message: "source commit unavailable", prData };
|
||||
const pipelineRun = options.pipelineRun ?? `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||||
const files = summarizePrFiles(readPullRequestFiles(options.prNumber));
|
||||
const rolloutGitopsRevision = typeof rollout.gitopsRevision === "string" && rollout.gitopsRevision.length > 0 ? rollout.gitopsRevision : null;
|
||||
const gitopsRevision = options.gitopsRevision ?? rolloutGitopsRevision ?? currentGitopsRevision();
|
||||
const prJson = record(prData.json);
|
||||
const prSummary = record(prData.pullRequest);
|
||||
const prMergedAt = typeof prJson.mergedAt === "string" && prJson.mergedAt.length > 0
|
||||
? prJson.mergedAt
|
||||
: typeof prSummary.mergedAt === "string" && prSummary.mergedAt.length > 0
|
||||
? prSummary.mergedAt
|
||||
: null;
|
||||
const rolloutPipelineSucceededAt = typeof rollout.pipelineSucceededAt === "string" && rollout.pipelineSucceededAt.length > 0 ? rollout.pipelineSucceededAt : null;
|
||||
const rolloutFinishedAt = typeof rollout.finishedAt === "string" && rollout.finishedAt.length > 0 ? rollout.finishedAt : null;
|
||||
const mergedAt = options.mergedAt ?? prMergedAt;
|
||||
const pipelineSucceededAt = options.pipelineSucceededAt ?? rolloutPipelineSucceededAt;
|
||||
const finishedAt = options.finishedAt ?? rolloutFinishedAt;
|
||||
const pr: OpenPullRequest = {
|
||||
number: options.prNumber,
|
||||
title: String(record(prData.json).title ?? ""),
|
||||
url: String(record(prData.json).url ?? ""),
|
||||
baseRefName: String(record(prData.json).baseRefName ?? ""),
|
||||
headRefName: String(record(prData.json).headRefName ?? ""),
|
||||
};
|
||||
const body = rolloutRecordBody({ pr, prData, fileSummary: files, sourceCommit, pipelineRun, gitopsRevision, mergedAt, pipelineSucceededAt, finishedAt, rollout });
|
||||
if (Number(brief.number) <= 0 || options.dryRun) {
|
||||
return { ok: true, dryRun: true, date: now.date, brief, wouldAppend: { bodyPreview: body.slice(0, 1000), bodyChars: body.length }, ensured };
|
||||
}
|
||||
const currentBrief = readIssue(Number(brief.number));
|
||||
if (!isCommandSuccess(currentBrief)) return { ok: false, phase: "read-daily-brief", currentBrief, brief };
|
||||
const currentBody = String(issueFromRead(currentBrief).body ?? "");
|
||||
if (currentBody.includes(`PR #${options.prNumber}`) && currentBody.includes(sourceCommit)) {
|
||||
return { ok: true, skipped: true, reason: "rollout-brief-already-recorded", brief, sourceCommit, ensured };
|
||||
}
|
||||
const bodyPath = writeStateFile(`rollout-pr-${options.prNumber}-${shortSha(sourceCommit)}.md`, `${body}\n`);
|
||||
const update = cliJson(["gh", "issue", "update", String(brief.number), "--repo", HWLAB_REPO, "--mode", "append", "--body-file", bodyPath, "--body-profile", "commander-brief"], 100_000);
|
||||
if (!isCommandSuccess(update)) return { ok: false, phase: "append-rollout-brief", update, bodyPath, brief, ensured };
|
||||
return { ok: true, date: now.date, brief, sourceCommit, pipelineRun, gitopsRevision, bodyPath, update: commandData(update), ensured };
|
||||
}
|
||||
|
||||
async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
const startedAt = new Date(started).toISOString();
|
||||
let pipelineText = "";
|
||||
let argoText = "";
|
||||
let healthOk = false;
|
||||
let workloadsReady: Record<string, unknown> = { ready: false };
|
||||
let pipelineSucceededAt: string | null = null;
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
const pipeline = getPipelineStatus(sourceCommit);
|
||||
pipelineText = statusText(pipeline);
|
||||
printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) });
|
||||
if (!pipelineText.startsWith("True\nSucceeded")) {
|
||||
if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText };
|
||||
await sleep(30_000);
|
||||
continue;
|
||||
}
|
||||
pipelineSucceededAt ??= new Date().toISOString();
|
||||
refreshArgoDev();
|
||||
const argo = getArgoStatus();
|
||||
argoText = statusText(argo);
|
||||
printEvent("g14.argo.status", { text: argoText.slice(0, 500) });
|
||||
const argoLines = argoText.split(/\r?\n/u);
|
||||
const synced = argoLines[1] === "Synced";
|
||||
const healthy = argoLines[2] === "Healthy";
|
||||
const gitopsRevision = argoLines[0] ?? null;
|
||||
const workloads = getDevWorkloads();
|
||||
const workloadsText = statusText(workloads);
|
||||
workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads));
|
||||
const health = getLiveHealth();
|
||||
healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout);
|
||||
if (synced && healthy && record(workloadsReady).ready === true && healthOk) {
|
||||
const finishedAt = new Date().toISOString();
|
||||
return { ok: true, sourceCommit, gitopsRevision, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk };
|
||||
}
|
||||
await sleep(30_000);
|
||||
}
|
||||
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
|
||||
}
|
||||
|
||||
async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||||
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||||
const precheck = precheckWorkspace();
|
||||
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
|
||||
const listed = listOpenG14PullRequests();
|
||||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||||
const prs = extractPullRequests(listed);
|
||||
printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||||
if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] };
|
||||
const merged: unknown[] = [];
|
||||
for (const pr of prs) {
|
||||
const preflight = preflightPullRequest(pr.number);
|
||||
printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) });
|
||||
if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) {
|
||||
return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight };
|
||||
}
|
||||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||||
printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||||
if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge };
|
||||
const sourceCommit = getG14Head();
|
||||
const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds);
|
||||
const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true
|
||||
? { skipped: true, dryRun: options.dryRun }
|
||||
: appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout);
|
||||
printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null });
|
||||
merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief });
|
||||
if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged };
|
||||
if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged };
|
||||
}
|
||||
return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged };
|
||||
}
|
||||
|
||||
async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<string, unknown>> {
|
||||
const maxCycles = options.once ? 1 : options.maxCycles;
|
||||
let cycle = 0;
|
||||
const results: unknown[] = [];
|
||||
while (maxCycles === 0 || cycle < maxCycles) {
|
||||
cycle += 1;
|
||||
const result = await monitorCycle(options, cycle);
|
||||
results.push(result);
|
||||
printEvent("g14.monitor.cycle.done", { cycle, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
||||
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
|
||||
if (options.once || record(result).action !== "none") return { ok: true, cycles: cycle, lastResult: result, results };
|
||||
printEvent("g14.monitor.sleep", { cycle, intervalSeconds: options.intervalSeconds });
|
||||
await sleep(options.intervalSeconds * 1000);
|
||||
}
|
||||
return { ok: true, cycles: cycle, results };
|
||||
}
|
||||
|
||||
export function hwlabG14Help(): Record<string, unknown> {
|
||||
return {
|
||||
command: "hwlab g14",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --once --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 record-rollout --pr <number> [--source-commit sha]",
|
||||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||||
],
|
||||
description: "G14 HWLAB PR monitor and DEV rollout command. The public command starts a fire-and-forget job; the worker uses UniDesk gh and ssh routes for every GitHub and k3s operation, then appends the rollout record to the #7-indexed daily brief.",
|
||||
defaults: {
|
||||
repo: HWLAB_REPO,
|
||||
base: G14_SOURCE_BRANCH,
|
||||
provider: G14_PROVIDER,
|
||||
workspace: G14_WORKSPACE,
|
||||
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
|
||||
devApplication: DEV_APP,
|
||||
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function runHwlabG14Command(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
||||
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return { ok: true, ...hwlabG14Help() };
|
||||
const [action] = args;
|
||||
if (action === "record-rollout") {
|
||||
const options = parseRecordRolloutOptions(args.slice(1));
|
||||
return appendRolloutBrief(options);
|
||||
}
|
||||
if (action !== "monitor-prs") {
|
||||
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 record-rollout" };
|
||||
}
|
||||
const options = parseOptions(args.slice(1));
|
||||
if (options.worker) return runMonitorWorker(options);
|
||||
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
|
||||
const job = startJob("hwlab_g14_pr_monitor", command, `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`);
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
|
||||
const stateDir = rootPath(".state", "hwlab-g14");
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
const latestPath = join(stateDir, "latest-monitor-job.json");
|
||||
const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null;
|
||||
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand }, null, 2)}\n`, "utf8");
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 monitor-prs",
|
||||
mode: "async-job",
|
||||
job,
|
||||
statusCommand,
|
||||
latestPath,
|
||||
previousLatest,
|
||||
next: {
|
||||
status: statusCommand,
|
||||
tail: `tail -f ${job.stdoutFile}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user