refactor: split control-plane cli modules
This commit is contained in:
Codex
@@ -0,0 +1,673 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. cd-trigger module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:9508-10162 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult, G14MonitorOptions, OpenPullRequest } from "./types";
|
||||
import { runRuntimeLaneControlPlane, runV02ControlPlane, runtimeLaneControlPlaneStatus, v02ControlPlaneStatus } from "./control-plane";
|
||||
import { runG14GitMirror, runGitMirrorStatus } from "./git-mirror";
|
||||
import { activeV02PipelineRuns, commentRuntimeLanePullRequest, commentV02PullRequest, durationSeconds, reportRuntimeLaneAutomationIssue, runtimeLaneCdFailed, runtimeLaneCdPassed, sourceCommitFromMergeResult, summarizeRuntimeLaneCdStatus, summarizeV02CdStatus, v02CdFailed, v02CdPassed, v02PipelineSucceeded, waitForV02Head } from "./pr-monitor";
|
||||
import { commandData, compactCommandResult, isCommandSuccess, nested, printEvent, printRuntimeLanePrMonitorProgress, printV02PrMonitorProgress, record, sleep, stringOrNull } from "./remote";
|
||||
import { resolveRuntimeLaneHead, runtimeLanePipelineRunName, v02PipelineRunName } from "./source";
|
||||
|
||||
export function triggerV02Current(timeoutSeconds: number): Record<string, unknown> {
|
||||
return runV02ControlPlane({
|
||||
action: "trigger-current",
|
||||
lane: "v02",
|
||||
dryRun: false,
|
||||
confirm: true,
|
||||
wait: true,
|
||||
allowLiveDbRead: false,
|
||||
timeoutSeconds,
|
||||
minAgeMinutes: 30,
|
||||
limit: 20,
|
||||
});
|
||||
}
|
||||
|
||||
export function flushV02GitMirrorIfNeeded(status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
|
||||
const pendingFlush = nested(status, ["gitMirror", "summary", "pendingFlush"]);
|
||||
if (pendingFlush !== true) return { ok: true, skipped: true, reason: "git-mirror-already-flushed" };
|
||||
return runG14GitMirror({
|
||||
action: "flush",
|
||||
lane: "v02",
|
||||
confirm: true,
|
||||
dryRun: false,
|
||||
wait: true,
|
||||
timeoutSeconds,
|
||||
});
|
||||
}
|
||||
|
||||
export function triggerRuntimeLaneCurrent(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
|
||||
return runRuntimeLaneControlPlane(spec, {
|
||||
action: "trigger-current",
|
||||
lane: spec.lane,
|
||||
dryRun: false,
|
||||
confirm: true,
|
||||
wait: true,
|
||||
allowLiveDbRead: false,
|
||||
timeoutSeconds,
|
||||
minAgeMinutes: 30,
|
||||
limit: 20,
|
||||
history: false,
|
||||
});
|
||||
}
|
||||
|
||||
export function runtimeLaneStatusWithGitMirror(spec: HwlabRuntimeLaneSpec, sourceCommit: string): Record<string, unknown> {
|
||||
const status = runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "source-commit" });
|
||||
const gitMirrorStatus = runGitMirrorStatus(spec.lane);
|
||||
return {
|
||||
...status,
|
||||
gitMirror: record(gitMirrorStatus.cache),
|
||||
gitMirrorStatus,
|
||||
};
|
||||
}
|
||||
|
||||
export function runtimeLaneGitopsSynced(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>): boolean {
|
||||
const summary = record(nested(status, ["gitMirror", "summary"]));
|
||||
const inSync = spec.lane === "v03" ? summary.v03GitopsInSync : summary.githubInSync;
|
||||
return summary.pendingFlush !== true && inSync === true;
|
||||
}
|
||||
|
||||
export function runtimeLaneCdPassedWithMirror(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>): boolean {
|
||||
return runtimeLaneCdPassed(status) && runtimeLaneGitopsSynced(spec, status);
|
||||
}
|
||||
|
||||
export function flushRuntimeLaneGitMirrorIfNeeded(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
|
||||
if (runtimeLaneGitopsSynced(spec, status)) return { ok: true, skipped: true, reason: `${spec.lane}-git-mirror-already-flushed` };
|
||||
return runG14GitMirror({
|
||||
action: "flush",
|
||||
lane: spec.lane,
|
||||
confirm: true,
|
||||
dryRun: false,
|
||||
wait: true,
|
||||
timeoutSeconds,
|
||||
});
|
||||
}
|
||||
|
||||
export function pullRequestFromMergeCommand(merge: CommandJsonResult): Record<string, unknown> {
|
||||
const data = commandData(merge);
|
||||
const direct = record(data.pullRequest);
|
||||
if (Object.keys(direct).length > 0) return direct;
|
||||
return record(nested(data, ["details", "details", "pullRequest"]));
|
||||
}
|
||||
|
||||
export function mergeCommandRaceState(merge: CommandJsonResult): "merged" | "closed" | null {
|
||||
const pullRequest = pullRequestFromMergeCommand(merge);
|
||||
if (pullRequest.merged === true || pullRequest.stateDetail === "merged") return "merged";
|
||||
if (pullRequest.closed === true || pullRequest.state === "closed") return "closed";
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function waitForRuntimeLaneHead(spec: HwlabRuntimeLaneSpec, expectedCommit: string | null, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
let head: string | null = null;
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
head = resolveRuntimeLaneHead(spec).sourceCommit;
|
||||
if (expectedCommit === null) {
|
||||
if (head !== null) return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||||
} else if (head === expectedCommit) {
|
||||
return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||||
}
|
||||
printEvent(`${spec.lane}.source.wait`, { lane: spec.lane, expectedCommit, head, waitedSeconds: Math.round((Date.now() - started) / 1000) });
|
||||
await sleep(5_000);
|
||||
}
|
||||
return { ok: false, sourceCommit: head, expectedCommit, matched: false, waitedSeconds: Math.round((Date.now() - started) / 1000), degradedReason: `${spec.lane}-source-head-not-aligned-after-merge` };
|
||||
}
|
||||
|
||||
export async function waitForV02Cd(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
const startedAt = new Date(started).toISOString();
|
||||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||||
let lastStatus: Record<string, unknown> = {};
|
||||
let firstPassedAt: string | null = null;
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
lastStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
const summary = summarizeV02CdStatus(lastStatus);
|
||||
printEvent("v02.cd.status", { sourceCommit, pipelineRun, summary });
|
||||
printV02PrMonitorProgress({ stage: "cd-status", status: "running", sourceCommit, pipelineRun, targetValidationState: summary.targetValidationState ?? null, pipelineStatus: summary.pipelineStatus ?? null, pendingFlush: summary.pendingFlush ?? null });
|
||||
if (v02PipelineSucceeded(lastStatus) && summary.pendingFlush === true) {
|
||||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||||
if (record(flush).ok !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "git-mirror-flush",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summary,
|
||||
rawStatus: lastStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
const finalSummary = summarizeV02CdStatus(finalStatus);
|
||||
if (v02CdPassed(finalStatus)) {
|
||||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: "succeeded", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||||
return {
|
||||
ok: true,
|
||||
phase: "cd-passed",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: finalSummary,
|
||||
rawStatus: finalStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
lastStatus = finalStatus;
|
||||
printEvent("v02.cd.after-flush", { sourceCommit, pipelineRun, flushOk: record(flush).ok, summary: finalSummary });
|
||||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: record(flush).ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||||
}
|
||||
if (v02CdPassed(lastStatus)) {
|
||||
firstPassedAt ??= new Date().toISOString();
|
||||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
printV02PrMonitorProgress({ stage: "cd-status", status: v02CdPassed(finalStatus) ? "succeeded" : "failed", sourceCommit, pipelineRun, targetValidationState: nested(finalStatus, ["targetValidation", "state"]) ?? null });
|
||||
return {
|
||||
ok: v02CdPassed(finalStatus),
|
||||
phase: v02CdPassed(finalStatus) ? "cd-passed" : "git-mirror-flush",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summarizeV02CdStatus(finalStatus),
|
||||
rawStatus: finalStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
if (v02CdFailed(lastStatus)) {
|
||||
printV02PrMonitorProgress({ stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
|
||||
return {
|
||||
ok: false,
|
||||
phase: "cd-failed",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summary,
|
||||
rawStatus: lastStatus,
|
||||
};
|
||||
}
|
||||
await sleep(30_000);
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
phase: "timeout",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
timeoutSeconds,
|
||||
firstPassedAt,
|
||||
status: summarizeV02CdStatus(lastStatus),
|
||||
rawStatus: lastStatus,
|
||||
};
|
||||
}
|
||||
|
||||
export async function waitForRuntimeLaneCd(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
const startedAt = new Date(started).toISOString();
|
||||
let pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit);
|
||||
let lastStatus: Record<string, unknown> = {};
|
||||
let firstPassedAt: string | null = null;
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
lastStatus = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
|
||||
const summary = summarizeRuntimeLaneCdStatus(spec, lastStatus);
|
||||
pipelineRun = stringOrNull(summary.pipelineRun) ?? pipelineRun;
|
||||
printEvent(`${spec.lane}.cd.status`, { lane: spec.lane, sourceCommit, pipelineRun, summary });
|
||||
printRuntimeLanePrMonitorProgress(spec, {
|
||||
stage: "cd-status",
|
||||
status: "running",
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
pipelineStatus: summary.pipelineStatus ?? null,
|
||||
pipelineReason: summary.pipelineReason ?? null,
|
||||
argoSync: summary.argoSync ?? null,
|
||||
argoHealth: summary.argoHealth ?? null,
|
||||
publicProbesOk: summary.publicProbesOk ?? null,
|
||||
pendingFlush: summary.pendingFlush ?? null,
|
||||
});
|
||||
if (runtimeLaneCdPassed(lastStatus)) {
|
||||
firstPassedAt ??= new Date().toISOString();
|
||||
const flush = flushRuntimeLaneGitMirrorIfNeeded(spec, lastStatus, Math.min(timeoutSeconds, 600));
|
||||
if (record(flush).ok !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "git-mirror-flush",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summary,
|
||||
rawStatus: lastStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
const finalStatus = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
|
||||
const finalSummary = summarizeRuntimeLaneCdStatus(spec, finalStatus);
|
||||
printRuntimeLanePrMonitorProgress(spec, {
|
||||
stage: "git-mirror-flush",
|
||||
status: runtimeLaneCdPassedWithMirror(spec, finalStatus) ? "succeeded" : "failed",
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
pendingFlush: finalSummary.pendingFlush ?? null,
|
||||
});
|
||||
return {
|
||||
ok: runtimeLaneCdPassedWithMirror(spec, finalStatus),
|
||||
phase: runtimeLaneCdPassedWithMirror(spec, finalStatus) ? "cd-passed" : "git-mirror-flush",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: finalSummary,
|
||||
rawStatus: finalStatus,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
if (runtimeLaneCdFailed(lastStatus)) {
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
|
||||
return {
|
||||
ok: false,
|
||||
phase: "cd-failed",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
status: summary,
|
||||
rawStatus: lastStatus,
|
||||
};
|
||||
}
|
||||
await sleep(30_000);
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
phase: "timeout",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
timeoutSeconds,
|
||||
firstPassedAt,
|
||||
status: summarizeRuntimeLaneCdStatus(spec, lastStatus),
|
||||
rawStatus: lastStatus,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
|
||||
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
|
||||
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
|
||||
printV02PrMonitorProgress({ stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "merge",
|
||||
state: "merge-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: compactCommandResult(merge),
|
||||
dryRun: options.dryRun,
|
||||
message: mergeRaceState === "closed"
|
||||
? "PR preflight 曾通过，但自动合并前 PR 已被关闭且未确认 merged；本轮不会触发 CD。"
|
||||
: "PR preflight 已通过，但自动合并失败；需要处理 GitHub 返回的 merge blocker 后重试。",
|
||||
});
|
||||
return { ok: mergeRaceState === "closed", phase: "merge", action: "merge-race-closed", pr, merge, comment };
|
||||
}
|
||||
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
|
||||
if (!options.dryRun && expectedCommit === null) {
|
||||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "merge-commit",
|
||||
state: "cd-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit: null,
|
||||
pipelineRun: null,
|
||||
dryRun: false,
|
||||
message: "PR 已合并，但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit；为避免触发错误 source head，本轮未启动 CD。",
|
||||
});
|
||||
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, degradedReason: "merge-commit-unresolved" };
|
||||
}
|
||||
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForV02Head(expectedCommit, 120);
|
||||
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
|
||||
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
|
||||
printV02PrMonitorProgress({ stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
|
||||
const startedComment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: options.dryRun ? "dry-run" : "cd-trigger",
|
||||
state: "cd-started",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
dryRun: options.dryRun,
|
||||
message: options.dryRun
|
||||
? "dry-run：PR 已达到自动合并条件；本轮不会写 GitHub merge 或 CD。"
|
||||
: mergeRaceState === "merged"
|
||||
? "PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 v0.2 CD。后续成功、失败或超时会继续在本 PR 下回复。"
|
||||
: "PR 已自动合并，v0.2 CD 准备开始；其他 commit 的运行中 PipelineRun 不会阻塞本轮 CI，旧 run 若发现 source head 已推进会以 superseded/no-op 收口。后续成功、superseded、失败或超时会继续在本 PR 下回复。",
|
||||
});
|
||||
if (!options.dryRun && record(startedComment).ok !== true) {
|
||||
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
|
||||
}
|
||||
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
|
||||
if (sourceCommit === null || record(headWait).ok !== true) {
|
||||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "v02-source-head",
|
||||
state: "cd-timeout",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
dryRun: false,
|
||||
message: "PR 已合并，但 G14 v0.2 CI/CD source repo 没有在等待窗口内对齐 merge commit；为避免触发旧 head，本轮未启动 CD。",
|
||||
});
|
||||
return { ok: false, phase: "v02-head", pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, degradedReason: "v02-head-unresolved-after-merge" };
|
||||
}
|
||||
const before = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||||
const beforeSummary = summarizeV02CdStatus(before);
|
||||
const activeRuns = activeV02PipelineRuns(before);
|
||||
if (activeRuns.length > 0) {
|
||||
printEvent("v02.cd.active-runs-observed", { pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, activeRuns: activeRuns.slice(0, 5), latestOnlyPolicy: "do-not-wait-or-cancel-old-runs" });
|
||||
}
|
||||
const trigger = v02CdPassed(before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerV02Current(Math.min(options.timeoutSeconds, 600));
|
||||
printEvent("v02.cd.trigger", { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||||
printV02PrMonitorProgress({ stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||||
if (record(trigger).ok !== true) {
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "cd-trigger",
|
||||
state: "cd-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: beforeSummary,
|
||||
dryRun: false,
|
||||
message: "PR 已合并，但 v0.2 CD 触发失败；需要查看 trigger 阶段的 degradedReason。",
|
||||
});
|
||||
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment };
|
||||
}
|
||||
const cd = await waitForV02Cd(sourceCommit, options.timeoutSeconds);
|
||||
const cdStatus = record(cd.status);
|
||||
const flush = record(cd.flush);
|
||||
const cdOk = cd.ok === true;
|
||||
const cdPhase = String(cd.phase ?? "");
|
||||
const cdSuperseded = cdOk && cdStatus.targetValidationState === "superseded";
|
||||
const finalComment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: cdPhase,
|
||||
state: cdOk ? (cdSuperseded ? "cd-superseded" : "cd-succeeded") : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: cdStatus,
|
||||
flush,
|
||||
dryRun: false,
|
||||
message: cdOk
|
||||
? cdSuperseded
|
||||
? "v0.2 自动 CD 已收口：本 merge head 的 PipelineRun 已完成，但 source head 已被后续提交推进，本轮按 latest-only 语义 superseded/no-op，未回写旧 GitOps revision。"
|
||||
: "v0.2 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。"
|
||||
: cdPhase === "timeout"
|
||||
? "v0.2 自动 CD 超时未收口；本评论保留最后一次 targetValidation / PipelineRun / Git mirror 状态用于接续排障。"
|
||||
: "v0.2 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。",
|
||||
});
|
||||
return {
|
||||
ok: cdOk && record(finalComment).ok === true,
|
||||
action: cdOk ? (cdSuperseded ? "merged-and-superseded-v02" : "merged-and-rolled-v02") : "v02-cd-failed",
|
||||
phase: cdOk ? "cd-passed" : cdPhase,
|
||||
pr,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
trigger,
|
||||
cd,
|
||||
comment: finalComment,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runRuntimeLanePrAutoCd(spec: HwlabRuntimeLaneSpec, pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
|
||||
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
|
||||
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
|
||||
const observedAt = new Date().toISOString();
|
||||
const comment = commentRuntimeLanePullRequest(spec, {
|
||||
pr,
|
||||
phase: "merge",
|
||||
state: "merge-failed",
|
||||
startedAt,
|
||||
observedAt,
|
||||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||||
preflight,
|
||||
merge: compactCommandResult(merge),
|
||||
dryRun: options.dryRun,
|
||||
message: mergeRaceState === "closed"
|
||||
? `PR preflight 曾通过，但自动合并前 PR 已被关闭且未确认 merged；本轮不会触发 ${spec.version} CD。`
|
||||
: "PR preflight 已通过，但自动合并失败；需要处理 GitHub 返回的 merge blocker 后重试。",
|
||||
});
|
||||
const issue = reportRuntimeLaneAutomationIssue({
|
||||
spec,
|
||||
pr,
|
||||
state: "merge-failed",
|
||||
phase: "merge",
|
||||
startedAt,
|
||||
observedAt,
|
||||
preflight,
|
||||
details: { merge: compactCommandResult(merge), mergeRaceState },
|
||||
message: "自动合并失败，v0.3 monitor 已暂停 CD 触发。",
|
||||
dryRun: options.dryRun,
|
||||
});
|
||||
return { ok: mergeRaceState === "closed" && record(comment).ok === true && record(issue).ok === true, phase: "merge", action: "merge-race-closed", pr, merge, comment, issue };
|
||||
}
|
||||
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
|
||||
if (!options.dryRun && expectedCommit === null) {
|
||||
const observedAt = new Date().toISOString();
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
|
||||
const comment = commentRuntimeLanePullRequest(spec, {
|
||||
pr,
|
||||
phase: "merge-commit",
|
||||
state: "cd-failed",
|
||||
startedAt,
|
||||
observedAt,
|
||||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit: null,
|
||||
pipelineRun: null,
|
||||
dryRun: false,
|
||||
message: "PR 已合并，但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit；为避免触发错误 source head，本轮未启动 CD。",
|
||||
});
|
||||
const issue = reportRuntimeLaneAutomationIssue({
|
||||
spec,
|
||||
pr,
|
||||
state: "cd-failed",
|
||||
phase: "merge-commit",
|
||||
startedAt,
|
||||
observedAt,
|
||||
preflight,
|
||||
details: { merge: commandData(merge) },
|
||||
message: "PR 已合并但 merge commit 不可解析，v0.3 CD 未启动。",
|
||||
dryRun: false,
|
||||
});
|
||||
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, issue, degradedReason: "merge-commit-unresolved" };
|
||||
}
|
||||
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForRuntimeLaneHead(spec, expectedCommit, 120);
|
||||
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
|
||||
const pipelineRun = sourceCommit === null ? null : runtimeLanePipelineRunName(spec, sourceCommit);
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
|
||||
const startedComment = commentRuntimeLanePullRequest(spec, {
|
||||
pr,
|
||||
phase: options.dryRun ? "dry-run" : "cd-trigger",
|
||||
state: "cd-started",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
dryRun: options.dryRun,
|
||||
message: options.dryRun
|
||||
? `dry-run：PR 已达到 ${spec.version} 自动合并条件；本轮不会写 GitHub merge 或 CD。`
|
||||
: mergeRaceState === "merged"
|
||||
? `PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 ${spec.version} CD。后续成功、失败或超时会继续在本 PR 下回复。`
|
||||
: `PR 已自动合并，${spec.version} CD 准备开始；后续成功、失败或超时会继续在本 PR 下回复。`,
|
||||
});
|
||||
if (!options.dryRun && record(startedComment).ok !== true) {
|
||||
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
|
||||
}
|
||||
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
|
||||
if (sourceCommit === null || record(headWait).ok !== true) {
|
||||
const observedAt = new Date().toISOString();
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
|
||||
const comment = commentRuntimeLanePullRequest(spec, {
|
||||
pr,
|
||||
phase: `${spec.lane}-source-head`,
|
||||
state: "cd-timeout",
|
||||
startedAt,
|
||||
observedAt,
|
||||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
dryRun: false,
|
||||
message: `PR 已合并，但 G14 ${spec.version} CI/CD source repo 没有在等待窗口内对齐 merge commit；为避免触发旧 head，本轮未启动 CD。`,
|
||||
});
|
||||
const issue = reportRuntimeLaneAutomationIssue({
|
||||
spec,
|
||||
pr,
|
||||
state: "cd-timeout",
|
||||
phase: `${spec.lane}-source-head`,
|
||||
startedAt,
|
||||
observedAt,
|
||||
preflight,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
details: { expectedCommit, headWait },
|
||||
message: `${spec.version} source head 未在等待窗口内对齐 merge commit，CD 未启动。`,
|
||||
dryRun: false,
|
||||
});
|
||||
return { ok: false, phase: `${spec.lane}-head`, pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, issue, degradedReason: `${spec.lane}-head-unresolved-after-merge` };
|
||||
}
|
||||
const before = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
|
||||
const beforeSummary = summarizeRuntimeLaneCdStatus(spec, before);
|
||||
const trigger = runtimeLaneCdPassedWithMirror(spec, before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerRuntimeLaneCurrent(spec, Math.min(options.timeoutSeconds, 600));
|
||||
printEvent(`${spec.lane}.cd.trigger`, { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||||
if (record(trigger).ok !== true) {
|
||||
const observedAt = new Date().toISOString();
|
||||
const comment = commentRuntimeLanePullRequest(spec, {
|
||||
pr,
|
||||
phase: "cd-trigger",
|
||||
state: "cd-failed",
|
||||
startedAt,
|
||||
observedAt,
|
||||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: beforeSummary,
|
||||
dryRun: false,
|
||||
message: `PR 已合并，但 ${spec.version} CD 触发失败；需要查看 trigger 阶段的 degradedReason。`,
|
||||
});
|
||||
const issue = reportRuntimeLaneAutomationIssue({
|
||||
spec,
|
||||
pr,
|
||||
state: "cd-failed",
|
||||
phase: "cd-trigger",
|
||||
startedAt,
|
||||
observedAt,
|
||||
preflight,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
cd: beforeSummary,
|
||||
details: { trigger },
|
||||
message: `${spec.version} CD 触发失败，PipelineRun 未进入可等待状态。`,
|
||||
dryRun: false,
|
||||
});
|
||||
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment, issue };
|
||||
}
|
||||
const cd = await waitForRuntimeLaneCd(spec, sourceCommit, options.timeoutSeconds);
|
||||
const cdStatus = record(cd.status);
|
||||
const flush = record(cd.flush);
|
||||
const cdOk = cd.ok === true;
|
||||
const cdPhase = String(cd.phase ?? "");
|
||||
const cdPipelineRun = stringOrNull(cdStatus.pipelineRun) ?? pipelineRun;
|
||||
const observedAt = new Date().toISOString();
|
||||
const finalComment = commentRuntimeLanePullRequest(spec, {
|
||||
pr,
|
||||
phase: cdPhase,
|
||||
state: cdOk ? "cd-succeeded" : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||||
startedAt,
|
||||
observedAt,
|
||||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||||
preflight,
|
||||
merge: commandData(merge),
|
||||
sourceCommit,
|
||||
pipelineRun: cdPipelineRun,
|
||||
cd: cdStatus,
|
||||
flush,
|
||||
dryRun: false,
|
||||
message: cdOk
|
||||
? `${spec.version} 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。`
|
||||
: cdPhase === "timeout"
|
||||
? `${spec.version} 自动 CD 超时未收口；本评论保留最后一次 PipelineRun / Argo / public probe / Git mirror 状态用于接续排障。`
|
||||
: `${spec.version} 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。`,
|
||||
});
|
||||
const issue = cdOk ? { ok: true, skipped: true, reason: "cd-succeeded" } : reportRuntimeLaneAutomationIssue({
|
||||
spec,
|
||||
pr,
|
||||
state: cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||||
phase: cdPhase || "cd",
|
||||
startedAt,
|
||||
observedAt,
|
||||
preflight,
|
||||
sourceCommit,
|
||||
pipelineRun: cdPipelineRun,
|
||||
cd: cdStatus,
|
||||
details: { cd },
|
||||
message: cdPhase === "timeout" ? `${spec.version} 自动 CD 超时。` : `${spec.version} 自动 CD 失败。`,
|
||||
dryRun: false,
|
||||
});
|
||||
return {
|
||||
ok: cdOk && record(finalComment).ok === true && record(issue).ok === true,
|
||||
action: cdOk ? `merged-and-rolled-${spec.lane}` : `${spec.lane}-cd-failed`,
|
||||
phase: cdOk ? "cd-passed" : cdPhase,
|
||||
pr,
|
||||
sourceCommit,
|
||||
pipelineRun: cdPipelineRun,
|
||||
trigger,
|
||||
cd,
|
||||
comment: finalComment,
|
||||
issue,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,576 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. cleanup module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:2788-3345 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult, G14ControlPlaneOptions } from "./types";
|
||||
import { g14HostScript } from "./images";
|
||||
import { statusText } from "./pr-monitor";
|
||||
import { g14K3s, isCommandSuccess, nested, shellQuote, shortSha, stringOrNull } from "./remote";
|
||||
import { runtimeLanePipelineRunName, v02PipelineRunName } from "./source";
|
||||
import { CI_NAMESPACE } from "./types";
|
||||
|
||||
export interface CleanupPipelineRunRow {
|
||||
name: string;
|
||||
createdAt: string;
|
||||
ageMinutes: number | null;
|
||||
status: string | null;
|
||||
reason: string | null;
|
||||
}
|
||||
|
||||
export function pipelinePrefixesForLane(lane: G14ControlPlaneOptions["lane"]): string[] {
|
||||
if (isHwlabRuntimeLane(lane)) return [`${hwlabRuntimeLaneSpec(lane).pipelineRunPrefix}-`];
|
||||
if (lane === "g14") return ["hwlab-g14-ci-poll-"];
|
||||
return [...hwlabRuntimeLaneIds().map((runtimeLane) => `${hwlabRuntimeLaneSpec(runtimeLane).pipelineRunPrefix}-`), "hwlab-g14-ci-poll-"];
|
||||
}
|
||||
|
||||
export function commandErrorSummary(result: CommandJsonResult): string {
|
||||
const remoteStderr = String(nested(result.parsed, ["data", "stderr"]) ?? "").trim();
|
||||
const remoteStdout = String(nested(result.parsed, ["data", "stdout"]) ?? "").trim();
|
||||
const text = [result.stderr.trim(), remoteStderr, remoteStdout, result.stdout.trim()].find((item) => item.length > 0) ?? "";
|
||||
return text.slice(0, 4000);
|
||||
}
|
||||
|
||||
export function tailText(value: unknown, maxBytes = 2000): string {
|
||||
const text = String(value ?? "").trim();
|
||||
return text.length <= maxBytes ? text : text.slice(-maxBytes);
|
||||
}
|
||||
|
||||
export function cleanupPipelineRunFieldsJsonPath(range: boolean): string {
|
||||
const body = [
|
||||
"{.metadata.name}",
|
||||
"{\"\\t\"}",
|
||||
"{.metadata.creationTimestamp}",
|
||||
"{\"\\t\"}",
|
||||
"{.status.conditions[0].status}",
|
||||
"{\"\\t\"}",
|
||||
"{.status.conditions[0].reason}",
|
||||
"{\"\\n\"}",
|
||||
].join("");
|
||||
return range ? `jsonpath={range .items[*]}${body}{end}` : `jsonpath=${body}`;
|
||||
}
|
||||
|
||||
export function parseCleanupPipelineRunLine(line: string, nowMs: number): CleanupPipelineRunRow | null {
|
||||
const [name = "", createdAt = "", status = "", reason = ""] = line.trim().split("\t");
|
||||
if (name.length === 0) return null;
|
||||
const createdMs = Date.parse(createdAt);
|
||||
const ageMinutes = Number.isFinite(createdMs) ? Math.floor((nowMs - createdMs) / 60000) : null;
|
||||
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
|
||||
}
|
||||
|
||||
export function formatRetentionBytes(bytes: number): string | null {
|
||||
if (!Number.isFinite(bytes)) return null;
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`;
|
||||
}
|
||||
|
||||
export function storageHostPathFromClaim(volume: string | null, namespace: string, claimName: string | null): string | null {
|
||||
if (volume === null || claimName === null) return null;
|
||||
if (!/^pvc-[a-z0-9-]+$/iu.test(volume) || !/^pvc-[a-z0-9]+$/iu.test(claimName)) return null;
|
||||
return `/var/lib/rancher/k3s/storage/${volume}_${namespace}_${claimName}`;
|
||||
}
|
||||
|
||||
export function safeStorageHostPath(value: unknown): string | null {
|
||||
const path = stringOrNull(value);
|
||||
return path !== null && path.startsWith("/var/lib/rancher/k3s/storage/") ? path : null;
|
||||
}
|
||||
|
||||
export function remoteStoragePathEstimates(paths: Array<string | null | undefined>): Map<string, number | null> {
|
||||
const uniquePaths = [...new Set(paths.map((item) => safeStorageHostPath(item)).filter((item): item is string => item !== null))];
|
||||
const estimates = new Map<string, number | null>(uniquePaths.map((path) => [path, null]));
|
||||
if (uniquePaths.length === 0) return estimates;
|
||||
const script = [
|
||||
"set -eu",
|
||||
`for path in ${uniquePaths.map((path) => shellQuote(path)).join(" ")}; do`,
|
||||
" bytes=\"\"",
|
||||
" if [ -e \"$path\" ]; then bytes=$(du -sB1 \"$path\" 2>/dev/null | awk '{print $1}' || true); fi",
|
||||
" printf '%s\\t%s\\n' \"$path\" \"$bytes\"",
|
||||
"done",
|
||||
].join("\n");
|
||||
const result = g14HostScript(script, 120_000);
|
||||
if (!isCommandSuccess(result)) return estimates;
|
||||
for (const line of statusText(result).split(/\r?\n/u)) {
|
||||
const [path = "", rawBytes = ""] = line.trim().split("\t");
|
||||
if (!estimates.has(path)) continue;
|
||||
const bytes = /^\d+$/u.test(rawBytes) ? Number(rawBytes) : null;
|
||||
estimates.set(path, Number.isFinite(bytes) ? bytes : null);
|
||||
}
|
||||
return estimates;
|
||||
}
|
||||
|
||||
export function sumEstimatedBytes(items: Record<string, unknown>[]): number {
|
||||
return items.reduce((sum, item) => {
|
||||
const bytes = item.estimatedBytes;
|
||||
return sum + (typeof bytes === "number" && Number.isFinite(bytes) ? bytes : 0);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
export function cleanupPipelineRunTargetCandidate(input: {
|
||||
targetPipelineRun: string;
|
||||
text: string;
|
||||
commandOk: boolean;
|
||||
exitCode: number | null;
|
||||
stderr: string;
|
||||
minAgeMinutes: number;
|
||||
nowMs: number;
|
||||
}): Record<string, unknown> {
|
||||
if (!input.commandOk) {
|
||||
const queryError = tailText(input.stderr);
|
||||
const notFound = /notfound|not found/i.test(queryError);
|
||||
return {
|
||||
name: input.targetPipelineRun,
|
||||
createdAt: null,
|
||||
ageMinutes: null,
|
||||
status: null,
|
||||
reason: notFound ? "target-pipelinerun-not-found" : "target-pipelinerun-query-failed",
|
||||
selected: false,
|
||||
queryExitCode: input.exitCode,
|
||||
...(queryError.length > 0 ? { queryError } : {}),
|
||||
};
|
||||
}
|
||||
const target = parseCleanupPipelineRunLine(input.text, input.nowMs);
|
||||
if (target === null || target.name !== input.targetPipelineRun) {
|
||||
return {
|
||||
name: input.targetPipelineRun,
|
||||
createdAt: null,
|
||||
ageMinutes: null,
|
||||
status: null,
|
||||
reason: "target-pipelinerun-query-empty",
|
||||
selected: false,
|
||||
queryOutputPreview: tailText(input.text),
|
||||
};
|
||||
}
|
||||
if (target.status !== "True" && target.status !== "False") {
|
||||
return {
|
||||
...target,
|
||||
selected: false,
|
||||
selectedReason: "target-pipelinerun-not-terminal",
|
||||
};
|
||||
}
|
||||
const ageMinutes = typeof target.ageMinutes === "number" ? target.ageMinutes : null;
|
||||
const belowMinAge = ageMinutes === null || ageMinutes < input.minAgeMinutes;
|
||||
return {
|
||||
...target,
|
||||
selected: !belowMinAge,
|
||||
...(belowMinAge
|
||||
? { selectedReason: ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function cleanupPipelineRunTargetCandidateFromTextForTest(input: {
|
||||
targetPipelineRun: string;
|
||||
text: string;
|
||||
commandOk?: boolean;
|
||||
exitCode?: number | null;
|
||||
stderr?: string;
|
||||
minAgeMinutes?: number;
|
||||
nowMs?: number;
|
||||
}): Record<string, unknown> {
|
||||
return cleanupPipelineRunTargetCandidate({
|
||||
targetPipelineRun: input.targetPipelineRun,
|
||||
text: input.text,
|
||||
commandOk: input.commandOk ?? true,
|
||||
exitCode: input.exitCode ?? 0,
|
||||
stderr: input.stderr ?? "",
|
||||
minAgeMinutes: input.minAgeMinutes ?? 60,
|
||||
nowMs: input.nowMs ?? Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
export function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||||
if (options.lane !== "g14" && options.lane !== "all" && !isHwlabRuntimeLane(options.lane)) {
|
||||
throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all");
|
||||
}
|
||||
const targetPipelineRun = options.pipelineRun ?? (options.sourceCommit === undefined
|
||||
? undefined
|
||||
: isHwlabRuntimeLane(options.lane)
|
||||
? runtimeLanePipelineRunName(hwlabRuntimeLaneSpec(options.lane), options.sourceCommit)
|
||||
: v02PipelineRunName(options.sourceCommit));
|
||||
const now = Date.now();
|
||||
if (targetPipelineRun !== undefined) {
|
||||
const targetResult = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pipelinerun",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
targetPipelineRun,
|
||||
"-o",
|
||||
cleanupPipelineRunFieldsJsonPath(false),
|
||||
], 60_000);
|
||||
return [cleanupPipelineRunTargetCandidate({
|
||||
targetPipelineRun,
|
||||
text: statusText(targetResult),
|
||||
commandOk: isCommandSuccess(targetResult),
|
||||
exitCode: targetResult.exitCode,
|
||||
stderr: commandErrorSummary(targetResult),
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
nowMs: now,
|
||||
})];
|
||||
}
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pipelinerun",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
cleanupPipelineRunFieldsJsonPath(true),
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const prefixes = pipelinePrefixesForLane(options.lane);
|
||||
const terminalRuns = statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => parseCleanupPipelineRunLine(line, now))
|
||||
.filter((item): item is CleanupPipelineRunRow => item !== null)
|
||||
.filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
|
||||
.filter((item) => item.status === "True" || item.status === "False")
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
|
||||
const protectedLatestByPrefix = new Map<string, string>();
|
||||
for (const prefix of prefixes) {
|
||||
const latest = terminalRuns
|
||||
.filter((item) => item.name.startsWith(prefix))
|
||||
.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0];
|
||||
if (latest !== undefined) protectedLatestByPrefix.set(prefix, latest.name);
|
||||
}
|
||||
const candidates = terminalRuns
|
||||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
||||
.map((item) => {
|
||||
const protectedLatest = [...protectedLatestByPrefix.values()].includes(item.name);
|
||||
return protectedLatest
|
||||
? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" }
|
||||
: item;
|
||||
})
|
||||
.slice(0, options.limit);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
export function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unknown>[] {
|
||||
if (pipelineRunNames.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pvc",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const pvResult = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pv",
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(pvResult)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRun PV backing paths: ${commandErrorSummary(pvResult)}`);
|
||||
}
|
||||
const podResult = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pod",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{","}{end}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(podResult)) {
|
||||
throw new Error(`failed to list hwlab-ci active pod PVC mounts: ${commandErrorSummary(podResult)}`);
|
||||
}
|
||||
const pvByName = new Map<string, Record<string, unknown>>();
|
||||
for (const line of statusText(pvResult).split(/\r?\n/u)) {
|
||||
const [name = "", storageClass = "", reclaimPolicy = "", localPath = "", hostPath = ""] = line.trim().split("\t");
|
||||
if (name.length === 0) continue;
|
||||
pvByName.set(name, {
|
||||
storageClass: storageClass || null,
|
||||
reclaimPolicy: reclaimPolicy || null,
|
||||
hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath),
|
||||
});
|
||||
}
|
||||
const activeClaimPods = new Map<string, string[]>();
|
||||
for (const line of statusText(podResult).split(/\r?\n/u)) {
|
||||
const [podName = "", phase = "", claims = ""] = line.trim().split("\t");
|
||||
if (podName.length === 0 || phase === "Succeeded" || phase === "Failed") continue;
|
||||
for (const claimName of claims.split(",").map((item) => item.trim()).filter(Boolean)) {
|
||||
const entry = activeClaimPods.get(claimName) ?? [];
|
||||
entry.push(podName);
|
||||
activeClaimPods.set(claimName, entry);
|
||||
}
|
||||
}
|
||||
const wanted = new Set(pipelineRunNames);
|
||||
const ownedPvcs = statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
|
||||
const pv = volume.length > 0 ? pvByName.get(volume) : undefined;
|
||||
const hostPath = safeStorageHostPath(pv?.hostPath) ?? storageHostPathFromClaim(volume || null, CI_NAMESPACE, name || null);
|
||||
return {
|
||||
name,
|
||||
volume: volume || null,
|
||||
phase: phase || null,
|
||||
ownerKind: ownerKind || null,
|
||||
owner: owner || null,
|
||||
storageClass: pv?.storageClass ?? null,
|
||||
reclaimPolicy: pv?.reclaimPolicy ?? null,
|
||||
hostPath,
|
||||
activeMountPods: activeClaimPods.get(name) ?? [],
|
||||
};
|
||||
})
|
||||
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
|
||||
const estimates = remoteStoragePathEstimates(ownedPvcs.map((item) => stringOrNull(item.hostPath)));
|
||||
return ownedPvcs.map((item) => ({
|
||||
...item,
|
||||
estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
export function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
if (names.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "no candidates",
|
||||
stderr: "",
|
||||
parsed: null,
|
||||
};
|
||||
}
|
||||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
|
||||
}
|
||||
|
||||
export function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pv",
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const candidates = statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = "", localPath = "", hostPath = ""] = line.split("\t");
|
||||
return {
|
||||
name,
|
||||
createdAt,
|
||||
phase,
|
||||
storageClass,
|
||||
reclaimPolicy,
|
||||
claimNamespace,
|
||||
claimName,
|
||||
capacity,
|
||||
hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath) ?? (claimNamespace === CI_NAMESPACE ? storageHostPathFromClaim(name || null, claimNamespace, claimName || null) : null),
|
||||
};
|
||||
})
|
||||
.filter((item) => item.phase === "Released")
|
||||
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
||||
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, options.limit);
|
||||
const estimates = remoteStoragePathEstimates(candidates.map((item) => stringOrNull(item.hostPath)));
|
||||
return candidates.map((item) => ({
|
||||
...item,
|
||||
estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
export function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
if (names.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "no candidates",
|
||||
stderr: "",
|
||||
parsed: null,
|
||||
};
|
||||
}
|
||||
return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs);
|
||||
}
|
||||
|
||||
export function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listReleasedCiWorkspacePvs(options);
|
||||
const candidateNames = candidates.map((item) => String(item.name));
|
||||
const estimatedReclaimBytes = sumEstimatedBytes(candidates);
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||||
mode: "dry-run",
|
||||
lane: options.lane,
|
||||
limit: options.limit,
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
selectedPersistentVolumes: candidateNames,
|
||||
selectedPersistentVolumeCount: candidateNames.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
mutation: false,
|
||||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
|
||||
};
|
||||
}
|
||||
const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000);
|
||||
return {
|
||||
ok: isCommandSuccess(deletion),
|
||||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||||
mode: "confirmed-cleanup",
|
||||
lane: options.lane,
|
||||
limit: options.limit,
|
||||
deletedPersistentVolumes: candidateNames,
|
||||
deletedPersistentVolumeCount: candidateNames.length,
|
||||
candidatesBefore: candidates,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
deletion,
|
||||
followUp: {
|
||||
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
storage: "trans G14 sh -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function runV02RuntimeMigration(options: G14ControlPlaneOptions, sourceCommit: string): Record<string, unknown> {
|
||||
const reportPath = `/tmp/hwlab-v02-runtime-migration-${shortSha(sourceCommit)}.json`;
|
||||
const migrationArgs = options.dryRun
|
||||
? [
|
||||
...(options.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
|
||||
"--report",
|
||||
reportPath,
|
||||
]
|
||||
: [
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--report",
|
||||
reportPath,
|
||||
];
|
||||
const command = [
|
||||
"exec",
|
||||
"-n",
|
||||
"hwlab-v02",
|
||||
"deployment/hwlab-cloud-api",
|
||||
"-c",
|
||||
"hwlab-cloud-api",
|
||||
"--",
|
||||
"bun",
|
||||
"cmd/hwlab-cloud-api/migrate.ts",
|
||||
...migrationArgs,
|
||||
];
|
||||
const result = g14K3s(["kubectl", ...command], options.timeoutSeconds * 1000);
|
||||
const ok = isCommandSuccess(result);
|
||||
return {
|
||||
ok,
|
||||
command: "hwlab g14 control-plane runtime-migration --lane v02",
|
||||
lane: "v02",
|
||||
mode: options.dryRun ? options.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
|
||||
sourceCommit,
|
||||
runtimeNamespace: "hwlab-v02",
|
||||
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
|
||||
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
|
||||
reportPath,
|
||||
mutation: !options.dryRun,
|
||||
result,
|
||||
next: options.dryRun
|
||||
? {
|
||||
liveReadDryRun: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||||
apply: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||||
}
|
||||
: {
|
||||
health: "curl -fsS --max-time 20 http://74.48.78.17:19667/health/live",
|
||||
triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listCleanupPipelineRuns(options);
|
||||
const candidateNames = candidates
|
||||
.filter((item) => item.selected !== false)
|
||||
.map((item) => String(item.name));
|
||||
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
|
||||
const estimatedReclaimBytes = sumEstimatedBytes(ownedPvcs);
|
||||
const followUpStatusLane = isHwlabRuntimeLane(options.lane) ? options.lane : "v02";
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 control-plane cleanup-runs",
|
||||
mode: "dry-run",
|
||||
lane: options.lane,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
sourceCommit: options.sourceCommit,
|
||||
pipelineRun: options.pipelineRun,
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
selectedPipelineRuns: candidateNames,
|
||||
selectedPipelineRunCount: candidateNames.length,
|
||||
ownedPvcs,
|
||||
ownedPvcCount: ownedPvcs.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
mutation: false,
|
||||
next: { confirm: [
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs",
|
||||
`--lane ${options.lane}`,
|
||||
`--min-age-minutes ${options.minAgeMinutes}`,
|
||||
`--limit ${options.limit}`,
|
||||
options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`,
|
||||
options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`,
|
||||
"--confirm",
|
||||
].filter(Boolean).join(" ") },
|
||||
};
|
||||
}
|
||||
const deletion = deletePipelineRuns(candidateNames, options.timeoutSeconds * 1000);
|
||||
return {
|
||||
ok: isCommandSuccess(deletion),
|
||||
command: "hwlab g14 control-plane cleanup-runs",
|
||||
mode: "confirmed-cleanup",
|
||||
lane: options.lane,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
sourceCommit: options.sourceCommit,
|
||||
pipelineRun: options.pipelineRun,
|
||||
deletedPipelineRuns: candidateNames,
|
||||
deletedPipelineRunCount: candidateNames.length,
|
||||
ownedPvcsBefore: ownedPvcs,
|
||||
ownedPvcCountBefore: ownedPvcs.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
deletion,
|
||||
followUp: {
|
||||
status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${followUpStatusLane}`,
|
||||
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
},
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,122 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. entry module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:10619-12000 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import { runRuntimeLaneControlPlane, runV02ControlPlane } from "./control-plane";
|
||||
import { runG14GitMirror } from "./git-mirror";
|
||||
import { hwlabG14Help, monitorStatus } from "./help";
|
||||
import { runG14ToolsImage, runG14UpstreamImage, startControlPlaneTriggerJob, startGitMirrorJob } from "./images";
|
||||
import { runMonitorWorker } from "./monitor";
|
||||
import { runG14Observability } from "./observability";
|
||||
import { hwlabG14MonitorStateFileName, hwlabG14MonitorStateRole, parseControlPlaneOptions, parseGitMirrorOptions, parseLegacyRetirementOptions, parseObservabilityOptions, parseOptions, parseSecretOptions, parseToolsImageOptions, parseUpstreamImageOptions } from "./options";
|
||||
import { monitorBaseBranch } from "./remote";
|
||||
import { legacyG14RecordRolloutRetired, legacyG14RetirementBlocksMonitor, runLegacyG14Retirement } from "./retirement";
|
||||
import { runG14Secret } from "./secrets";
|
||||
import { G14_SOURCE_BRANCH, HWLAB_REPO } from "./types";
|
||||
|
||||
export async function runHwlabG14Command(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
||||
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return { ok: true, ...hwlabG14Help() };
|
||||
const [action] = args;
|
||||
if (action === "record-rollout") {
|
||||
return legacyG14RecordRolloutRetired();
|
||||
}
|
||||
if (action === "control-plane") {
|
||||
const options = parseControlPlaneOptions(args.slice(1));
|
||||
if (options.action === "trigger-current" && options.confirm && !options.dryRun && !options.wait) {
|
||||
return startControlPlaneTriggerJob(options);
|
||||
}
|
||||
if (isHwlabRuntimeLane(options.lane) && options.lane !== "v02") {
|
||||
return runRuntimeLaneControlPlane(hwlabRuntimeLaneSpec(options.lane), options);
|
||||
}
|
||||
return runV02ControlPlane(options);
|
||||
}
|
||||
if (action === "retirement") {
|
||||
const options = parseLegacyRetirementOptions(args.slice(1));
|
||||
return runLegacyG14Retirement(options);
|
||||
}
|
||||
if (action === "secret") {
|
||||
const options = parseSecretOptions(args.slice(1));
|
||||
return runG14Secret(options);
|
||||
}
|
||||
if (action === "tools-image") {
|
||||
const options = parseToolsImageOptions(args.slice(1));
|
||||
return runG14ToolsImage(options);
|
||||
}
|
||||
if (action === "upstream-image") {
|
||||
const options = parseUpstreamImageOptions(args.slice(1));
|
||||
return runG14UpstreamImage(options);
|
||||
}
|
||||
if (action === "git-mirror") {
|
||||
const options = parseGitMirrorOptions(args.slice(1));
|
||||
if ((options.action === "sync" || options.action === "flush") && options.confirm && !options.dryRun && !options.wait) {
|
||||
return startGitMirrorJob(options);
|
||||
}
|
||||
return runG14GitMirror(options);
|
||||
}
|
||||
if (action === "observability") {
|
||||
const options = parseObservabilityOptions(args.slice(1));
|
||||
return runG14Observability(options);
|
||||
}
|
||||
if (action !== "monitor-prs") {
|
||||
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 retirement, hwlab g14 control-plane, hwlab g14 secret, hwlab g14 git-mirror, hwlab g14 observability, hwlab g14 tools-image, hwlab g14 upstream-image" };
|
||||
}
|
||||
const options = parseOptions(args.slice(1));
|
||||
if (options.worker) return runMonitorWorker(options);
|
||||
if (args.includes("--status")) return monitorStatus(options);
|
||||
const retirement = options.lane === "g14" ? legacyG14RetirementBlocksMonitor() : null;
|
||||
if (retirement !== null) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "hwlab g14 monitor-prs",
|
||||
lane: options.lane,
|
||||
baseBranch: monitorBaseBranch(options.lane),
|
||||
degradedReason: "legacy-g14-dev-prod-retired",
|
||||
message: "Legacy G14 DEV/PROD has a retirement marker; start a runtime lane monitor with --lane v02/v03 or inspect retirement status.",
|
||||
retirement,
|
||||
next: {
|
||||
retirementStatus: "bun scripts/cli.ts hwlab g14 retirement status",
|
||||
v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
|
||||
v03Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
|
||||
},
|
||||
};
|
||||
}
|
||||
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--lane", options.lane, "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
|
||||
const jobName = options.lane === "g14" ? "hwlab_g14_pr_monitor" : `hwlab_g14_${options.lane}_pr_monitor`;
|
||||
const jobNote = options.lane === "g14"
|
||||
? `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`
|
||||
: `Monitor ${HWLAB_REPO} PRs targeting ${monitorBaseBranch(options.lane)}, merge ready PRs, trigger ${hwlabRuntimeLaneSpec(options.lane).version} CD, comment PR progress, and file failure issues`;
|
||||
const job = startJob(jobName, command, jobNote);
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
|
||||
const stateDir = rootPath(".state", "hwlab-g14");
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
const stateFileName = hwlabG14MonitorStateFileName(options);
|
||||
const stateFileRole = hwlabG14MonitorStateRole(options);
|
||||
const latestPath = join(stateDir, stateFileName);
|
||||
const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null;
|
||||
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole, lane: options.lane, baseBranch: monitorBaseBranch(options.lane) }, null, 2)}\n`, "utf8");
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 monitor-prs",
|
||||
lane: options.lane,
|
||||
baseBranch: monitorBaseBranch(options.lane),
|
||||
mode: "async-job",
|
||||
job,
|
||||
statusCommand,
|
||||
latestPath,
|
||||
stateFileName,
|
||||
stateFileRole,
|
||||
previousLatest,
|
||||
next: {
|
||||
status: statusCommand,
|
||||
tail: `tail -f ${job.stdoutFile}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,952 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. git-mirror module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:5846-6777 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult, G14ControlPlaneOptions, G14GitMirrorOptions } from "./types";
|
||||
import { tailText } from "./cleanup";
|
||||
import { statusText } from "./pr-monitor";
|
||||
import { compactCommandResult, g14K3s, isCommandSuccess, nested, printProgressEvent, record, shellQuote, shortSha, stringOrNull } from "./remote";
|
||||
import { cleanupRuntimeLaneRenderDir, runRuntimeLaneRenderToTemp } from "./render";
|
||||
import { resolveRuntimeLaneHead } from "./source";
|
||||
import { GIT_MIRROR_LEGACY_CRONJOB, GIT_MIRROR_MANIFEST_FIELD_MANAGER, GIT_MIRROR_NAMESPACE, GIT_MIRROR_SYNC_JOB_PREFIX, V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS, V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, V02_GIT_MIRROR_PRESYNC_POLL_MS, V02_GIT_READ_URL, V02_GIT_WRITE_URL } from "./types";
|
||||
import { parseShellSections, shellSectionOk, timestampMs } from "./v02-status";
|
||||
|
||||
export function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"delete",
|
||||
"cronjob",
|
||||
"-n",
|
||||
GIT_MIRROR_NAMESPACE,
|
||||
GIT_MIRROR_LEGACY_CRONJOB,
|
||||
"--ignore-not-found=true",
|
||||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||||
], 60_000);
|
||||
}
|
||||
|
||||
export function applyGitMirrorManifestFile(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"apply",
|
||||
"--server-side",
|
||||
"--force-conflicts",
|
||||
`--field-manager=${GIT_MIRROR_MANIFEST_FIELD_MANAGER}`,
|
||||
...(dryRun ? ["--dry-run=server"] : []),
|
||||
"-f",
|
||||
`${renderDir}/devops-infra/git-mirror.yaml`,
|
||||
], timeoutSeconds * 1000);
|
||||
}
|
||||
|
||||
export function gitMirrorSyncJobName(): string {
|
||||
return `${GIT_MIRROR_SYNC_JOB_PREFIX}-${Date.now().toString(36)}`.slice(0, 63);
|
||||
}
|
||||
|
||||
export function gitMirrorFlushJobName(): string {
|
||||
return `git-mirror-hwlab-flush-manual-${Date.now().toString(36)}`.slice(0, 63);
|
||||
}
|
||||
|
||||
export function gitMirrorSyncJobManifest(name: string): Record<string, unknown> {
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name,
|
||||
namespace: GIT_MIRROR_NAMESPACE,
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "git-mirror",
|
||||
"app.kubernetes.io/part-of": "devops-infra",
|
||||
"app.kubernetes.io/component": "sync-controller",
|
||||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
activeDeadlineSeconds: 300,
|
||||
ttlSecondsAfterFinished: 3600,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "git-mirror",
|
||||
"app.kubernetes.io/part-of": "devops-infra",
|
||||
"app.kubernetes.io/component": "sync-controller",
|
||||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
restartPolicy: "Never",
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
volumes: [
|
||||
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
|
||||
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
|
||||
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } },
|
||||
],
|
||||
containers: [{
|
||||
name: "sync",
|
||||
image: "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1",
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["/script/sync.sh"],
|
||||
volumeMounts: [
|
||||
{ name: "cache", mountPath: "/cache" },
|
||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||||
{ name: "script", mountPath: "/script", readOnly: true },
|
||||
],
|
||||
}],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function gitMirrorFlushJobManifest(name: string): Record<string, unknown> {
|
||||
const manifest = gitMirrorSyncJobManifest(name);
|
||||
record(record(manifest.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||||
const template = record(record(manifest.spec).template);
|
||||
record(record(template.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||||
const podSpec = record(template.spec);
|
||||
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
|
||||
const first = record(containers[0]);
|
||||
first.name = "flush";
|
||||
first.command = ["/script/flush.sh"];
|
||||
podSpec.containers = [first];
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function parseGitMirrorStatusRefs(raw: string): { refs: Record<string, string | null>; pendingFlush: boolean | null } {
|
||||
const line = raw.split(/\r?\n/u).find((item) => item.startsWith("refs="));
|
||||
if (line === undefined) return { refs: {}, pendingFlush: null };
|
||||
try {
|
||||
const parsed = record(JSON.parse(line.slice("refs=".length)) as unknown);
|
||||
const refs = record(parsed.refs);
|
||||
const parsedRefs: Record<string, string | null> = {};
|
||||
for (const [key, value] of Object.entries(refs)) parsedRefs[key] = stringOrNull(value);
|
||||
return {
|
||||
refs: {
|
||||
...parsedRefs,
|
||||
localV02: stringOrNull(refs.localV02),
|
||||
githubV02: stringOrNull(refs.githubV02),
|
||||
localG14: stringOrNull(refs.localG14),
|
||||
githubG14: stringOrNull(refs.githubG14),
|
||||
localGitops: stringOrNull(refs.localGitops),
|
||||
githubGitops: stringOrNull(refs.githubGitops),
|
||||
localV03: stringOrNull(refs.localV03),
|
||||
githubV03: stringOrNull(refs.githubV03),
|
||||
localV03Gitops: stringOrNull(refs.localV03Gitops),
|
||||
githubV03Gitops: stringOrNull(refs.githubV03Gitops),
|
||||
},
|
||||
pendingFlush: typeof parsed.pendingFlush === "boolean" ? parsed.pendingFlush : null,
|
||||
};
|
||||
} catch {
|
||||
return { refs: {}, pendingFlush: null };
|
||||
}
|
||||
}
|
||||
|
||||
export function parseLabeledJsonLine(raw: string, label: string): Record<string, unknown> {
|
||||
const line = raw.split(/\r?\n/u).find((item) => item.startsWith(`${label}=`));
|
||||
if (line === undefined) return {};
|
||||
const value = line.slice(label.length + 1).trim();
|
||||
if (value.length === 0) return {};
|
||||
try {
|
||||
return record(JSON.parse(value) as unknown);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
|
||||
const refs = parseGitMirrorStatusRefs(raw);
|
||||
const lastSync = parseLabeledJsonLine(raw, "lastSync");
|
||||
const lastWrite = parseLabeledJsonLine(raw, "lastWrite");
|
||||
const lastFlush = parseLabeledJsonLine(raw, "lastFlush");
|
||||
const localV02 = refs.refs.localV02 ?? null;
|
||||
const githubV02 = refs.refs.githubV02 ?? null;
|
||||
const localGitops = refs.refs.localGitops ?? null;
|
||||
const githubGitops = refs.refs.githubGitops ?? null;
|
||||
const localV03 = refs.refs.localV03 ?? null;
|
||||
const githubV03 = refs.refs.githubV03 ?? null;
|
||||
const localV03Gitops = refs.refs.localV03Gitops ?? null;
|
||||
const githubV03Gitops = refs.refs.githubV03Gitops ?? null;
|
||||
const sourceInSync = Boolean(localV02 && githubV02 && localV02 === githubV02);
|
||||
const gitopsInSync = Boolean(localGitops && githubGitops && localGitops === githubGitops);
|
||||
const runtimeLanes = Object.fromEntries(hwlabRuntimeLaneIds().map((lane) => {
|
||||
const normalized = lane.slice(0, 1).toUpperCase() + lane.slice(1).toLowerCase();
|
||||
const localSource = refs.refs[`local${normalized}`] ?? null;
|
||||
const githubSource = refs.refs[`github${normalized}`] ?? null;
|
||||
const localLaneGitops = refs.refs[`local${normalized}Gitops`] ?? null;
|
||||
const githubLaneGitops = refs.refs[`github${normalized}Gitops`] ?? null;
|
||||
return [lane, {
|
||||
localSource,
|
||||
githubSource,
|
||||
localGitops: localLaneGitops,
|
||||
githubGitops: githubLaneGitops,
|
||||
sourceInSync: Boolean(localSource && githubSource && localSource === githubSource),
|
||||
gitopsInSync: Boolean(localLaneGitops && githubLaneGitops && localLaneGitops === githubLaneGitops),
|
||||
}];
|
||||
}));
|
||||
const allRuntimeLanesInSync = Object.values(runtimeLanes).every((item) => {
|
||||
const laneSummary = record(item);
|
||||
return laneSummary.sourceInSync === true && laneSummary.gitopsInSync === true;
|
||||
});
|
||||
return {
|
||||
localV02,
|
||||
githubV02,
|
||||
localV03,
|
||||
githubV03,
|
||||
localG14: refs.refs.localG14 ?? null,
|
||||
githubG14: refs.refs.githubG14 ?? null,
|
||||
localGitops,
|
||||
githubGitops,
|
||||
localV03Gitops,
|
||||
githubV03Gitops,
|
||||
pendingFlush: refs.pendingFlush,
|
||||
lastSync: {
|
||||
status: stringOrNull(lastSync.status) ?? null,
|
||||
at: stringOrNull(lastSync.publishedAt) ?? stringOrNull(lastSync.syncedAt) ?? null,
|
||||
pendingFlush: typeof lastSync.pendingFlush === "boolean" ? lastSync.pendingFlush : null,
|
||||
},
|
||||
lastWrite: {
|
||||
status: stringOrNull(lastWrite.status) ?? null,
|
||||
at: stringOrNull(lastWrite.writtenAt) ?? null,
|
||||
pendingFlush: typeof lastWrite.pendingFlush === "boolean" ? lastWrite.pendingFlush : null,
|
||||
},
|
||||
lastFlush: {
|
||||
status: stringOrNull(lastFlush.status) ?? null,
|
||||
at: stringOrNull(lastFlush.flushedAt) ?? null,
|
||||
pendingFlush: typeof lastFlush.pendingFlush === "boolean" ? lastFlush.pendingFlush : null,
|
||||
},
|
||||
flushNeeded: refs.pendingFlush === true,
|
||||
flushCommand: refs.pendingFlush === true ? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" : null,
|
||||
sourceInSync,
|
||||
gitopsInSync,
|
||||
v03SourceInSync: Boolean(localV03 && githubV03 && localV03 === githubV03),
|
||||
v03GitopsInSync: Boolean(localV03Gitops && githubV03Gitops && localV03Gitops === githubV03Gitops),
|
||||
runtimeLanes,
|
||||
allRuntimeLanesInSync,
|
||||
githubInSync: sourceInSync && gitopsInSync,
|
||||
};
|
||||
}
|
||||
|
||||
export function gitMirrorV02SyncRequirement(sourceCommit: string, rawStatus: string): Record<string, unknown> {
|
||||
const parsed = parseGitMirrorStatusRefs(rawStatus);
|
||||
const localV02 = parsed.refs.localV02 ?? null;
|
||||
return {
|
||||
required: localV02 !== sourceCommit,
|
||||
sourceCommit,
|
||||
localV02,
|
||||
pendingFlush: parsed.pendingFlush,
|
||||
refs: parsed.refs,
|
||||
reason: localV02 === sourceCommit ? "local-v02-current" : localV02 === null ? "local-v02-unresolved" : "local-v02-stale",
|
||||
};
|
||||
}
|
||||
|
||||
export function runtimeLaneGitMirrorSourceInSyncForTest(lane: string, sourceCommit: string, summary: Record<string, unknown>): boolean {
|
||||
const localKey = lane === "v02" ? "localV02" : lane === "v03" ? "localV03" : null;
|
||||
return localKey !== null && typeof summary[localKey] === "string" && summary[localKey] === sourceCommit;
|
||||
}
|
||||
|
||||
export function gitMirrorStatusCacheRaw(status: Record<string, unknown>): string {
|
||||
return String(nested(status, ["cache", "raw"]) ?? "");
|
||||
}
|
||||
|
||||
export function runtimeLaneGitMirrorSourceInSync(spec: HwlabRuntimeLaneSpec, sourceCommit: string, status: Record<string, unknown>): boolean {
|
||||
return runtimeLaneGitMirrorSourceInSyncForTest(spec.lane, sourceCommit, record(nested(status, ["cache", "summary"])));
|
||||
}
|
||||
|
||||
export function compactGitMirrorStatus(status: Record<string, unknown>, sourceCommit: string): Record<string, unknown> {
|
||||
const requirement = gitMirrorV02SyncRequirement(sourceCommit, gitMirrorStatusCacheRaw(status));
|
||||
return {
|
||||
ok: status.ok === true,
|
||||
readUrl: status.readUrl ?? V02_GIT_READ_URL,
|
||||
writeUrl: status.writeUrl ?? V02_GIT_WRITE_URL,
|
||||
elapsedMs: status.elapsedMs ?? null,
|
||||
legacyCronJobExists: nested(status, ["legacyCronJob", "exists"]) === true,
|
||||
required: requirement.required,
|
||||
sourceCommit,
|
||||
localV02: requirement.localV02 ?? null,
|
||||
githubV02: requirement.refs.githubV02 ?? null,
|
||||
pendingFlush: requirement.pendingFlush ?? null,
|
||||
sourceInSync: Boolean(requirement.refs.localV02 && requirement.refs.githubV02 && requirement.refs.localV02 === requirement.refs.githubV02),
|
||||
reason: requirement.reason,
|
||||
cacheOk: nested(status, ["cache", "ok"]) === true,
|
||||
cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000),
|
||||
resourcesOk: nested(status, ["resources", "ok"]) === true,
|
||||
};
|
||||
}
|
||||
|
||||
export function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
ok: sync.ok === true,
|
||||
command: sync.command ?? "hwlab g14 git-mirror sync",
|
||||
mode: sync.mode ?? null,
|
||||
skipped: sync.skipped === true,
|
||||
reason: sync.reason ?? null,
|
||||
namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE,
|
||||
jobName: sync.jobName ?? null,
|
||||
elapsedMs: sync.elapsedMs ?? null,
|
||||
exitCode: nested(sync, ["result", "exitCode"]) ?? null,
|
||||
stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600),
|
||||
stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200),
|
||||
stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null,
|
||||
stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export interface V02GitMirrorPreSyncMarker {
|
||||
ok: boolean;
|
||||
sourceCommit: string;
|
||||
syncedAt: string;
|
||||
localV02?: string | null;
|
||||
githubV02?: string | null;
|
||||
reason?: string | null;
|
||||
}
|
||||
|
||||
export function v02GitMirrorPreSyncStateDir(): string {
|
||||
return rootPath(".state", "hwlab-g14", "v02-git-mirror-presync");
|
||||
}
|
||||
|
||||
export function v02GitMirrorPreSyncMarkerPath(sourceCommit: string): string {
|
||||
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.json`);
|
||||
}
|
||||
|
||||
export function v02GitMirrorPreSyncLockDir(sourceCommit: string): string {
|
||||
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.lock`);
|
||||
}
|
||||
|
||||
export function v02GitMirrorPreSyncWaitMs(timeoutSeconds: number): number {
|
||||
const requestedMs = Math.max(0, Math.floor(timeoutSeconds * 1000));
|
||||
if (requestedMs === 0) return V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS;
|
||||
return Math.min(V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, requestedMs);
|
||||
}
|
||||
|
||||
export function v02ReusableGitMirrorPreSyncMarker(marker: unknown, sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
|
||||
const candidate = record(marker) as V02GitMirrorPreSyncMarker;
|
||||
const syncedAtMs = timestampMs(candidate.syncedAt);
|
||||
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit) return null;
|
||||
if (syncedAtMs === null || nowMs - syncedAtMs > V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS) return null;
|
||||
if (syncedAtMs < minSyncedAtMs) return null;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
export function readV02GitMirrorPreSyncMarker(sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
|
||||
const path = v02GitMirrorPreSyncMarkerPath(sourceCommit);
|
||||
if (!existsSync(path)) return null;
|
||||
try {
|
||||
return v02ReusableGitMirrorPreSyncMarker(JSON.parse(readFileSync(path, "utf8")) as unknown, sourceCommit, nowMs, minSyncedAtMs);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function writeV02GitMirrorPreSyncMarker(sourceCommit: string, summary: Record<string, unknown>): V02GitMirrorPreSyncMarker {
|
||||
const marker = {
|
||||
ok: true,
|
||||
sourceCommit,
|
||||
syncedAt: new Date().toISOString(),
|
||||
localV02: stringOrNull(summary.localV02),
|
||||
githubV02: stringOrNull(summary.githubV02),
|
||||
reason: stringOrNull(summary.reason),
|
||||
};
|
||||
const dir = v02GitMirrorPreSyncStateDir();
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(v02GitMirrorPreSyncMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
||||
return marker;
|
||||
}
|
||||
|
||||
export function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number, minSyncedAtMs = 0): { acquired: boolean; lockDir: string; waitedMs: number; marker?: V02GitMirrorPreSyncMarker } {
|
||||
const stateDir = v02GitMirrorPreSyncStateDir();
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
const lockDir = v02GitMirrorPreSyncLockDir(sourceCommit);
|
||||
const startedAtMs = Date.now();
|
||||
for (;;) {
|
||||
const marker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), minSyncedAtMs);
|
||||
if (marker !== null) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs, marker };
|
||||
try {
|
||||
mkdirSync(lockDir);
|
||||
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
||||
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
} catch {
|
||||
const ageMs = (() => {
|
||||
try {
|
||||
return Date.now() - statSync(lockDir).mtimeMs;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
})();
|
||||
if (ageMs > V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS) {
|
||||
try {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
continue;
|
||||
} catch {
|
||||
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
}
|
||||
}
|
||||
if (Date.now() - startedAtMs >= waitMs) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
||||
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function releaseV02GitMirrorPreSyncLock(lockDir: string): void {
|
||||
try {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
||||
}
|
||||
}
|
||||
|
||||
export function waitForV02GitMirrorPreSync(sourceCommit: string, waitMs: number): Record<string, unknown> {
|
||||
const startedAtMs = Date.now();
|
||||
const observations: Record<string, unknown>[] = [];
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
attempt += 1;
|
||||
const statusStartMs = Date.now();
|
||||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||||
stage: "git-mirror-pre-sync-wait",
|
||||
status: "polling",
|
||||
sourceCommit,
|
||||
attempt,
|
||||
elapsedMs: statusStartMs - startedAtMs,
|
||||
});
|
||||
const status = runGitMirrorStatus();
|
||||
const summary = compactGitMirrorStatus(status, sourceCommit);
|
||||
observations.push({
|
||||
attempt,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
statusElapsedMs: Date.now() - statusStartMs,
|
||||
ok: summary.ok,
|
||||
required: summary.required,
|
||||
localV02: summary.localV02,
|
||||
githubV02: summary.githubV02,
|
||||
pendingFlush: summary.pendingFlush,
|
||||
reason: summary.reason,
|
||||
});
|
||||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||||
stage: "git-mirror-pre-sync-wait",
|
||||
status: summary.required === false ? "succeeded" : "waiting",
|
||||
sourceCommit,
|
||||
attempt,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
statusElapsedMs: Date.now() - statusStartMs,
|
||||
required: summary.required,
|
||||
localV02: summary.localV02,
|
||||
githubV02: summary.githubV02,
|
||||
pendingFlush: summary.pendingFlush,
|
||||
reason: summary.reason,
|
||||
});
|
||||
if (summary.ok === true && summary.required === false) {
|
||||
return {
|
||||
ok: true,
|
||||
sourceCommit,
|
||||
attempts: attempt,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
final: summary,
|
||||
observations: observations.slice(-6),
|
||||
};
|
||||
}
|
||||
if (Date.now() - startedAtMs >= waitMs) {
|
||||
return {
|
||||
ok: false,
|
||||
sourceCommit,
|
||||
attempts: attempt,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
final: summary,
|
||||
observations: observations.slice(-6),
|
||||
degradedReason: "git-mirror-local-v02-not-current-after-wait",
|
||||
};
|
||||
}
|
||||
const remainingMs = waitMs - (Date.now() - startedAtMs);
|
||||
const sleepMs = Math.max(500, Math.min(V02_GIT_MIRROR_PRESYNC_POLL_MS, remainingMs));
|
||||
const wait = runCommand(["sleep", String(Math.ceil(sleepMs / 1000))], repoRoot, { timeoutMs: sleepMs + 2_000 });
|
||||
if (wait.exitCode !== 0 && wait.timedOut) {
|
||||
return {
|
||||
ok: false,
|
||||
sourceCommit,
|
||||
attempts: attempt,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
final: summary,
|
||||
observations: observations.slice(-6),
|
||||
degradedReason: "git-mirror-pre-sync-wait-sleep-timeout",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function gitMirrorCacheProbeScript(): string {
|
||||
const runtimeLanesJson = JSON.stringify(hwlabRuntimeLaneIds().map((lane) => {
|
||||
const spec = hwlabRuntimeLaneSpec(lane);
|
||||
return { lane, sourceBranch: spec.sourceBranch, gitopsBranch: spec.gitopsBranch };
|
||||
}));
|
||||
return [
|
||||
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
|
||||
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
|
||||
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
|
||||
"printf 'refs='",
|
||||
"node - <<'NODE' 2>/dev/null || true",
|
||||
"const { execFileSync } = require('node:child_process');",
|
||||
"const repo = '/cache/pikasTech/HWLAB.git';",
|
||||
"function rev(ref) {",
|
||||
" try {",
|
||||
" return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim();",
|
||||
" } catch {",
|
||||
" return null;",
|
||||
" }",
|
||||
"}",
|
||||
`const runtimeLanes = ${runtimeLanesJson};`,
|
||||
"const refs = {};",
|
||||
"for (const spec of runtimeLanes) {",
|
||||
" const key = spec.lane.toUpperCase();",
|
||||
" const normalized = key.slice(0, 1) + key.slice(1).toLowerCase();",
|
||||
" const localSource = rev('refs/heads/' + spec.sourceBranch);",
|
||||
" const githubSource = rev('refs/mirror-stage/heads/' + spec.sourceBranch);",
|
||||
" const localGitops = rev('refs/heads/' + spec.gitopsBranch);",
|
||||
" const githubGitops = rev('refs/mirror-stage/heads/' + spec.gitopsBranch);",
|
||||
" refs['local' + normalized] = localSource;",
|
||||
" refs['github' + normalized] = githubSource;",
|
||||
" refs['local' + normalized + 'Gitops'] = localGitops;",
|
||||
" refs['github' + normalized + 'Gitops'] = githubGitops;",
|
||||
" if (spec.lane === 'v02') {",
|
||||
" refs.localV02 = localSource;",
|
||||
" refs.githubV02 = githubSource;",
|
||||
" refs.localGitops = localGitops;",
|
||||
" refs.githubGitops = githubGitops;",
|
||||
" }",
|
||||
"}",
|
||||
"refs.localG14 = rev('refs/heads/G14');",
|
||||
"refs.githubG14 = rev('refs/mirror-stage/heads/G14');",
|
||||
"const pendingFlush = runtimeLanes.some((spec) => {",
|
||||
" const localGitops = rev('refs/heads/' + spec.gitopsBranch);",
|
||||
" const githubGitops = rev('refs/mirror-stage/heads/' + spec.gitopsBranch);",
|
||||
" return Boolean(localGitops && githubGitops && localGitops !== githubGitops);",
|
||||
"});",
|
||||
"console.log(JSON.stringify({ refs, pendingFlush }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
|
||||
const waitMs = v02GitMirrorPreSyncWaitMs(options.timeoutSeconds);
|
||||
const statusStartMs = Date.now();
|
||||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
|
||||
const before = runGitMirrorStatus();
|
||||
const beforeSummary = compactGitMirrorStatus(before, sourceCommit);
|
||||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||||
stage: "git-mirror-pre-sync-status",
|
||||
status: beforeSummary.ok === true ? "succeeded" : "failed",
|
||||
sourceCommit,
|
||||
durationMs: Date.now() - statusStartMs,
|
||||
required: beforeSummary.required,
|
||||
localV02: beforeSummary.localV02,
|
||||
pendingFlush: beforeSummary.pendingFlush,
|
||||
reason: beforeSummary.reason,
|
||||
});
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
mode: "dry-run",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
action: beforeSummary.required === true ? "would-sync-before-trigger" : "already-current",
|
||||
next: beforeSummary.required === true ? { syncAndTrigger: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" } : undefined,
|
||||
};
|
||||
}
|
||||
if (beforeSummary.required !== true) {
|
||||
return {
|
||||
ok: true,
|
||||
mode: "already-current",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
};
|
||||
}
|
||||
const existingMarker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
||||
if (existingMarker !== null) {
|
||||
return {
|
||||
ok: true,
|
||||
mode: "reused-recent-git-mirror-presync-marker",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
marker: existingMarker,
|
||||
waitMs,
|
||||
};
|
||||
}
|
||||
const lock = acquireV02GitMirrorPreSyncLock(sourceCommit, waitMs, statusStartMs);
|
||||
if (!lock.acquired) {
|
||||
const markerAfterLockTimeout = lock.marker ?? readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
||||
if (markerAfterLockTimeout !== null) {
|
||||
return {
|
||||
ok: true,
|
||||
mode: "waited-for-recent-git-mirror-presync-marker",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
marker: markerAfterLockTimeout,
|
||||
waitedMs: lock.waitedMs,
|
||||
waitMs,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
mode: "local-git-mirror-presync-lock",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
lockDir: lock.lockDir,
|
||||
waitedMs: lock.waitedMs,
|
||||
waitMs,
|
||||
degradedReason: "git-mirror-pre-sync-lock-timeout",
|
||||
};
|
||||
}
|
||||
try {
|
||||
const markerAfterWait = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
||||
if (markerAfterWait !== null) {
|
||||
return {
|
||||
ok: true,
|
||||
mode: "waited-for-recent-git-mirror-presync-marker",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
marker: markerAfterWait,
|
||||
waitedMs: lock.waitedMs,
|
||||
waitMs,
|
||||
};
|
||||
}
|
||||
const recheckStartMs = Date.now();
|
||||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||||
stage: "git-mirror-pre-sync-recheck",
|
||||
status: "started",
|
||||
sourceCommit,
|
||||
waitedMs: lock.waitedMs,
|
||||
});
|
||||
const recheck = runGitMirrorStatus();
|
||||
const recheckSummary = compactGitMirrorStatus(recheck, sourceCommit);
|
||||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||||
stage: "git-mirror-pre-sync-recheck",
|
||||
status: recheckSummary.ok === true ? "succeeded" : "failed",
|
||||
sourceCommit,
|
||||
durationMs: Date.now() - recheckStartMs,
|
||||
required: recheckSummary.required,
|
||||
localV02: recheckSummary.localV02,
|
||||
githubV02: recheckSummary.githubV02,
|
||||
pendingFlush: recheckSummary.pendingFlush,
|
||||
reason: recheckSummary.reason,
|
||||
});
|
||||
if (recheckSummary.ok === true && recheckSummary.required === false) {
|
||||
const marker = writeV02GitMirrorPreSyncMarker(sourceCommit, recheckSummary);
|
||||
return {
|
||||
ok: true,
|
||||
mode: "became-current-after-lock-wait",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
recheck: recheckSummary,
|
||||
marker,
|
||||
waitedMs: lock.waitedMs,
|
||||
waitMs,
|
||||
};
|
||||
}
|
||||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||||
stage: "git-mirror-pre-sync-sync",
|
||||
status: "started",
|
||||
sourceCommit,
|
||||
reason: recheckSummary.reason,
|
||||
localV02: recheckSummary.localV02,
|
||||
pendingFlush: recheckSummary.pendingFlush,
|
||||
});
|
||||
const sync = runGitMirrorSync({
|
||||
action: "sync",
|
||||
lane: "v02",
|
||||
confirm: true,
|
||||
dryRun: false,
|
||||
wait: true,
|
||||
timeoutSeconds: options.timeoutSeconds,
|
||||
});
|
||||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||||
stage: "git-mirror-pre-sync-sync",
|
||||
status: sync.ok === true ? "succeeded" : "failed",
|
||||
sourceCommit,
|
||||
durationMs: sync.elapsedMs ?? null,
|
||||
jobName: sync.jobName ?? null,
|
||||
});
|
||||
const syncStatus = record(sync.status);
|
||||
const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
|
||||
const afterSummary = compactGitMirrorStatus(after, sourceCommit);
|
||||
const wait = sync.ok === true && afterSummary.required === true ? waitForV02GitMirrorPreSync(sourceCommit, waitMs) : null;
|
||||
const waitFinal = wait === null ? null : record(record(wait).final);
|
||||
const finalSummary = wait !== null && wait.ok === true ? waitFinal : afterSummary;
|
||||
const ok = sync.ok === true && finalSummary.required === false;
|
||||
const marker = ok ? writeV02GitMirrorPreSyncMarker(sourceCommit, finalSummary) : null;
|
||||
return {
|
||||
ok,
|
||||
mode: "auto-sync-before-trigger",
|
||||
sourceCommit,
|
||||
before: beforeSummary,
|
||||
recheck: recheckSummary,
|
||||
sync: compactGitMirrorSync(sync),
|
||||
after: afterSummary,
|
||||
wait,
|
||||
final: finalSummary,
|
||||
marker,
|
||||
waitedMs: lock.waitedMs,
|
||||
waitMs,
|
||||
degradedReason: ok ? undefined : stringOrNull(record(wait).degradedReason) ?? "git-mirror-local-v02-not-current-after-sync",
|
||||
};
|
||||
} finally {
|
||||
releaseV02GitMirrorPreSyncLock(lock.lockDir);
|
||||
}
|
||||
}
|
||||
|
||||
export function runGitMirrorStatus(lane: HwlabRuntimeLane = "v02"): Record<string, unknown> {
|
||||
const startedAtMs = Date.now();
|
||||
const spec = hwlabRuntimeLaneSpec(lane);
|
||||
const script = [
|
||||
"set +e",
|
||||
"section() {",
|
||||
" name=\"$1\"",
|
||||
" shift",
|
||||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||||
" \"$@\"",
|
||||
" code=$?",
|
||||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||||
"}",
|
||||
[
|
||||
"section resources kubectl get deploy,svc,pvc,cm",
|
||||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||||
"-l app.kubernetes.io/name=git-mirror",
|
||||
"-o name",
|
||||
].join(" "),
|
||||
[
|
||||
"section legacyCronJob kubectl get cronjob",
|
||||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||||
shellQuote(GIT_MIRROR_LEGACY_CRONJOB),
|
||||
"--ignore-not-found",
|
||||
"-o name",
|
||||
].join(" "),
|
||||
[
|
||||
`section jobs kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||||
"-l app.kubernetes.io/name=git-mirror",
|
||||
"--sort-by=.metadata.creationTimestamp",
|
||||
"-o custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime",
|
||||
"--no-headers",
|
||||
].join(" "),
|
||||
[
|
||||
"section cache kubectl exec",
|
||||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||||
"deploy/git-mirror-http",
|
||||
"-- sh -lc",
|
||||
shellQuote(gitMirrorCacheProbeScript()),
|
||||
].join(" "),
|
||||
].join("\n");
|
||||
const bundle = g14K3s(["sh", "--", script], 60_000);
|
||||
const sections = parseShellSections(statusText(bundle));
|
||||
const resources = sections.resources;
|
||||
const legacyCronJob = sections.legacyCronJob;
|
||||
const jobs = sections.jobs;
|
||||
const cache = sections.cache;
|
||||
const bundleStderr = bundle.stderr.trim().slice(0, 2000);
|
||||
return {
|
||||
ok: isCommandSuccess(bundle) && shellSectionOk(resources),
|
||||
command: `hwlab g14 git-mirror status --lane ${lane}`,
|
||||
lane,
|
||||
namespace: GIT_MIRROR_NAMESPACE,
|
||||
readUrl: V02_GIT_READ_URL,
|
||||
writeUrl: V02_GIT_WRITE_URL,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
resources: {
|
||||
ok: shellSectionOk(resources),
|
||||
names: String(resources?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||||
stderr: shellSectionOk(resources) ? "" : bundleStderr,
|
||||
},
|
||||
legacyCronJob: {
|
||||
ok: shellSectionOk(legacyCronJob),
|
||||
exists: String(legacyCronJob?.stdout ?? "").trim().length > 0,
|
||||
name: String(legacyCronJob?.stdout ?? "").trim() || null,
|
||||
cleanupCommand: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||||
},
|
||||
jobs: {
|
||||
ok: shellSectionOk(jobs),
|
||||
raw: String(jobs?.stdout ?? "").trim(),
|
||||
stderr: shellSectionOk(jobs) ? "" : bundleStderr,
|
||||
},
|
||||
cache: {
|
||||
ok: shellSectionOk(cache),
|
||||
summary: gitMirrorStatusSummary(String(cache?.stdout ?? "").trim()),
|
||||
raw: String(cache?.stdout ?? "").trim(),
|
||||
stderr: shellSectionOk(cache) ? "" : bundleStderr,
|
||||
},
|
||||
next: {
|
||||
apply: `bun scripts/cli.ts hwlab g14 git-mirror apply --lane ${spec.lane} --confirm`,
|
||||
sync: `bun scripts/cli.ts hwlab g14 git-mirror sync --lane ${spec.lane} --confirm`,
|
||||
flush: `bun scripts/cli.ts hwlab g14 git-mirror flush --lane ${spec.lane} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
|
||||
const spec = hwlabRuntimeLaneSpec(options.lane);
|
||||
const head = resolveRuntimeLaneHead(spec);
|
||||
const sourceCommit = head.sourceCommit;
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: `hwlab g14 git-mirror apply --lane ${spec.lane}`, lane: spec.lane, degradedReason: `${spec.lane}-head-unresolved`, sourceRepo: spec.cicdRepo, workspace: spec.workspace, headProbe: compactCommandResult(head.result) };
|
||||
}
|
||||
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
|
||||
if (!isCommandSuccess(render.result)) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab g14 git-mirror apply --lane ${spec.lane}`,
|
||||
lane: spec.lane,
|
||||
phase: "source-render",
|
||||
sourceCommit,
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
};
|
||||
}
|
||||
const apply = applyGitMirrorManifestFile(render.renderDir, options.dryRun, options.timeoutSeconds);
|
||||
const cleanupLegacyCronJob = isCommandSuccess(apply)
|
||||
? deleteLegacyGitMirrorCronJob(options.dryRun)
|
||||
: {
|
||||
ok: false,
|
||||
command: [],
|
||||
exitCode: null,
|
||||
stdout: "",
|
||||
stderr: "skipped because apply failed",
|
||||
parsed: null,
|
||||
};
|
||||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
|
||||
return {
|
||||
ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob),
|
||||
command: `hwlab g14 git-mirror apply --lane ${spec.lane}`,
|
||||
lane: spec.lane,
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||||
sourceCommit,
|
||||
manifest: `${render.renderDir}/devops-infra/git-mirror.yaml`,
|
||||
render: compactCommandResult(render.result),
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
|
||||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||||
status: runGitMirrorStatus(spec.lane),
|
||||
next: options.dryRun
|
||||
? { apply: `bun scripts/cli.ts hwlab g14 git-mirror apply --lane ${spec.lane} --confirm` }
|
||||
: { sync: `bun scripts/cli.ts hwlab g14 git-mirror sync --lane ${spec.lane} --confirm` },
|
||||
};
|
||||
}
|
||||
|
||||
export function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown> {
|
||||
const startedAtMs = Date.now();
|
||||
const spec = hwlabRuntimeLaneSpec(options.lane);
|
||||
const jobName = gitMirrorSyncJobName();
|
||||
const manifest = gitMirrorSyncJobManifest(jobName);
|
||||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`job=${shellQuote(jobName)}`,
|
||||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||||
"manifest_path=\"/tmp/$job.json\"",
|
||||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||||
options.dryRun
|
||||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||||
: [
|
||||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||||
"kubectl create -f \"$manifest_path\"",
|
||||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||||
"while :; do",
|
||||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||||
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||||
" exit 44",
|
||||
" fi",
|
||||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||||
" exit 45",
|
||||
" fi",
|
||||
" sleep 2",
|
||||
"done",
|
||||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(`cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/${spec.sourceBranch} refs/mirror-stage/heads/${spec.sourceBranch} refs/heads/${spec.gitopsBranch} refs/mirror-stage/heads/${spec.gitopsBranch} 2>/dev/null || true`)}`,
|
||||
].join("\n"),
|
||||
].join("\n");
|
||||
const result = g14K3s(["sh", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||||
const syncCommand = spec.lane === "v02"
|
||||
? "bun scripts/cli.ts hwlab g14 git-mirror sync --lane v02 --confirm"
|
||||
: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`;
|
||||
const triggerCommand = spec.lane === "v02"
|
||||
? "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm"
|
||||
: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`;
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
command: spec.lane === "v02" ? "hwlab g14 git-mirror sync" : "hwlab nodes git-mirror sync",
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-sync",
|
||||
namespace: GIT_MIRROR_NAMESPACE,
|
||||
jobName,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
manifest: options.dryRun ? manifest : undefined,
|
||||
result: compactCommandResult(result),
|
||||
status: options.dryRun ? undefined : runGitMirrorStatus(spec.lane),
|
||||
next: options.dryRun ? { sync: syncCommand } : { triggerCurrent: triggerCommand },
|
||||
};
|
||||
}
|
||||
|
||||
export function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown> {
|
||||
const jobName = gitMirrorFlushJobName();
|
||||
const manifest = gitMirrorFlushJobManifest(jobName);
|
||||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`job=${shellQuote(jobName)}`,
|
||||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||||
"manifest_path=\"/tmp/$job.json\"",
|
||||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||||
options.dryRun
|
||||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||||
: [
|
||||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||||
"kubectl create -f \"$manifest_path\"",
|
||||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||||
"while :; do",
|
||||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||||
" succeeded=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||||
" failed=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||||
" exit 54",
|
||||
" fi",
|
||||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||||
" exit 55",
|
||||
" fi",
|
||||
" sleep 2",
|
||||
"done",
|
||||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(`cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/${options.lane === "v02" ? "v0.2-gitops" : hwlabRuntimeLaneSpec(options.lane).gitopsBranch} refs/mirror-stage/heads/${options.lane === "v02" ? "v0.2-gitops" : hwlabRuntimeLaneSpec(options.lane).gitopsBranch} 2>/dev/null || true`)}`,
|
||||
].join("\n"),
|
||||
].join("\n");
|
||||
const result = g14K3s(["sh", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
command: "hwlab g14 git-mirror flush",
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-flush",
|
||||
namespace: GIT_MIRROR_NAMESPACE,
|
||||
jobName,
|
||||
manifest: options.dryRun ? manifest : undefined,
|
||||
result: compactCommandResult(result),
|
||||
status: options.dryRun ? undefined : runGitMirrorStatus(options.lane),
|
||||
next: options.dryRun
|
||||
? { flush: `bun scripts/cli.ts hwlab g14 git-mirror flush --lane ${options.lane} --confirm` }
|
||||
: { status: `bun scripts/cli.ts hwlab g14 git-mirror status --lane ${options.lane}` },
|
||||
};
|
||||
}
|
||||
|
||||
export function runG14GitMirror(options: G14GitMirrorOptions): Record<string, unknown> {
|
||||
if (options.action === "status") return runGitMirrorStatus(options.lane);
|
||||
if (options.action === "apply") return runGitMirrorApply(options);
|
||||
if (options.action === "flush") return runGitMirrorFlush(options);
|
||||
return runGitMirrorSync(options);
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. help module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:10421-10618 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { G14MonitorOptions } from "./types";
|
||||
import { hwlabG14MonitorStateFileName, hwlabG14MonitorStateRole } from "./options";
|
||||
import { monitorBaseBranch, record } from "./remote";
|
||||
import { legacyG14RetirementBlocksMonitor, legacyG14RetirementStatePath } from "./retirement";
|
||||
import { DEFAULT_INTERVAL_SECONDS, DEV_APP, G14_BRIEF_INDEX_ISSUE, G14_CI_TOOLS_IMAGE_REPO, G14_OBSERVABILITY_NAMESPACE, G14_PROMETHEUS_OPERATOR_VERSION, G14_PROMETHEUS_VERSION, G14_PROVIDER, G14_SOURCE_BRANCH, G14_WORKSPACE, HWLAB_REPO, PROD_APP, V02_APP, V02_OBSERVABILITY_EXPECTED_TARGET_COUNT, V02_SOURCE_BRANCH, V02_WORKSPACE } from "./types";
|
||||
|
||||
export function hwlabG14Help(): Record<string, unknown> {
|
||||
return {
|
||||
command: "hwlab g14",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --status",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03 --status",
|
||||
"bun scripts/cli.ts hwlab g14 retirement status",
|
||||
"bun scripts/cli.ts hwlab g14 retirement plan",
|
||||
"bun scripts/cli.ts hwlab g14 retirement execute --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --once --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03 --once --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --history",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit <full-sha>",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit <full-sha>",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm --wait",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha> --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --source-commit <full-sha> --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run hwlab-v03-ci-poll-<short-sha> --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --source-commit <full-sha> --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-openfga",
|
||||
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
|
||||
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-master-server-admin-api-key",
|
||||
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-master-server-admin-api-key --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key",
|
||||
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
|
||||
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
|
||||
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 git-mirror status",
|
||||
"bun scripts/cli.ts hwlab g14 git-mirror apply --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes git-mirror apply --node G14 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait",
|
||||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm --wait",
|
||||
"bun scripts/cli.ts hwlab g14 observability status",
|
||||
"bun scripts/cli.ts hwlab g14 observability apply --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 observability apply --confirm",
|
||||
`bun scripts/cli.ts hwlab g14 observability query --promql 'up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
||||
`bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
||||
`bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_health_probe_success{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
||||
"bun scripts/cli.ts hwlab g14 observability targets --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 observability boundary --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 observability closeout --lane v02",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 upstream-image status --name openfga --tag v1.17.0",
|
||||
"bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --confirm",
|
||||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||||
],
|
||||
description: "G14 HWLAB PR monitor, legacy DEV/PROD retirement status/plan/execute command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, node-scoped runtime lane v03 control-plane apply/status/refresh/trigger entry, runtime lane SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The legacy base=G14 monitor is blocked by the retirement contract; the local retirement marker records live execution evidence. `--lane v02` monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. `--lane v03` monitors base=v0.3 PRs through the runtime lane control-plane, waits for GitHub preflight/CI readiness, auto-merges ready non-conflicting PRs, triggers v0.3 CD, verifies PipelineRun/Argo/runtime public probes/Git mirror flush, posts deduplicated PR comments, and creates or updates failure issues for failed checks, conflicts, merge blockers, CD failures, and timeouts. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ is advertised through `hwlab nodes ... --node <node-id> --lane vNN` so node identity remains configuration data instead of a command family. secret status/ensure is the standard runtime lane SecretRef bootstrap path for OpenFGA, cloud-api DB, master admin API key, and code-agent provider refs; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.",
|
||||
defaults: {
|
||||
repo: HWLAB_REPO,
|
||||
legacyBase: G14_SOURCE_BRANCH,
|
||||
v02Base: V02_SOURCE_BRANCH,
|
||||
runtimeLaneConfig: hwlabRuntimeLaneConfigPath(),
|
||||
runtimeLanes: hwlabRuntimeLaneIds(),
|
||||
provider: G14_PROVIDER,
|
||||
legacyWorkspace: G14_WORKSPACE,
|
||||
v02Workspace: V02_WORKSPACE,
|
||||
v03Workspace: hwlabRuntimeLaneSpec("v03").workspace,
|
||||
ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO,
|
||||
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
|
||||
devApplication: DEV_APP,
|
||||
prodApplication: PROD_APP,
|
||||
legacyRetirementState: legacyG14RetirementStatePath(),
|
||||
v02Application: V02_APP,
|
||||
v03Application: hwlabRuntimeLaneSpec("v03").app,
|
||||
v03Node: hwlabRuntimeLaneSpec("v03").nodeId,
|
||||
v03NetworkProfile: hwlabRuntimeLaneSpec("v03").networkProfileId,
|
||||
v03DownloadProfile: hwlabRuntimeLaneSpec("v03").downloadProfileId,
|
||||
requiredNoProxy: hwlabRequiredNoProxyEntries(),
|
||||
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
|
||||
observabilityNamespace: G14_OBSERVABILITY_NAMESPACE,
|
||||
prometheusOperatorVersion: G14_PROMETHEUS_OPERATOR_VERSION,
|
||||
prometheusVersion: G14_PROMETHEUS_VERSION,
|
||||
},
|
||||
stateFiles: {
|
||||
monitor: ".state/hwlab-g14/latest-monitor-job.json",
|
||||
once: ".state/hwlab-g14/latest-once-job.json",
|
||||
dryRun: ".state/hwlab-g14/latest-dry-run-job.json",
|
||||
onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json",
|
||||
v02Monitor: ".state/hwlab-g14/latest-v02-monitor-job.json",
|
||||
v02Once: ".state/hwlab-g14/latest-v02-once-job.json",
|
||||
v02DryRun: ".state/hwlab-g14/latest-v02-dry-run-job.json",
|
||||
v02OnceDryRun: ".state/hwlab-g14/latest-v02-once-dry-run-job.json",
|
||||
v02PrCommentSignatures: ".state/hwlab-g14/v02-pr-comment-signatures.json",
|
||||
v03Monitor: ".state/hwlab-g14/latest-v03-monitor-job.json",
|
||||
v03Once: ".state/hwlab-g14/latest-v03-once-job.json",
|
||||
v03DryRun: ".state/hwlab-g14/latest-v03-dry-run-job.json",
|
||||
v03OnceDryRun: ".state/hwlab-g14/latest-v03-once-dry-run-job.json",
|
||||
v03PrCommentSignatures: ".state/hwlab-g14/v03-pr-comment-signatures.json",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function monitorStatus(options: G14MonitorOptions): Record<string, unknown> {
|
||||
const stateDir = rootPath(".state", "hwlab-g14");
|
||||
const stateFileName = hwlabG14MonitorStateFileName(options);
|
||||
const stateFileRole = hwlabG14MonitorStateRole(options);
|
||||
const latestPath = join(stateDir, stateFileName);
|
||||
const retirement = options.lane === "g14" ? legacyG14RetirementBlocksMonitor() : null;
|
||||
const exists = existsSync(latestPath);
|
||||
let latest: Record<string, unknown> | null = null;
|
||||
let job: Record<string, unknown> | null = null;
|
||||
let statusCommand: string | null = null;
|
||||
let degradedReason: string | null = null;
|
||||
if (exists) {
|
||||
try {
|
||||
latest = record(JSON.parse(readFileSync(latestPath, "utf8")) as unknown);
|
||||
const jobId = typeof latest.jobId === "string" ? latest.jobId : null;
|
||||
if (jobId !== null) {
|
||||
statusCommand = `bun scripts/cli.ts job status ${jobId} --tail-bytes 30000`;
|
||||
try {
|
||||
const current = readJob(jobId);
|
||||
job = {
|
||||
id: current.id,
|
||||
name: current.name,
|
||||
status: current.status,
|
||||
createdAt: current.createdAt,
|
||||
startedAt: current.startedAt,
|
||||
finishedAt: current.finishedAt,
|
||||
exitCode: current.exitCode,
|
||||
runnerPid: current.runnerPid ?? null,
|
||||
stdoutFile: current.stdoutFile,
|
||||
stderrFile: current.stderrFile,
|
||||
note: current.note,
|
||||
};
|
||||
} catch (error) {
|
||||
degradedReason = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
degradedReason = `failed to parse ${latestPath}: ${error instanceof Error ? error.message : String(error)}`;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: degradedReason === null,
|
||||
command: "hwlab g14 monitor-prs --status",
|
||||
lane: options.lane,
|
||||
baseBranch: monitorBaseBranch(options.lane),
|
||||
mode: "status",
|
||||
latestPath,
|
||||
stateFileName,
|
||||
stateFileRole,
|
||||
exists,
|
||||
latest,
|
||||
job,
|
||||
statusCommand,
|
||||
degradedReason,
|
||||
retirement,
|
||||
next: retirement !== null ? {
|
||||
retirementStatus: "bun scripts/cli.ts hwlab g14 retirement status",
|
||||
v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
|
||||
v03Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
|
||||
} : statusCommand === null ? {
|
||||
start: `bun scripts/cli.ts hwlab g14 monitor-prs --lane ${options.lane}`,
|
||||
} : {
|
||||
status: statusCommand,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. images module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:8075-8386 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult, G14ControlPlaneOptions, G14GitMirrorOptions, G14ToolsImageOptions, G14UpstreamImageOptions } from "./types";
|
||||
import { statusText } from "./pr-monitor";
|
||||
import { cliJson, isCommandSuccess, record, shellQuote } from "./remote";
|
||||
import { G14_CI_TOOLS_IMAGE_REPO, G14_PROVIDER, GIT_MIRROR_NAMESPACE, V02_REGISTRY_PREFIX, V02_WORKSPACE } from "./types";
|
||||
|
||||
export function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record<string, unknown> {
|
||||
const job = startJob(name, command, note);
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
||||
return {
|
||||
ok: true,
|
||||
mode: "async-job",
|
||||
job,
|
||||
statusCommand,
|
||||
tailCommand: `tail -f ${job.stdoutFile}`,
|
||||
next: {
|
||||
status: statusCommand,
|
||||
tail: `tail -f ${job.stdoutFile}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function startControlPlaneTriggerJob(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const command = [
|
||||
"bun",
|
||||
"scripts/cli.ts",
|
||||
"hwlab",
|
||||
"g14",
|
||||
"control-plane",
|
||||
"trigger-current",
|
||||
"--lane",
|
||||
options.lane,
|
||||
"--confirm",
|
||||
"--timeout-seconds",
|
||||
String(options.timeoutSeconds),
|
||||
"--wait",
|
||||
];
|
||||
if (options.rerun) command.splice(command.length - 1, 0, "--rerun");
|
||||
return {
|
||||
command: `hwlab g14 control-plane trigger-current --lane ${options.lane}`,
|
||||
lane: options.lane,
|
||||
reason: "confirmed trigger can spend tens of seconds syncing git mirror and creating PipelineRun; default is fire-and-forget to avoid silent blocking",
|
||||
waitCommand: command.join(" "),
|
||||
...startAsyncHwlabG14Job(
|
||||
`hwlab_g14_${options.lane}_trigger_current`,
|
||||
command,
|
||||
`Trigger HWLAB ${options.lane} current commit PipelineRun with git mirror pre-sync through G14 control-plane`,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function startGitMirrorJob(options: G14GitMirrorOptions): Record<string, unknown> {
|
||||
const command = [
|
||||
"bun",
|
||||
"scripts/cli.ts",
|
||||
"hwlab",
|
||||
"g14",
|
||||
"git-mirror",
|
||||
options.action,
|
||||
"--lane",
|
||||
options.lane,
|
||||
"--confirm",
|
||||
"--timeout-seconds",
|
||||
String(options.timeoutSeconds),
|
||||
"--wait",
|
||||
];
|
||||
return {
|
||||
command: `hwlab g14 git-mirror ${options.action} --lane ${options.lane}`,
|
||||
lane: options.lane,
|
||||
namespace: GIT_MIRROR_NAMESPACE,
|
||||
reason: "manual git mirror sync/flush waits for a Kubernetes Job; default is fire-and-forget to keep CLI output immediately visible",
|
||||
waitCommand: command.join(" "),
|
||||
...startAsyncHwlabG14Job(
|
||||
`hwlab_g14_git_mirror_${options.action}`,
|
||||
command,
|
||||
`Run HWLAB devops-infra git mirror ${options.action} through a bounded manual Kubernetes Job`,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||||
return cliJson(["ssh", G14_PROVIDER, "sh", "--", script], timeoutMs);
|
||||
}
|
||||
|
||||
export function g14CiToolsImage(tag: string): string {
|
||||
return `${G14_CI_TOOLS_IMAGE_REPO}:${tag}`;
|
||||
}
|
||||
|
||||
export function runG14ToolsImageStatus(options: G14ToolsImageOptions): Record<string, unknown> {
|
||||
const image = g14CiToolsImage(options.tag);
|
||||
const script = [
|
||||
"set -u",
|
||||
`image=${shellQuote(image)}`,
|
||||
"exists=false",
|
||||
"image_id=",
|
||||
"created=",
|
||||
"size=",
|
||||
"node_version=",
|
||||
"npm_version=",
|
||||
"bun_version=",
|
||||
"git_version=",
|
||||
"docker_version=",
|
||||
"python_version=",
|
||||
"registry_tags=",
|
||||
"if docker image inspect \"$image\" >/tmp/hwlab-tools-image-inspect.json 2>/tmp/hwlab-tools-image-inspect.err; then",
|
||||
" exists=true",
|
||||
" image_id=$(docker image inspect \"$image\" --format '{{.Id}}' 2>/dev/null || true)",
|
||||
" created=$(docker image inspect \"$image\" --format '{{.Created}}' 2>/dev/null || true)",
|
||||
" size=$(docker image inspect \"$image\" --format '{{.Size}}' 2>/dev/null || true)",
|
||||
" node_version=$(docker run --rm \"$image\" node --version 2>/dev/null || true)",
|
||||
" npm_version=$(docker run --rm \"$image\" npm --version 2>/dev/null || true)",
|
||||
" bun_version=$(docker run --rm \"$image\" bun --version 2>/dev/null || true)",
|
||||
" git_version=$(docker run --rm \"$image\" git --version 2>/dev/null || true)",
|
||||
" docker_version=$(docker run --rm \"$image\" docker --version 2>/dev/null || true)",
|
||||
" python_version=$(docker run --rm \"$image\" python3 --version 2>/dev/null || true)",
|
||||
"fi",
|
||||
`registry_tags=$(curl -fsS --max-time 10 http://127.0.0.1:5000/v2/hwlab/hwlab-ci-node-tools/tags/list 2>/dev/null || true)`,
|
||||
"export image exists image_id created size node_version npm_version bun_version git_version docker_version python_version registry_tags",
|
||||
"node - <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const env = process.env;",
|
||||
"const payload = {",
|
||||
" image: env.image,",
|
||||
" exists: env.exists === 'true',",
|
||||
" imageId: env.image_id || null,",
|
||||
" created: env.created || null,",
|
||||
" sizeBytes: env.size ? Number(env.size) : null,",
|
||||
" tools: {",
|
||||
" node: env.node_version || null,",
|
||||
" npm: env.npm_version || null,",
|
||||
" bun: env.bun_version || null,",
|
||||
" git: env.git_version || null,",
|
||||
" docker: env.docker_version || null,",
|
||||
" python: env.python_version || null",
|
||||
" },",
|
||||
" registryTagsRaw: env.registry_tags || null",
|
||||
"};",
|
||||
"console.log(JSON.stringify(payload, null, 2));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = g14HostScript(script, 180_000);
|
||||
let parsedStatus: unknown = null;
|
||||
const text = statusText(result);
|
||||
if (text.length > 0) {
|
||||
try {
|
||||
parsedStatus = JSON.parse(text) as unknown;
|
||||
} catch {
|
||||
parsedStatus = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: isCommandSuccess(result) && record(parsedStatus).exists === true,
|
||||
command: "hwlab g14 tools-image status --name ci-node-tools",
|
||||
name: options.name,
|
||||
image,
|
||||
status: parsedStatus ?? text,
|
||||
result,
|
||||
};
|
||||
}
|
||||
|
||||
export function runG14ToolsImageBuild(options: G14ToolsImageOptions): Record<string, unknown> {
|
||||
const image = g14CiToolsImage(options.tag);
|
||||
const script = [
|
||||
"set -eu",
|
||||
`cd ${shellQuote(V02_WORKSPACE)}`,
|
||||
"git fetch origin v0.2 --prune",
|
||||
"git checkout v0.2 >/dev/null 2>&1 || true",
|
||||
"git merge --ff-only origin/v0.2",
|
||||
`test -f ${shellQuote(options.dockerfile)}`,
|
||||
`image=${shellQuote(image)}`,
|
||||
`dockerfile=${shellQuote(options.dockerfile)}`,
|
||||
"echo \"{\\\"phase\\\":\\\"preflight\\\",\\\"image\\\":\\\"$image\\\",\\\"dockerfile\\\":\\\"$dockerfile\\\",\\\"commit\\\":\\\"$(git rev-parse HEAD)\\\"}\"",
|
||||
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
|
||||
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
|
||||
"export no_proxy=$NO_PROXY",
|
||||
"docker build --pull --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg http_proxy --build-arg https_proxy --build-arg NO_PROXY --build-arg no_proxy -f \"$dockerfile\" -t \"$image\" .",
|
||||
"docker run --rm \"$image\" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V'",
|
||||
"docker push \"$image\"",
|
||||
"digest=$(docker image inspect \"$image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
|
||||
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"image\\\":\\\"$image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
|
||||
].join("\n");
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||||
mode: "dry-run",
|
||||
image,
|
||||
workspace: V02_WORKSPACE,
|
||||
dockerfile: options.dockerfile,
|
||||
buildScriptPreview: script.split(/\n/u),
|
||||
next: { build: `bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag ${options.tag} --confirm` },
|
||||
};
|
||||
}
|
||||
const result = g14HostScript(script, options.timeoutSeconds * 1000);
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||||
mode: "confirmed-build",
|
||||
image,
|
||||
workspace: V02_WORKSPACE,
|
||||
dockerfile: options.dockerfile,
|
||||
result,
|
||||
status: runG14ToolsImageStatus({ ...options, action: "status", dryRun: true, confirm: false }),
|
||||
};
|
||||
}
|
||||
|
||||
export function runG14ToolsImage(options: G14ToolsImageOptions): Record<string, unknown> {
|
||||
if (options.action === "status") return runG14ToolsImageStatus(options);
|
||||
return runG14ToolsImageBuild(options);
|
||||
}
|
||||
|
||||
export function g14UpstreamImageTarget(options: G14UpstreamImageOptions): string {
|
||||
return `${V02_REGISTRY_PREFIX}/${options.name}:${options.tag}`;
|
||||
}
|
||||
|
||||
export function g14UpstreamImageSource(options: G14UpstreamImageOptions): string {
|
||||
if (options.name === "openfga") return `docker.io/openfga/openfga:${options.tag}`;
|
||||
return `${options.name}:${options.tag}`;
|
||||
}
|
||||
|
||||
export function runG14UpstreamImageStatus(options: G14UpstreamImageOptions): Record<string, unknown> {
|
||||
const sourceImage = g14UpstreamImageSource(options);
|
||||
const targetImage = g14UpstreamImageTarget(options);
|
||||
const script = [
|
||||
"set +e",
|
||||
`source_image=${shellQuote(sourceImage)}`,
|
||||
`target_image=${shellQuote(targetImage)}`,
|
||||
`repo_path=${shellQuote(`hwlab/${options.name}`)}`,
|
||||
`tag=${shellQuote(options.tag)}`,
|
||||
"local_id=$(docker image inspect \"$target_image\" --format '{{.Id}}' 2>/dev/null || true)",
|
||||
"local_created=$(docker image inspect \"$target_image\" --format '{{.Created}}' 2>/dev/null || true)",
|
||||
"registry_tags=$(curl -fsS --max-time 10 \"http://127.0.0.1:5000/v2/$repo_path/tags/list\" 2>/dev/null || true)",
|
||||
"registry_has_tag=false",
|
||||
"if printf '%s' \"$registry_tags\" | grep -F '\"'$tag'\"' >/dev/null 2>&1; then registry_has_tag=true; fi",
|
||||
"export source_image target_image local_id local_created registry_tags registry_has_tag",
|
||||
"node <<'NODE'",
|
||||
"const env = process.env;",
|
||||
"console.log(JSON.stringify({",
|
||||
" sourceImage: env.source_image,",
|
||||
" targetImage: env.target_image,",
|
||||
" localExists: Boolean(env.local_id),",
|
||||
" localImageId: env.local_id || null,",
|
||||
" localCreated: env.local_created || null,",
|
||||
" registryHasTag: env.registry_has_tag === 'true',",
|
||||
" registryTagsRaw: env.registry_tags || null",
|
||||
"}, null, 2));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = g14HostScript(script, 120_000);
|
||||
let parsedStatus: unknown = null;
|
||||
const text = statusText(result);
|
||||
if (text.length > 0) {
|
||||
try {
|
||||
parsedStatus = JSON.parse(text) as unknown;
|
||||
} catch {
|
||||
parsedStatus = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: isCommandSuccess(result) && record(parsedStatus).registryHasTag === true,
|
||||
command: "hwlab g14 upstream-image status --name openfga",
|
||||
name: options.name,
|
||||
tag: options.tag,
|
||||
sourceImage,
|
||||
targetImage,
|
||||
status: parsedStatus ?? text,
|
||||
result,
|
||||
};
|
||||
}
|
||||
|
||||
export function runG14UpstreamImageEnsure(options: G14UpstreamImageOptions): Record<string, unknown> {
|
||||
const sourceImage = g14UpstreamImageSource(options);
|
||||
const targetImage = g14UpstreamImageTarget(options);
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 upstream-image ensure --name openfga",
|
||||
mode: "dry-run",
|
||||
name: options.name,
|
||||
tag: options.tag,
|
||||
sourceImage,
|
||||
targetImage,
|
||||
mutation: false,
|
||||
next: { confirm: `bun scripts/cli.ts hwlab g14 upstream-image ensure --name ${options.name} --tag ${options.tag} --confirm` },
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`source_image=${shellQuote(sourceImage)}`,
|
||||
`target_image=${shellQuote(targetImage)}`,
|
||||
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
|
||||
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
|
||||
"export no_proxy=$NO_PROXY",
|
||||
"echo \"{\\\"phase\\\":\\\"pull\\\",\\\"sourceImage\\\":\\\"$source_image\\\"}\"",
|
||||
"docker pull \"$source_image\"",
|
||||
"docker tag \"$source_image\" \"$target_image\"",
|
||||
"echo \"{\\\"phase\\\":\\\"push\\\",\\\"targetImage\\\":\\\"$target_image\\\"}\"",
|
||||
"docker push \"$target_image\"",
|
||||
"digest=$(docker image inspect \"$target_image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
|
||||
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"targetImage\\\":\\\"$target_image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
|
||||
].join("\n");
|
||||
const result = g14HostScript(script, options.timeoutSeconds * 1000);
|
||||
const status = runG14UpstreamImageStatus(options);
|
||||
return {
|
||||
ok: isCommandSuccess(result) && status.ok === true,
|
||||
command: "hwlab g14 upstream-image ensure --name openfga",
|
||||
mode: "confirmed-ensure",
|
||||
name: options.name,
|
||||
tag: options.tag,
|
||||
sourceImage,
|
||||
targetImage,
|
||||
mutation: true,
|
||||
result,
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
export function runG14UpstreamImage(options: G14UpstreamImageOptions): Record<string, unknown> {
|
||||
if (options.action === "status") return runG14UpstreamImageStatus(options);
|
||||
return runG14UpstreamImageEnsure(options);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. Domain barrel for scripts/src/hwlab-g14.ts.
|
||||
export * from "./types";
|
||||
export * from "./options";
|
||||
export * from "./remote";
|
||||
export * from "./source";
|
||||
export * from "./v02-status";
|
||||
export * from "./cleanup";
|
||||
export * from "./render";
|
||||
export * from "./control-plane";
|
||||
export * from "./secrets";
|
||||
export * from "./retirement";
|
||||
export * from "./git-mirror";
|
||||
export * from "./observability";
|
||||
export * from "./images";
|
||||
export * from "./pr-monitor";
|
||||
export * from "./cd-trigger";
|
||||
export * from "./monitor";
|
||||
export * from "./help";
|
||||
export * from "./entry";
|
||||
@@ -0,0 +1,276 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. monitor module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:10163-10420 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult, G14MonitorOptions } from "./types";
|
||||
import { runRuntimeLanePrAutoCd, runV02PrAutoCd } from "./cd-trigger";
|
||||
import { appendRolloutBrief, collectPipelineMetrics, commentRuntimeLanePullRequest, commentV02PullRequest, durationSeconds, getArgoStatus, getDevWorkloads, getG14Head, getLiveHealth, getPipelineStatus, listOpenG14PullRequests, mergePullRequest, preflightPullRequest, preflightSummary, refreshArgoDev, reportRuntimeLaneAutomationIssue, statusText, v02PrConflictState, workloadReadiness } from "./pr-monitor";
|
||||
import { commandData, compactCommandResult, extractPullRequests, isCommandSuccess, nested, precheckWorkspace, printEvent, printRuntimeLanePrMonitorProgress, printV02PrMonitorProgress, record, shortSha, sleep } from "./remote";
|
||||
import { legacyG14RetirementBlocksMonitor } from "./retirement";
|
||||
import { V02_SOURCE_BRANCH } from "./types";
|
||||
|
||||
export async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const started = Date.now();
|
||||
const startedAt = new Date(started).toISOString();
|
||||
let pipelineText = "";
|
||||
let argoText = "";
|
||||
let healthOk = false;
|
||||
let workloadsReady: Record<string, unknown> = { ready: false };
|
||||
let pipelineSucceededAt: string | null = null;
|
||||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||||
const pipeline = getPipelineStatus(sourceCommit);
|
||||
pipelineText = statusText(pipeline);
|
||||
printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) });
|
||||
if (!pipelineText.startsWith("True\nSucceeded")) {
|
||||
if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText };
|
||||
await sleep(30_000);
|
||||
continue;
|
||||
}
|
||||
pipelineSucceededAt ??= new Date().toISOString();
|
||||
refreshArgoDev();
|
||||
const argo = getArgoStatus();
|
||||
argoText = statusText(argo);
|
||||
printEvent("g14.argo.status", { text: argoText.slice(0, 500) });
|
||||
const argoLines = argoText.split(/\r?\n/u);
|
||||
const synced = argoLines[1] === "Synced";
|
||||
const healthy = argoLines[2] === "Healthy";
|
||||
const gitopsRevision = argoLines[0] ?? null;
|
||||
const workloads = getDevWorkloads();
|
||||
const workloadsText = statusText(workloads);
|
||||
workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads));
|
||||
const health = getLiveHealth();
|
||||
healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout);
|
||||
if (synced && healthy && record(workloadsReady).ready === true && healthOk) {
|
||||
const finishedAt = new Date().toISOString();
|
||||
const pipelineRun = `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||||
const ciMetrics = collectPipelineMetrics(pipelineRun);
|
||||
return { ok: true, sourceCommit, gitopsRevision, pipelineRun, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk, ciMetrics };
|
||||
}
|
||||
await sleep(30_000);
|
||||
}
|
||||
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
|
||||
}
|
||||
|
||||
export async function monitorV02Cycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||||
printEvent("v02.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||||
printV02PrMonitorProgress({ stage: "cycle", status: "started", cycle, dryRun: options.dryRun });
|
||||
const listed = listOpenG14PullRequests();
|
||||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||||
const prs = extractPullRequests(listed, V02_SOURCE_BRANCH);
|
||||
printEvent("v02.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||||
if (prs.length === 0) {
|
||||
printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, pullRequests: 0, intervalSeconds: options.intervalSeconds });
|
||||
return { ok: true, cycle, action: "none", lane: "v02", pullRequests: [] };
|
||||
}
|
||||
const observations: unknown[] = [];
|
||||
for (const pr of prs) {
|
||||
const startedAt = new Date().toISOString();
|
||||
const preflightResult = preflightPullRequest(pr.number);
|
||||
const preflight = preflightSummary(preflightResult);
|
||||
printEvent("v02.pr.preflight", {
|
||||
cycle,
|
||||
number: pr.number,
|
||||
ok: isCommandSuccess(preflightResult),
|
||||
conclusion: preflight.conclusion,
|
||||
readyForCommanderMerge: preflight.readyForCommanderMerge,
|
||||
conflict: v02PrConflictState(preflight),
|
||||
});
|
||||
printV02PrMonitorProgress({
|
||||
stage: "preflight",
|
||||
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
|
||||
pr: pr.number,
|
||||
conclusion: preflight.conclusion,
|
||||
conflict: v02PrConflictState(preflight),
|
||||
});
|
||||
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
|
||||
const conclusion = String(preflight.conclusion ?? "unknown");
|
||||
const blocked = conclusion === "blocked" || v02PrConflictState(preflight) === "conflict" || !isCommandSuccess(preflightResult);
|
||||
const comment = commentV02PullRequest({
|
||||
pr,
|
||||
phase: "preflight",
|
||||
state: blocked ? "blocked" : "waiting-ci",
|
||||
startedAt,
|
||||
observedAt: new Date().toISOString(),
|
||||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||||
preflight,
|
||||
dryRun: options.dryRun,
|
||||
message: blocked
|
||||
? "v0.2 自动化已暂停在 PR preflight：存在 blocker 或冲突，需要先修复后再继续。"
|
||||
: "v0.2 自动化正在等待 GitHub CI / mergeability 收敛；CI 通过且无冲突后会自动合并并触发 CD。",
|
||||
});
|
||||
observations.push({ pullRequest: pr, preflight, comment });
|
||||
printV02PrMonitorProgress({ stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
|
||||
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
|
||||
continue;
|
||||
}
|
||||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||||
printEvent("v02.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||||
printV02PrMonitorProgress({ stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
|
||||
const result = await runV02PrAutoCd(pr, preflight, merge, options, startedAt);
|
||||
observations.push(result);
|
||||
if (record(result).ok !== true) return { ok: false, cycle, lane: "v02", phase: record(result).phase ?? "v02-auto-cd", pullRequest: pr, result, observations };
|
||||
return { ok: true, cycle, lane: "v02", action: record(result).action ?? (options.dryRun ? "dry-run-merge" : "merged-and-rolled-v02"), result, observations };
|
||||
}
|
||||
return { ok: true, cycle, lane: "v02", action: "none", observations };
|
||||
}
|
||||
|
||||
export function runtimeLanePreflightBlocked(preflightResult: CommandJsonResult, preflight: Record<string, unknown>): boolean {
|
||||
const blockers = Array.isArray(preflight.blockers) ? preflight.blockers : [];
|
||||
const pending = Array.isArray(preflight.pending) ? preflight.pending : [];
|
||||
const conclusion = String(preflight.conclusion ?? "unknown").toLowerCase();
|
||||
return !isCommandSuccess(preflightResult)
|
||||
|| v02PrConflictState(preflight) === "conflict"
|
||||
|| conclusion === "blocked"
|
||||
|| conclusion === "failed"
|
||||
|| conclusion === "failure"
|
||||
|| conclusion === "error"
|
||||
|| (blockers.length > 0 && pending.length === 0);
|
||||
}
|
||||
|
||||
export async function monitorRuntimeLaneCycle(spec: HwlabRuntimeLaneSpec, options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||||
printEvent(`${spec.lane}.monitor.cycle.start`, { cycle, dryRun: options.dryRun });
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "cycle", status: "started", cycle, dryRun: options.dryRun });
|
||||
const listed = listOpenG14PullRequests();
|
||||
if (!isCommandSuccess(listed)) return { ok: false, cycle, lane: spec.lane, phase: "list-prs", listed };
|
||||
const prs = extractPullRequests(listed, spec.sourceBranch);
|
||||
printEvent(`${spec.lane}.monitor.prs`, { cycle, count: prs.length, pullRequests: prs });
|
||||
if (prs.length === 0) {
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "idle", status: "waiting", cycle, pullRequests: 0, intervalSeconds: options.intervalSeconds });
|
||||
return { ok: true, cycle, action: "none", lane: spec.lane, pullRequests: [] };
|
||||
}
|
||||
const observations: unknown[] = [];
|
||||
for (const pr of prs) {
|
||||
const startedAt = new Date().toISOString();
|
||||
const preflightResult = preflightPullRequest(pr.number);
|
||||
const preflight = preflightSummary(preflightResult);
|
||||
printEvent(`${spec.lane}.pr.preflight`, {
|
||||
cycle,
|
||||
number: pr.number,
|
||||
ok: isCommandSuccess(preflightResult),
|
||||
conclusion: preflight.conclusion,
|
||||
readyForCommanderMerge: preflight.readyForCommanderMerge,
|
||||
conflict: v02PrConflictState(preflight),
|
||||
});
|
||||
printRuntimeLanePrMonitorProgress(spec, {
|
||||
stage: "preflight",
|
||||
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
|
||||
pr: pr.number,
|
||||
conclusion: preflight.conclusion,
|
||||
conflict: v02PrConflictState(preflight),
|
||||
});
|
||||
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
|
||||
const blocked = runtimeLanePreflightBlocked(preflightResult, preflight);
|
||||
const observedAt = new Date().toISOString();
|
||||
const comment = commentRuntimeLanePullRequest(spec, {
|
||||
pr,
|
||||
phase: "preflight",
|
||||
state: blocked ? "blocked" : "waiting-ci",
|
||||
startedAt,
|
||||
observedAt,
|
||||
elapsedSeconds: durationSeconds(startedAt, observedAt),
|
||||
preflight,
|
||||
dryRun: options.dryRun,
|
||||
message: blocked
|
||||
? `${spec.version} 自动化已暂停在 PR preflight：存在 blocker、失败 check 或冲突，需要先修复后再继续。`
|
||||
: `${spec.version} 自动化正在等待 GitHub CI / mergeability 收敛；CI 通过且无冲突后会自动合并并触发 CD。`,
|
||||
});
|
||||
const issue = blocked ? reportRuntimeLaneAutomationIssue({
|
||||
spec,
|
||||
pr,
|
||||
state: "blocked",
|
||||
phase: "preflight",
|
||||
startedAt,
|
||||
observedAt,
|
||||
preflight,
|
||||
details: { preflightResult: compactCommandResult(preflightResult) },
|
||||
message: `${spec.version} PR preflight 未通过或存在冲突，自动合并/CD 已暂停。`,
|
||||
dryRun: options.dryRun,
|
||||
}) : { ok: true, skipped: true, reason: "waiting-ci-without-blocker" };
|
||||
observations.push({ pullRequest: pr, preflight, comment, issue });
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
|
||||
if (record(comment).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: "pr-comment", pullRequest: pr, preflight, comment, issue, observations };
|
||||
if (record(issue).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: "failure-issue", pullRequest: pr, preflight, comment, issue, observations };
|
||||
continue;
|
||||
}
|
||||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||||
printEvent(`${spec.lane}.pr.merge`, { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||||
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
|
||||
const result = await runRuntimeLanePrAutoCd(spec, pr, preflight, merge, options, startedAt);
|
||||
observations.push(result);
|
||||
if (record(result).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: record(result).phase ?? `${spec.lane}-auto-cd`, pullRequest: pr, result, observations };
|
||||
return { ok: true, cycle, lane: spec.lane, action: record(result).action ?? (options.dryRun ? "dry-run-merge" : `merged-and-rolled-${spec.lane}`), result, observations };
|
||||
}
|
||||
return { ok: true, cycle, lane: spec.lane, action: "none", observations };
|
||||
}
|
||||
|
||||
export async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||||
if (options.lane === "v02") return monitorV02Cycle(options, cycle);
|
||||
if (options.lane !== "g14") return monitorRuntimeLaneCycle(hwlabRuntimeLaneSpec(options.lane), options, cycle);
|
||||
const retirement = legacyG14RetirementBlocksMonitor();
|
||||
if (retirement !== null) {
|
||||
return {
|
||||
ok: false,
|
||||
cycle,
|
||||
phase: "retired",
|
||||
degradedReason: "legacy-g14-dev-prod-retired",
|
||||
retirement,
|
||||
next: { status: "bun scripts/cli.ts hwlab g14 retirement status", v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02" },
|
||||
};
|
||||
}
|
||||
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||||
const precheck = precheckWorkspace();
|
||||
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
|
||||
const listed = listOpenG14PullRequests();
|
||||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||||
const prs = extractPullRequests(listed);
|
||||
printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||||
if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] };
|
||||
const merged: unknown[] = [];
|
||||
for (const pr of prs) {
|
||||
const preflight = preflightPullRequest(pr.number);
|
||||
printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) });
|
||||
if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) {
|
||||
return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight };
|
||||
}
|
||||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||||
printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||||
if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge };
|
||||
const sourceCommit = getG14Head();
|
||||
const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds);
|
||||
const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true
|
||||
? { skipped: true, dryRun: options.dryRun }
|
||||
: appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout);
|
||||
printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null });
|
||||
merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief });
|
||||
if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged };
|
||||
if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged };
|
||||
}
|
||||
return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged };
|
||||
}
|
||||
|
||||
export async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<string, unknown>> {
|
||||
const maxCycles = options.once ? 1 : options.maxCycles;
|
||||
let cycle = 0;
|
||||
const results: unknown[] = [];
|
||||
while (maxCycles === 0 || cycle < maxCycles) {
|
||||
cycle += 1;
|
||||
const result = await monitorCycle(options, cycle);
|
||||
results.push(result);
|
||||
printEvent(`${options.lane}.monitor.cycle.done`, { cycle, lane: options.lane, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
||||
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
|
||||
if (options.once || (options.lane === "g14" && record(result).action !== "none")) return { ok: true, cycles: cycle, lastResult: result, results };
|
||||
printEvent(`${options.lane}.monitor.sleep`, { cycle, lane: options.lane, intervalSeconds: options.intervalSeconds });
|
||||
if (options.lane === "v02") printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, intervalSeconds: options.intervalSeconds });
|
||||
else if (options.lane !== "g14") printRuntimeLanePrMonitorProgress(hwlabRuntimeLaneSpec(options.lane), { stage: "idle", status: "waiting", cycle, intervalSeconds: options.intervalSeconds });
|
||||
await sleep(options.intervalSeconds * 1000);
|
||||
}
|
||||
return { ok: true, cycles: cycle, results };
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,365 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. options module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:289-638 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { G14ControlPlaneOptions, G14GitMirrorOptions, G14LegacyRetirementOptions, G14MonitorLane, G14MonitorOptions, G14ObservabilityOptions, G14RecordRolloutOptions, G14SecretOptions, G14ToolsImageOptions, G14UpstreamImageOptions } from "./types";
|
||||
import { positiveIntegerOption, runtimeLaneMasterAdminApiKeySecretName, runtimeLaneOpenFgaSecretName } from "./remote";
|
||||
import { DEFAULT_INTERVAL_SECONDS, DEFAULT_MAX_CYCLES, DEFAULT_TIMEOUT_SECONDS, G14_CI_TOOLS_BASE_TAG, V02_MASTER_ADMIN_API_KEY_SECRET, V02_MASTER_ADMIN_API_KEY_SECRET_KEY, V02_OPENFGA_AUTHN_SECRET_KEY, V02_OPENFGA_DATASTORE_URI_SECRET_KEY, V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY, V02_OPENFGA_SECRET } from "./types";
|
||||
|
||||
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||||
const prefix = options.lane !== undefined && options.lane !== "g14" ? `latest-${options.lane}-` : "latest-";
|
||||
if (options.once && options.dryRun) return `${prefix}once-dry-run-job.json`;
|
||||
if (options.once) return `${prefix}once-job.json`;
|
||||
if (options.dryRun) return `${prefix}dry-run-job.json`;
|
||||
return `${prefix}monitor-job.json`;
|
||||
}
|
||||
|
||||
export function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||||
const lanePrefix = options.lane !== undefined && options.lane !== "g14" ? `${options.lane}-` : "";
|
||||
if (options.once && options.dryRun) return `${lanePrefix}once-dry-run`;
|
||||
if (options.once) return `${lanePrefix}once`;
|
||||
if (options.dryRun) return `${lanePrefix}dry-run-monitor`;
|
||||
return `${lanePrefix}monitor`;
|
||||
}
|
||||
|
||||
export function parseMonitorLane(args: string[]): G14MonitorLane {
|
||||
const lane = optionValue(args, "--lane") ?? "g14";
|
||||
if (lane !== "g14" && !isHwlabRuntimeLane(lane)) throw new Error(`monitor-prs --lane must be g14 or ${hwlabRuntimeLaneIds().join("|")}`);
|
||||
return lane;
|
||||
}
|
||||
|
||||
export function parseOptions(args: string[]): G14MonitorOptions {
|
||||
return {
|
||||
lane: parseMonitorLane(args),
|
||||
intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400),
|
||||
maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000),
|
||||
once: args.includes("--once"),
|
||||
dryRun: args.includes("--dry-run"),
|
||||
worker: args.includes("--worker"),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", DEFAULT_TIMEOUT_SECONDS, 86400),
|
||||
};
|
||||
}
|
||||
|
||||
export function optionValue(args: string[], name: string): string | undefined {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return undefined;
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function validateFullShaOption(value: string, name: string): string {
|
||||
if (!/^[0-9a-f]{40}$/iu.test(value)) throw new Error(`${name} must be a full 40-character git SHA`);
|
||||
return value.toLowerCase();
|
||||
}
|
||||
|
||||
export function validateV02PipelineRunOption(value: string): string {
|
||||
if (!/^hwlab-v02-ci-poll-[0-9a-f]{7,40}$/iu.test(value)) throw new Error("--pipeline-run must be a hwlab-v02-ci-poll-<sha> PipelineRun name");
|
||||
return value.toLowerCase();
|
||||
}
|
||||
|
||||
export function validateRuntimeLanePipelineRunOption(lane: HwlabRuntimeLane, value: string): string {
|
||||
const spec = hwlabRuntimeLaneSpec(lane);
|
||||
const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||||
if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) {
|
||||
throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-<sha>[-rerun] PipelineRun name for --lane ${lane}`);
|
||||
}
|
||||
return value.toLowerCase();
|
||||
}
|
||||
|
||||
export function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||||
const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number");
|
||||
const prNumber = Number(prRaw);
|
||||
if (!Number.isInteger(prNumber) || prNumber <= 0) throw new Error("record-rollout requires --pr <number>");
|
||||
return {
|
||||
prNumber,
|
||||
sourceCommit: optionValue(args, "--source-commit"),
|
||||
pipelineRun: optionValue(args, "--pipeline-run"),
|
||||
gitopsRevision: optionValue(args, "--gitops-revision"),
|
||||
mergedAt: optionValue(args, "--merged-at"),
|
||||
pipelineSucceededAt: optionValue(args, "--pipeline-succeeded-at"),
|
||||
finishedAt: optionValue(args, "--finished-at"),
|
||||
dryRun: args.includes("--dry-run"),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||||
const [rawAction] = args;
|
||||
const actionRaw = rawAction === "rerun-current" ? "trigger-current" : rawAction;
|
||||
if (
|
||||
actionRaw !== "status" &&
|
||||
actionRaw !== "closeout" &&
|
||||
actionRaw !== "apply" &&
|
||||
actionRaw !== "trigger-current" &&
|
||||
actionRaw !== "refresh" &&
|
||||
actionRaw !== "cleanup-runs" &&
|
||||
actionRaw !== "cleanup-released-pvs" &&
|
||||
actionRaw !== "runtime-migration"
|
||||
) {
|
||||
throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|v03|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
|
||||
}
|
||||
const laneRaw = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
|
||||
let lane: G14ControlPlaneOptions["lane"];
|
||||
if (actionRaw === "cleanup-runs") {
|
||||
if (laneRaw !== "g14" && laneRaw !== "all" && !isHwlabRuntimeLane(laneRaw)) throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all");
|
||||
lane = laneRaw;
|
||||
} else if (actionRaw === "cleanup-released-pvs") {
|
||||
if (laneRaw !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
|
||||
lane = laneRaw;
|
||||
} else if (actionRaw === "closeout" || actionRaw === "runtime-migration") {
|
||||
if (laneRaw !== "v02") throw new Error("control-plane closeout/runtime-migration currently requires --lane v02");
|
||||
lane = laneRaw;
|
||||
} else if (!isHwlabRuntimeLane(laneRaw)) {
|
||||
throw new Error(`control-plane ${actionRaw} requires --lane ${hwlabRuntimeLaneIds().join("|")}`);
|
||||
} else {
|
||||
lane = laneRaw;
|
||||
}
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run");
|
||||
const allowLiveDbRead = args.includes("--allow-live-db-read");
|
||||
if (allowLiveDbRead && confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
|
||||
if (allowLiveDbRead && actionRaw !== "runtime-migration") throw new Error("--allow-live-db-read is only valid for control-plane runtime-migration");
|
||||
const sourceCommitRaw = optionValue(args, "--source-commit");
|
||||
const pipelineRunRaw = optionValue(args, "--pipeline-run");
|
||||
if ((sourceCommitRaw !== undefined || pipelineRunRaw !== undefined) && actionRaw !== "status" && actionRaw !== "closeout" && actionRaw !== "cleanup-runs") {
|
||||
throw new Error("--source-commit and --pipeline-run are only valid for control-plane status/closeout/cleanup-runs");
|
||||
}
|
||||
if (actionRaw === "closeout" && sourceCommitRaw === undefined && pipelineRunRaw === undefined) {
|
||||
throw new Error("control-plane closeout requires --source-commit <full-sha> or --pipeline-run hwlab-v02-ci-poll-<sha>");
|
||||
}
|
||||
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) throw new Error("control-plane status/closeout/cleanup-runs accepts only one of --source-commit or --pipeline-run");
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane,
|
||||
confirm,
|
||||
wait: args.includes("--wait"),
|
||||
rerun: args.includes("--rerun") || rawAction === "rerun-current",
|
||||
allowLiveDbRead,
|
||||
dryRun: actionRaw === "status" || actionRaw === "closeout" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
||||
limit: positiveIntegerOption(args, "--limit", 20, 200),
|
||||
sourceCommit: sourceCommitRaw === undefined ? undefined : validateFullShaOption(sourceCommitRaw, "--source-commit"),
|
||||
pipelineRun: pipelineRunRaw === undefined
|
||||
? undefined
|
||||
: isHwlabRuntimeLane(lane)
|
||||
? validateRuntimeLanePipelineRunOption(lane, pipelineRunRaw)
|
||||
: validateV02PipelineRunOption(pipelineRunRaw),
|
||||
history: args.includes("--history"),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseLegacyRetirementOptions(args: string[]): G14LegacyRetirementOptions {
|
||||
const [actionRaw = "status"] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "plan" && actionRaw !== "execute") {
|
||||
throw new Error("retirement usage: status|plan|execute [--dry-run|--confirm] [--wait] [--reason text] [--timeout-seconds N]");
|
||||
}
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("retirement accepts only one of --confirm or --dry-run");
|
||||
if (confirm && actionRaw !== "execute") throw new Error("--confirm is only valid for retirement execute");
|
||||
if (args.includes("--wait") && actionRaw !== "execute") throw new Error("--wait is only valid for retirement execute");
|
||||
const reason = optionValue(args, "--reason") ?? "user requested immediate retirement of legacy G14 DEV/PROD";
|
||||
if (reason.length > 240) throw new Error("--reason is limited to 240 characters");
|
||||
return {
|
||||
action: actionRaw,
|
||||
confirm,
|
||||
wait: args.includes("--wait"),
|
||||
dryRun: actionRaw !== "execute" || explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "execute" ? 60 : 30, 300),
|
||||
reason,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseToolsImageOptions(args: string[]): G14ToolsImageOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "build") {
|
||||
throw new Error("tools-image usage: status|build --name ci-node-tools --tag <tag> [--dockerfile path] [--dry-run|--confirm]");
|
||||
}
|
||||
const name = optionValue(args, "--name") ?? "ci-node-tools";
|
||||
if (name !== "ci-node-tools") throw new Error("tools-image currently supports --name ci-node-tools");
|
||||
const tag = optionValue(args, "--tag") ?? G14_CI_TOOLS_BASE_TAG;
|
||||
if (!/^[A-Za-z0-9_.-]+$/u.test(tag)) throw new Error("--tag may only contain letters, numbers, dot, underscore, and dash");
|
||||
const dockerfile = optionValue(args, "--dockerfile") ?? "deploy/ci/hwlab-ci-node-tools.Dockerfile";
|
||||
if (dockerfile.startsWith("/") || dockerfile.includes("..")) throw new Error("--dockerfile must be a repo-relative path without '..'");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run");
|
||||
return {
|
||||
action: actionRaw,
|
||||
name,
|
||||
tag,
|
||||
dockerfile,
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseUpstreamImageOptions(args: string[]): G14UpstreamImageOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure") {
|
||||
throw new Error("upstream-image usage: status|ensure --name openfga --tag v1.17.0 [--dry-run|--confirm]");
|
||||
}
|
||||
const name = optionValue(args, "--name") ?? "openfga";
|
||||
if (name !== "openfga") throw new Error("upstream-image currently supports --name openfga");
|
||||
const tag = optionValue(args, "--tag") ?? "v1.17.0";
|
||||
if (tag !== "v1.17.0") throw new Error("upstream-image currently supports OpenFGA tag v1.17.0");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("upstream-image accepts only one of --confirm or --dry-run");
|
||||
return {
|
||||
action: actionRaw,
|
||||
name,
|
||||
tag,
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseGitMirrorOptions(args: string[]): G14GitMirrorOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "sync" && actionRaw !== "flush") {
|
||||
throw new Error("git-mirror usage: status|apply|sync|flush [--lane v02|v03] [--dry-run|--confirm]");
|
||||
}
|
||||
const laneRaw = optionValue(args, "--lane") ?? "v02";
|
||||
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`git-mirror --lane must be ${hwlabRuntimeLaneIds().join("|")}`);
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("git-mirror accepts only one of --confirm or --dry-run");
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane: laneRaw,
|
||||
confirm,
|
||||
wait: args.includes("--wait"),
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "sync" || actionRaw === "flush" ? 300 : 120, 900),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseObservabilityOptions(args: string[]): G14ObservabilityOptions {
|
||||
const [actionRaw] = args;
|
||||
if (
|
||||
actionRaw !== "status" &&
|
||||
actionRaw !== "apply" &&
|
||||
actionRaw !== "query" &&
|
||||
actionRaw !== "targets" &&
|
||||
actionRaw !== "boundary" &&
|
||||
actionRaw !== "closeout"
|
||||
) {
|
||||
throw new Error("observability usage: status|apply|query|targets|boundary|closeout [--lane v02] [--promql <expr>] [--expect-count N] [--expect-value V] [--dry-run|--confirm]");
|
||||
}
|
||||
const valueOptions = new Set(["--lane", "--promql", "--query", "--expect-count", "--expect-value", "--timeout-seconds"]);
|
||||
const booleanOptions = new Set(["--confirm", "--dry-run"]);
|
||||
for (let i = 1; i < args.length; i += 1) {
|
||||
const arg = args[i];
|
||||
if (valueOptions.has(arg)) {
|
||||
i += 1;
|
||||
if (i >= args.length || args[i].startsWith("--")) throw new Error(`${arg} requires a value`);
|
||||
continue;
|
||||
}
|
||||
if (booleanOptions.has(arg)) continue;
|
||||
if (arg === "--wait") throw new Error("observability does not support --wait; commands are bounded short CLI calls");
|
||||
if (arg.startsWith("--")) throw new Error(`unsupported observability option: ${arg}`);
|
||||
throw new Error(`unexpected observability argument: ${arg}`);
|
||||
}
|
||||
const lane = optionValue(args, "--lane") ?? "v02";
|
||||
if (lane !== "v02") throw new Error("observability currently supports --lane v02");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("observability accepts only one of --confirm or --dry-run");
|
||||
if ((confirm || explicitDryRun) && actionRaw !== "apply") throw new Error("--confirm and --dry-run are only supported by observability apply");
|
||||
const query = optionValue(args, "--promql") ?? optionValue(args, "--query") ?? 'up{namespace="hwlab-v02"}';
|
||||
if (query.length > 500) throw new Error("--promql is limited to 500 characters");
|
||||
if (query.includes("\n") || query.includes("\r")) throw new Error("--promql must be a single-line expression");
|
||||
const expectCountRaw = optionValue(args, "--expect-count");
|
||||
const expectCount = expectCountRaw === undefined ? undefined : Number(expectCountRaw);
|
||||
if (expectCountRaw !== undefined && (!Number.isInteger(expectCount) || expectCount < 0)) throw new Error("--expect-count must be a non-negative integer");
|
||||
const expectValue = optionValue(args, "--expect-value");
|
||||
if ((expectCount !== undefined || expectValue !== undefined) && actionRaw !== "query") throw new Error("--expect-count and --expect-value are only supported by observability query");
|
||||
if (expectValue !== undefined && expectValue.trim().length === 0) throw new Error("--expect-value must not be blank");
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane,
|
||||
confirm,
|
||||
wait: args.includes("--wait"),
|
||||
dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "apply" ? 240 : 120, 900),
|
||||
query,
|
||||
expectCount,
|
||||
expectValue,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseSecretOptions(args: string[]): G14SecretOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "delete") {
|
||||
throw new Error("secret usage: status|ensure --lane v02|v03 --name hwlab-v0x-openfga|hwlab-v0x-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name <obsolete-secret> [--dry-run|--confirm]");
|
||||
}
|
||||
const laneRaw = optionValue(args, "--lane") ?? "v02";
|
||||
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`secret --lane must be one of ${hwlabRuntimeLaneIds().join(", ")}`);
|
||||
const lane = laneRaw;
|
||||
const spec = hwlabRuntimeLaneSpec(lane);
|
||||
const openFgaSecret = runtimeLaneOpenFgaSecretName(spec);
|
||||
const masterAdminSecret = runtimeLaneMasterAdminApiKeySecretName(spec);
|
||||
const name = optionValue(args, "--name") ?? openFgaSecret;
|
||||
const key = optionValue(args, "--key");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
||||
if (actionRaw === "delete") {
|
||||
if (lane !== "v02") throw new Error("secret delete is currently limited to --lane v02 obsolete cleanup");
|
||||
if (key !== undefined) throw new Error("secret delete does not accept --key; it deletes a whole obsolete Secret object");
|
||||
if (!/^hwlab-v02-[a-z0-9-]+$/u.test(name)) throw new Error("secret delete requires a hwlab-v02-* Secret name");
|
||||
if (name === V02_OPENFGA_SECRET || name === V02_MASTER_ADMIN_API_KEY_SECRET || name === "hwlab-v02-postgres") throw new Error(`secret delete refuses required v0.2 Secret ${name}`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane,
|
||||
confirm,
|
||||
dryRun: explicitDryRun || !confirm,
|
||||
name,
|
||||
preset: "generic-delete",
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
};
|
||||
}
|
||||
if (name === masterAdminSecret) {
|
||||
if (key !== undefined && key !== V02_MASTER_ADMIN_API_KEY_SECRET_KEY) throw new Error(`secret ${masterAdminSecret} supports only key ${V02_MASTER_ADMIN_API_KEY_SECRET_KEY}`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane,
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
name,
|
||||
key: key ?? V02_MASTER_ADMIN_API_KEY_SECRET_KEY,
|
||||
preset: "master-server-admin-api-key",
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
};
|
||||
}
|
||||
if (name !== openFgaSecret) {
|
||||
throw new Error(`secret status/ensure currently supports only --name ${openFgaSecret} or ${masterAdminSecret} for --lane ${lane}; use secret delete for obsolete v02 Secret objects`);
|
||||
}
|
||||
if (key !== undefined && key !== V02_OPENFGA_AUTHN_SECRET_KEY && key !== V02_OPENFGA_DATASTORE_URI_SECRET_KEY && key !== V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) {
|
||||
throw new Error(`secret ${openFgaSecret} supports keys ${V02_OPENFGA_AUTHN_SECRET_KEY}, ${V02_OPENFGA_DATASTORE_URI_SECRET_KEY}, and ${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY}`);
|
||||
}
|
||||
if (lane !== "v02" && key === V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) throw new Error(`${openFgaSecret}/${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY} is derived from ${postgresSecret}/POSTGRES_PASSWORD`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane,
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
name,
|
||||
key,
|
||||
preset: "openfga",
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,581 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:639-1204 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult, G14MonitorLane, OpenPullRequest, RemoteAsyncCommandSpec } from "./types";
|
||||
import { tailText } from "./cleanup";
|
||||
import { G14_PROVIDER, G14_SOURCE_BRANCH, G14_WORKSPACE, V02_MASTER_ADMIN_API_KEY_LOCAL_ENV } from "./types";
|
||||
|
||||
export function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return defaultValue;
|
||||
const raw = args[index + 1];
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||||
return Math.min(value, maxValue);
|
||||
}
|
||||
|
||||
export function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
export function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
|
||||
const startedAtMs = Date.now();
|
||||
const result = runCommand(command, repoRoot, { timeoutMs });
|
||||
const durationMs = Date.now() - startedAtMs;
|
||||
let parsed: unknown | null = null;
|
||||
if (result.stdout.trim().length > 0) {
|
||||
try {
|
||||
parsed = JSON.parse(result.stdout) as unknown;
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: result.exitCode === 0,
|
||||
command,
|
||||
exitCode: result.exitCode,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
parsed,
|
||||
durationMs,
|
||||
timedOut: result.timedOut,
|
||||
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
||||
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
export function commandJsonWithInput(command: string[], input: string, timeoutMs = 60_000): CommandJsonResult {
|
||||
const startedAtMs = Date.now();
|
||||
const result = runCommand(command, repoRoot, { timeoutMs, input });
|
||||
const durationMs = Date.now() - startedAtMs;
|
||||
let parsed: unknown | null = null;
|
||||
if (result.stdout.trim().length > 0) {
|
||||
try {
|
||||
parsed = JSON.parse(result.stdout) as unknown;
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ok: result.exitCode === 0,
|
||||
command,
|
||||
exitCode: result.exitCode,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
parsed,
|
||||
durationMs,
|
||||
timedOut: result.timedOut,
|
||||
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
||||
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
export function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||||
return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs);
|
||||
}
|
||||
|
||||
export function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
export function stringOrNull(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
export function fieldBoolean(value: unknown): boolean | null {
|
||||
if (value === true || value === false) return value;
|
||||
if (typeof value !== "string") return null;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "true" || normalized === "yes" || normalized === "1") return true;
|
||||
if (normalized === "false" || normalized === "no" || normalized === "0") return false;
|
||||
return null;
|
||||
}
|
||||
|
||||
export function nested(value: unknown, keys: string[]): unknown {
|
||||
let current = value;
|
||||
for (const key of keys) current = record(current)[key];
|
||||
return current;
|
||||
}
|
||||
|
||||
export function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown> {
|
||||
const dumpPath = nested(result.parsed, ["data", "dump", "path"]);
|
||||
if (typeof dumpPath === "string" && existsSync(dumpPath)) {
|
||||
try {
|
||||
return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown);
|
||||
} catch {
|
||||
return record(result.parsed);
|
||||
}
|
||||
}
|
||||
return record(result.parsed);
|
||||
}
|
||||
|
||||
export function commandData(result: CommandJsonResult): Record<string, unknown> {
|
||||
return record(sanitizeCliOutput(record(expandedParsedRoot(result).data)));
|
||||
}
|
||||
|
||||
export function compactCommandMetadata(result: CommandJsonResult, includeTail = false, includeCommand = false): Record<string, unknown> {
|
||||
return {
|
||||
ok: result.ok,
|
||||
exitCode: result.exitCode,
|
||||
command: includeCommand ? redactedCommand(result.command) : undefined,
|
||||
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
||||
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
||||
stdoutTail: includeTail ? compactInlineText(result.stdout, 4000) : undefined,
|
||||
stderrTail: includeTail ? compactInlineText(result.stderr, 4000) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function textHash(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
||||
}
|
||||
|
||||
export function generateHwlabApiKey(): string {
|
||||
return `hwl_live_${randomBytes(32).toString("base64url")}`;
|
||||
}
|
||||
|
||||
export function hwlabApiKeyPrefix(value: string): string {
|
||||
const text = value.trim();
|
||||
if (!text.startsWith("hwl_live_")) return "";
|
||||
const remainder = text.slice("hwl_live_".length);
|
||||
const dot = remainder.indexOf(".");
|
||||
const segment = dot === -1 ? remainder : remainder.slice(0, dot);
|
||||
return `hwl_live_${segment}`.slice(0, 24);
|
||||
}
|
||||
|
||||
export function runtimeLaneSecretFieldManager(spec: HwlabRuntimeLaneSpec): string {
|
||||
return `unidesk-hwlab-${spec.lane}-secret`;
|
||||
}
|
||||
|
||||
export function runtimeLanePostgresSecretName(spec: HwlabRuntimeLaneSpec): string {
|
||||
return `${spec.runtimeNamespace}-postgres`;
|
||||
}
|
||||
|
||||
export function runtimeLanePrimaryDbName(spec: HwlabRuntimeLaneSpec): string {
|
||||
return `hwlab_${spec.lane}`;
|
||||
}
|
||||
|
||||
export function runtimeLaneOpenFgaSecretName(spec: HwlabRuntimeLaneSpec): string {
|
||||
return `hwlab-${spec.lane}-openfga`;
|
||||
}
|
||||
|
||||
export function runtimeLaneMasterAdminApiKeySecretName(spec: HwlabRuntimeLaneSpec): string {
|
||||
return `hwlab-${spec.lane}-master-server-admin-api-key`;
|
||||
}
|
||||
|
||||
export function runtimeLaneMasterAdminApiKeyLocalEnv(spec: HwlabRuntimeLaneSpec): string {
|
||||
if (spec.lane === "v02") return V02_MASTER_ADMIN_API_KEY_LOCAL_ENV;
|
||||
return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`;
|
||||
}
|
||||
|
||||
export function localMasterAdminApiKeyStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
|
||||
if (!existsSync(envPath)) {
|
||||
return { exists: false, path: envPath, mode: null, valueBytes: 0, keyPrefix: null };
|
||||
}
|
||||
const stat = statSync(envPath);
|
||||
const content = readFileSync(envPath, "utf8");
|
||||
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
|
||||
const value = match?.[1]?.trim() ?? "";
|
||||
return {
|
||||
exists: true,
|
||||
path: envPath,
|
||||
mode: `0${(stat.mode & 0o777).toString(8)}`,
|
||||
valueBytes: Buffer.byteLength(value, "utf8"),
|
||||
keyPrefix: value ? hwlabApiKeyPrefix(value) : null,
|
||||
validPrefix: value.startsWith("hwl_live_"),
|
||||
};
|
||||
}
|
||||
|
||||
export function readOrCreateLocalMasterAdminApiKey(spec: HwlabRuntimeLaneSpec, dryRun: boolean): { key: string | null; created: boolean; status: Record<string, unknown> } {
|
||||
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
|
||||
const existing = localMasterAdminApiKeyStatus(spec);
|
||||
if (existing.exists === true) {
|
||||
const content = readFileSync(envPath, "utf8");
|
||||
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
|
||||
const key = match?.[1]?.trim() ?? "";
|
||||
if (!key.startsWith("hwl_live_")) throw new Error(`${envPath} exists but does not contain a hwl_live_ HWLAB_API_KEY`);
|
||||
if (!dryRun && existing.mode !== "0600") chmodSync(envPath, 0o600);
|
||||
return { key, created: false, status: localMasterAdminApiKeyStatus(spec) };
|
||||
}
|
||||
if (dryRun) return { key: null, created: false, status: existing };
|
||||
const key = generateHwlabApiKey();
|
||||
mkdirSync(dirname(envPath), { recursive: true, mode: 0o700 });
|
||||
writeFileSync(envPath, `# HWLAB ${spec.version} master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 });
|
||||
return { key, created: true, status: localMasterAdminApiKeyStatus(spec) };
|
||||
}
|
||||
|
||||
export function redactLargePayloads(value: string): string {
|
||||
return value
|
||||
.replace(/hwl_live_[A-Za-z0-9._:-]{12,}/gu, (payload: string) => {
|
||||
return `hwl_live_<redacted chars=${payload.length} sha256=${textHash(payload)}>`;
|
||||
})
|
||||
.replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => {
|
||||
return `manifest_b64='<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>'`;
|
||||
})
|
||||
.replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => {
|
||||
return `manifest_b64="<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>"`;
|
||||
})
|
||||
.replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => {
|
||||
return `manifest_b64=<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||||
})
|
||||
.replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => {
|
||||
return `<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||||
});
|
||||
}
|
||||
|
||||
export function compactInlineText(value: string, maxChars = 4000): string {
|
||||
const redacted = redactLargePayloads(value);
|
||||
if (redacted.length <= maxChars) return redacted;
|
||||
const headChars = Math.floor(maxChars * 0.35);
|
||||
const tailChars = maxChars - headChars;
|
||||
const omittedChars = redacted.length - headChars - tailChars;
|
||||
return [
|
||||
redacted.slice(0, headChars),
|
||||
`<truncated chars=${omittedChars} sha256=${textHash(redacted)}>`,
|
||||
redacted.slice(-tailChars),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function sanitizeCliOutput(value: unknown): unknown {
|
||||
if (typeof value === "string") return compactInlineText(value);
|
||||
if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item));
|
||||
if (typeof value !== "object" || value === null) return value;
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item);
|
||||
return output;
|
||||
}
|
||||
|
||||
export function redactedCommand(command: string[]): string[] {
|
||||
return command.map((arg) => compactInlineText(arg, 1200));
|
||||
}
|
||||
|
||||
export function compactCommandResult(result: CommandJsonResult): Record<string, unknown> {
|
||||
const data = record(expandedParsedRoot(result).data);
|
||||
const stdout = String(data.stdout ?? result.stdout ?? "");
|
||||
const stderr = String(data.stderr ?? result.stderr ?? "");
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
exitCode: result.exitCode,
|
||||
command: redactedCommand(result.command),
|
||||
stdout: compactInlineText(stdout.trim(), 4000),
|
||||
stderr: compactInlineText(stderr.trim(), 4000),
|
||||
stdoutBytes: Buffer.byteLength(stdout),
|
||||
stderrBytes: Buffer.byteLength(stderr),
|
||||
};
|
||||
}
|
||||
|
||||
export function compactControlPlaneRefresh(value: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
ok: value.ok === true,
|
||||
phase: value.phase ?? null,
|
||||
sourceCommit: value.sourceCommit ?? null,
|
||||
renderDir: value.renderDir ?? null,
|
||||
applyOk: nested(value, ["apply", "ok"]) ?? null,
|
||||
cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null,
|
||||
applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200),
|
||||
degradedReason: value.degradedReason ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function compactTriggerCommandResult(value: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
if (value === null) return null;
|
||||
return {
|
||||
ok: value.ok ?? null,
|
||||
exitCode: value.exitCode ?? null,
|
||||
stdout: tailText(value.stdout, 1200),
|
||||
stderr: tailText(value.stderr, 1200),
|
||||
stdoutBytes: value.stdoutBytes ?? null,
|
||||
stderrBytes: value.stderrBytes ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function isCommandSuccess(result: CommandJsonResult): boolean {
|
||||
if (!result.ok) return false;
|
||||
const topOk = record(result.parsed).ok;
|
||||
if (topOk === false) return false;
|
||||
const dataOk = nested(result.parsed, ["data", "ok"]);
|
||||
return dataOk !== false;
|
||||
}
|
||||
|
||||
export function monitorBaseBranch(lane: G14MonitorLane): string {
|
||||
return lane === "g14" ? G14_SOURCE_BRANCH : hwlabRuntimeLaneSpec(lane).sourceBranch;
|
||||
}
|
||||
|
||||
export function extractPullRequests(result: CommandJsonResult, baseBranch = G14_SOURCE_BRANCH): OpenPullRequest[] {
|
||||
const prs = nested(result.parsed, ["data", "pullRequests"]);
|
||||
if (!Array.isArray(prs)) return [];
|
||||
return prs
|
||||
.map((item) => record(item))
|
||||
.filter((item) => item.baseRefName === baseBranch || record(item.base).ref === baseBranch)
|
||||
.map((item) => ({
|
||||
number: Number(item.number),
|
||||
title: typeof item.title === "string" ? item.title : undefined,
|
||||
url: typeof item.url === "string" ? item.url : undefined,
|
||||
baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined,
|
||||
headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined,
|
||||
mergeable: typeof item.mergeable === "string" ? item.mergeable : null,
|
||||
mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null,
|
||||
draft: item.draft === true,
|
||||
}))
|
||||
.filter((item) => Number.isInteger(item.number) && item.number > 0);
|
||||
}
|
||||
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function printEvent(event: string, data: Record<string, unknown> = {}): void {
|
||||
process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||||
}
|
||||
|
||||
export function printProgressEvent(event: string, data: Record<string, unknown> = {}): void {
|
||||
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||||
}
|
||||
|
||||
export function printV02PrMonitorProgress(data: Record<string, unknown> = {}): void {
|
||||
printProgressEvent("hwlab.v02.pr-monitor.progress", data);
|
||||
}
|
||||
|
||||
export function printRuntimeLanePrMonitorProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
||||
printProgressEvent("hwlab.runtime-lane.pr-monitor.progress", {
|
||||
lane: spec.lane,
|
||||
node: spec.nodeId,
|
||||
...data,
|
||||
});
|
||||
}
|
||||
|
||||
export function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
||||
printProgressEvent("hwlab.runtime-lane.trigger.progress", {
|
||||
lane: spec.lane,
|
||||
node: spec.nodeId,
|
||||
...data,
|
||||
});
|
||||
}
|
||||
|
||||
export function shortSha(sha: string): string {
|
||||
return sha.slice(0, 12);
|
||||
}
|
||||
|
||||
export function precheckWorkspace(): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "sh", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
|
||||
}
|
||||
|
||||
export function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||||
return cliJson(["ssh", G14_PROVIDER, "sh", "--", script], timeoutMs);
|
||||
}
|
||||
|
||||
export function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs);
|
||||
}
|
||||
|
||||
export function g14K3sInlineScriptWithInput(script: string, input: string, timeoutMs = 60_000): CommandJsonResult {
|
||||
return commandJsonWithInput(["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "sh", "--", script], input, timeoutMs);
|
||||
}
|
||||
|
||||
export function remoteAsyncStatusDir(label: string): string {
|
||||
return `/tmp/hwlab-${label}-async`;
|
||||
}
|
||||
|
||||
export function remoteAsyncStatusPath(label: string, token: string): string {
|
||||
return `${remoteAsyncStatusDir(label)}/${token}.json`;
|
||||
}
|
||||
|
||||
export function remoteAsyncStartScript(spec: RemoteAsyncCommandSpec): string {
|
||||
const statusPath = remoteAsyncStatusPath(spec.label, spec.token);
|
||||
const donePath = `${statusPath}.done`;
|
||||
const stdoutPath = `${statusPath}.stdout`;
|
||||
const stderrPath = `${statusPath}.stderr`;
|
||||
const commandB64 = Buffer.from(spec.script, "utf8").toString("base64");
|
||||
return [
|
||||
"set -eu",
|
||||
`status_dir=${shellQuote(remoteAsyncStatusDir(spec.label))}`,
|
||||
`status_path=${shellQuote(statusPath)}`,
|
||||
`done_path=${shellQuote(donePath)}`,
|
||||
`stdout_path=${shellQuote(stdoutPath)}`,
|
||||
`stderr_path=${shellQuote(stderrPath)}`,
|
||||
`command_b64=${shellQuote(commandB64)}`,
|
||||
"mkdir -p \"$status_dir\"",
|
||||
"rm -f \"$status_path\" \"$done_path\" \"$stdout_path\" \"$stderr_path\"",
|
||||
"command_path=\"$status_path.command.sh\"",
|
||||
"printf '%s' \"$command_b64\" | base64 -d > \"$command_path\"",
|
||||
"chmod +x \"$command_path\"",
|
||||
"node -e 'const fs=require(\"fs\"); const p=process.argv[1]; const payload={ok:null,state:\"running\",pid:Number(process.argv[2]),startedAt:new Date().toISOString(),finishedAt:null,exitCode:null,stdout:\"\",stderr:\"\"}; fs.writeFileSync(p, JSON.stringify(payload));' \"$status_path\" $$",
|
||||
"(",
|
||||
" set +e",
|
||||
" sh \"$command_path\" >\"$stdout_path\" 2>\"$stderr_path\"",
|
||||
" code=$?",
|
||||
" node - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'NODE'",
|
||||
"const fs = require('fs');",
|
||||
"const [statusPath, stdoutPath, stderrPath, codeRaw] = process.argv.slice(2);",
|
||||
"const code = Number(codeRaw);",
|
||||
"const read = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };",
|
||||
"fs.writeFileSync(statusPath, JSON.stringify({",
|
||||
" ok: code === 0,",
|
||||
" state: 'finished',",
|
||||
" pid: null,",
|
||||
" startedAt: null,",
|
||||
" finishedAt: new Date().toISOString(),",
|
||||
" exitCode: code,",
|
||||
" stdout: read(stdoutPath),",
|
||||
" stderr: read(stderrPath),",
|
||||
"}));",
|
||||
"NODE",
|
||||
" touch \"$done_path\"",
|
||||
") >/dev/null 2>&1 &",
|
||||
"pid=$!",
|
||||
"node - \"$status_path\" \"$pid\" <<'NODE'",
|
||||
"const fs = require('fs');",
|
||||
"const [statusPath, pidRaw] = process.argv.slice(2);",
|
||||
"const current = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
|
||||
"current.pid = Number(pidRaw);",
|
||||
"fs.writeFileSync(statusPath, JSON.stringify(current));",
|
||||
"console.log(JSON.stringify({ ok: true, state: current.state, pid: current.pid, statusPath }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function remoteAsyncStatusScript(label: string, token: string): string {
|
||||
const statusPath = remoteAsyncStatusPath(label, token);
|
||||
const donePath = `${statusPath}.done`;
|
||||
return [
|
||||
"set -eu",
|
||||
`status_path=${shellQuote(statusPath)}`,
|
||||
`done_path=${shellQuote(donePath)}`,
|
||||
"if [ ! -f \"$status_path\" ]; then",
|
||||
" printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing\"}'",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"if [ -f \"$done_path\" ]; then",
|
||||
" cat \"$status_path\"",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"node - \"$status_path\" <<'NODE'",
|
||||
"const fs = require('fs');",
|
||||
"const statusPath = process.argv[2];",
|
||||
"const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
|
||||
"payload.state = payload.state || 'running';",
|
||||
"payload.ok = payload.ok ?? null;",
|
||||
"payload.stdout = payload.stdout || '';",
|
||||
"payload.stderr = payload.stderr || '';",
|
||||
"console.log(JSON.stringify(payload));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function remoteAsyncKillScript(label: string, token: string): string {
|
||||
const statusPath = remoteAsyncStatusPath(label, token);
|
||||
const donePath = `${statusPath}.done`;
|
||||
return [
|
||||
"set -eu",
|
||||
`status_path=${shellQuote(statusPath)}`,
|
||||
`done_path=${shellQuote(donePath)}`,
|
||||
"if [ ! -f \"$status_path\" ]; then",
|
||||
" printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing during kill\"}'",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"node - \"$status_path\" <<'NODE'",
|
||||
"const fs = require('fs');",
|
||||
"const statusPath = process.argv[2];",
|
||||
"const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
|
||||
"if (Number.isInteger(payload.pid) && payload.pid > 0) {",
|
||||
" try { process.kill(payload.pid, 'TERM'); } catch {}",
|
||||
"}",
|
||||
"payload.ok = false;",
|
||||
"payload.state = 'timeout-killed';",
|
||||
"payload.exitCode = 124;",
|
||||
"payload.finishedAt = new Date().toISOString();",
|
||||
"payload.stderr = `${payload.stderr || ''}\\nremote async command exceeded local poll timeout and was terminated`;",
|
||||
"fs.writeFileSync(statusPath, JSON.stringify(payload));",
|
||||
"console.log(JSON.stringify(payload));",
|
||||
"NODE",
|
||||
"touch \"$done_path\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function parseRemoteAsyncPayload(result: CommandJsonResult): Record<string, unknown> {
|
||||
const text = result.stdout.trim();
|
||||
if (text.length === 0) return {};
|
||||
try {
|
||||
return record(JSON.parse(text) as unknown);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export function runG14K3sRemoteAsync(spec: RemoteAsyncCommandSpec): CommandJsonResult {
|
||||
const startedAtMs = Date.now();
|
||||
const start = g14K3s(["sh", "--", remoteAsyncStartScript(spec)], 55_000);
|
||||
const startPayload = parseRemoteAsyncPayload(start);
|
||||
if (!isCommandSuccess(start) || startPayload.ok !== true) return start;
|
||||
const deadlineMs = startedAtMs + Math.max(1, spec.timeoutSeconds) * 1000;
|
||||
let attempts = 0;
|
||||
let lastStatus: CommandJsonResult | null = null;
|
||||
while (Date.now() <= deadlineMs) {
|
||||
attempts += 1;
|
||||
const wait = runCommand(["sleep", attempts <= 1 ? "1" : "2"], repoRoot, { timeoutMs: 3_000 });
|
||||
if (wait.exitCode !== 0 && wait.timedOut) break;
|
||||
const status = g14K3s(["sh", "--", remoteAsyncStatusScript(spec.label, spec.token)], 55_000);
|
||||
lastStatus = status;
|
||||
const payload = parseRemoteAsyncPayload(status);
|
||||
const state = typeof payload.state === "string" ? payload.state : null;
|
||||
if (state === "finished" || payload.ok === true || payload.ok === false) {
|
||||
return {
|
||||
ok: payload.ok === true,
|
||||
command: spec.command,
|
||||
exitCode: typeof payload.exitCode === "number" ? payload.exitCode : status.exitCode,
|
||||
stdout: typeof payload.stdout === "string" ? payload.stdout : status.stdout,
|
||||
stderr: typeof payload.stderr === "string" ? payload.stderr : status.stderr,
|
||||
parsed: {
|
||||
ok: payload.ok === true,
|
||||
data: {
|
||||
mode: "remote-async",
|
||||
label: spec.label,
|
||||
token: spec.token,
|
||||
statusPath: remoteAsyncStatusPath(spec.label, spec.token),
|
||||
attempts,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
state,
|
||||
pid: payload.pid ?? null,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
const killed = g14K3s(["sh", "--", remoteAsyncKillScript(spec.label, spec.token)], 55_000);
|
||||
const payload = parseRemoteAsyncPayload(killed);
|
||||
return {
|
||||
ok: false,
|
||||
command: spec.command,
|
||||
exitCode: 124,
|
||||
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
|
||||
stderr: [
|
||||
`remote async command timed out after ${spec.timeoutSeconds}s`,
|
||||
typeof payload.stderr === "string" ? payload.stderr : "",
|
||||
lastStatus === null ? "" : `last status stderr: ${lastStatus.stderr.trim()}`,
|
||||
killed.stderr.trim(),
|
||||
].filter(Boolean).join("\n"),
|
||||
parsed: {
|
||||
ok: false,
|
||||
data: {
|
||||
mode: "remote-async",
|
||||
label: spec.label,
|
||||
token: spec.token,
|
||||
statusPath: remoteAsyncStatusPath(spec.label, spec.token),
|
||||
attempts,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
state: typeof payload.state === "string" ? payload.state : "timeout",
|
||||
pid: payload.pid ?? null,
|
||||
killed: compactCommandMetadata(killed, true, false),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:3346-3736 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult } from "./types";
|
||||
import { g14HostScript } from "./images";
|
||||
import { compactCommandResult, g14K3s, isCommandSuccess, record, runG14K3sRemoteAsync, shellQuote, shortSha, textHash } from "./remote";
|
||||
import { runtimeLaneCicdRepoEnsureScript } from "./source";
|
||||
import { CI_NAMESPACE, G14_PROVIDER, V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS, V02_CONTROL_PLANE_REFRESH_TTL_MS, V02_LANE_SPEC, V02_POLLER, V02_RECONCILER } from "./types";
|
||||
import { timestampMs } from "./v02-status";
|
||||
|
||||
export function runtimeLaneControlPlaneRenderScript(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token = runtimeLaneRenderToken()): string {
|
||||
const renderDir = runtimeLaneRenderDir(spec, sourceCommit, token);
|
||||
const worktreeDir = runtimeLaneRenderWorktreeDir(spec, sourceCommit, token);
|
||||
return [
|
||||
"set -eu",
|
||||
runtimeLaneCicdRepoEnsureScript(spec),
|
||||
`source_commit=${shellQuote(sourceCommit)}`,
|
||||
`render_dir=${shellQuote(renderDir)}`,
|
||||
`worktree_dir=${shellQuote(worktreeDir)}`,
|
||||
`render_lane=${shellQuote(spec.lane)}`,
|
||||
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
|
||||
"trap cleanup_render_worktree EXIT",
|
||||
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = ${shellQuote(sourceCommit)}`,
|
||||
"cleanup_render_worktree",
|
||||
"rm -rf \"$render_dir\"",
|
||||
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
||||
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
||||
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
||||
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
||||
"cd \"$worktree_dir\"",
|
||||
"if [ ! -d node_modules/yaml ]; then",
|
||||
" if [ -f package-lock.json ]; then",
|
||||
" npm ci --ignore-scripts --no-audit --prefer-offline",
|
||||
" elif [ -f bun.lock ] || [ -f bun.lockb ]; then",
|
||||
" bun install --frozen-lockfile --ignore-scripts",
|
||||
" else",
|
||||
` echo "${spec.version} control-plane render cannot install dependencies: no lockfile found" >&2`,
|
||||
" exit 42",
|
||||
" fi",
|
||||
"fi",
|
||||
"if [ -f scripts/gitops-render.mjs ]; then",
|
||||
" render_script=scripts/gitops-render.mjs",
|
||||
"elif [ \"$render_lane\" = v02 ] && [ -f scripts/g14-gitops-render.mjs ]; then",
|
||||
" render_script=scripts/g14-gitops-render.mjs",
|
||||
"else",
|
||||
` echo "${spec.version} control-plane render script missing: scripts/gitops-render.mjs" >&2`,
|
||||
" exit 43",
|
||||
"fi",
|
||||
"if [ -f scripts/run-bun.mjs ]; then",
|
||||
` node scripts/run-bun.mjs "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
||||
"else",
|
||||
` node "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
||||
"fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function v02ControlPlaneRenderScript(sourceCommit: string, token = v02RenderToken()): string {
|
||||
return runtimeLaneControlPlaneRenderScript(V02_LANE_SPEC, sourceCommit, token);
|
||||
}
|
||||
|
||||
export interface V02RenderTempResult {
|
||||
result: CommandJsonResult;
|
||||
renderDir: string;
|
||||
worktreeDir: string;
|
||||
token: string;
|
||||
}
|
||||
|
||||
export function runRuntimeLaneRenderToTemp(spec: HwlabRuntimeLaneSpec, sourceCommit: string): V02RenderTempResult {
|
||||
const token = runtimeLaneRenderToken();
|
||||
return {
|
||||
result: g14HostScript(runtimeLaneControlPlaneRenderScript(spec, sourceCommit, token), 180_000),
|
||||
renderDir: runtimeLaneRenderDir(spec, sourceCommit, token),
|
||||
worktreeDir: runtimeLaneRenderWorktreeDir(spec, sourceCommit, token),
|
||||
token,
|
||||
};
|
||||
}
|
||||
|
||||
export function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult {
|
||||
return runRuntimeLaneRenderToTemp(V02_LANE_SPEC, sourceCommit);
|
||||
}
|
||||
|
||||
export function runtimeLaneRenderToken(): string {
|
||||
const random = Math.random().toString(16).slice(2, 10);
|
||||
return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-");
|
||||
}
|
||||
|
||||
export function v02RenderToken(): string {
|
||||
return runtimeLaneRenderToken();
|
||||
}
|
||||
|
||||
export function runtimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
|
||||
return `/tmp/hwlab-${spec.lane}-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
||||
}
|
||||
|
||||
export function v02RenderDir(sourceCommit: string, token?: string): string {
|
||||
return runtimeLaneRenderDir(V02_LANE_SPEC, sourceCommit, token);
|
||||
}
|
||||
|
||||
export function runtimeLaneRenderWorktreeDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
|
||||
return `/tmp/hwlab-${spec.lane}-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
||||
}
|
||||
|
||||
export function v02RenderWorktreeDir(sourceCommit: string, token?: string): string {
|
||||
return runtimeLaneRenderWorktreeDir(V02_LANE_SPEC, sourceCommit, token);
|
||||
}
|
||||
|
||||
export function cleanupRuntimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandJsonResult {
|
||||
return g14HostScript([
|
||||
"set -eu",
|
||||
`render_dir=${shellQuote(renderDir)}`,
|
||||
"case \"$render_dir\" in",
|
||||
` /tmp/hwlab-${spec.lane}-control-plane-*) rm -rf "$render_dir" ;;`,
|
||||
` *) echo "refusing to cleanup unexpected ${spec.version} render dir: $render_dir" >&2; exit 44 ;;`,
|
||||
"esac",
|
||||
].join("\n"), 60_000);
|
||||
}
|
||||
|
||||
export function cleanupV02RenderDir(renderDir: string): CommandJsonResult {
|
||||
return cleanupRuntimeLaneRenderDir(V02_LANE_SPEC, renderDir);
|
||||
}
|
||||
|
||||
export function legacyRuntimeLaneArgoApplications(spec: HwlabRuntimeLaneSpec): string[] {
|
||||
if (spec.lane === "v02") return [];
|
||||
return [`hwlab-g14-${spec.lane}`].filter((name) => name !== spec.app);
|
||||
}
|
||||
|
||||
export function applyRuntimeLaneControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
const namespaceFile = `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`;
|
||||
const rbacFile = `${renderDir}/${spec.tektonDir}/rbac.yaml`;
|
||||
const pipelineFile = `${renderDir}/${spec.tektonDir}/pipeline.yaml`;
|
||||
const projectFile = `${renderDir}/argocd/project.yaml`;
|
||||
const applicationFile = `${renderDir}/argocd/${spec.argoApplicationFile}`;
|
||||
const resourceFiles = [namespaceFile, rbacFile, pipelineFile, projectFile, applicationFile];
|
||||
const legacyApplications = legacyRuntimeLaneArgoApplications(spec);
|
||||
const serverSideArgs = [
|
||||
"kubectl",
|
||||
"apply",
|
||||
"--server-side",
|
||||
"--force-conflicts",
|
||||
`--field-manager=${spec.controlPlaneFieldManager}`,
|
||||
...(dryRun ? ["--dry-run=server"] : []),
|
||||
...resourceFiles.flatMap((file) => ["-f", file]),
|
||||
];
|
||||
const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...serverSideArgs];
|
||||
const shellCommand = dryRun && spec.lane !== "v02"
|
||||
? [
|
||||
"set -eu",
|
||||
[
|
||||
...serverSideArgs.slice(0, 6),
|
||||
"-f",
|
||||
namespaceFile,
|
||||
"-f",
|
||||
pipelineFile,
|
||||
"-f",
|
||||
projectFile,
|
||||
"-f",
|
||||
applicationFile,
|
||||
].map(shellQuote).join(" "),
|
||||
[
|
||||
"kubectl",
|
||||
"apply",
|
||||
"--dry-run=client",
|
||||
...resourceFiles.flatMap((file) => ["-f", file]),
|
||||
].map(shellQuote).join(" "),
|
||||
].join("\n")
|
||||
: `exec ${serverSideArgs.map(shellQuote).join(" ")}`;
|
||||
const cleanupCommands = legacyApplications.map((name) => [
|
||||
"kubectl",
|
||||
"delete",
|
||||
"application",
|
||||
"-n",
|
||||
"argocd",
|
||||
name,
|
||||
"--ignore-not-found=true",
|
||||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||||
].map(shellQuote).join(" "));
|
||||
const script = cleanupCommands.length > 0
|
||||
? ["set -eu", shellCommand.replace(/^exec /, ""), ...cleanupCommands].join("\n")
|
||||
: shellCommand;
|
||||
const visibleCommand = cleanupCommands.length > 0
|
||||
? ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "sh", "--", script]
|
||||
: command;
|
||||
return runG14K3sRemoteAsync({
|
||||
script,
|
||||
timeoutSeconds,
|
||||
label: `${spec.lane}-control-plane-apply`,
|
||||
token: runtimeLaneRenderToken(),
|
||||
command: visibleCommand,
|
||||
});
|
||||
}
|
||||
|
||||
export function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
return applyRuntimeLaneControlPlaneFiles(V02_LANE_SPEC, renderDir, dryRun, timeoutSeconds);
|
||||
}
|
||||
|
||||
export interface V02RefreshMarker {
|
||||
ok: boolean;
|
||||
sourceCommit: string;
|
||||
refreshedAt: string;
|
||||
renderScriptHash: string;
|
||||
renderDir?: string | null;
|
||||
}
|
||||
|
||||
export function v02ControlPlaneRefreshStateDir(): string {
|
||||
return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh");
|
||||
}
|
||||
|
||||
export function v02ControlPlaneRefreshScriptHash(sourceCommit: string): string {
|
||||
return textHash(v02ControlPlaneRenderScript(sourceCommit, "contract-token"));
|
||||
}
|
||||
|
||||
export function v02ControlPlaneRefreshMarkerPath(sourceCommit: string): string {
|
||||
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`);
|
||||
}
|
||||
|
||||
export function v02ControlPlaneRefreshLockDir(sourceCommit: string): string {
|
||||
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`);
|
||||
}
|
||||
|
||||
export function readV02RefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
||||
const path = v02ControlPlaneRefreshMarkerPath(sourceCommit);
|
||||
if (!existsSync(path)) return null;
|
||||
try {
|
||||
const marker = JSON.parse(readFileSync(path, "utf8")) as V02RefreshMarker;
|
||||
return v02ReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function v02ReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
||||
const candidate = record(marker) as V02RefreshMarker;
|
||||
const refreshedAtMs = timestampMs(candidate.refreshedAt);
|
||||
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null;
|
||||
if (refreshedAtMs === null || nowMs - refreshedAtMs > V02_CONTROL_PLANE_REFRESH_TTL_MS) return null;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
export function writeV02RefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): V02RefreshMarker {
|
||||
const marker = {
|
||||
ok: true,
|
||||
sourceCommit,
|
||||
refreshedAt: new Date().toISOString(),
|
||||
renderScriptHash: scriptHash,
|
||||
renderDir,
|
||||
};
|
||||
const dir = v02ControlPlaneRefreshStateDir();
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(v02ControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
||||
return marker;
|
||||
}
|
||||
|
||||
export function acquireV02RefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } {
|
||||
const stateDir = v02ControlPlaneRefreshStateDir();
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
const lockDir = v02ControlPlaneRefreshLockDir(sourceCommit);
|
||||
const startedAtMs = Date.now();
|
||||
for (;;) {
|
||||
try {
|
||||
mkdirSync(lockDir);
|
||||
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
||||
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
} catch {
|
||||
const ageMs = (() => {
|
||||
try {
|
||||
return Date.now() - statSync(lockDir).mtimeMs;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
})();
|
||||
if (ageMs > V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) {
|
||||
try {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
continue;
|
||||
} catch {
|
||||
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
}
|
||||
}
|
||||
if (Date.now() - startedAtMs >= V02_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
||||
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function releaseV02RefreshLock(lockDir: string): void {
|
||||
try {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
||||
}
|
||||
}
|
||||
|
||||
export function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
|
||||
const scriptHash = v02ControlPlaneRefreshScriptHash(sourceCommit);
|
||||
const existingMarker = readV02RefreshMarker(sourceCommit, scriptHash);
|
||||
if (existingMarker !== null) {
|
||||
return {
|
||||
ok: true,
|
||||
phase: "control-plane-refresh",
|
||||
mode: "reused-recent-refresh-marker",
|
||||
sourceCommit,
|
||||
renderDir: existingMarker.renderDir ?? null,
|
||||
marker: existingMarker,
|
||||
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
||||
};
|
||||
}
|
||||
const lock = acquireV02RefreshLock(sourceCommit);
|
||||
if (!lock.acquired) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "control-plane-refresh-lock",
|
||||
mode: "local-refresh-lock",
|
||||
sourceCommit,
|
||||
lockDir: lock.lockDir,
|
||||
waitedMs: lock.waitedMs,
|
||||
degradedReason: "control-plane-refresh-lock-timeout",
|
||||
};
|
||||
}
|
||||
try {
|
||||
const markerAfterWait = readV02RefreshMarker(sourceCommit, scriptHash);
|
||||
if (markerAfterWait !== null) {
|
||||
return {
|
||||
ok: true,
|
||||
phase: "control-plane-refresh",
|
||||
mode: "waited-for-recent-refresh-marker",
|
||||
sourceCommit,
|
||||
renderDir: markerAfterWait.renderDir ?? null,
|
||||
marker: markerAfterWait,
|
||||
waitedMs: lock.waitedMs,
|
||||
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
||||
};
|
||||
}
|
||||
const render = runV02RenderToTemp(sourceCommit);
|
||||
if (!isCommandSuccess(render.result)) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "source-render",
|
||||
sourceCommit,
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
degradedReason: "control-plane-render-failed",
|
||||
};
|
||||
}
|
||||
const apply = applyV02ControlPlaneFiles(render.renderDir, false, timeoutSeconds);
|
||||
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
|
||||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
||||
const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs));
|
||||
const marker = ok ? writeV02RefreshMarker(sourceCommit, scriptHash, render.renderDir) : null;
|
||||
return {
|
||||
ok,
|
||||
phase: "control-plane-refresh",
|
||||
mode: "refreshed",
|
||||
sourceCommit,
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
||||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||||
marker,
|
||||
waitedMs: lock.waitedMs,
|
||||
degradedReason: !isCommandSuccess(apply)
|
||||
? "control-plane-apply-failed"
|
||||
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
||||
? "obsolete-cronjob-cleanup-failed"
|
||||
: undefined,
|
||||
};
|
||||
} finally {
|
||||
releaseV02RefreshLock(lock.lockDir);
|
||||
}
|
||||
}
|
||||
|
||||
export function getV02ObsoleteCronJobs(): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"cronjob",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
V02_POLLER,
|
||||
V02_RECONCILER,
|
||||
"--ignore-not-found",
|
||||
"-o",
|
||||
"name",
|
||||
], 60_000);
|
||||
}
|
||||
|
||||
export function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"delete",
|
||||
"cronjob",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
V02_POLLER,
|
||||
V02_RECONCILER,
|
||||
"--ignore-not-found=true",
|
||||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||||
], 60_000);
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. retirement module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:5525-5845 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { G14LegacyRetirementOptions } from "./types";
|
||||
import { k8sMetadataName, parseSectionJsonArray } from "./observability";
|
||||
import { statusText } from "./pr-monitor";
|
||||
import { compactCommandResult, g14K3s, isCommandSuccess, nested, record, shellQuote } from "./remote";
|
||||
import { ARGO_NAMESPACE, DEV_APP, DEV_NAMESPACE, G14_PROVIDER, PROD_APP, PROD_NAMESPACE, V02_APP, V02_RUNTIME_NAMESPACE } from "./types";
|
||||
import { parseShellSections, shellSectionOk } from "./v02-status";
|
||||
|
||||
export function legacyG14RetirementStatePath(): string {
|
||||
return rootPath(".state", "hwlab-g14", "legacy-g14-retirement.json");
|
||||
}
|
||||
|
||||
export function legacyG14RetirementApplications(): string[] {
|
||||
return [DEV_APP, PROD_APP];
|
||||
}
|
||||
|
||||
export function legacyG14RetirementNamespaces(): string[] {
|
||||
return [DEV_NAMESPACE, PROD_NAMESPACE];
|
||||
}
|
||||
|
||||
export function protectedG14RuntimeApplications(): string[] {
|
||||
return [V02_APP, hwlabRuntimeLaneSpec("v03").app];
|
||||
}
|
||||
|
||||
export function protectedG14RuntimeNamespaces(): string[] {
|
||||
return [V02_RUNTIME_NAMESPACE, hwlabRuntimeLaneSpec("v03").runtimeNamespace];
|
||||
}
|
||||
|
||||
export function readLegacyG14RetirementMarker(): Record<string, unknown> | null {
|
||||
const path = legacyG14RetirementStatePath();
|
||||
if (!existsSync(path)) return null;
|
||||
try {
|
||||
return record(JSON.parse(readFileSync(path, "utf8")) as unknown);
|
||||
} catch (error) {
|
||||
return { status: "unreadable", path, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
export function writeLegacyG14RetirementMarker(status: string, reason: string, details: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
const path = legacyG14RetirementStatePath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const previous = readLegacyG14RetirementMarker();
|
||||
const marker = {
|
||||
status,
|
||||
reason,
|
||||
requestedAt: typeof previous?.requestedAt === "string" ? previous.requestedAt : new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
legacyApplications: legacyG14RetirementApplications(),
|
||||
legacyNamespaces: legacyG14RetirementNamespaces(),
|
||||
protectedApplications: protectedG14RuntimeApplications(),
|
||||
protectedNamespaces: protectedG14RuntimeNamespaces(),
|
||||
...details,
|
||||
};
|
||||
writeFileSync(path, `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
||||
return { ...marker, path };
|
||||
}
|
||||
|
||||
export function legacyG14RetirementBlocksMonitor(): Record<string, unknown> | null {
|
||||
const marker = readLegacyG14RetirementMarker();
|
||||
if (marker === null) {
|
||||
return {
|
||||
status: "retired-by-contract",
|
||||
path: legacyG14RetirementStatePath(),
|
||||
reason: "legacy G14 DEV/PROD monitor is retired; inspect retirement status for live cluster evidence",
|
||||
next: { status: "bun scripts/cli.ts hwlab g14 retirement status", v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02", v03Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v03" },
|
||||
};
|
||||
}
|
||||
const status = String(marker.status ?? "");
|
||||
return status === "retiring" || status === "retired" || status === "failed" || status === "unreadable" ? marker : null;
|
||||
}
|
||||
|
||||
export function k8sMetadataNamespace(item: Record<string, unknown>): string | null {
|
||||
const namespace = record(item.metadata).namespace;
|
||||
return typeof namespace === "string" && namespace.length > 0 ? namespace : null;
|
||||
}
|
||||
|
||||
export function applicationSummary(item: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
name: k8sMetadataName(item),
|
||||
namespace: k8sMetadataNamespace(item) ?? ARGO_NAMESPACE,
|
||||
deletionTimestamp: record(item.metadata).deletionTimestamp ?? null,
|
||||
sync: nested(item, ["status", "sync", "status"]) ?? null,
|
||||
health: nested(item, ["status", "health", "status"]) ?? null,
|
||||
revision: nested(item, ["status", "sync", "revision"]) ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function namespaceSummary(item: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
name: k8sMetadataName(item),
|
||||
phase: nested(item, ["status", "phase"]) ?? null,
|
||||
deletionTimestamp: record(item.metadata).deletionTimestamp ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseNamespacedResourcePreview(text: string): Record<string, unknown>[] {
|
||||
const groups: Record<string, string[]> = {};
|
||||
let current = "";
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0) continue;
|
||||
const namespaceMatch = /^__namespace\t(.+)$/u.exec(trimmed);
|
||||
if (namespaceMatch !== null) {
|
||||
current = namespaceMatch[1] ?? "";
|
||||
if (current.length > 0) groups[current] = [];
|
||||
continue;
|
||||
}
|
||||
if (current.length > 0) groups[current]?.push(trimmed);
|
||||
}
|
||||
return Object.entries(groups).map(([namespace, items]) => ({
|
||||
namespace,
|
||||
count: items.length,
|
||||
preview: items.slice(0, 40),
|
||||
truncated: items.length > 40,
|
||||
}));
|
||||
}
|
||||
|
||||
export function legacyG14RetirementStatusScript(): string {
|
||||
const legacyApps = legacyG14RetirementApplications().map(shellQuote).join(" ");
|
||||
const legacyNamespaces = legacyG14RetirementNamespaces().map(shellQuote).join(" ");
|
||||
const protectedApps = protectedG14RuntimeApplications().map(shellQuote).join(" ");
|
||||
const protectedNamespaces = protectedG14RuntimeNamespaces().map(shellQuote).join(" ");
|
||||
return [
|
||||
"set +e",
|
||||
"section() {",
|
||||
" name=\"$1\"",
|
||||
" shift",
|
||||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||||
" \"$@\"",
|
||||
" code=$?",
|
||||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||||
"}",
|
||||
`section legacyApplications kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${legacyApps} --ignore-not-found=true -o json`,
|
||||
`section legacyNamespaces kubectl get namespace ${legacyNamespaces} --ignore-not-found=true -o json`,
|
||||
`section protectedApplications kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${protectedApps} --ignore-not-found=true -o json`,
|
||||
`section protectedNamespaces kubectl get namespace ${protectedNamespaces} --ignore-not-found=true -o json`,
|
||||
"section legacyResources sh -lc " + shellQuote([
|
||||
`for ns in ${legacyNamespaces}; do`,
|
||||
" printf '__namespace\\t%s\\n' \"$ns\"",
|
||||
" kubectl get deploy,statefulset,svc,pod,pvc,ingress,configmap,secret -n \"$ns\" -o name --ignore-not-found=true 2>/dev/null | sed -n '1,160p'",
|
||||
"done",
|
||||
].join("\n")),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function legacyG14RetirementStatus(): Record<string, unknown> {
|
||||
const result = g14K3s(["sh", "--", legacyG14RetirementStatusScript()], 60_000);
|
||||
const sections = parseShellSections(statusText(result));
|
||||
const legacyApplications = parseSectionJsonArray(sections.legacyApplications).map(applicationSummary);
|
||||
const legacyNamespaces = parseSectionJsonArray(sections.legacyNamespaces).map(namespaceSummary);
|
||||
const protectedApplications = parseSectionJsonArray(sections.protectedApplications).map(applicationSummary);
|
||||
const protectedNamespaces = parseSectionJsonArray(sections.protectedNamespaces).map(namespaceSummary);
|
||||
const terminating = legacyApplications.some((item) => item.deletionTimestamp !== null)
|
||||
|| legacyNamespaces.some((item) => item.deletionTimestamp !== null || item.phase === "Terminating");
|
||||
const retired = legacyApplications.length === 0 && legacyNamespaces.length === 0;
|
||||
const retirementState = retired ? "retired" : terminating ? "terminating" : "active";
|
||||
const protectedAppNames = new Set(protectedApplications.map((item) => item.name).filter((name): name is string => typeof name === "string"));
|
||||
const protectedNamespaceNames = new Set(protectedNamespaces.map((item) => item.name).filter((name): name is string => typeof name === "string"));
|
||||
const missingProtectedApplications = protectedG14RuntimeApplications().filter((name) => !protectedAppNames.has(name));
|
||||
const missingProtectedNamespaces = protectedG14RuntimeNamespaces().filter((name) => !protectedNamespaceNames.has(name));
|
||||
const sectionsOk =
|
||||
shellSectionOk(sections.legacyApplications) &&
|
||||
shellSectionOk(sections.legacyNamespaces) &&
|
||||
shellSectionOk(sections.protectedApplications) &&
|
||||
shellSectionOk(sections.protectedNamespaces) &&
|
||||
shellSectionOk(sections.legacyResources);
|
||||
return {
|
||||
ok: isCommandSuccess(result) && sectionsOk,
|
||||
command: "hwlab g14 retirement status",
|
||||
provider: G14_PROVIDER,
|
||||
argoNamespace: ARGO_NAMESPACE,
|
||||
retirementState,
|
||||
retired,
|
||||
legacy: {
|
||||
applications: legacyApplications,
|
||||
namespaces: legacyNamespaces,
|
||||
resources: parseNamespacedResourcePreview(String(sections.legacyResources?.stdout ?? "")),
|
||||
},
|
||||
protected: {
|
||||
applications: protectedApplications,
|
||||
namespaces: protectedNamespaces,
|
||||
missingApplications: missingProtectedApplications,
|
||||
missingNamespaces: missingProtectedNamespaces,
|
||||
ok: missingProtectedApplications.length === 0 && missingProtectedNamespaces.length === 0,
|
||||
},
|
||||
marker: readLegacyG14RetirementMarker(),
|
||||
result: compactCommandResult(result),
|
||||
next: retired
|
||||
? { verifyV02: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02", verifyV03: "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03" }
|
||||
: { plan: "bun scripts/cli.ts hwlab g14 retirement plan", execute: "bun scripts/cli.ts hwlab g14 retirement execute --confirm" },
|
||||
};
|
||||
}
|
||||
|
||||
export function legacyG14RetirementPlan(status = legacyG14RetirementStatus()): Record<string, unknown> {
|
||||
return {
|
||||
ok: status.ok === true,
|
||||
command: "hwlab g14 retirement plan",
|
||||
mode: "dry-run",
|
||||
mutation: false,
|
||||
reason: "retire legacy G14 DEV/PROD runtime and support surface without touching v0.2/v0.3",
|
||||
destructiveTargets: {
|
||||
applications: legacyG14RetirementApplications().map((name) => ({ apiVersion: "argoproj.io/v1alpha1", kind: "Application", namespace: ARGO_NAMESPACE, name })),
|
||||
namespaces: legacyG14RetirementNamespaces().map((name) => ({ apiVersion: "v1", kind: "Namespace", name })),
|
||||
},
|
||||
protectedTargets: {
|
||||
applications: protectedG14RuntimeApplications().map((name) => ({ apiVersion: "argoproj.io/v1alpha1", kind: "Application", namespace: ARGO_NAMESPACE, name })),
|
||||
namespaces: protectedG14RuntimeNamespaces().map((name) => ({ apiVersion: "v1", kind: "Namespace", name })),
|
||||
},
|
||||
localSupportActions: [
|
||||
{ action: "write-retirement-marker", path: legacyG14RetirementStatePath() },
|
||||
{ action: "cancel-active-local-job", jobName: "hwlab_g14_pr_monitor" },
|
||||
{ action: "block-new-monitor-starts", command: "bun scripts/cli.ts hwlab g14 monitor-prs --lane g14" },
|
||||
],
|
||||
status,
|
||||
next: { execute: "bun scripts/cli.ts hwlab g14 retirement execute --confirm" },
|
||||
};
|
||||
}
|
||||
|
||||
export function legacyG14RecordRolloutRetired(): Record<string, unknown> {
|
||||
return {
|
||||
ok: false,
|
||||
command: "hwlab g14 record-rollout",
|
||||
phase: "retired",
|
||||
degradedReason: "legacy-g14-dev-prod-retired",
|
||||
message: "Legacy G14 DEV rollout recording is retired. Use active runtime lane closeout/status evidence instead.",
|
||||
retirement: readLegacyG14RetirementMarker(),
|
||||
next: {
|
||||
retirementStatus: "bun scripts/cli.ts hwlab g14 retirement status",
|
||||
v02Status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||||
v03Status: "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function cancelLegacyG14MonitorJobs(dryRun: boolean): Record<string, unknown> {
|
||||
const candidates = listJobs()
|
||||
.filter((job) => job.name === "hwlab_g14_pr_monitor" && (job.status === "queued" || job.status === "running"))
|
||||
.map((job) => ({ id: job.id, name: job.name, status: job.status, createdAt: job.createdAt, runnerPid: job.runnerPid ?? null }));
|
||||
if (dryRun) return { ok: true, dryRun: true, candidates, actions: candidates.map((job) => ({ action: "would-cancel", ...job })) };
|
||||
const actions = candidates.map((job) => cancelJob(job.id));
|
||||
return { ok: actions.every((action) => record(action).ok !== false), dryRun: false, candidates, actions };
|
||||
}
|
||||
|
||||
export function legacyG14RetirementDeleteScript(): string {
|
||||
const legacyApps = legacyG14RetirementApplications();
|
||||
const legacyNamespaces = legacyG14RetirementNamespaces();
|
||||
return [
|
||||
"set +e",
|
||||
"overall=0",
|
||||
"printf '__UNIDESK_SECTION_BEGIN__ deleteApplications\\n'",
|
||||
...legacyApps.map((app) => [
|
||||
`kubectl delete application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(app)} --ignore-not-found=true --wait=false`,
|
||||
"code=$?",
|
||||
"[ \"$code\" -eq 0 ] || overall=1",
|
||||
`printf 'application\\t%s\\texit=%s\\n' ${shellQuote(app)} "$code"`,
|
||||
].join("\n")),
|
||||
"printf '__UNIDESK_SECTION_END__ deleteApplications exit=0\\n'",
|
||||
"printf '__UNIDESK_SECTION_BEGIN__ deleteNamespaces\\n'",
|
||||
...legacyNamespaces.map((namespace) => [
|
||||
`kubectl delete namespace ${shellQuote(namespace)} --ignore-not-found=true --wait=false`,
|
||||
"code=$?",
|
||||
"[ \"$code\" -eq 0 ] || overall=1",
|
||||
`printf 'namespace\\t%s\\texit=%s\\n' ${shellQuote(namespace)} "$code"`,
|
||||
].join("\n")),
|
||||
"printf '__UNIDESK_SECTION_END__ deleteNamespaces exit=0\\n'",
|
||||
"exit \"$overall\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function waitForLegacyG14Retirement(timeoutSeconds: number): Record<string, unknown> {
|
||||
const startedAtMs = Date.now();
|
||||
let lastStatus: Record<string, unknown> = {};
|
||||
let attempts = 0;
|
||||
while (Date.now() - startedAtMs <= timeoutSeconds * 1000) {
|
||||
attempts += 1;
|
||||
lastStatus = legacyG14RetirementStatus();
|
||||
if (lastStatus.retired === true) {
|
||||
return { ok: true, attempts, elapsedMs: Date.now() - startedAtMs, status: lastStatus };
|
||||
}
|
||||
const wait = runCommand(["sleep", "5"], repoRoot, { timeoutMs: 7_000 });
|
||||
if (wait.exitCode !== 0) break;
|
||||
}
|
||||
return { ok: false, attempts, elapsedMs: Date.now() - startedAtMs, status: lastStatus, degradedReason: "legacy-g14-retirement-not-complete-before-timeout" };
|
||||
}
|
||||
|
||||
export function runLegacyG14Retirement(options: G14LegacyRetirementOptions): Record<string, unknown> {
|
||||
const before = legacyG14RetirementStatus();
|
||||
if (options.action === "status") return before;
|
||||
if (options.action === "plan" || options.dryRun) return { ...legacyG14RetirementPlan(before), command: `hwlab g14 retirement ${options.action}`, reason: options.reason };
|
||||
|
||||
const markerBeforeDelete = writeLegacyG14RetirementMarker("retiring", options.reason, { command: "hwlab g14 retirement execute --confirm" });
|
||||
const localJobs = cancelLegacyG14MonitorJobs(false);
|
||||
const deleteResult = g14K3s(["sh", "--", legacyG14RetirementDeleteScript()], Math.max(60_000, options.timeoutSeconds * 1000));
|
||||
const deleteSections = parseShellSections(statusText(deleteResult));
|
||||
const afterDelete = legacyG14RetirementStatus();
|
||||
const wait = options.wait ? waitForLegacyG14Retirement(options.timeoutSeconds) : null;
|
||||
const finalStatus = wait === null ? afterDelete : record(wait.status);
|
||||
const liveMutationOk = isCommandSuccess(deleteResult);
|
||||
const finalRetired = finalStatus.retired === true;
|
||||
const marker = writeLegacyG14RetirementMarker(finalRetired ? "retired" : liveMutationOk ? "retiring" : "failed", options.reason, {
|
||||
command: "hwlab g14 retirement execute --confirm",
|
||||
deleteExitCode: deleteResult.exitCode,
|
||||
finalRetirementState: finalStatus.retirementState ?? null,
|
||||
});
|
||||
return {
|
||||
ok: liveMutationOk && (options.wait ? finalRetired : finalStatus.retirementState !== "active"),
|
||||
command: "hwlab g14 retirement execute",
|
||||
mode: "confirmed-retirement",
|
||||
mutation: true,
|
||||
reason: options.reason,
|
||||
before,
|
||||
markerBeforeDelete,
|
||||
localJobs,
|
||||
delete: {
|
||||
ok: liveMutationOk,
|
||||
applications: String(deleteSections.deleteApplications?.stdout ?? "").split(/\r?\n/u).filter(Boolean),
|
||||
namespaces: String(deleteSections.deleteNamespaces?.stdout ?? "").split(/\r?\n/u).filter(Boolean),
|
||||
result: compactCommandResult(deleteResult),
|
||||
},
|
||||
afterDelete,
|
||||
wait,
|
||||
finalStatus,
|
||||
marker,
|
||||
next: finalRetired
|
||||
? { verifyV02: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02", verifyV03: "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03" }
|
||||
: { status: "bun scripts/cli.ts hwlab g14 retirement status" },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. secrets module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:5055-5524 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { G14SecretOptions } from "./types";
|
||||
import { statusText } from "./pr-monitor";
|
||||
import { compactCommandResult, g14K3s, g14K3sInlineScriptWithInput, isCommandSuccess, readOrCreateLocalMasterAdminApiKey, runtimeLaneMasterAdminApiKeySecretName, runtimeLaneOpenFgaSecretName, runtimeLanePostgresSecretName, runtimeLanePrimaryDbName, runtimeLaneSecretFieldManager, shellQuote } from "./remote";
|
||||
import { V02_MASTER_ADMIN_API_KEY_SECRET, V02_MASTER_ADMIN_API_KEY_SECRET_KEY, V02_OPENFGA_AUTHN_SECRET_KEY, V02_OPENFGA_DATASTORE_URI_SECRET_KEY, V02_OPENFGA_DB_NAME, V02_OPENFGA_DB_USER, V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY, V02_OPENFGA_SECRET, V02_RUNTIME_NAMESPACE } from "./types";
|
||||
import { keyValueLinesFromText, numericField } from "./v02-status";
|
||||
|
||||
export function v02SecretScript(options: G14SecretOptions): string {
|
||||
const spec = hwlabRuntimeLaneSpec(options.lane);
|
||||
if (options.preset === "generic-delete") return v02DeleteSecretScript(options);
|
||||
if (options.preset === "master-server-admin-api-key") return v02MasterAdminApiKeySecretScript(options, spec);
|
||||
return [
|
||||
v02OpenFgaSecretScript(options, spec),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function v02DeleteSecretScript(options: G14SecretOptions): string {
|
||||
return [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(V02_RUNTIME_NAMESPACE)}`,
|
||||
`name=${shellQuote(options.name)}`,
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
"preset=generic-delete",
|
||||
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||||
"before_exists=$(secret_exists_flag)",
|
||||
"delete_exit=",
|
||||
"action=observed",
|
||||
"mutation=false",
|
||||
"if [ \"$dry_run\" = true ]; then",
|
||||
" if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
|
||||
"else",
|
||||
" kubectl -n \"$namespace\" delete secret \"$name\" --ignore-not-found=true >/tmp/hwlab-secret-delete.out 2>/tmp/hwlab-secret-delete.err",
|
||||
" delete_exit=$?",
|
||||
" if [ \"$delete_exit\" -eq 0 ]; then",
|
||||
" if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
|
||||
" else",
|
||||
" action=delete-failed",
|
||||
" fi",
|
||||
"fi",
|
||||
"after_exists=$(secret_exists_flag)",
|
||||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||||
"printf 'secret\\t%s\\n' \"$name\"",
|
||||
"printf 'preset\\t%s\\n' \"$preset\"",
|
||||
"printf 'action\\t%s\\n' \"$action\"",
|
||||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||||
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
||||
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
||||
"printf 'deleteExitCode\\t%s\\n' \"$delete_exit\"",
|
||||
"if [ -n \"$delete_exit\" ] && [ \"$delete_exit\" != 0 ]; then exit \"$delete_exit\"; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function v02OpenFgaSecretScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string {
|
||||
const selectedKey = options.key ?? "";
|
||||
const namespace = spec.runtimeNamespace;
|
||||
const openFgaSecret = runtimeLaneOpenFgaSecretName(spec);
|
||||
const postgresSecret = runtimeLanePostgresSecretName(spec);
|
||||
const primaryDbName = runtimeLanePrimaryDbName(spec);
|
||||
const dedicatedDb = spec.lane === "v02";
|
||||
const dbMode = dedicatedDb ? "dedicated" : "primary";
|
||||
const dbName = dedicatedDb ? V02_OPENFGA_DB_NAME : primaryDbName;
|
||||
const dbUser = dedicatedDb ? V02_OPENFGA_DB_USER : primaryDbName;
|
||||
const dbHost = `${postgresSecret}.${namespace}.svc.cluster.local`;
|
||||
const fieldManager = runtimeLaneSecretFieldManager(spec);
|
||||
return [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`name=${shellQuote(openFgaSecret)}`,
|
||||
`selected_key=${shellQuote(selectedKey)}`,
|
||||
`authn_key=${shellQuote(V02_OPENFGA_AUTHN_SECRET_KEY)}`,
|
||||
`datastore_uri_key=${shellQuote(V02_OPENFGA_DATASTORE_URI_SECRET_KEY)}`,
|
||||
`postgres_password_key=${shellQuote(V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY)}`,
|
||||
`db_mode=${shellQuote(dbMode)}`,
|
||||
`db_name=${shellQuote(dbName)}`,
|
||||
`db_user=${shellQuote(dbUser)}`,
|
||||
`postgres_secret=${shellQuote(postgresSecret)}`,
|
||||
`postgres_statefulset=${shellQuote(postgresSecret)}`,
|
||||
`postgres_admin_user=${shellQuote(primaryDbName)}`,
|
||||
`db_host=${shellQuote(dbHost)}`,
|
||||
`action_request=${shellQuote(options.action)}`,
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
`field_manager=${shellQuote(fieldManager)}`,
|
||||
"preset=openfga",
|
||||
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||||
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
||||
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
|
||||
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
||||
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
|
||||
"probe_db() {",
|
||||
" if [ \"$db_mode\" = primary ]; then",
|
||||
" role_result=primary",
|
||||
" database_result=primary",
|
||||
" probe_exit=not-required",
|
||||
" return",
|
||||
" fi",
|
||||
" role_result=unknown",
|
||||
" database_result=unknown",
|
||||
" probe_exit=missing-postgres-admin-secret",
|
||||
" if [ -n \"$postgres_admin_password\" ]; then",
|
||||
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
|
||||
" role_exit=$?",
|
||||
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
|
||||
" database_exit=$?",
|
||||
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
|
||||
" fi",
|
||||
"}",
|
||||
"before_exists=$(secret_exists_flag)",
|
||||
"before_authn_b64=$(secret_b64_key \"$authn_key\")",
|
||||
"before_uri_b64=$(secret_b64_key \"$datastore_uri_key\")",
|
||||
"before_pg_password_b64=$(secret_b64_key \"$postgres_password_key\")",
|
||||
"before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)",
|
||||
"before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)",
|
||||
"before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)",
|
||||
"before_authn_bytes=$(decoded_length \"$before_authn_b64\")",
|
||||
"before_uri_bytes=$(decoded_length \"$before_uri_b64\")",
|
||||
"before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")",
|
||||
"authn_value=$(decoded_value \"$before_authn_b64\")",
|
||||
"datastore_uri=$(decoded_value \"$before_uri_b64\")",
|
||||
"pg_password=$(decoded_value \"$before_pg_password_b64\")",
|
||||
"postgres_admin_b64=$(kubectl -n \"$namespace\" get secret \"$postgres_secret\" -o 'go-template={{ index .data \"POSTGRES_PASSWORD\" }}' 2>/dev/null || true)",
|
||||
"postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)",
|
||||
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
|
||||
"probe_db",
|
||||
"db_role_exists_before=$role_result",
|
||||
"db_database_exists_before=$database_result",
|
||||
"db_probe_exit_before=$probe_exit",
|
||||
"action=observed",
|
||||
"mutation=false",
|
||||
"apply_exit=",
|
||||
"db_ensure_exit=",
|
||||
"if [ \"$action_request\" = ensure ]; then",
|
||||
" missing_secret=false",
|
||||
" [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true",
|
||||
" [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true",
|
||||
" [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true",
|
||||
" missing_db=false",
|
||||
" if [ \"$db_mode\" = dedicated ]; then",
|
||||
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
|
||||
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
|
||||
" fi",
|
||||
" if [ \"$dry_run\" = true ]; then",
|
||||
" if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
|
||||
" elif ! command -v openssl >/dev/null 2>&1; then",
|
||||
" action=openssl-missing",
|
||||
" apply_exit=127",
|
||||
" elif [ -z \"$postgres_admin_password\" ]; then",
|
||||
" action=postgres-admin-secret-missing",
|
||||
" apply_exit=44",
|
||||
" else",
|
||||
" [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)",
|
||||
" if [ \"$db_mode\" = primary ]; then",
|
||||
" [ -n \"$pg_password\" ] || pg_password=\"$postgres_admin_password\"",
|
||||
" else",
|
||||
" [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
|
||||
" fi",
|
||||
" [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
|
||||
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
||||
" apply_exit=$?",
|
||||
" if [ \"$apply_exit\" -eq 0 ]; then",
|
||||
" if [ \"$db_mode\" = primary ]; then",
|
||||
" action=ensured",
|
||||
" mutation=true",
|
||||
" else",
|
||||
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'",
|
||||
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
|
||||
"\\gexec",
|
||||
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
|
||||
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
|
||||
"\\gexec",
|
||||
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
|
||||
"SQL",
|
||||
" db_ensure_exit=$?",
|
||||
" if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi",
|
||||
" fi",
|
||||
" else",
|
||||
" action=apply-failed",
|
||||
" fi",
|
||||
" fi",
|
||||
"fi",
|
||||
"after_exists=$(secret_exists_flag)",
|
||||
"after_authn_b64=$(secret_b64_key \"$authn_key\")",
|
||||
"after_uri_b64=$(secret_b64_key \"$datastore_uri_key\")",
|
||||
"after_pg_password_b64=$(secret_b64_key \"$postgres_password_key\")",
|
||||
"after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)",
|
||||
"after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)",
|
||||
"after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)",
|
||||
"after_authn_bytes=$(decoded_length \"$after_authn_b64\")",
|
||||
"after_uri_bytes=$(decoded_length \"$after_uri_b64\")",
|
||||
"after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")",
|
||||
"probe_db",
|
||||
"db_role_exists_after=$role_result",
|
||||
"db_database_exists_after=$database_result",
|
||||
"db_probe_exit_after=$probe_exit",
|
||||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||||
"printf 'secret\\t%s\\n' \"$name\"",
|
||||
"printf 'key\\t%s\\n' \"$selected_key\"",
|
||||
"printf 'preset\\t%s\\n' \"$preset\"",
|
||||
"printf 'action\\t%s\\n' \"$action\"",
|
||||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||||
"printf 'dbMode\\t%s\\n' \"$db_mode\"",
|
||||
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
||||
"printf 'beforeAuthnPresent\\t%s\\n' \"$before_authn_present\"",
|
||||
"printf 'beforeAuthnBytes\\t%s\\n' \"$before_authn_bytes\"",
|
||||
"printf 'beforeDatastoreUriPresent\\t%s\\n' \"$before_uri_present\"",
|
||||
"printf 'beforeDatastoreUriBytes\\t%s\\n' \"$before_uri_bytes\"",
|
||||
"printf 'beforePostgresPasswordPresent\\t%s\\n' \"$before_pg_password_present\"",
|
||||
"printf 'beforePostgresPasswordBytes\\t%s\\n' \"$before_pg_password_bytes\"",
|
||||
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
||||
"printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"",
|
||||
"printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"",
|
||||
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"",
|
||||
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"",
|
||||
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"",
|
||||
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"",
|
||||
"printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"",
|
||||
"printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"",
|
||||
"printf 'dbName\\t%s\\n' \"$db_name\"",
|
||||
"printf 'dbUser\\t%s\\n' \"$db_user\"",
|
||||
"printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"",
|
||||
"printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"",
|
||||
"printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"",
|
||||
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
|
||||
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
|
||||
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
|
||||
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
||||
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
|
||||
"authn_value=",
|
||||
"datastore_uri=",
|
||||
"pg_password=",
|
||||
"postgres_admin_password=",
|
||||
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
||||
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function v02MasterAdminApiKeySecretScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string {
|
||||
const namespace = spec.runtimeNamespace;
|
||||
const name = runtimeLaneMasterAdminApiKeySecretName(spec);
|
||||
const fieldManager = runtimeLaneSecretFieldManager(spec);
|
||||
return [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`name=${shellQuote(name)}`,
|
||||
`api_key_name=${shellQuote(V02_MASTER_ADMIN_API_KEY_SECRET_KEY)}`,
|
||||
`action_request=${shellQuote(options.action)}`,
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
`field_manager=${shellQuote(fieldManager)}`,
|
||||
"preset=master-server-admin-api-key",
|
||||
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||||
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
||||
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
||||
"decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-24; value=; fi; }",
|
||||
"before_exists=$(secret_exists_flag)",
|
||||
"before_api_key_b64=$(secret_b64_key \"$api_key_name\")",
|
||||
"before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)",
|
||||
"before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")",
|
||||
"before_api_key_prefix=$(decoded_prefix \"$before_api_key_b64\")",
|
||||
"action=observed",
|
||||
"mutation=false",
|
||||
"apply_exit=",
|
||||
"if [ \"$action_request\" = ensure ]; then",
|
||||
" missing_secret=false",
|
||||
" [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true",
|
||||
" if [ \"$dry_run\" = true ]; then",
|
||||
" if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi",
|
||||
" else",
|
||||
" api_key=$(cat)",
|
||||
" case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac",
|
||||
" if [ -z \"$apply_exit\" ]; then",
|
||||
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
||||
" apply_exit=$?",
|
||||
" if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
|
||||
" fi",
|
||||
" api_key=",
|
||||
" fi",
|
||||
"fi",
|
||||
"after_exists=$(secret_exists_flag)",
|
||||
"after_api_key_b64=$(secret_b64_key \"$api_key_name\")",
|
||||
"after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)",
|
||||
"after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")",
|
||||
"after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")",
|
||||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||||
"printf 'secret\\t%s\\n' \"$name\"",
|
||||
"printf 'key\\t%s\\n' \"$api_key_name\"",
|
||||
"printf 'preset\\t%s\\n' \"$preset\"",
|
||||
"printf 'action\\t%s\\n' \"$action\"",
|
||||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||||
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
||||
"printf 'beforeApiKeyPresent\\t%s\\n' \"$before_api_key_present\"",
|
||||
"printf 'beforeApiKeyBytes\\t%s\\n' \"$before_api_key_bytes\"",
|
||||
"printf 'beforeApiKeyPrefix\\t%s\\n' \"$before_api_key_prefix\"",
|
||||
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
||||
"printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"",
|
||||
"printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"",
|
||||
"printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"",
|
||||
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
||||
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||||
const fields = keyValueLinesFromText(text);
|
||||
if (fields.preset === "generic-delete") {
|
||||
const absent = fields.afterExists !== "yes";
|
||||
return {
|
||||
ok: commandOk && absent,
|
||||
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
|
||||
secret: fields.secret || null,
|
||||
preset: "generic-delete",
|
||||
action: fields.action || null,
|
||||
dryRun: fields.dryRun === "true",
|
||||
mutation: fields.mutation === "true",
|
||||
before: {
|
||||
exists: fields.beforeExists === "yes",
|
||||
},
|
||||
after: {
|
||||
exists: fields.afterExists === "yes",
|
||||
},
|
||||
deleteExitCode: numericField(fields.deleteExitCode),
|
||||
exitCode,
|
||||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||||
valuesRedacted: true,
|
||||
summary: absent
|
||||
? `${fields.secret || "secret"} absent`
|
||||
: `${fields.secret || "secret"} still exists`,
|
||||
};
|
||||
}
|
||||
if (fields.preset === "master-server-admin-api-key") {
|
||||
const afterBytes = numericField(fields.afterApiKeyBytes);
|
||||
const healthy =
|
||||
fields.afterExists === "yes" &&
|
||||
fields.afterApiKeyPresent === "yes" &&
|
||||
typeof afterBytes === "number" && afterBytes > 0 &&
|
||||
String(fields.afterApiKeyPrefix ?? "").startsWith("hwl_live_");
|
||||
return {
|
||||
ok: commandOk && healthy,
|
||||
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
|
||||
secret: fields.secret || V02_MASTER_ADMIN_API_KEY_SECRET,
|
||||
key: fields.key || V02_MASTER_ADMIN_API_KEY_SECRET_KEY,
|
||||
preset: "master-server-admin-api-key",
|
||||
action: fields.action || null,
|
||||
dryRun: fields.dryRun === "true",
|
||||
mutation: fields.mutation === "true",
|
||||
before: {
|
||||
exists: fields.beforeExists === "yes",
|
||||
apiKey: { keyPresent: fields.beforeApiKeyPresent === "yes", valueBytes: numericField(fields.beforeApiKeyBytes), keyPrefix: fields.beforeApiKeyPrefix || null },
|
||||
},
|
||||
after: {
|
||||
exists: fields.afterExists === "yes",
|
||||
apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null },
|
||||
},
|
||||
applyExitCode: numericField(fields.applyExitCode),
|
||||
exitCode,
|
||||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||||
valuesRedacted: true,
|
||||
summary: healthy
|
||||
? `${fields.secret || V02_MASTER_ADMIN_API_KEY_SECRET}/${fields.key || V02_MASTER_ADMIN_API_KEY_SECRET_KEY} exists`
|
||||
: `${fields.secret || V02_MASTER_ADMIN_API_KEY_SECRET}/${fields.key || V02_MASTER_ADMIN_API_KEY_SECRET_KEY} missing`,
|
||||
};
|
||||
}
|
||||
if (fields.preset === "openfga") {
|
||||
const afterAuthnBytes = numericField(fields.afterAuthnBytes);
|
||||
const afterUriBytes = numericField(fields.afterDatastoreUriBytes);
|
||||
const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes);
|
||||
const keysHealthy =
|
||||
fields.afterExists === "yes" &&
|
||||
fields.afterAuthnPresent === "yes" &&
|
||||
fields.afterDatastoreUriPresent === "yes" &&
|
||||
fields.afterPostgresPasswordPresent === "yes" &&
|
||||
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
|
||||
typeof afterUriBytes === "number" && afterUriBytes > 0 &&
|
||||
typeof afterPasswordBytes === "number" && afterPasswordBytes > 0;
|
||||
const databaseHealthy = fields.dbMode === "primary" || (fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t");
|
||||
const healthy = keysHealthy && databaseHealthy;
|
||||
return {
|
||||
ok: commandOk && healthy,
|
||||
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
|
||||
secret: fields.secret || V02_OPENFGA_SECRET,
|
||||
key: fields.key || null,
|
||||
preset: "openfga",
|
||||
action: fields.action || null,
|
||||
dryRun: fields.dryRun === "true",
|
||||
mutation: fields.mutation === "true",
|
||||
dbMode: fields.dbMode || "dedicated",
|
||||
before: {
|
||||
exists: fields.beforeExists === "yes",
|
||||
authnPresharedKey: { keyPresent: fields.beforeAuthnPresent === "yes", valueBytes: numericField(fields.beforeAuthnBytes) },
|
||||
datastoreUri: { keyPresent: fields.beforeDatastoreUriPresent === "yes", valueBytes: numericField(fields.beforeDatastoreUriBytes) },
|
||||
postgresPassword: { keyPresent: fields.beforePostgresPasswordPresent === "yes", valueBytes: numericField(fields.beforePostgresPasswordBytes) },
|
||||
database: {
|
||||
roleExists: fields.dbRoleExistsBefore || "unknown",
|
||||
databaseExists: fields.dbDatabaseExistsBefore || "unknown",
|
||||
probeExitCode: fields.dbProbeExitCodeBefore || null,
|
||||
},
|
||||
},
|
||||
after: {
|
||||
exists: fields.afterExists === "yes",
|
||||
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
|
||||
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
|
||||
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
|
||||
database: {
|
||||
roleExists: fields.dbRoleExistsAfter || "unknown",
|
||||
databaseExists: fields.dbDatabaseExistsAfter || "unknown",
|
||||
probeExitCode: fields.dbProbeExitCodeAfter || null,
|
||||
},
|
||||
},
|
||||
postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes",
|
||||
postgresSecret: fields.postgresSecret || null,
|
||||
dbName: fields.dbName || V02_OPENFGA_DB_NAME,
|
||||
dbUser: fields.dbUser || V02_OPENFGA_DB_USER,
|
||||
applyExitCode: numericField(fields.applyExitCode),
|
||||
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
|
||||
exitCode,
|
||||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||||
valuesRedacted: true,
|
||||
summary: healthy
|
||||
? `${fields.secret || V02_OPENFGA_SECRET} keys and Postgres database exist`
|
||||
: `${fields.secret || V02_OPENFGA_SECRET} keys or Postgres database missing`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
|
||||
secret: fields.secret || null,
|
||||
preset: fields.preset || null,
|
||||
action: fields.action || null,
|
||||
dryRun: fields.dryRun === "true",
|
||||
mutation: fields.mutation === "true",
|
||||
exitCode,
|
||||
stderr: stderr.trim().slice(0, 2000),
|
||||
valuesRedacted: true,
|
||||
summary: "unrecognized secret status output",
|
||||
};
|
||||
}
|
||||
|
||||
export function runG14Secret(options: G14SecretOptions): Record<string, unknown> {
|
||||
const spec = hwlabRuntimeLaneSpec(options.lane);
|
||||
const script = v02SecretScript(options);
|
||||
const localAdminApiKey = options.preset === "master-server-admin-api-key"
|
||||
? readOrCreateLocalMasterAdminApiKey(spec, options.action !== "ensure" || options.dryRun)
|
||||
: null;
|
||||
const result = options.preset === "master-server-admin-api-key"
|
||||
? g14K3sInlineScriptWithInput(script, localAdminApiKey?.key ?? "", options.timeoutSeconds * 1000 + 2000)
|
||||
: g14K3s(["sh", "--", script], options.timeoutSeconds * 1000);
|
||||
const status = v02SecretStatusFromText(statusText(result), isCommandSuccess(result), result.exitCode, result.stderr);
|
||||
const dryRunOk = options.action === "ensure" && options.dryRun && isCommandSuccess(result);
|
||||
const deleteDryRunOk = options.action === "delete" && options.dryRun && isCommandSuccess(result);
|
||||
const ok = dryRunOk || deleteDryRunOk ? true : status.ok === true;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab g14 secret ${options.action} --lane ${options.lane}`,
|
||||
lane: options.lane,
|
||||
namespace: spec.runtimeNamespace,
|
||||
secret: options.name,
|
||||
key: options.key ?? null,
|
||||
preset: options.preset,
|
||||
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "delete" ? "confirmed-delete" : "confirmed-ensure",
|
||||
status,
|
||||
localMasterServer: localAdminApiKey
|
||||
? { created: localAdminApiKey.created, envFile: localAdminApiKey.status, valuesRedacted: true }
|
||||
: undefined,
|
||||
mutation: status.mutation === true,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
next: ok && options.action === "status"
|
||||
? undefined
|
||||
: options.action === "delete"
|
||||
? undefined
|
||||
: { ensure: `bun scripts/cli.ts hwlab g14 secret ensure --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. source module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:1205-1308 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import type { CommandJsonResult } from "./types";
|
||||
import { g14HostScript } from "./images";
|
||||
import { statusText } from "./pr-monitor";
|
||||
import { commandJson, isCommandSuccess, nested, shellQuote, shortSha } from "./remote";
|
||||
import { V02_GIT_URL, V02_LANE_SPEC } from "./types";
|
||||
|
||||
export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
|
||||
return [
|
||||
`cicd_repo=${shellQuote(spec.cicdRepo)}`,
|
||||
`cicd_url=${shellQuote(spec.gitUrl)}`,
|
||||
`cicd_branch=${shellQuote(spec.sourceBranch)}`,
|
||||
`cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`,
|
||||
"cicd_repo_ensure_fetch() {",
|
||||
" mkdir -p \"$(dirname \"$cicd_repo\")\" || return $?",
|
||||
" if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
||||
" :",
|
||||
" elif [ -e \"$cicd_repo\" ]; then",
|
||||
` echo "${spec.version} CI/CD repo path exists but is not a bare git repo: $cicd_repo" >&2`,
|
||||
" return 41",
|
||||
" else",
|
||||
" git clone --bare \"$cicd_url\" \"$cicd_repo\" || return $?",
|
||||
" fi",
|
||||
" git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\" || return $?",
|
||||
" git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' || return $?",
|
||||
" git --git-dir=\"$cicd_repo\" fetch origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune || return $?",
|
||||
"}",
|
||||
"cicd_repo_lock_mode=none",
|
||||
"if command -v flock >/dev/null 2>&1; then",
|
||||
" exec 9>\"$cicd_repo_lock\"",
|
||||
` flock -w 120 9 || { echo "timed out waiting for ${spec.version} CI/CD repo lock: $cicd_repo_lock" >&2; exit 42; }`,
|
||||
" cicd_repo_lock_mode=flock",
|
||||
"else",
|
||||
" cicd_repo_lock_dir=\"$cicd_repo_lock.d\"",
|
||||
" cicd_repo_lock_attempt=0",
|
||||
" while ! mkdir \"$cicd_repo_lock_dir\" 2>/dev/null; do",
|
||||
" cicd_repo_lock_attempt=$((cicd_repo_lock_attempt + 1))",
|
||||
` if [ "$cicd_repo_lock_attempt" -ge 120 ]; then echo "timed out waiting for ${spec.version} CI/CD repo lock: $cicd_repo_lock_dir" >&2; exit 42; fi`,
|
||||
" sleep 1",
|
||||
" done",
|
||||
" cicd_repo_lock_mode=dir",
|
||||
"fi",
|
||||
"cicd_repo_status=0",
|
||||
"cicd_repo_ensure_fetch || cicd_repo_status=$?",
|
||||
"if [ \"$cicd_repo_lock_mode\" = flock ]; then flock -u 9 >/dev/null 2>&1 || true; fi",
|
||||
"if [ \"$cicd_repo_lock_mode\" = dir ]; then rmdir \"$cicd_repo_lock_dir\" >/dev/null 2>&1 || true; fi",
|
||||
`if [ "$cicd_repo_status" -ne 0 ]; then echo "${spec.version} CI/CD repo refresh failed status=$cicd_repo_status repo=$cicd_repo" >&2; exit "$cicd_repo_status"; fi`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function v02CicdRepoEnsureScript(): string {
|
||||
return runtimeLaneCicdRepoEnsureScript(V02_LANE_SPEC);
|
||||
}
|
||||
|
||||
export function resolveV02Head(): { sourceCommit: string | null; result: CommandJsonResult } {
|
||||
const result = g14HostScript([
|
||||
"set -eu",
|
||||
v02CicdRepoEnsureScript(),
|
||||
"git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2",
|
||||
].join("\n"), 180_000);
|
||||
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
||||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||||
return { sourceCommit: match?.[0] ?? null, result };
|
||||
}
|
||||
|
||||
export function getV02Head(): string | null {
|
||||
return resolveV02Head().sourceCommit;
|
||||
}
|
||||
|
||||
export function resolveV02LatestRemoteHead(): { sourceCommit: string | null; result: CommandJsonResult } {
|
||||
const result = commandJson(["git", "ls-remote", V02_GIT_URL, "refs/heads/v0.2"], 30_000);
|
||||
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
||||
const output = statusText(result);
|
||||
const match = /^[0-9a-f]{40}/imu.exec(output);
|
||||
return { sourceCommit: match?.[0] ?? null, result };
|
||||
}
|
||||
|
||||
export function resolveRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandJsonResult } {
|
||||
const result = g14HostScript([
|
||||
"set -eu",
|
||||
runtimeLaneCicdRepoEnsureScript(spec),
|
||||
`git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch}`,
|
||||
].join("\n"), 180_000);
|
||||
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
||||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||||
return { sourceCommit: match?.[0] ?? null, result };
|
||||
}
|
||||
|
||||
export function resolveRuntimeLaneLatestRemoteHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandJsonResult } {
|
||||
const result = commandJson(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], 30_000);
|
||||
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
||||
const output = statusText(result);
|
||||
const match = /^[0-9a-f]{40}/imu.exec(output);
|
||||
return { sourceCommit: match?.[0] ?? null, result };
|
||||
}
|
||||
|
||||
export function runtimeLanePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
|
||||
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
|
||||
}
|
||||
|
||||
export function runtimeLaneRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string, now = new Date()): string {
|
||||
const stamp = now.toISOString().replace(/\D/gu, "").slice(0, 14);
|
||||
return `${runtimeLanePipelineRunName(spec, sourceCommit)}-r${stamp}`;
|
||||
}
|
||||
|
||||
export function v02PipelineRunName(sourceCommit: string): string {
|
||||
return runtimeLanePipelineRunName(V02_LANE_SPEC, sourceCommit);
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. types module for scripts/src/hwlab-g14.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-g14.ts:1-288 for #903.
|
||||
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand } from "../command";
|
||||
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
||||
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
|
||||
import { durationSeconds } from "./pr-monitor";
|
||||
|
||||
export const HWLAB_REPO = "pikasTech/HWLAB";
|
||||
|
||||
export const G14_SOURCE_BRANCH = "G14";
|
||||
|
||||
export const G14_PROVIDER = "G14";
|
||||
|
||||
export const G14_WORKSPACE = "/root/hwlab";
|
||||
|
||||
export const V02_LANE_SPEC = hwlabRuntimeLaneSpec("v02");
|
||||
|
||||
export const V02_SOURCE_BRANCH = V02_LANE_SPEC.sourceBranch;
|
||||
|
||||
export const V02_WORKSPACE = V02_LANE_SPEC.workspace;
|
||||
|
||||
export const V02_CICD_REPO = V02_LANE_SPEC.cicdRepo;
|
||||
|
||||
export const DEV_NAMESPACE = "hwlab-dev";
|
||||
|
||||
export const PROD_NAMESPACE = "hwlab-prod";
|
||||
|
||||
export const CI_NAMESPACE = "hwlab-ci";
|
||||
|
||||
export const ARGO_NAMESPACE = "argocd";
|
||||
|
||||
export const DEV_APP = "hwlab-g14-dev";
|
||||
|
||||
export const PROD_APP = "hwlab-g14-prod";
|
||||
|
||||
export const V02_APP = V02_LANE_SPEC.app;
|
||||
|
||||
export const V02_PIPELINE = V02_LANE_SPEC.pipeline;
|
||||
|
||||
export const V02_POLLER = "hwlab-v02-branch-poller";
|
||||
|
||||
export const V02_RECONCILER = "hwlab-v02-control-plane-reconciler";
|
||||
|
||||
export const V02_PIPELINERUN_PREFIX = V02_LANE_SPEC.pipelineRunPrefix;
|
||||
|
||||
export const V02_CONTROL_PLANE_FIELD_MANAGER = V02_LANE_SPEC.controlPlaneFieldManager;
|
||||
|
||||
export const V02_GIT_URL = V02_LANE_SPEC.gitUrl;
|
||||
|
||||
export const V02_GIT_READ_URL = V02_LANE_SPEC.gitReadUrl;
|
||||
|
||||
export const V02_GIT_WRITE_URL = V02_LANE_SPEC.gitWriteUrl;
|
||||
|
||||
export const V02_GITOPS_BRANCH = V02_LANE_SPEC.gitopsBranch;
|
||||
|
||||
export const V02_CATALOG_PATH = V02_LANE_SPEC.catalogPath;
|
||||
|
||||
export const V02_RUNTIME_PATH = V02_LANE_SPEC.runtimePath;
|
||||
|
||||
export const V02_RUNTIME_NAMESPACE = V02_LANE_SPEC.runtimeNamespace;
|
||||
|
||||
export const V02_OPENFGA_SECRET = "hwlab-v02-openfga";
|
||||
|
||||
export const V02_OPENFGA_AUTHN_SECRET_KEY = "authn-preshared-key";
|
||||
|
||||
export const V02_OPENFGA_DATASTORE_URI_SECRET_KEY = "datastore-uri";
|
||||
|
||||
export const V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY = "postgres-password";
|
||||
|
||||
export const V02_OPENFGA_DB_NAME = "hwlab_openfga";
|
||||
|
||||
export const V02_OPENFGA_DB_USER = "hwlab_openfga";
|
||||
|
||||
export const V02_MASTER_ADMIN_API_KEY_SECRET = "hwlab-v02-master-server-admin-api-key";
|
||||
|
||||
export const V02_MASTER_ADMIN_API_KEY_SECRET_KEY = "api-key";
|
||||
|
||||
export const V02_MASTER_ADMIN_API_KEY_LOCAL_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env";
|
||||
|
||||
export const V02_REGISTRY_PREFIX = V02_LANE_SPEC.registryPrefix;
|
||||
|
||||
export const V02_BASE_IMAGE = V02_LANE_SPEC.baseImage;
|
||||
|
||||
export const GIT_MIRROR_NAMESPACE = "devops-infra";
|
||||
|
||||
export const GIT_MIRROR_MANIFEST_FIELD_MANAGER = "unidesk-hwlab-git-mirror";
|
||||
|
||||
export const GIT_MIRROR_SYNC_JOB_PREFIX = "git-mirror-hwlab-sync-manual";
|
||||
|
||||
export const GIT_MIRROR_LEGACY_CRONJOB = "git-mirror-hwlab-sync";
|
||||
|
||||
export const G14_OBSERVABILITY_NAMESPACE = "devops-infra";
|
||||
|
||||
export const G14_OBSERVABILITY_FIELD_MANAGER = "unidesk-g14-observability";
|
||||
|
||||
export const G14_PROMETHEUS_OPERATOR_VERSION = "v0.91.0";
|
||||
|
||||
export const G14_PROMETHEUS_VERSION = "v3.12.0";
|
||||
|
||||
export const G14_PROMETHEUS_NAME = "g14-shared";
|
||||
|
||||
export const G14_PROMETHEUS_SERVICE = "prometheus-g14-shared";
|
||||
|
||||
export const G14_PROMETHEUS_SERVICE_ACCOUNT = "g14-observability-prometheus";
|
||||
|
||||
export const G14_PROMETHEUS_OPERATOR_RELEASE_ASSET = `https://github.com/prometheus-operator/prometheus-operator/releases/download/${G14_PROMETHEUS_OPERATOR_VERSION}/bundle.yaml`;
|
||||
|
||||
export const V02_SERVICE_IDS = [...V02_LANE_SPEC.serviceIds];
|
||||
|
||||
export const V02_CLOUD_WEB_URL = V02_LANE_SPEC.publicWebUrl;
|
||||
|
||||
export const V02_CLOUD_API_URL = V02_LANE_SPEC.publicApiUrl;
|
||||
|
||||
export const V02_OBSERVABILITY_SERVICE_IDS = [
|
||||
"hwlab-agent-skills",
|
||||
"hwlab-cloud-api",
|
||||
"hwlab-cloud-web",
|
||||
"hwlab-deepseek-proxy",
|
||||
"hwlab-edge-proxy",
|
||||
];
|
||||
|
||||
export const V02_OBSERVABILITY_EXPECTED_TARGET_COUNT = V02_OBSERVABILITY_SERVICE_IDS.length;
|
||||
|
||||
export const V02_OBSERVABILITY_QUERIES = {
|
||||
scrapeReachable: 'up{namespace="hwlab-v02"}',
|
||||
sidecarServing: 'hwlab_service_up{namespace="hwlab-v02"}',
|
||||
businessHealthProbe: 'hwlab_service_health_probe_success{namespace="hwlab-v02"}',
|
||||
healthProbeConfigured: 'hwlab_service_health_probe_configured{namespace="hwlab-v02"}',
|
||||
healthProbeStatusCode: 'hwlab_service_health_probe_status_code{namespace="hwlab-v02"}',
|
||||
healthProbeDurationSeconds: 'hwlab_service_health_probe_duration_seconds{namespace="hwlab-v02"}',
|
||||
processUptimeSeconds: 'hwlab_service_process_uptime_seconds{namespace="hwlab-v02"}',
|
||||
scrapeDurationSeconds: 'scrape_duration_seconds{namespace="hwlab-v02"}',
|
||||
scrapeSamplesScraped: 'scrape_samples_scraped{namespace="hwlab-v02"}',
|
||||
};
|
||||
|
||||
export const V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES = ["scrapeReachable", "sidecarServing", "businessHealthProbe"] as const;
|
||||
|
||||
export const V02_OBSERVABILITY_BOOLEAN_QUERY_NAMES = [...V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES, "healthProbeConfigured"];
|
||||
|
||||
export function v02PipelineServiceIds(): string[] {
|
||||
return [...V02_SERVICE_IDS];
|
||||
}
|
||||
|
||||
export const G14_CI_TOOLS_IMAGE_REPO = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools";
|
||||
|
||||
export const G14_CI_TOOLS_BASE_TAG = "node22-alpine-v1";
|
||||
|
||||
export const DEFAULT_INTERVAL_SECONDS = 600;
|
||||
|
||||
export const DEFAULT_MAX_CYCLES = 0;
|
||||
|
||||
export const DEFAULT_TIMEOUT_SECONDS = 1800;
|
||||
|
||||
export const G14_BRIEF_INDEX_ISSUE = 7;
|
||||
|
||||
export const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||||
|
||||
export const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
|
||||
|
||||
export const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
|
||||
|
||||
export const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
export const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000;
|
||||
|
||||
export const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000;
|
||||
|
||||
export const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000;
|
||||
|
||||
export const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000;
|
||||
|
||||
export type G14MonitorLane = "g14" | HwlabRuntimeLane;
|
||||
|
||||
export interface G14MonitorOptions {
|
||||
lane: G14MonitorLane;
|
||||
intervalSeconds: number;
|
||||
maxCycles: number;
|
||||
once: boolean;
|
||||
dryRun: boolean;
|
||||
worker: boolean;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface G14RecordRolloutOptions {
|
||||
prNumber: number;
|
||||
sourceCommit?: string;
|
||||
pipelineRun?: string;
|
||||
gitopsRevision?: string;
|
||||
mergedAt?: string;
|
||||
pipelineSucceededAt?: string;
|
||||
finishedAt?: string;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
export interface G14ControlPlaneOptions {
|
||||
action: "status" | "closeout" | "apply" | "trigger-current" | "refresh" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration";
|
||||
lane: HwlabRuntimeLane | "g14" | "all";
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
wait: boolean;
|
||||
rerun: boolean;
|
||||
allowLiveDbRead: boolean;
|
||||
timeoutSeconds: number;
|
||||
minAgeMinutes: number;
|
||||
limit: number;
|
||||
sourceCommit?: string;
|
||||
pipelineRun?: string;
|
||||
history: boolean;
|
||||
}
|
||||
|
||||
export interface G14LegacyRetirementOptions {
|
||||
action: "status" | "plan" | "execute";
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
wait: boolean;
|
||||
timeoutSeconds: number;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export type V02StatusTargetMode = "latest-source-head" | "source-commit" | "pipeline-run";
|
||||
|
||||
export interface V02ControlPlaneStatusTarget {
|
||||
sourceCommit?: string | null;
|
||||
pipelineRun?: string | null;
|
||||
mode?: V02StatusTargetMode;
|
||||
includeHistory?: boolean;
|
||||
}
|
||||
|
||||
export interface G14ToolsImageOptions {
|
||||
action: "status" | "build";
|
||||
name: "ci-node-tools";
|
||||
tag: string;
|
||||
dockerfile: string;
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface G14UpstreamImageOptions {
|
||||
action: "status" | "ensure";
|
||||
name: "openfga";
|
||||
tag: string;
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface G14GitMirrorOptions {
|
||||
action: "status" | "apply" | "sync" | "flush";
|
||||
lane: HwlabRuntimeLane;
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
wait: boolean;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface G14ObservabilityOptions {
|
||||
action: "status" | "apply" | "query" | "targets" | "boundary" | "closeout";
|
||||
lane: "v02";
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
wait: boolean;
|
||||
timeoutSeconds: number;
|
||||
query: string;
|
||||
expectCount?: number;
|
||||
expectValue?: string;
|
||||
}
|
||||
|
||||
export interface G14SecretOptions {
|
||||
action: "status" | "ensure" | "delete";
|
||||
lane: HwlabRuntimeLane;
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
name: string;
|
||||
key?: string;
|
||||
preset: "openfga" | "master-server-admin-api-key" | "generic-delete";
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface CommandJsonResult {
|
||||
ok: boolean;
|
||||
command: string[];
|
||||
exitCode: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
parsed: unknown | null;
|
||||
durationMs?: number;
|
||||
timedOut?: boolean;
|
||||
stdoutBytes?: number;
|
||||
stderrBytes?: number;
|
||||
}
|
||||
|
||||
export interface RemoteAsyncCommandSpec {
|
||||
script: string;
|
||||
timeoutSeconds: number;
|
||||
label: string;
|
||||
token: string;
|
||||
command: string[];
|
||||
}
|
||||
|
||||
export interface ShellSection {
|
||||
stdout: string;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
export interface OpenPullRequest {
|
||||
number: number;
|
||||
title?: string;
|
||||
url?: string;
|
||||
baseRefName?: string;
|
||||
headRefName?: string;
|
||||
mergeable?: string | null;
|
||||
mergeStateStatus?: string | null;
|
||||
draft?: boolean;
|
||||
}
|
||||
|
||||
export interface CiServiceMetric {
|
||||
taskRun: string;
|
||||
serviceId: string;
|
||||
status: string;
|
||||
durationSeconds: number | null;
|
||||
buildBackend: string | null;
|
||||
reusedFrom: string | null;
|
||||
imageTag: string | null;
|
||||
sourceCommitId: string | null;
|
||||
}
|
||||
|
||||
export interface CiPipelineMetrics {
|
||||
ok: boolean;
|
||||
pipelineRun: string;
|
||||
serviceTotal: number;
|
||||
reusedCount: number;
|
||||
rebuildCount: number;
|
||||
reusedServices: string[];
|
||||
rebuildServices: string[];
|
||||
services: CiServiceMetric[];
|
||||
degradedReason?: string;
|
||||
rawText?: string;
|
||||
}
|
||||
|
||||
export interface V02PrCommentInput {
|
||||
pr: OpenPullRequest;
|
||||
phase: string;
|
||||
state: "waiting-ci" | "blocked" | "merge-failed" | "cd-started" | "cd-succeeded" | "cd-superseded" | "cd-failed" | "cd-timeout" | "cd-blocked";
|
||||
startedAt: string;
|
||||
observedAt: string;
|
||||
elapsedSeconds: number | null;
|
||||
preflight?: Record<string, unknown>;
|
||||
merge?: Record<string, unknown>;
|
||||
sourceCommit?: string | null;
|
||||
pipelineRun?: string | null;
|
||||
cd?: Record<string, unknown>;
|
||||
flush?: Record<string, unknown>;
|
||||
dryRun?: boolean;
|
||||
message?: string;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user