Codex
3017 lines
127 KiB
TypeScript
3017 lines
127 KiB
TypeScript
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { createHash } from "node:crypto";
|
||
import { repoRoot, rootPath, type Config } from "./config";
|
||
import { runCommand } from "./command";
|
||
import { startJob } from "./jobs";
|
||
|
||
const HWLAB_REPO = "pikasTech/HWLAB";
|
||
const G14_SOURCE_BRANCH = "G14";
|
||
const G14_PROVIDER = "G14";
|
||
const G14_WORKSPACE = "/root/hwlab";
|
||
const V02_SOURCE_BRANCH = "v0.2";
|
||
const V02_WORKSPACE = "/root/hwlab-v02";
|
||
const DEV_NAMESPACE = "hwlab-dev";
|
||
const CI_NAMESPACE = "hwlab-ci";
|
||
const ARGO_NAMESPACE = "argocd";
|
||
const DEV_APP = "hwlab-g14-dev";
|
||
const V02_APP = "hwlab-g14-v02";
|
||
const V02_PIPELINE = "hwlab-v02-ci-image-publish";
|
||
const V02_POLLER = "hwlab-v02-branch-poller";
|
||
const V02_RECONCILER = "hwlab-v02-control-plane-reconciler";
|
||
const V02_PIPELINERUN_PREFIX = "hwlab-v02-ci-poll";
|
||
const V02_CONTROL_PLANE_FIELD_MANAGER = "unidesk-hwlab-v02-control-plane";
|
||
const V02_GIT_URL = "git@github.com:pikasTech/HWLAB.git";
|
||
const V02_GIT_READ_URL = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git";
|
||
const V02_GIT_WRITE_URL = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git";
|
||
const V02_GITOPS_BRANCH = "v0.2-gitops";
|
||
const V02_CATALOG_PATH = "deploy/artifact-catalog.v02.json";
|
||
const V02_RUNTIME_PATH = "deploy/gitops/g14/runtime-v02";
|
||
const V02_REGISTRY_PREFIX = "127.0.0.1:5000/hwlab";
|
||
const V02_BASE_IMAGE = "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim";
|
||
const GIT_MIRROR_NAMESPACE = "devops-infra";
|
||
const GIT_MIRROR_MANIFEST_FIELD_MANAGER = "unidesk-hwlab-git-mirror";
|
||
const GIT_MIRROR_SYNC_JOB_PREFIX = "git-mirror-hwlab-sync-manual";
|
||
const GIT_MIRROR_LEGACY_CRONJOB = "git-mirror-hwlab-sync";
|
||
const V02_SERVICE_IDS = [
|
||
"hwlab-cloud-api",
|
||
"hwlab-cloud-web",
|
||
"hwlab-agent-mgr",
|
||
"hwlab-agent-worker",
|
||
"hwlab-device-pod",
|
||
"hwlab-gateway",
|
||
"hwlab-edge-proxy",
|
||
"hwlab-agent-skills",
|
||
];
|
||
const V02_CLOUD_WEB_URL = "http://74.48.78.17:19666";
|
||
const V02_CLOUD_API_URL = "http://74.48.78.17:19667";
|
||
|
||
export function v02PipelineServiceIds(): string[] {
|
||
return [...V02_SERVICE_IDS];
|
||
}
|
||
|
||
const G14_CI_TOOLS_IMAGE_REPO = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools";
|
||
const G14_CI_TOOLS_BASE_TAG = "node22-alpine-v1";
|
||
const DEFAULT_INTERVAL_SECONDS = 600;
|
||
const DEFAULT_MAX_CYCLES = 0;
|
||
const DEFAULT_TIMEOUT_SECONDS = 1800;
|
||
const G14_BRIEF_INDEX_ISSUE = 7;
|
||
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||
|
||
interface G14MonitorOptions {
|
||
intervalSeconds: number;
|
||
maxCycles: number;
|
||
once: boolean;
|
||
dryRun: boolean;
|
||
worker: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface G14RecordRolloutOptions {
|
||
prNumber: number;
|
||
sourceCommit?: string;
|
||
pipelineRun?: string;
|
||
gitopsRevision?: string;
|
||
mergedAt?: string;
|
||
pipelineSucceededAt?: string;
|
||
finishedAt?: string;
|
||
dryRun: boolean;
|
||
}
|
||
|
||
interface G14ControlPlaneOptions {
|
||
action: "status" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration";
|
||
lane: "v02" | "g14" | "all";
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
wait: boolean;
|
||
allowLiveDbRead: boolean;
|
||
timeoutSeconds: number;
|
||
minAgeMinutes: number;
|
||
limit: number;
|
||
}
|
||
|
||
interface G14ToolsImageOptions {
|
||
action: "status" | "build";
|
||
name: "ci-node-tools";
|
||
tag: string;
|
||
dockerfile: string;
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface G14GitMirrorOptions {
|
||
action: "status" | "apply" | "sync" | "flush";
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
wait: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface CommandJsonResult {
|
||
ok: boolean;
|
||
command: string[];
|
||
exitCode: number | null;
|
||
stdout: string;
|
||
stderr: string;
|
||
parsed: unknown | null;
|
||
}
|
||
|
||
interface ShellSection {
|
||
stdout: string;
|
||
exitCode: number | null;
|
||
}
|
||
|
||
interface OpenPullRequest {
|
||
number: number;
|
||
title?: string;
|
||
url?: string;
|
||
baseRefName?: string;
|
||
headRefName?: string;
|
||
mergeable?: string | null;
|
||
mergeStateStatus?: string | null;
|
||
draft?: boolean;
|
||
}
|
||
|
||
interface CiServiceMetric {
|
||
taskRun: string;
|
||
serviceId: string;
|
||
status: string;
|
||
durationSeconds: number | null;
|
||
buildBackend: string | null;
|
||
reusedFrom: string | null;
|
||
imageTag: string | null;
|
||
sourceCommitId: string | null;
|
||
}
|
||
|
||
interface CiPipelineMetrics {
|
||
ok: boolean;
|
||
pipelineRun: string;
|
||
serviceTotal: number;
|
||
reusedCount: number;
|
||
rebuildCount: number;
|
||
reusedServices: string[];
|
||
rebuildServices: string[];
|
||
services: CiServiceMetric[];
|
||
degradedReason?: string;
|
||
rawText?: string;
|
||
}
|
||
|
||
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun">): string {
|
||
if (options.once && options.dryRun) return "latest-once-dry-run-job.json";
|
||
if (options.once) return "latest-once-job.json";
|
||
if (options.dryRun) return "latest-dry-run-job.json";
|
||
return "latest-monitor-job.json";
|
||
}
|
||
|
||
function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun">): string {
|
||
if (options.once && options.dryRun) return "once-dry-run";
|
||
if (options.once) return "once";
|
||
if (options.dryRun) return "dry-run-monitor";
|
||
return "monitor";
|
||
}
|
||
|
||
function parseOptions(args: string[]): G14MonitorOptions {
|
||
return {
|
||
intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400),
|
||
maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000),
|
||
once: args.includes("--once"),
|
||
dryRun: args.includes("--dry-run"),
|
||
worker: args.includes("--worker"),
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", DEFAULT_TIMEOUT_SECONDS, 86400),
|
||
};
|
||
}
|
||
|
||
function optionValue(args: string[], name: string): string | undefined {
|
||
const index = args.indexOf(name);
|
||
if (index === -1) return undefined;
|
||
const value = args[index + 1];
|
||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||
return value;
|
||
}
|
||
|
||
function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||
const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number");
|
||
const prNumber = Number(prRaw);
|
||
if (!Number.isInteger(prNumber) || prNumber <= 0) throw new Error("record-rollout requires --pr <number>");
|
||
return {
|
||
prNumber,
|
||
sourceCommit: optionValue(args, "--source-commit"),
|
||
pipelineRun: optionValue(args, "--pipeline-run"),
|
||
gitopsRevision: optionValue(args, "--gitops-revision"),
|
||
mergedAt: optionValue(args, "--merged-at"),
|
||
pipelineSucceededAt: optionValue(args, "--pipeline-succeeded-at"),
|
||
finishedAt: optionValue(args, "--finished-at"),
|
||
dryRun: args.includes("--dry-run"),
|
||
};
|
||
}
|
||
|
||
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||
const [rawAction] = args;
|
||
const actionRaw = rawAction === "rerun-current" ? "trigger-current" : rawAction;
|
||
if (
|
||
actionRaw !== "status" &&
|
||
actionRaw !== "apply" &&
|
||
actionRaw !== "trigger-current" &&
|
||
actionRaw !== "cleanup-runs" &&
|
||
actionRaw !== "cleanup-released-pvs" &&
|
||
actionRaw !== "runtime-migration"
|
||
) {
|
||
throw new Error("control-plane usage: status|apply|trigger-current|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
|
||
}
|
||
const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
|
||
if (actionRaw === "cleanup-runs") {
|
||
if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
|
||
} else if (actionRaw === "cleanup-released-pvs") {
|
||
if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
|
||
} else if (lane !== "v02") {
|
||
throw new Error("control-plane status/apply/trigger-current/runtime-migration currently requires --lane v02");
|
||
}
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run");
|
||
const allowLiveDbRead = args.includes("--allow-live-db-read");
|
||
if (allowLiveDbRead && confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
|
||
if (allowLiveDbRead && actionRaw !== "runtime-migration") throw new Error("--allow-live-db-read is only valid for control-plane runtime-migration");
|
||
return {
|
||
action: actionRaw,
|
||
lane,
|
||
confirm,
|
||
wait: args.includes("--wait"),
|
||
allowLiveDbRead,
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
||
limit: positiveIntegerOption(args, "--limit", 20, 200),
|
||
};
|
||
}
|
||
|
||
function parseToolsImageOptions(args: string[]): G14ToolsImageOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "status" && actionRaw !== "build") {
|
||
throw new Error("tools-image usage: status|build --name ci-node-tools --tag <tag> [--dockerfile path] [--dry-run|--confirm]");
|
||
}
|
||
const name = optionValue(args, "--name") ?? "ci-node-tools";
|
||
if (name !== "ci-node-tools") throw new Error("tools-image currently supports --name ci-node-tools");
|
||
const tag = optionValue(args, "--tag") ?? G14_CI_TOOLS_BASE_TAG;
|
||
if (!/^[A-Za-z0-9_.-]+$/u.test(tag)) throw new Error("--tag may only contain letters, numbers, dot, underscore, and dash");
|
||
const dockerfile = optionValue(args, "--dockerfile") ?? "deploy/ci/hwlab-ci-node-tools.Dockerfile";
|
||
if (dockerfile.startsWith("/") || dockerfile.includes("..")) throw new Error("--dockerfile must be a repo-relative path without '..'");
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run");
|
||
return {
|
||
action: actionRaw,
|
||
name,
|
||
tag,
|
||
dockerfile,
|
||
confirm,
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
|
||
};
|
||
}
|
||
|
||
function parseGitMirrorOptions(args: string[]): G14GitMirrorOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "sync" && actionRaw !== "flush") {
|
||
throw new Error("git-mirror usage: status|apply|sync|flush [--dry-run|--confirm]");
|
||
}
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("git-mirror accepts only one of --confirm or --dry-run");
|
||
return {
|
||
action: actionRaw,
|
||
confirm,
|
||
wait: args.includes("--wait"),
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "sync" || actionRaw === "flush" ? 300 : 120, 900),
|
||
};
|
||
}
|
||
|
||
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
||
const index = args.indexOf(name);
|
||
if (index === -1) return defaultValue;
|
||
const raw = args[index + 1];
|
||
const value = Number(raw);
|
||
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||
return Math.min(value, maxValue);
|
||
}
|
||
|
||
function shellQuote(value: string): string {
|
||
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
||
}
|
||
|
||
function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
const result = runCommand(command, repoRoot, { timeoutMs });
|
||
let parsed: unknown | null = null;
|
||
if (result.stdout.trim().length > 0) {
|
||
try {
|
||
parsed = JSON.parse(result.stdout) as unknown;
|
||
} catch {
|
||
parsed = null;
|
||
}
|
||
}
|
||
return {
|
||
ok: result.exitCode === 0,
|
||
command,
|
||
exitCode: result.exitCode,
|
||
stdout: result.stdout,
|
||
stderr: result.stderr,
|
||
parsed,
|
||
};
|
||
}
|
||
|
||
function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs);
|
||
}
|
||
|
||
function record(value: unknown): Record<string, unknown> {
|
||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||
}
|
||
|
||
function stringOrNull(value: unknown): string | null {
|
||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||
}
|
||
|
||
function nested(value: unknown, keys: string[]): unknown {
|
||
let current = value;
|
||
for (const key of keys) current = record(current)[key];
|
||
return current;
|
||
}
|
||
|
||
function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown> {
|
||
const dumpPath = nested(result.parsed, ["data", "dump", "path"]);
|
||
if (typeof dumpPath === "string" && existsSync(dumpPath)) {
|
||
try {
|
||
return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown);
|
||
} catch {
|
||
return record(result.parsed);
|
||
}
|
||
}
|
||
return record(result.parsed);
|
||
}
|
||
|
||
function commandData(result: CommandJsonResult): Record<string, unknown> {
|
||
return record(sanitizeCliOutput(record(expandedParsedRoot(result).data)));
|
||
}
|
||
|
||
function textHash(value: string): string {
|
||
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
||
}
|
||
|
||
function redactLargePayloads(value: string): string {
|
||
return value
|
||
.replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => {
|
||
return `manifest_b64='<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>'`;
|
||
})
|
||
.replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => {
|
||
return `manifest_b64="<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>"`;
|
||
})
|
||
.replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => {
|
||
return `manifest_b64=<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||
})
|
||
.replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => {
|
||
return `<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||
});
|
||
}
|
||
|
||
function compactInlineText(value: string, maxChars = 4000): string {
|
||
const redacted = redactLargePayloads(value);
|
||
if (redacted.length <= maxChars) return redacted;
|
||
const headChars = Math.floor(maxChars * 0.35);
|
||
const tailChars = maxChars - headChars;
|
||
const omittedChars = redacted.length - headChars - tailChars;
|
||
return [
|
||
redacted.slice(0, headChars),
|
||
`<truncated chars=${omittedChars} sha256=${textHash(redacted)}>`,
|
||
redacted.slice(-tailChars),
|
||
].join("\n");
|
||
}
|
||
|
||
function sanitizeCliOutput(value: unknown): unknown {
|
||
if (typeof value === "string") return compactInlineText(value);
|
||
if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item));
|
||
if (typeof value !== "object" || value === null) return value;
|
||
const output: Record<string, unknown> = {};
|
||
for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item);
|
||
return output;
|
||
}
|
||
|
||
function redactedCommand(command: string[]): string[] {
|
||
return command.map((arg) => compactInlineText(arg, 1200));
|
||
}
|
||
|
||
function compactCommandResult(result: CommandJsonResult): Record<string, unknown> {
|
||
const data = record(expandedParsedRoot(result).data);
|
||
const stdout = String(data.stdout ?? result.stdout ?? "");
|
||
const stderr = String(data.stderr ?? result.stderr ?? "");
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
exitCode: result.exitCode,
|
||
command: redactedCommand(result.command),
|
||
stdout: compactInlineText(stdout.trim(), 4000),
|
||
stderr: compactInlineText(stderr.trim(), 4000),
|
||
stdoutBytes: Buffer.byteLength(stdout),
|
||
stderrBytes: Buffer.byteLength(stderr),
|
||
};
|
||
}
|
||
|
||
function compactControlPlaneRefresh(value: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
ok: value.ok === true,
|
||
phase: value.phase ?? null,
|
||
sourceCommit: value.sourceCommit ?? null,
|
||
renderDir: value.renderDir ?? null,
|
||
applyOk: nested(value, ["apply", "ok"]) ?? null,
|
||
cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null,
|
||
applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200),
|
||
degradedReason: value.degradedReason ?? null,
|
||
};
|
||
}
|
||
|
||
function compactTriggerCommandResult(value: Record<string, unknown> | null): Record<string, unknown> | null {
|
||
if (value === null) return null;
|
||
return {
|
||
ok: value.ok ?? null,
|
||
exitCode: value.exitCode ?? null,
|
||
stdout: tailText(value.stdout, 1200),
|
||
stderr: tailText(value.stderr, 1200),
|
||
stdoutBytes: value.stdoutBytes ?? null,
|
||
stderrBytes: value.stderrBytes ?? null,
|
||
};
|
||
}
|
||
|
||
function isCommandSuccess(result: CommandJsonResult): boolean {
|
||
if (!result.ok) return false;
|
||
const topOk = record(result.parsed).ok;
|
||
if (topOk === false) return false;
|
||
const dataOk = nested(result.parsed, ["data", "ok"]);
|
||
return dataOk !== false;
|
||
}
|
||
|
||
function extractPullRequests(result: CommandJsonResult): OpenPullRequest[] {
|
||
const prs = nested(result.parsed, ["data", "pullRequests"]);
|
||
if (!Array.isArray(prs)) return [];
|
||
return prs
|
||
.map((item) => record(item))
|
||
.filter((item) => item.baseRefName === G14_SOURCE_BRANCH || record(item.base).ref === G14_SOURCE_BRANCH)
|
||
.map((item) => ({
|
||
number: Number(item.number),
|
||
title: typeof item.title === "string" ? item.title : undefined,
|
||
url: typeof item.url === "string" ? item.url : undefined,
|
||
baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined,
|
||
headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined,
|
||
mergeable: typeof item.mergeable === "string" ? item.mergeable : null,
|
||
mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null,
|
||
draft: item.draft === true,
|
||
}))
|
||
.filter((item) => Number.isInteger(item.number) && item.number > 0);
|
||
}
|
||
|
||
function sleep(ms: number): Promise<void> {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
function printEvent(event: string, data: Record<string, unknown> = {}): void {
|
||
process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||
}
|
||
|
||
function printProgressEvent(event: string, data: Record<string, unknown> = {}): void {
|
||
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||
}
|
||
|
||
function shortSha(sha: string): string {
|
||
return sha.slice(0, 12);
|
||
}
|
||
|
||
function precheckWorkspace(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
|
||
}
|
||
|
||
function v02WorkspaceScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:${V02_WORKSPACE}`, "script", "--", script], timeoutMs);
|
||
}
|
||
|
||
function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs);
|
||
}
|
||
|
||
function getV02Head(): string | null {
|
||
const result = v02WorkspaceScript("git fetch origin v0.2 --prune >/dev/null 2>&1; git rev-parse origin/v0.2", 120_000);
|
||
if (!isCommandSuccess(result)) return null;
|
||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||
return match?.[0] ?? null;
|
||
}
|
||
|
||
function v02PipelineRunName(sourceCommit: string): string {
|
||
return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`;
|
||
}
|
||
|
||
function getPipelineRunCompact(name: string): Record<string, unknown> {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
name,
|
||
"-o",
|
||
"jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}",
|
||
], 60_000);
|
||
return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr);
|
||
}
|
||
|
||
function pipelineRunCompactFromText(
|
||
name: string,
|
||
text: string,
|
||
commandOk: boolean,
|
||
command: string[] | string,
|
||
exitCode: number | null,
|
||
stderr: string,
|
||
): Record<string, unknown> {
|
||
const [status = "", reason = "", message = ""] = text.split(/\r?\n/u);
|
||
const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`);
|
||
return {
|
||
ok: commandOk,
|
||
exists: commandOk || !notFound,
|
||
pipelineRun: name,
|
||
status: status || null,
|
||
reason: reason || null,
|
||
message: message || null,
|
||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
function timestampMs(value: unknown): number | null {
|
||
if (typeof value !== "string" || value.trim().length === 0) return null;
|
||
const parsed = Date.parse(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function secondsBetween(start: unknown, end: unknown): number | null {
|
||
const startMs = timestampMs(start);
|
||
const endMs = timestampMs(end);
|
||
if (startMs === null || endMs === null || endMs < startMs) return null;
|
||
return Math.round((endMs - startMs) / 1000);
|
||
}
|
||
|
||
function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record<string, unknown> | null {
|
||
const [
|
||
name = "",
|
||
sourceCommit = "",
|
||
createdAt = "",
|
||
startTime = "",
|
||
completionTime = "",
|
||
status = "",
|
||
reason = "",
|
||
] = line.split("\t");
|
||
if (!name.startsWith(V02_PIPELINERUN_PREFIX)) return null;
|
||
const startMs = timestampMs(startTime);
|
||
const completion = completionTime.trim().length > 0 ? completionTime : null;
|
||
const conditionStatus = status.trim().length > 0 ? status : null;
|
||
return {
|
||
name,
|
||
sourceCommit: sourceCommit || null,
|
||
status: conditionStatus,
|
||
reason: reason || null,
|
||
createdAt: createdAt || null,
|
||
startTime: startTime || null,
|
||
completionTime: completion,
|
||
durationSeconds: secondsBetween(startTime, completion),
|
||
elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)),
|
||
active: completion === null && conditionStatus !== "True" && conditionStatus !== "False",
|
||
};
|
||
}
|
||
|
||
function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now()): Record<string, unknown> {
|
||
const rows = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs))
|
||
.filter((item): item is Record<string, unknown> => item !== null)
|
||
.sort((left, right) => {
|
||
return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0);
|
||
});
|
||
const activeItems = rows.filter((item) => item.active === true);
|
||
return {
|
||
ok: true,
|
||
count: rows.length,
|
||
activeCount: activeItems.length,
|
||
items: rows.slice(0, limit),
|
||
activeItems: activeItems.slice(0, limit),
|
||
disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete",
|
||
};
|
||
}
|
||
|
||
function pipelineRunRowsJsonPath(): string {
|
||
return [
|
||
"jsonpath={range .items[*]}",
|
||
"{.metadata.name}",
|
||
"{\"\\t\"}",
|
||
"{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}",
|
||
"{\"\\t\"}",
|
||
"{.metadata.creationTimestamp}",
|
||
"{\"\\t\"}",
|
||
"{.status.startTime}",
|
||
"{\"\\t\"}",
|
||
"{.status.completionTime}",
|
||
"{\"\\t\"}",
|
||
"{.status.conditions[0].status}",
|
||
"{\"\\t\"}",
|
||
"{.status.conditions[0].reason}",
|
||
"{\"\\n\"}",
|
||
"{end}",
|
||
].join("");
|
||
}
|
||
|
||
function listV02PipelineRunsCompact(limit = 8): Record<string, unknown> {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-l",
|
||
"hwlab.pikastech.local/gitops-target=v02",
|
||
"-o",
|
||
pipelineRunRowsJsonPath(),
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
return {
|
||
ok: false,
|
||
command: redactedCommand(result.command),
|
||
exitCode: result.exitCode,
|
||
stderr: commandErrorSummary(result),
|
||
items: [],
|
||
activeItems: [],
|
||
};
|
||
}
|
||
return parsePipelineRunRows(statusText(result), limit);
|
||
}
|
||
|
||
function parseShellSections(output: string): Record<string, ShellSection> {
|
||
const sections: Record<string, ShellSection> = {};
|
||
let currentName: string | null = null;
|
||
let currentLines: string[] = [];
|
||
let currentExitCode: number | null = null;
|
||
const flush = (): void => {
|
||
if (currentName === null) return;
|
||
sections[currentName] = {
|
||
stdout: currentLines.join("\n").trim(),
|
||
exitCode: currentExitCode,
|
||
};
|
||
};
|
||
for (const line of output.split(/\r?\n/u)) {
|
||
const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line);
|
||
if (begin !== null) {
|
||
flush();
|
||
currentName = begin[1] ?? null;
|
||
currentLines = [];
|
||
currentExitCode = null;
|
||
continue;
|
||
}
|
||
const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line);
|
||
if (end !== null && currentName === end[1]) {
|
||
currentExitCode = Number(end[2]);
|
||
flush();
|
||
currentName = null;
|
||
currentLines = [];
|
||
currentExitCode = null;
|
||
continue;
|
||
}
|
||
if (currentName !== null) currentLines.push(line);
|
||
}
|
||
flush();
|
||
return sections;
|
||
}
|
||
|
||
function shellSectionOk(section: ShellSection | undefined): boolean {
|
||
return section?.exitCode === 0;
|
||
}
|
||
|
||
function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): CommandJsonResult {
|
||
const sourceCommitLine = sourceCommit === undefined
|
||
? `source_commit=$(git -C ${shellQuote(V02_WORKSPACE)} rev-parse origin/v0.2 2>/dev/null || true)`
|
||
: `source_commit=${shellQuote(sourceCommit ?? "")}`;
|
||
const script = [
|
||
"set +e",
|
||
sourceCommitLine,
|
||
"pipeline_run=",
|
||
`if [ -n "$source_commit" ]; then pipeline_run="${V02_PIPELINERUN_PREFIX}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`,
|
||
"section() {",
|
||
" name=\"$1\"",
|
||
" shift",
|
||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||
" \"$@\"",
|
||
" code=$?",
|
||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||
"}",
|
||
"section sourceCommit printf '%s\\n' \"$source_commit\"",
|
||
"section queryNow date -u +%Y-%m-%dT%H:%M:%SZ",
|
||
`section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`,
|
||
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
|
||
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
||
`if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
|
||
`if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
|
||
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
|
||
`section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
|
||
].join("\n");
|
||
return g14K3s(["script", "--", script], 60_000);
|
||
}
|
||
|
||
function v02WebAssetsProbeScript(): string {
|
||
return [
|
||
"set +e",
|
||
`base=${shellQuote(V02_CLOUD_WEB_URL)}`,
|
||
`api=${shellQuote(V02_CLOUD_API_URL)}`,
|
||
"fetch_url() {",
|
||
" if command -v curl >/dev/null 2>&1; then",
|
||
" curl -fsS --connect-timeout 2 --max-time 5 \"$1\"",
|
||
" elif command -v wget >/dev/null 2>&1; then",
|
||
" wget -q -T 5 -O - \"$1\"",
|
||
" else",
|
||
" return 127",
|
||
" fi",
|
||
"}",
|
||
"printf 'baseUrl\\t%s\\n' \"$base\"",
|
||
"printf 'apiUrl\\t%s\\n' \"$api\"",
|
||
"html=$(fetch_url \"$base/\" 2>/dev/null)",
|
||
"html_code=$?",
|
||
"printf 'htmlOk\\t%s\\n' \"$html_code\"",
|
||
"printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"",
|
||
"css=$(fetch_url \"$base/styles.css\" 2>/dev/null)",
|
||
"css_code=$?",
|
||
"printf 'cssOk\\t%s\\n' \"$css_code\"",
|
||
"printf 'sidebarFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||
"printf 'workspaceFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||
"printf 'eventPanelFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(132px,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||
"health=$(fetch_url \"$api/health/live\" 2>/dev/null)",
|
||
"health_code=$?",
|
||
"printf 'apiHealthOk\\t%s\\n' \"$health_code\"",
|
||
"printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"",
|
||
].join("\n");
|
||
}
|
||
|
||
function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
|
||
items: [],
|
||
};
|
||
}
|
||
const items = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
status: status || null,
|
||
reason: reason || null,
|
||
startTime: startTime || null,
|
||
completionTime: completionTime || null,
|
||
durationSeconds: secondsBetween(startTime, completionTime),
|
||
};
|
||
});
|
||
const counts = {
|
||
succeeded: items.filter((item) => item.status === "True").length,
|
||
failed: items.filter((item) => item.status === "False").length,
|
||
running: items.filter((item) => item.status === "Unknown").length,
|
||
unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
|
||
};
|
||
return {
|
||
ok: true,
|
||
pipelineRun,
|
||
counts,
|
||
items,
|
||
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}`,
|
||
disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
|
||
};
|
||
}
|
||
|
||
function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: string | null, argoSyncRevision: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
const fields: Record<string, string> = {};
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const [key = "", ...rest] = line.split("\t");
|
||
if (key.length > 0) fields[key] = rest.join("\t");
|
||
}
|
||
const htmlOk = fields.htmlOk === "0";
|
||
const cssOk = fields.cssOk === "0";
|
||
const apiHealthOk = fields.apiHealthOk === "0";
|
||
const readonlyNoteAbsent = fields.readonlyNote === "1";
|
||
const sidebarFitCss = fields.sidebarFitCss === "0";
|
||
const workspaceFitCss = fields.workspaceFitCss === "0";
|
||
const eventPanelFitCss = fields.eventPanelFitCss === "0";
|
||
const apiRevision = fields.apiRevision || null;
|
||
const webChecksPass = htmlOk && cssOk && readonlyNoteAbsent && sidebarFitCss && workspaceFitCss && eventPanelFitCss && apiHealthOk;
|
||
const failedChecks = Object.entries({
|
||
htmlOk,
|
||
cssOk,
|
||
readonlyNoteAbsent,
|
||
sidebarFitCss,
|
||
workspaceFitCss,
|
||
eventPanelFitCss,
|
||
apiHealthOk,
|
||
}).filter(([, ok]) => !ok).map(([name]) => name);
|
||
return {
|
||
ok: commandOk && webChecksPass,
|
||
summary: commandOk && webChecksPass ? "19666/19667 probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`,
|
||
baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL,
|
||
apiUrl: fields.apiUrl || V02_CLOUD_API_URL,
|
||
sourceCommit,
|
||
argoSyncRevision: argoSyncRevision || null,
|
||
checks: {
|
||
htmlOk,
|
||
cssOk,
|
||
readonlyNoteAbsent,
|
||
sidebarFitCss,
|
||
workspaceFitCss,
|
||
eventPanelFitCss,
|
||
apiHealthOk,
|
||
},
|
||
probeExitCodes: {
|
||
html: numericField(fields.htmlOk),
|
||
css: numericField(fields.cssOk),
|
||
readonlyNoteGrep: numericField(fields.readonlyNote),
|
||
sidebarFitCssGrep: numericField(fields.sidebarFitCss),
|
||
workspaceFitCssGrep: numericField(fields.workspaceFitCss),
|
||
eventPanelFitCssGrep: numericField(fields.eventPanelFitCss),
|
||
apiHealth: numericField(fields.apiHealthOk),
|
||
},
|
||
apiRevision,
|
||
note: apiRevision && sourceCommit && apiRevision !== sourceCommit
|
||
? "cloud-api image revision can differ when a change only republishes Cloud Web static assets; use webAssets.checks for 19666 frontend asset readiness."
|
||
: null,
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
function numericField(value: string | undefined): number | null {
|
||
if (value === undefined || value.trim().length === 0) return null;
|
||
const parsed = Number(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 4000),
|
||
items: [],
|
||
activeItems: [],
|
||
};
|
||
}
|
||
return parsePipelineRunRows(text, limit, nowMs);
|
||
}
|
||
|
||
function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] {
|
||
if (lane === "v02") return ["hwlab-v02-ci-poll-"];
|
||
if (lane === "g14") return ["hwlab-g14-ci-poll-"];
|
||
return ["hwlab-v02-ci-poll-", "hwlab-g14-ci-poll-"];
|
||
}
|
||
|
||
function commandErrorSummary(result: CommandJsonResult): string {
|
||
const remoteStderr = String(nested(result.parsed, ["data", "stderr"]) ?? "").trim();
|
||
const remoteStdout = String(nested(result.parsed, ["data", "stdout"]) ?? "").trim();
|
||
const text = [result.stderr.trim(), remoteStderr, remoteStdout, result.stdout.trim()].find((item) => item.length > 0) ?? "";
|
||
return text.slice(0, 4000);
|
||
}
|
||
|
||
function tailText(value: unknown, maxBytes = 2000): string {
|
||
const text = String(value ?? "").trim();
|
||
return text.length <= maxBytes ? text : text.slice(-maxBytes);
|
||
}
|
||
|
||
function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
|
||
}
|
||
const prefixes = pipelinePrefixesForLane(options.lane);
|
||
const now = Date.now();
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", createdAt = "", status = "", reason = ""] = line.split("\t");
|
||
const createdMs = Date.parse(createdAt);
|
||
const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
||
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
|
||
})
|
||
.filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
|
||
.filter((item) => item.status === "True" || item.status === "False")
|
||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||
.slice(0, options.limit);
|
||
}
|
||
|
||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unknown>[] {
|
||
if (pipelineRunNames.length === 0) {
|
||
return [];
|
||
}
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pvc",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
|
||
}
|
||
const wanted = new Set(pipelineRunNames);
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
volume: volume || null,
|
||
phase: phase || null,
|
||
ownerKind: ownerKind || null,
|
||
owner: owner || null,
|
||
};
|
||
})
|
||
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
|
||
}
|
||
|
||
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||
if (names.length === 0) {
|
||
return {
|
||
ok: true,
|
||
command: [],
|
||
exitCode: 0,
|
||
stdout: "no candidates",
|
||
stderr: "",
|
||
parsed: null,
|
||
};
|
||
}
|
||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
|
||
}
|
||
|
||
function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pv",
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
|
||
}
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
createdAt,
|
||
phase,
|
||
storageClass,
|
||
reclaimPolicy,
|
||
claimNamespace,
|
||
claimName,
|
||
capacity,
|
||
hostPath: claimNamespace === CI_NAMESPACE && claimName.length > 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null,
|
||
};
|
||
})
|
||
.filter((item) => item.phase === "Released")
|
||
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
||
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||
.slice(0, options.limit);
|
||
}
|
||
|
||
function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
|
||
if (names.length === 0) {
|
||
return {
|
||
ok: true,
|
||
command: [],
|
||
exitCode: 0,
|
||
stdout: "no candidates",
|
||
stderr: "",
|
||
parsed: null,
|
||
};
|
||
}
|
||
return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs);
|
||
}
|
||
|
||
function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const candidates = listReleasedCiWorkspacePvs(options);
|
||
const candidateNames = candidates.map((item) => String(item.name));
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||
mode: "dry-run",
|
||
lane: options.lane,
|
||
limit: options.limit,
|
||
candidates,
|
||
candidateCount: candidates.length,
|
||
mutation: false,
|
||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
|
||
};
|
||
}
|
||
const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(deletion),
|
||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||
mode: "confirmed-cleanup",
|
||
lane: options.lane,
|
||
limit: options.limit,
|
||
deletedPersistentVolumes: candidateNames,
|
||
deletedPersistentVolumeCount: candidateNames.length,
|
||
candidatesBefore: candidates,
|
||
deletion,
|
||
followUp: {
|
||
diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||
storage: "bun scripts/cli.ts ssh G14 script -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runV02RuntimeMigration(options: G14ControlPlaneOptions, sourceCommit: string): Record<string, unknown> {
|
||
const reportPath = `/tmp/hwlab-v02-runtime-migration-${shortSha(sourceCommit)}.json`;
|
||
const migrationArgs = options.dryRun
|
||
? [
|
||
...(options.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
|
||
"--report",
|
||
reportPath,
|
||
]
|
||
: [
|
||
"--apply",
|
||
"--confirm-dev",
|
||
"--confirmed-non-production",
|
||
"--report",
|
||
reportPath,
|
||
];
|
||
const command = [
|
||
"exec",
|
||
"-n",
|
||
"hwlab-v02",
|
||
"deployment/hwlab-cloud-api",
|
||
"-c",
|
||
"hwlab-cloud-api",
|
||
"--",
|
||
"bun",
|
||
"cmd/hwlab-cloud-api/migrate.ts",
|
||
...migrationArgs,
|
||
];
|
||
const result = g14K3s(["kubectl", ...command], options.timeoutSeconds * 1000);
|
||
const ok = isCommandSuccess(result);
|
||
return {
|
||
ok,
|
||
command: "hwlab g14 control-plane runtime-migration --lane v02",
|
||
lane: "v02",
|
||
mode: options.dryRun ? options.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
|
||
sourceCommit,
|
||
runtimeNamespace: "hwlab-v02",
|
||
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
|
||
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
|
||
reportPath,
|
||
mutation: !options.dryRun,
|
||
result,
|
||
next: options.dryRun
|
||
? {
|
||
liveReadDryRun: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||
apply: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||
}
|
||
: {
|
||
health: "curl -fsS --max-time 20 http://74.48.78.17:19667/health/live",
|
||
triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const candidates = listCleanupPipelineRuns(options);
|
||
const candidateNames = candidates.map((item) => String(item.name));
|
||
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane cleanup-runs",
|
||
mode: "dry-run",
|
||
lane: options.lane,
|
||
minAgeMinutes: options.minAgeMinutes,
|
||
limit: options.limit,
|
||
candidates,
|
||
candidateCount: candidates.length,
|
||
ownedPvcs,
|
||
ownedPvcCount: ownedPvcs.length,
|
||
mutation: false,
|
||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane ${options.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
|
||
};
|
||
}
|
||
const deletion = deletePipelineRuns(candidateNames, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(deletion),
|
||
command: "hwlab g14 control-plane cleanup-runs",
|
||
mode: "confirmed-cleanup",
|
||
lane: options.lane,
|
||
minAgeMinutes: options.minAgeMinutes,
|
||
limit: options.limit,
|
||
deletedPipelineRuns: candidateNames,
|
||
deletedPipelineRunCount: candidateNames.length,
|
||
ownedPvcsBefore: ownedPvcs,
|
||
ownedPvcCountBefore: ownedPvcs.length,
|
||
deletion,
|
||
followUp: {
|
||
status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||
diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runV02RenderToTemp(sourceCommit: string): CommandJsonResult {
|
||
const renderDir = v02RenderDir(sourceCommit);
|
||
return v02WorkspaceScript([
|
||
"set -eu",
|
||
"git fetch origin v0.2 --prune",
|
||
"git checkout v0.2 >/dev/null 2>&1 || true",
|
||
"git merge --ff-only origin/v0.2",
|
||
`test "$(git rev-parse HEAD)" = ${shellQuote(sourceCommit)}`,
|
||
`rm -rf ${shellQuote(renderDir)}`,
|
||
`mkdir -p ${shellQuote(renderDir)}`,
|
||
`node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out ${shellQuote(renderDir)}`,
|
||
].join("\n"), 180_000);
|
||
}
|
||
|
||
function v02RenderDir(sourceCommit: string): string {
|
||
return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}`;
|
||
}
|
||
|
||
function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||
const renderDir = v02RenderDir(sourceCommit);
|
||
return g14K3s([
|
||
"kubectl",
|
||
"apply",
|
||
"--server-side",
|
||
"--force-conflicts",
|
||
`--field-manager=${V02_CONTROL_PLANE_FIELD_MANAGER}`,
|
||
...(dryRun ? ["--dry-run=server"] : []),
|
||
"-f",
|
||
`${renderDir}/tekton-v02/rbac.yaml`,
|
||
"-f",
|
||
`${renderDir}/tekton-v02/pipeline.yaml`,
|
||
"-f",
|
||
`${renderDir}/argocd/project.yaml`,
|
||
"-f",
|
||
`${renderDir}/argocd/application-v02.yaml`,
|
||
], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render)) {
|
||
return {
|
||
ok: false,
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
renderDir: v02RenderDir(sourceCommit),
|
||
render,
|
||
degradedReason: "control-plane-render-failed",
|
||
};
|
||
}
|
||
const apply = applyV02ControlPlaneFiles(sourceCommit, false, timeoutSeconds);
|
||
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
|
||
return {
|
||
ok: isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs)),
|
||
phase: "control-plane-refresh",
|
||
sourceCommit,
|
||
renderDir: v02RenderDir(sourceCommit),
|
||
render: commandData(render),
|
||
apply: compactCommandResult(apply),
|
||
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
||
degradedReason: !isCommandSuccess(apply)
|
||
? "control-plane-apply-failed"
|
||
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
||
? "obsolete-cronjob-cleanup-failed"
|
||
: undefined,
|
||
};
|
||
}
|
||
|
||
function getV02ObsoleteCronJobs(): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"cronjob",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
V02_POLLER,
|
||
V02_RECONCILER,
|
||
"--ignore-not-found",
|
||
"-o",
|
||
"name",
|
||
], 60_000);
|
||
}
|
||
|
||
function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"delete",
|
||
"cronjob",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
V02_POLLER,
|
||
V02_RECONCILER,
|
||
"--ignore-not-found=true",
|
||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||
], 60_000);
|
||
}
|
||
|
||
function v02PipelineRunManifest(sourceCommit: string): Record<string, unknown> {
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
return {
|
||
apiVersion: "tekton.dev/v1",
|
||
kind: "PipelineRun",
|
||
metadata: {
|
||
name: pipelineRun,
|
||
namespace: CI_NAMESPACE,
|
||
labels: {
|
||
"app.kubernetes.io/part-of": "hwlab",
|
||
"hwlab.pikastech.local/gitops-target": "v02",
|
||
"hwlab.pikastech.local/source-commit": sourceCommit,
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
annotations: {
|
||
"hwlab.pikastech.local/source-branch": V02_SOURCE_BRANCH,
|
||
"hwlab.pikastech.local/gitops-branch": V02_GITOPS_BRANCH,
|
||
"hwlab.pikastech.local/triggered-by": "unidesk-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
pipelineRef: { name: V02_PIPELINE },
|
||
taskRunTemplate: {
|
||
serviceAccountName: "hwlab-v02-tekton-runner",
|
||
podTemplate: {
|
||
hostNetwork: true,
|
||
dnsPolicy: "ClusterFirstWithHostNet",
|
||
securityContext: { fsGroup: 1000 },
|
||
},
|
||
},
|
||
params: [
|
||
{ name: "git-url", value: V02_GIT_URL },
|
||
{ name: "git-read-url", value: V02_GIT_READ_URL },
|
||
{ name: "source-branch", value: V02_SOURCE_BRANCH },
|
||
{ name: "gitops-branch", value: V02_GITOPS_BRANCH },
|
||
{ name: "lane", value: "v02" },
|
||
{ name: "catalog-path", value: V02_CATALOG_PATH },
|
||
{ name: "image-tag-mode", value: "full" },
|
||
{ name: "runtime-path", value: V02_RUNTIME_PATH },
|
||
{ name: "revision", value: sourceCommit },
|
||
{ name: "registry-prefix", value: V02_REGISTRY_PREFIX },
|
||
{ name: "services", value: V02_SERVICE_IDS.join(",") },
|
||
{ name: "base-image", value: V02_BASE_IMAGE },
|
||
],
|
||
workspaces: [
|
||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
|
||
],
|
||
},
|
||
};
|
||
}
|
||
|
||
function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult {
|
||
const manifest = JSON.stringify(v02PipelineRunManifest(sourceCommit));
|
||
const manifestB64 = Buffer.from(manifest, "utf8").toString("base64");
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
const script = [
|
||
"set -eu",
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
`manifest_path=/tmp/${pipelineRun}.json`,
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`,
|
||
].join("\n");
|
||
return g14K3s(["script", "--", script], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function deleteV02PipelineRun(pipelineRun: string): CommandJsonResult {
|
||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, pipelineRun, "--ignore-not-found=true"], 60_000);
|
||
}
|
||
|
||
function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string, unknown> {
|
||
const bundle = v02ControlPlaneStatusBundle(sourceCommitInput);
|
||
const sections = parseShellSections(statusText(bundle));
|
||
const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null;
|
||
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
|
||
const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now();
|
||
const controlPlane = sections.controlPlane;
|
||
const obsoleteCronJobs = sections.obsoleteCronJobs;
|
||
const argo = sections.argo;
|
||
const pipelineRunSection = sections.pipelineRun;
|
||
const taskRunsSection = sections.taskRuns;
|
||
const webAssetsSection = sections.webAssets;
|
||
const recentPipelineRuns = listV02PipelineRunsCompactFromText(
|
||
sections.recentPipelineRuns?.stdout ?? "",
|
||
shellSectionOk(sections.recentPipelineRuns),
|
||
"kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=<summary-fields>",
|
||
sections.recentPipelineRuns?.exitCode ?? null,
|
||
bundle.stderr,
|
||
8,
|
||
queryNowMs,
|
||
);
|
||
const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
|
||
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
|
||
return {
|
||
ok: sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo),
|
||
command: "hwlab g14 control-plane status --lane v02",
|
||
lane: "v02",
|
||
sourceCommit,
|
||
sourceCommitSource: "cached origin/v0.2; write operations fetch before acting",
|
||
expected: {
|
||
workspace: V02_WORKSPACE,
|
||
branch: V02_SOURCE_BRANCH,
|
||
namespace: CI_NAMESPACE,
|
||
runtimeNamespace: "hwlab-v02",
|
||
pipeline: V02_PIPELINE,
|
||
trigger: "manual UniDesk CLI PipelineRun creation",
|
||
argoApplication: V02_APP,
|
||
},
|
||
controlPlane: {
|
||
ok: shellSectionOk(controlPlane),
|
||
names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
exitCode: controlPlane?.exitCode ?? null,
|
||
},
|
||
obsoleteCronJobs: {
|
||
ok: shellSectionOk(obsoleteCronJobs),
|
||
names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
exitCode: obsoleteCronJobs?.exitCode ?? null,
|
||
cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||
},
|
||
argo: {
|
||
ok: shellSectionOk(argo),
|
||
raw: argo?.stdout ?? "",
|
||
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
||
exitCode: argo?.exitCode ?? null,
|
||
},
|
||
pipelineRun: pipelineRun === null
|
||
? null
|
||
: pipelineRunCompactFromText(
|
||
pipelineRun,
|
||
pipelineRunSection?.stdout ?? "",
|
||
shellSectionOk(pipelineRunSection),
|
||
`kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
|
||
pipelineRunSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
),
|
||
taskRuns: taskRunsCompactFromText(
|
||
taskRunsSection?.stdout ?? "",
|
||
shellSectionOk(taskRunsSection),
|
||
pipelineRun,
|
||
taskRunsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
),
|
||
webAssets: v02WebAssetsFromText(
|
||
webAssetsSection?.stdout ?? "",
|
||
shellSectionOk(webAssetsSection),
|
||
sourceCommit,
|
||
syncRevision,
|
||
webAssetsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
),
|
||
activePipelineRuns,
|
||
recentPipelineRuns,
|
||
query: {
|
||
ok: isCommandSuccess(bundle),
|
||
exitCode: bundle.exitCode,
|
||
stderr: bundle.stderr.trim().slice(0, 2000),
|
||
},
|
||
visibilityHint: activePipelineRuns.length > 0
|
||
? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger"
|
||
: "no active v02 PipelineRun observed",
|
||
};
|
||
}
|
||
|
||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
||
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
||
const sourceCommit = options.action === "status" ? undefined : getV02Head();
|
||
if (sourceCommit === null) {
|
||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
|
||
}
|
||
if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit);
|
||
if (options.action === "status") return v02ControlPlaneStatus();
|
||
if (options.action === "apply") {
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render)) {
|
||
return {
|
||
ok: false,
|
||
command: `hwlab g14 control-plane ${options.action} --lane v02`,
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
render: compactCommandResult(render),
|
||
};
|
||
}
|
||
const apply = applyV02ControlPlaneFiles(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||
return {
|
||
ok: isCommandSuccess(apply),
|
||
command: "hwlab g14 control-plane apply --lane v02",
|
||
lane: "v02",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||
sourceCommit,
|
||
renderDir: v02RenderDir(sourceCommit),
|
||
render: commandData(render),
|
||
apply: compactCommandResult(apply),
|
||
cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false)),
|
||
status: v02ControlPlaneStatus(sourceCommit),
|
||
next: options.dryRun
|
||
? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" }
|
||
: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
const before = getPipelineRunCompact(v02PipelineRunName(sourceCommit));
|
||
if (options.dryRun) {
|
||
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "dry-run",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
gitMirrorPreSync,
|
||
controlPlaneRefresh: {
|
||
mode: "skipped-dry-run",
|
||
wouldRefreshBeforeCreate: true,
|
||
resources: ["tekton-v02/rbac.yaml", "tekton-v02/pipeline.yaml", "argocd/project.yaml", "argocd/application-v02.yaml"],
|
||
},
|
||
manifest: v02PipelineRunManifest(sourceCommit),
|
||
next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
if (before.status === "True" || before.status === "Unknown") {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
degradedReason: "refuse-active-or-successful-pipelinerun",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const controlPlaneRefresh = refreshV02ControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds);
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: controlPlaneRefresh.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), degradedReason: record(controlPlaneRefresh).degradedReason ?? null });
|
||
if (controlPlaneRefresh.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "control-plane-refresh",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh,
|
||
degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: gitMirrorPreSync.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), mode: record(gitMirrorPreSync).mode ?? null, degradedReason: record(gitMirrorPreSync).degradedReason ?? null });
|
||
if (gitMirrorPreSync.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "git-mirror-pre-sync",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh,
|
||
gitMirrorPreSync,
|
||
degradedReason: "git-mirror-pre-sync-failed",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const deletePipelineRun = deleteV02PipelineRun(v02PipelineRunName(sourceCommit));
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: isCommandSuccess(deletePipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: deletePipelineRun.exitCode });
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const createPipelineRun = isCommandSuccess(deletePipelineRun)
|
||
? createV02PipelineRun(sourceCommit, options.timeoutSeconds)
|
||
: null;
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: createPipelineRun !== null && isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: createPipelineRun?.exitCode ?? null });
|
||
return {
|
||
ok: isCommandSuccess(deletePipelineRun) && createPipelineRun !== null && isCommandSuccess(createPipelineRun),
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh),
|
||
gitMirrorPreSync,
|
||
deletePipelineRun: compactTriggerCommandResult(compactCommandResult(deletePipelineRun)),
|
||
createPipelineRun: createPipelineRun === null ? null : compactTriggerCommandResult(compactCommandResult(createPipelineRun)),
|
||
after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)),
|
||
disclosure: {
|
||
fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.",
|
||
},
|
||
};
|
||
}
|
||
|
||
function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"delete",
|
||
"cronjob",
|
||
"-n",
|
||
GIT_MIRROR_NAMESPACE,
|
||
GIT_MIRROR_LEGACY_CRONJOB,
|
||
"--ignore-not-found=true",
|
||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||
], 60_000);
|
||
}
|
||
|
||
function applyGitMirrorManifestFile(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||
const renderDir = v02RenderDir(sourceCommit);
|
||
return g14K3s([
|
||
"kubectl",
|
||
"apply",
|
||
"--server-side",
|
||
"--force-conflicts",
|
||
`--field-manager=${GIT_MIRROR_MANIFEST_FIELD_MANAGER}`,
|
||
...(dryRun ? ["--dry-run=server"] : []),
|
||
"-f",
|
||
`${renderDir}/devops-infra/git-mirror.yaml`,
|
||
], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function gitMirrorSyncJobName(): string {
|
||
return `${GIT_MIRROR_SYNC_JOB_PREFIX}-${Date.now().toString(36)}`.slice(0, 63);
|
||
}
|
||
|
||
function gitMirrorFlushJobName(): string {
|
||
return `git-mirror-hwlab-flush-manual-${Date.now().toString(36)}`.slice(0, 63);
|
||
}
|
||
|
||
export function gitMirrorSyncJobManifest(name: string): Record<string, unknown> {
|
||
return {
|
||
apiVersion: "batch/v1",
|
||
kind: "Job",
|
||
metadata: {
|
||
name,
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
labels: {
|
||
"app.kubernetes.io/name": "git-mirror",
|
||
"app.kubernetes.io/part-of": "devops-infra",
|
||
"app.kubernetes.io/component": "sync-controller",
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
backoffLimit: 0,
|
||
activeDeadlineSeconds: 300,
|
||
ttlSecondsAfterFinished: 3600,
|
||
template: {
|
||
metadata: {
|
||
labels: {
|
||
"app.kubernetes.io/name": "git-mirror",
|
||
"app.kubernetes.io/part-of": "devops-infra",
|
||
"app.kubernetes.io/component": "sync-controller",
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
restartPolicy: "Never",
|
||
hostNetwork: true,
|
||
dnsPolicy: "ClusterFirstWithHostNet",
|
||
volumes: [
|
||
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
|
||
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
|
||
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } },
|
||
],
|
||
containers: [{
|
||
name: "sync",
|
||
image: "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1",
|
||
imagePullPolicy: "IfNotPresent",
|
||
command: ["/script/sync.sh"],
|
||
volumeMounts: [
|
||
{ name: "cache", mountPath: "/cache" },
|
||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||
{ name: "script", mountPath: "/script", readOnly: true },
|
||
],
|
||
}],
|
||
},
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
export function gitMirrorFlushJobManifest(name: string): Record<string, unknown> {
|
||
const manifest = gitMirrorSyncJobManifest(name);
|
||
record(record(manifest.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||
const template = record(record(manifest.spec).template);
|
||
record(record(template.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||
const podSpec = record(template.spec);
|
||
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
|
||
const first = record(containers[0]);
|
||
first.name = "flush";
|
||
first.command = ["/script/flush.sh"];
|
||
podSpec.containers = [first];
|
||
return manifest;
|
||
}
|
||
|
||
export function parseGitMirrorStatusRefs(raw: string): { refs: Record<string, string | null>; pendingFlush: boolean | null } {
|
||
const line = raw.split(/\r?\n/u).find((item) => item.startsWith("refs="));
|
||
if (line === undefined) return { refs: {}, pendingFlush: null };
|
||
try {
|
||
const parsed = record(JSON.parse(line.slice("refs=".length)) as unknown);
|
||
const refs = record(parsed.refs);
|
||
return {
|
||
refs: {
|
||
localV02: stringOrNull(refs.localV02),
|
||
localG14: stringOrNull(refs.localG14),
|
||
localGitops: stringOrNull(refs.localGitops),
|
||
githubGitops: stringOrNull(refs.githubGitops),
|
||
},
|
||
pendingFlush: typeof parsed.pendingFlush === "boolean" ? parsed.pendingFlush : null,
|
||
};
|
||
} catch {
|
||
return { refs: {}, pendingFlush: null };
|
||
}
|
||
}
|
||
|
||
export function gitMirrorV02SyncRequirement(sourceCommit: string, rawStatus: string): Record<string, unknown> {
|
||
const parsed = parseGitMirrorStatusRefs(rawStatus);
|
||
const localV02 = parsed.refs.localV02 ?? null;
|
||
return {
|
||
required: localV02 !== sourceCommit,
|
||
sourceCommit,
|
||
localV02,
|
||
pendingFlush: parsed.pendingFlush,
|
||
refs: parsed.refs,
|
||
reason: localV02 === sourceCommit ? "local-v02-current" : localV02 === null ? "local-v02-unresolved" : "local-v02-stale",
|
||
};
|
||
}
|
||
|
||
function gitMirrorStatusCacheRaw(status: Record<string, unknown>): string {
|
||
return String(nested(status, ["cache", "raw"]) ?? "");
|
||
}
|
||
|
||
function compactGitMirrorStatus(status: Record<string, unknown>, sourceCommit: string): Record<string, unknown> {
|
||
const requirement = gitMirrorV02SyncRequirement(sourceCommit, gitMirrorStatusCacheRaw(status));
|
||
return {
|
||
ok: status.ok === true,
|
||
readUrl: status.readUrl ?? V02_GIT_READ_URL,
|
||
writeUrl: status.writeUrl ?? V02_GIT_WRITE_URL,
|
||
legacyCronJobExists: nested(status, ["legacyCronJob", "exists"]) === true,
|
||
required: requirement.required,
|
||
sourceCommit,
|
||
localV02: requirement.localV02 ?? null,
|
||
pendingFlush: requirement.pendingFlush ?? null,
|
||
reason: requirement.reason,
|
||
cacheOk: nested(status, ["cache", "ok"]) === true,
|
||
cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000),
|
||
resourcesOk: nested(status, ["resources", "ok"]) === true,
|
||
};
|
||
}
|
||
|
||
function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
ok: sync.ok === true,
|
||
command: sync.command ?? "hwlab g14 git-mirror sync",
|
||
mode: sync.mode ?? null,
|
||
namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE,
|
||
jobName: sync.jobName ?? null,
|
||
exitCode: nested(sync, ["result", "exitCode"]) ?? null,
|
||
stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600),
|
||
stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200),
|
||
stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null,
|
||
stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null,
|
||
};
|
||
}
|
||
|
||
function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
|
||
const before = runGitMirrorStatus();
|
||
const beforeSummary = compactGitMirrorStatus(before, sourceCommit);
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
mode: "dry-run",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
action: beforeSummary.required === true ? "would-sync-before-trigger" : "already-current",
|
||
next: beforeSummary.required === true ? { syncAndTrigger: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" } : undefined,
|
||
};
|
||
}
|
||
if (beforeSummary.required !== true) {
|
||
return {
|
||
ok: true,
|
||
mode: "already-current",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
};
|
||
}
|
||
const sync = runGitMirrorSync({
|
||
action: "sync",
|
||
confirm: true,
|
||
dryRun: false,
|
||
timeoutSeconds: options.timeoutSeconds,
|
||
});
|
||
const after = runGitMirrorStatus();
|
||
const afterSummary = compactGitMirrorStatus(after, sourceCommit);
|
||
const ok = sync.ok === true && afterSummary.required === false;
|
||
return {
|
||
ok,
|
||
mode: "auto-sync-before-trigger",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
sync: compactGitMirrorSync(sync),
|
||
after: afterSummary,
|
||
degradedReason: ok ? undefined : "git-mirror-local-v02-not-current-after-sync",
|
||
};
|
||
}
|
||
|
||
function runGitMirrorStatus(): Record<string, unknown> {
|
||
const resources = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"deploy,svc,pvc,cm",
|
||
"-n",
|
||
GIT_MIRROR_NAMESPACE,
|
||
"-l",
|
||
"app.kubernetes.io/name=git-mirror",
|
||
"-o",
|
||
"name",
|
||
], 60_000);
|
||
const legacyCronJob = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"cronjob",
|
||
"-n",
|
||
GIT_MIRROR_NAMESPACE,
|
||
GIT_MIRROR_LEGACY_CRONJOB,
|
||
"--ignore-not-found",
|
||
"-o",
|
||
"name",
|
||
], 60_000);
|
||
const jobs = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"job",
|
||
"-n",
|
||
GIT_MIRROR_NAMESPACE,
|
||
"-l",
|
||
"app.kubernetes.io/name=git-mirror",
|
||
"--sort-by=.metadata.creationTimestamp",
|
||
"-o",
|
||
"custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime",
|
||
"--no-headers",
|
||
], 60_000);
|
||
const cache = g14K3s([
|
||
"kubectl",
|
||
"exec",
|
||
"-n",
|
||
GIT_MIRROR_NAMESPACE,
|
||
"deploy/git-mirror-http",
|
||
"--",
|
||
"sh",
|
||
"-lc",
|
||
[
|
||
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'refs='",
|
||
"node - <<'NODE' 2>/dev/null || true",
|
||
"const { execFileSync } = require('node:child_process');",
|
||
"const repo = '/cache/pikasTech/HWLAB.git';",
|
||
"function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim(); } catch { return null; } }",
|
||
"const localGitops = rev('refs/heads/v0.2-gitops');",
|
||
"const githubGitops = rev('refs/mirror-stage/heads/v0.2-gitops');",
|
||
"console.log(JSON.stringify({ refs: { localV02: rev('refs/heads/v0.2'), localG14: rev('refs/heads/G14'), localGitops, githubGitops }, pendingFlush: Boolean(localGitops && githubGitops && localGitops !== githubGitops) }));",
|
||
"NODE",
|
||
].join("\n"),
|
||
], 60_000);
|
||
return {
|
||
ok: isCommandSuccess(resources),
|
||
command: "hwlab g14 git-mirror status",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
readUrl: V02_GIT_READ_URL,
|
||
writeUrl: V02_GIT_WRITE_URL,
|
||
resources: {
|
||
ok: isCommandSuccess(resources),
|
||
names: statusText(resources).split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
stderr: resources.stderr.trim().slice(0, 2000),
|
||
},
|
||
legacyCronJob: {
|
||
ok: isCommandSuccess(legacyCronJob),
|
||
exists: statusText(legacyCronJob).trim().length > 0,
|
||
name: statusText(legacyCronJob).trim() || null,
|
||
cleanupCommand: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
},
|
||
jobs: {
|
||
ok: isCommandSuccess(jobs),
|
||
raw: statusText(jobs),
|
||
stderr: jobs.stderr.trim().slice(0, 2000),
|
||
},
|
||
cache: {
|
||
ok: isCommandSuccess(cache),
|
||
raw: statusText(cache),
|
||
stderr: cache.stderr.trim().slice(0, 2000),
|
||
},
|
||
next: {
|
||
apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
||
flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const sourceCommit = getV02Head();
|
||
if (sourceCommit === null) {
|
||
return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
|
||
}
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render)) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 git-mirror apply",
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
render: compactCommandResult(render),
|
||
};
|
||
}
|
||
const apply = applyGitMirrorManifestFile(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||
const cleanupLegacyCronJob = isCommandSuccess(apply)
|
||
? deleteLegacyGitMirrorCronJob(options.dryRun)
|
||
: {
|
||
ok: false,
|
||
command: [],
|
||
exitCode: null,
|
||
stdout: "",
|
||
stderr: "skipped because apply failed",
|
||
parsed: null,
|
||
};
|
||
return {
|
||
ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob),
|
||
command: "hwlab g14 git-mirror apply",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||
sourceCommit,
|
||
manifest: `${v02RenderDir(sourceCommit)}/devops-infra/git-mirror.yaml`,
|
||
render: commandData(render),
|
||
apply: compactCommandResult(apply),
|
||
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
|
||
status: runGitMirrorStatus(),
|
||
next: options.dryRun
|
||
? { apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm" }
|
||
: { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" },
|
||
};
|
||
}
|
||
|
||
function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const jobName = gitMirrorSyncJobName();
|
||
const manifest = gitMirrorSyncJobManifest(jobName);
|
||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||
const script = [
|
||
"set -eu",
|
||
`job=${shellQuote(jobName)}`,
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
"manifest_path=\"/tmp/$job.json\"",
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
options.dryRun
|
||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||
: [
|
||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||
"while :; do",
|
||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='{.status.succeeded} {.status.failed}' 2>/dev/null || true)`,
|
||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{print $1}')",
|
||
" failed=$(printf '%s\\n' \"$status\" | awk '{print $2}')",
|
||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
" exit 44",
|
||
" fi",
|
||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||
" exit 45",
|
||
" fi",
|
||
" sleep 2",
|
||
"done",
|
||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2 refs/heads/v0.2-gitops refs/heads/G14 2>/dev/null || true'`,
|
||
].join("\n"),
|
||
].join("\n");
|
||
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 git-mirror sync",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-sync",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
jobName,
|
||
manifest: options.dryRun ? manifest : undefined,
|
||
result: compactCommandResult(result),
|
||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||
next: options.dryRun ? { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
|
||
function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const jobName = gitMirrorFlushJobName();
|
||
const manifest = gitMirrorFlushJobManifest(jobName);
|
||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||
const script = [
|
||
"set -eu",
|
||
`job=${shellQuote(jobName)}`,
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
"manifest_path=\"/tmp/$job.json\"",
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
options.dryRun
|
||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||
: [
|
||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||
"while :; do",
|
||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='{.status.succeeded} {.status.failed}' 2>/dev/null || true)`,
|
||
" succeeded=$(printf '%s\n' \"$status\" | awk '{print $1}')",
|
||
" failed=$(printf '%s\n' \"$status\" | awk '{print $2}')",
|
||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
" exit 54",
|
||
" fi",
|
||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||
" exit 55",
|
||
" fi",
|
||
" sleep 2",
|
||
"done",
|
||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2-gitops refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true'`,
|
||
].join("\n"),
|
||
].join("\n");
|
||
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 git-mirror flush",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-flush",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
jobName,
|
||
manifest: options.dryRun ? manifest : undefined,
|
||
result: compactCommandResult(result),
|
||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||
next: options.dryRun ? { flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" } : { status: "bun scripts/cli.ts hwlab g14 git-mirror status" },
|
||
};
|
||
}
|
||
|
||
function runG14GitMirror(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
if (options.action === "status") return runGitMirrorStatus();
|
||
if (options.action === "apply") return runGitMirrorApply(options);
|
||
if (options.action === "flush") return runGitMirrorFlush(options);
|
||
return runGitMirrorSync(options);
|
||
}
|
||
|
||
function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record<string, unknown> {
|
||
const job = startJob(name, command, note);
|
||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
||
return {
|
||
ok: true,
|
||
mode: "async-job",
|
||
job,
|
||
statusCommand,
|
||
tailCommand: `tail -f ${job.stdoutFile}`,
|
||
next: {
|
||
status: statusCommand,
|
||
tail: `tail -f ${job.stdoutFile}`,
|
||
},
|
||
};
|
||
}
|
||
|
||
function startControlPlaneTriggerJob(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const command = [
|
||
"bun",
|
||
"scripts/cli.ts",
|
||
"hwlab",
|
||
"g14",
|
||
"control-plane",
|
||
"trigger-current",
|
||
"--lane",
|
||
options.lane,
|
||
"--confirm",
|
||
"--timeout-seconds",
|
||
String(options.timeoutSeconds),
|
||
"--wait",
|
||
];
|
||
return {
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: options.lane,
|
||
reason: "confirmed trigger can spend tens of seconds syncing git mirror and creating PipelineRun; default is fire-and-forget to avoid silent blocking",
|
||
waitCommand: command.join(" "),
|
||
...startAsyncHwlabG14Job(
|
||
"hwlab_g14_v02_trigger_current",
|
||
command,
|
||
"Trigger HWLAB v0.2 current commit PipelineRun with git mirror pre-sync through G14 control-plane",
|
||
),
|
||
};
|
||
}
|
||
|
||
function startGitMirrorJob(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const command = [
|
||
"bun",
|
||
"scripts/cli.ts",
|
||
"hwlab",
|
||
"g14",
|
||
"git-mirror",
|
||
options.action,
|
||
"--confirm",
|
||
"--timeout-seconds",
|
||
String(options.timeoutSeconds),
|
||
"--wait",
|
||
];
|
||
return {
|
||
command: `hwlab g14 git-mirror ${options.action}`,
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
reason: "manual git mirror sync/flush waits for a Kubernetes Job; default is fire-and-forget to keep CLI output immediately visible",
|
||
waitCommand: command.join(" "),
|
||
...startAsyncHwlabG14Job(
|
||
`hwlab_g14_git_mirror_${options.action}`,
|
||
command,
|
||
`Run HWLAB devops-infra git mirror ${options.action} through a bounded manual Kubernetes Job`,
|
||
),
|
||
};
|
||
}
|
||
|
||
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
|
||
}
|
||
|
||
function g14CiToolsImage(tag: string): string {
|
||
return `${G14_CI_TOOLS_IMAGE_REPO}:${tag}`;
|
||
}
|
||
|
||
function runG14ToolsImageStatus(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
const image = g14CiToolsImage(options.tag);
|
||
const script = [
|
||
"set -u",
|
||
`image=${shellQuote(image)}`,
|
||
"exists=false",
|
||
"image_id=",
|
||
"created=",
|
||
"size=",
|
||
"node_version=",
|
||
"npm_version=",
|
||
"bun_version=",
|
||
"git_version=",
|
||
"docker_version=",
|
||
"python_version=",
|
||
"registry_tags=",
|
||
"if docker image inspect \"$image\" >/tmp/hwlab-tools-image-inspect.json 2>/tmp/hwlab-tools-image-inspect.err; then",
|
||
" exists=true",
|
||
" image_id=$(docker image inspect \"$image\" --format '{{.Id}}' 2>/dev/null || true)",
|
||
" created=$(docker image inspect \"$image\" --format '{{.Created}}' 2>/dev/null || true)",
|
||
" size=$(docker image inspect \"$image\" --format '{{.Size}}' 2>/dev/null || true)",
|
||
" node_version=$(docker run --rm \"$image\" node --version 2>/dev/null || true)",
|
||
" npm_version=$(docker run --rm \"$image\" npm --version 2>/dev/null || true)",
|
||
" bun_version=$(docker run --rm \"$image\" bun --version 2>/dev/null || true)",
|
||
" git_version=$(docker run --rm \"$image\" git --version 2>/dev/null || true)",
|
||
" docker_version=$(docker run --rm \"$image\" docker --version 2>/dev/null || true)",
|
||
" python_version=$(docker run --rm \"$image\" python3 --version 2>/dev/null || true)",
|
||
"fi",
|
||
`registry_tags=$(curl -fsS --max-time 10 http://127.0.0.1:5000/v2/hwlab/hwlab-ci-node-tools/tags/list 2>/dev/null || true)`,
|
||
"export image exists image_id created size node_version npm_version bun_version git_version docker_version python_version registry_tags",
|
||
"node - <<'NODE'",
|
||
"const fs = require('node:fs');",
|
||
"const env = process.env;",
|
||
"const payload = {",
|
||
" image: env.image,",
|
||
" exists: env.exists === 'true',",
|
||
" imageId: env.image_id || null,",
|
||
" created: env.created || null,",
|
||
" sizeBytes: env.size ? Number(env.size) : null,",
|
||
" tools: {",
|
||
" node: env.node_version || null,",
|
||
" npm: env.npm_version || null,",
|
||
" bun: env.bun_version || null,",
|
||
" git: env.git_version || null,",
|
||
" docker: env.docker_version || null,",
|
||
" python: env.python_version || null",
|
||
" },",
|
||
" registryTagsRaw: env.registry_tags || null",
|
||
"};",
|
||
"console.log(JSON.stringify(payload, null, 2));",
|
||
"NODE",
|
||
].join("\n");
|
||
const result = g14HostScript(script, 180_000);
|
||
let parsedStatus: unknown = null;
|
||
const text = statusText(result);
|
||
if (text.length > 0) {
|
||
try {
|
||
parsedStatus = JSON.parse(text) as unknown;
|
||
} catch {
|
||
parsedStatus = null;
|
||
}
|
||
}
|
||
return {
|
||
ok: isCommandSuccess(result) && record(parsedStatus).exists === true,
|
||
command: "hwlab g14 tools-image status --name ci-node-tools",
|
||
name: options.name,
|
||
image,
|
||
status: parsedStatus ?? text,
|
||
result,
|
||
};
|
||
}
|
||
|
||
function runG14ToolsImageBuild(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
const image = g14CiToolsImage(options.tag);
|
||
const script = [
|
||
"set -eu",
|
||
`cd ${shellQuote(V02_WORKSPACE)}`,
|
||
"git fetch origin v0.2 --prune",
|
||
"git checkout v0.2 >/dev/null 2>&1 || true",
|
||
"git merge --ff-only origin/v0.2",
|
||
`test -f ${shellQuote(options.dockerfile)}`,
|
||
`image=${shellQuote(image)}`,
|
||
`dockerfile=${shellQuote(options.dockerfile)}`,
|
||
"echo \"{\\\"phase\\\":\\\"preflight\\\",\\\"image\\\":\\\"$image\\\",\\\"dockerfile\\\":\\\"$dockerfile\\\",\\\"commit\\\":\\\"$(git rev-parse HEAD)\\\"}\"",
|
||
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
|
||
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
|
||
"export no_proxy=$NO_PROXY",
|
||
"docker build --pull --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg http_proxy --build-arg https_proxy --build-arg NO_PROXY --build-arg no_proxy -f \"$dockerfile\" -t \"$image\" .",
|
||
"docker run --rm \"$image\" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V'",
|
||
"docker push \"$image\"",
|
||
"digest=$(docker image inspect \"$image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
|
||
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"image\\\":\\\"$image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
|
||
].join("\n");
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||
mode: "dry-run",
|
||
image,
|
||
workspace: V02_WORKSPACE,
|
||
dockerfile: options.dockerfile,
|
||
buildScriptPreview: script.split(/\n/u),
|
||
next: { build: `bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag ${options.tag} --confirm` },
|
||
};
|
||
}
|
||
const result = g14HostScript(script, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||
mode: "confirmed-build",
|
||
image,
|
||
workspace: V02_WORKSPACE,
|
||
dockerfile: options.dockerfile,
|
||
result,
|
||
status: runG14ToolsImageStatus({ ...options, action: "status", dryRun: true, confirm: false }),
|
||
};
|
||
}
|
||
|
||
function runG14ToolsImage(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
if (options.action === "status") return runG14ToolsImageStatus(options);
|
||
return runG14ToolsImageBuild(options);
|
||
}
|
||
|
||
function listOpenG14PullRequests(): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "30", "--json", "number,title,state,url,head,base,draft,headRefName,baseRefName"], 60_000);
|
||
}
|
||
|
||
function preflightPullRequest(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "preflight", String(number), "--repo", HWLAB_REPO], 80_000);
|
||
}
|
||
|
||
function mergePullRequest(number: number, dryRun: boolean): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "merge", String(number), "--repo", HWLAB_REPO, "--merge", "--delete-branch", ...(dryRun ? ["--dry-run"] : [])], 100_000);
|
||
}
|
||
|
||
function getG14Head(): string | null {
|
||
const result = cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "git fetch origin G14 --prune >/dev/null 2>&1; git rev-parse origin/G14"], 120_000);
|
||
if (!isCommandSuccess(result)) return null;
|
||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||
return match?.[0] ?? null;
|
||
}
|
||
|
||
function refreshArgoDev(): void {
|
||
cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "annotate", "application", "-n", ARGO_NAMESPACE, DEV_APP, "argocd.argoproj.io/refresh=hard", "--overwrite"], 60_000);
|
||
}
|
||
|
||
function getPipelineStatus(sourceCommit: string): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}"], 60_000);
|
||
}
|
||
|
||
function getArgoStatus(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "application", "-n", ARGO_NAMESPACE, DEV_APP, "-o", "jsonpath={.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.operationState.phase}{\"\\n\"}{.status.operationState.syncResult.revision}{\"\\n\"}"], 60_000);
|
||
}
|
||
|
||
function getDevWorkloads(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "deploy,statefulset,pod", "-n", DEV_NAMESPACE, "-o", "wide"], 60_000);
|
||
}
|
||
|
||
function getLiveHealth(): CommandJsonResult {
|
||
return commandJson(["curl", "-fsS", "--max-time", "20", "http://74.48.78.17:17667/health/live"], 30_000);
|
||
}
|
||
|
||
function statusText(result: CommandJsonResult): string {
|
||
return String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
}
|
||
|
||
function workloadReadiness(workloadsText: string, commandOk: boolean): Record<string, unknown> {
|
||
const blockers: string[] = [];
|
||
const ignoredPods: string[] = [];
|
||
const readyWorkloads: string[] = [];
|
||
const allowedZeroDeployments = new Set(["deployment.apps/hwlab-gateway", "deployment.apps/hwlab-tunnel-client"]);
|
||
for (const line of workloadsText.split(/\r?\n/u)) {
|
||
const trimmed = line.trim();
|
||
if (trimmed.length === 0 || trimmed.startsWith("NAME ")) continue;
|
||
const fields = trimmed.split(/\s+/u);
|
||
const name = fields[0] ?? "";
|
||
const ready = fields[1] ?? "";
|
||
if (name.startsWith("deployment.apps/") || name.startsWith("statefulset.apps/")) {
|
||
if (ready === "0/0" && allowedZeroDeployments.has(name)) {
|
||
readyWorkloads.push(`${name}:${ready}:scaled-zero`);
|
||
continue;
|
||
}
|
||
const match = /^(\d+)\/(\d+)$/u.exec(ready);
|
||
if (match === null) {
|
||
blockers.push(`${name}:unparseable-ready:${ready}`);
|
||
continue;
|
||
}
|
||
const readyCount = Number(match[1]);
|
||
const desiredCount = Number(match[2]);
|
||
if (desiredCount > 0 && readyCount < desiredCount) blockers.push(`${name}:not-ready:${ready}`);
|
||
else readyWorkloads.push(`${name}:${ready}`);
|
||
continue;
|
||
}
|
||
if (name.startsWith("pod/")) {
|
||
const status = fields[2] ?? "";
|
||
if (status === "Completed" || status === "Succeeded") ignoredPods.push(`${name}:${status}`);
|
||
if (/^(CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError|RunContainerError)$/u.test(status)) {
|
||
blockers.push(`${name}:pod-status:${status}`);
|
||
}
|
||
}
|
||
}
|
||
return {
|
||
ready: commandOk && blockers.length === 0,
|
||
blockers,
|
||
readyWorkloads: readyWorkloads.slice(0, 20),
|
||
ignoredPods: ignoredPods.slice(0, 20),
|
||
};
|
||
}
|
||
|
||
function beijingParts(date = new Date()): { date: string; time: string; iso: string } {
|
||
const shifted = new Date(date.getTime() + BEIJING_OFFSET_MS).toISOString();
|
||
return { date: shifted.slice(0, 10), time: shifted.slice(11, 19), iso: shifted };
|
||
}
|
||
|
||
function durationSeconds(start?: string | null, end?: string | null): number | null {
|
||
if (start === undefined || start === null || end === undefined || end === null) return null;
|
||
const startMs = Date.parse(start);
|
||
const endMs = Date.parse(end);
|
||
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) return null;
|
||
return Math.round((endMs - startMs) / 1000);
|
||
}
|
||
|
||
function formatDuration(seconds: number | null): string {
|
||
if (seconds === null) return "n/a";
|
||
const minutes = Math.floor(seconds / 60);
|
||
const rest = seconds % 60;
|
||
if (minutes === 0) return `${rest}s`;
|
||
return `${minutes}m${String(rest).padStart(2, "0")}s`;
|
||
}
|
||
|
||
function pipelineTaskRunResultsJsonPath(): string {
|
||
return [
|
||
"jsonpath=",
|
||
"{range .items[*]}",
|
||
"{.metadata.name}{\"\\t\"}",
|
||
"{.metadata.labels.tekton\\.dev/pipelineTask}{\"\\t\"}",
|
||
"{.status.startTime}{\"\\t\"}",
|
||
"{.status.completionTime}{\"\\t\"}",
|
||
"{range .status.results[*]}{.name}{\"=\"}{.value}{\";\"}{end}",
|
||
"{\"\\n\"}",
|
||
"{end}",
|
||
].join("");
|
||
}
|
||
|
||
function getPipelineTaskRunResults(pipelineRun: string): CommandJsonResult {
|
||
return cliJson([
|
||
"ssh",
|
||
`${G14_PROVIDER}:k3s`,
|
||
"kubectl",
|
||
"get",
|
||
"taskrun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-l",
|
||
`tekton.dev/pipelineRun=${pipelineRun}`,
|
||
"-o",
|
||
pipelineTaskRunResultsJsonPath(),
|
||
], 80_000);
|
||
}
|
||
|
||
function parseResultMap(raw: string): Record<string, string> {
|
||
const values: Record<string, string> = {};
|
||
for (const item of raw.split(";")) {
|
||
if (item.length === 0) continue;
|
||
const separator = item.indexOf("=");
|
||
if (separator <= 0) continue;
|
||
values[item.slice(0, separator)] = item.slice(separator + 1);
|
||
}
|
||
return values;
|
||
}
|
||
|
||
function serviceMetricSort(left: CiServiceMetric, right: CiServiceMetric): number {
|
||
return left.serviceId.localeCompare(right.serviceId);
|
||
}
|
||
|
||
export function parsePipelineTaskRunMetrics(pipelineRun: string, text: string): CiPipelineMetrics {
|
||
const services: CiServiceMetric[] = [];
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const trimmed = line.trimEnd();
|
||
if (trimmed.length === 0) continue;
|
||
const [taskRunName = "", pipelineTask = "", startTime = "", completionTime = "", resultsRaw = ""] = trimmed.split("\t");
|
||
if (!pipelineTask.startsWith("build-")) continue;
|
||
const results = parseResultMap(resultsRaw);
|
||
const serviceId = results["service-id"] ?? pipelineTask.replace(/^build-/u, "");
|
||
if (serviceId.length === 0) continue;
|
||
services.push({
|
||
taskRun: taskRunName,
|
||
serviceId,
|
||
status: results.status ?? "unknown",
|
||
durationSeconds: durationSeconds(startTime, completionTime),
|
||
buildBackend: results["build-backend"] ?? null,
|
||
reusedFrom: results["reused-from"] ?? null,
|
||
imageTag: results["image-tag"] ?? null,
|
||
sourceCommitId: results["source-commit-id"] ?? null,
|
||
});
|
||
}
|
||
services.sort(serviceMetricSort);
|
||
const reusedServices = services.filter((service) => service.status === "reused").map((service) => service.serviceId);
|
||
const rebuildServices = services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
|
||
return {
|
||
ok: services.length > 0,
|
||
pipelineRun,
|
||
serviceTotal: services.length,
|
||
reusedCount: reusedServices.length,
|
||
rebuildCount: rebuildServices.length,
|
||
reusedServices,
|
||
rebuildServices,
|
||
services,
|
||
...(services.length === 0 ? { degradedReason: "no-build-task-results", rawText: text.slice(0, 2000) } : {}),
|
||
};
|
||
}
|
||
|
||
function ciMetricsFromValue(value: unknown): CiPipelineMetrics | null {
|
||
const data = record(value);
|
||
if (data.ok !== true || typeof data.pipelineRun !== "string") return null;
|
||
const services = Array.isArray(data.services) ? data.services.map((item) => {
|
||
const service = record(item);
|
||
return {
|
||
taskRun: String(service.taskRun ?? ""),
|
||
serviceId: String(service.serviceId ?? ""),
|
||
status: String(service.status ?? "unknown"),
|
||
durationSeconds: typeof service.durationSeconds === "number" ? service.durationSeconds : null,
|
||
buildBackend: typeof service.buildBackend === "string" ? service.buildBackend : null,
|
||
reusedFrom: typeof service.reusedFrom === "string" ? service.reusedFrom : null,
|
||
imageTag: typeof service.imageTag === "string" ? service.imageTag : null,
|
||
sourceCommitId: typeof service.sourceCommitId === "string" ? service.sourceCommitId : null,
|
||
};
|
||
}).filter((service) => service.serviceId.length > 0).sort(serviceMetricSort) : [];
|
||
const reusedServices = Array.isArray(data.reusedServices) ? data.reusedServices.map(String) : services.filter((service) => service.status === "reused").map((service) => service.serviceId);
|
||
const rebuildServices = Array.isArray(data.rebuildServices) ? data.rebuildServices.map(String) : services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
|
||
return {
|
||
ok: true,
|
||
pipelineRun: data.pipelineRun,
|
||
serviceTotal: typeof data.serviceTotal === "number" ? data.serviceTotal : services.length,
|
||
reusedCount: typeof data.reusedCount === "number" ? data.reusedCount : reusedServices.length,
|
||
rebuildCount: typeof data.rebuildCount === "number" ? data.rebuildCount : rebuildServices.length,
|
||
reusedServices,
|
||
rebuildServices,
|
||
services,
|
||
};
|
||
}
|
||
|
||
function collectPipelineMetrics(pipelineRun: string): CiPipelineMetrics {
|
||
const result = getPipelineTaskRunResults(pipelineRun);
|
||
if (!isCommandSuccess(result)) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
serviceTotal: 0,
|
||
reusedCount: 0,
|
||
rebuildCount: 0,
|
||
reusedServices: [],
|
||
rebuildServices: [],
|
||
services: [],
|
||
degradedReason: "taskrun-query-failed",
|
||
rawText: `${result.stderr}\n${result.stdout}`.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
return parsePipelineTaskRunMetrics(pipelineRun, statusText(result));
|
||
}
|
||
|
||
function ciMetricLines(metrics: CiPipelineMetrics | null): string[] {
|
||
if (metrics === null) {
|
||
return [
|
||
"- CI/CD 关键指标:",
|
||
" - lazy build reused: n/a",
|
||
" - reused services: n/a",
|
||
" - rebuild services: n/a",
|
||
" - service durations: n/a",
|
||
];
|
||
}
|
||
if (metrics.ok !== true) {
|
||
return [
|
||
"- CI/CD 关键指标:",
|
||
` - lazy build reused: unavailable (${metrics.degradedReason ?? "unknown"})`,
|
||
" - reused services: n/a",
|
||
" - rebuild services: n/a",
|
||
" - service durations: n/a",
|
||
];
|
||
}
|
||
const lines = [
|
||
"- CI/CD 关键指标:",
|
||
` - lazy build reused: ${metrics.reusedCount}/${metrics.serviceTotal}`,
|
||
` - reused services: ${metrics.reusedServices.length > 0 ? metrics.reusedServices.join(", ") : "none"}`,
|
||
` - rebuild services: ${metrics.rebuildServices.length > 0 ? metrics.rebuildServices.join(", ") : "none"}`,
|
||
" - service durations:",
|
||
];
|
||
for (const service of metrics.services) {
|
||
const backend = service.buildBackend === null ? "" : `, backend=${service.buildBackend}`;
|
||
lines.push(` - ${service.serviceId}: ${service.status}, ${formatDuration(service.durationSeconds)}${backend}`);
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
function stripMarkdownInline(value: string): string {
|
||
return value
|
||
.replace(/\[([^\]]+)\]\([^)]+\)/gu, "$1")
|
||
.replace(/\*\*([^*]+)\*\*/gu, "$1")
|
||
.replace(/\*([^*]+)\*/gu, "$1")
|
||
.replace(/__([^_]+)__/gu, "$1")
|
||
.replace(/_([^_]+)_/gu, "$1")
|
||
.replace(/\s+/gu, " ")
|
||
.trim();
|
||
}
|
||
|
||
function markdownSection(body: string, headings: string[]): string {
|
||
const normalizedHeadings = new Set(headings.map((heading) => heading.toLowerCase()));
|
||
const lines = body.split(/\r?\n/u);
|
||
let capturing = false;
|
||
const captured: string[] = [];
|
||
for (const line of lines) {
|
||
const heading = /^(#{1,6})\s+(.+?)\s*$/u.exec(line);
|
||
if (heading !== null) {
|
||
const title = stripMarkdownInline(heading[2] ?? "").replace(/[：:]\s*$/u, "").toLowerCase();
|
||
if (capturing) break;
|
||
capturing = normalizedHeadings.has(title);
|
||
continue;
|
||
}
|
||
if (capturing) captured.push(line);
|
||
}
|
||
return captured.join("\n").trim();
|
||
}
|
||
|
||
function firstParagraph(section: string): string | null {
|
||
for (const paragraph of section.split(/\n\s*\n/u)) {
|
||
const trimmed = stripMarkdownInline(paragraph.replace(/^[-*]\s+/gmu, ""));
|
||
if (trimmed.length > 0) return trimmed;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function topLevelBullets(section: string, limit = 8): string[] {
|
||
const bullets: string[] = [];
|
||
for (const line of section.split(/\r?\n/u)) {
|
||
const match = /^[-*]\s+(.+)$/u.exec(line);
|
||
if (match === null) continue;
|
||
const text = stripMarkdownInline(match[1] ?? "");
|
||
if (text.length > 0) bullets.push(text);
|
||
if (bullets.length >= limit) break;
|
||
}
|
||
return bullets;
|
||
}
|
||
|
||
function conventionalTitleSummary(title: string): string {
|
||
const match = /^(feat|fix|docs|test|refactor|perf|ci|build|chore)(?:\([^)]+\))?:\s*(.+)$/iu.exec(title.trim());
|
||
if (match === null) return stripMarkdownInline(title);
|
||
const kind = String(match[1] ?? "").toLowerCase();
|
||
const subject = stripMarkdownInline(match[2] ?? title);
|
||
const labels: Record<string, string> = {
|
||
feat: "功能",
|
||
fix: "修复",
|
||
docs: "文档",
|
||
test: "测试",
|
||
refactor: "重构",
|
||
perf: "性能",
|
||
ci: "CI/CD",
|
||
build: "构建",
|
||
chore: "维护",
|
||
};
|
||
return `${labels[kind] ?? "变更"}：${subject}`;
|
||
}
|
||
|
||
function fileHeuristicBullets(fileSummary: Record<string, unknown>): string[] {
|
||
const keyFiles = Array.isArray(fileSummary.keyFiles) ? fileSummary.keyFiles.map(String) : [];
|
||
const bullets: string[] = [];
|
||
if (keyFiles.some((file) => file.includes("web/hwlab-cloud-web/"))) {
|
||
bullets.push("前端：更新 HWLAB Cloud Web / Workbench 交互行为。");
|
||
}
|
||
if (keyFiles.some((file) => /\.(test|spec)\./u.test(file) || file.includes("/test"))) {
|
||
bullets.push("验证：补充或更新回归测试，覆盖本次变更路径。");
|
||
}
|
||
if (keyFiles.some((file) => file.includes("deploy/") || file.includes("k8s") || file.includes("gitops"))) {
|
||
bullets.push("部署：更新 G14 Kubernetes/GitOps 发布声明或发布计划。");
|
||
}
|
||
if (keyFiles.some((file) => file.includes("internal/cloud/") || file.includes("code-agent"))) {
|
||
bullets.push("运行时：调整 Cloud API / Code Agent 运行链路。");
|
||
}
|
||
return bullets;
|
||
}
|
||
|
||
export function semanticChangelogBullets(title: string, body: string, fileSummary: Record<string, unknown>): string[] {
|
||
const bullets: string[] = [];
|
||
const background = firstParagraph(markdownSection(body, ["背景", "问题", "problem", "background", "context"]));
|
||
if (background !== null && /^(fix|perf)(?:\([^)]+\))?:/iu.test(title)) {
|
||
bullets.push(`修复目标：${background}`);
|
||
}
|
||
const changeSection = markdownSection(body, ["上线 changelog", "changelog", "摘要", "概述", "总结", "修改", "变更", "改动", "changes", "what changed", "summary"]);
|
||
bullets.push(...topLevelBullets(changeSection));
|
||
if (bullets.length === 0) bullets.push(conventionalTitleSummary(title));
|
||
for (const bullet of fileHeuristicBullets(fileSummary)) {
|
||
if (bullets.length >= 8) break;
|
||
if (!bullets.includes(bullet)) bullets.push(bullet);
|
||
}
|
||
return bullets.slice(0, 8);
|
||
}
|
||
|
||
function readIssue(issueNumber: number): CommandJsonResult {
|
||
return cliJson(["gh", "issue", "read", String(issueNumber), "--repo", HWLAB_REPO, "--json", "title,body,state,updatedAt", "--raw"], 80_000);
|
||
}
|
||
|
||
function issueFromRead(result: CommandJsonResult): Record<string, unknown> {
|
||
return record(commandData(result).issue);
|
||
}
|
||
|
||
function parseBriefIssueFromIndex(indexBody: string, date: string): { number: number; url: string } | null {
|
||
const escapedDate = date.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||
const linePattern = new RegExp(`^\\|\\s*${escapedDate}\\s*\\|([^\\n]+)$`, "mu");
|
||
const line = linePattern.exec(indexBody)?.[0];
|
||
if (line === undefined) return null;
|
||
const match = /\[#(\d+)\]\((https:\/\/github\.com\/pikasTech\/HWLAB\/issues\/\d+)\)/u.exec(line);
|
||
if (match === null) return null;
|
||
return { number: Number(match[1]), url: match[2] };
|
||
}
|
||
|
||
function dailyBriefTitle(date: string): string {
|
||
return `${date} 指挥简报（北京时间）`;
|
||
}
|
||
|
||
function writeStateFile(name: string, content: string): string {
|
||
const dir = rootPath(".state", "hwlab-g14");
|
||
mkdirSync(dir, { recursive: true });
|
||
const path = join(dir, name);
|
||
writeFileSync(path, content, "utf8");
|
||
return path;
|
||
}
|
||
|
||
function listIssues(): CommandJsonResult {
|
||
return cliJson(["gh", "issue", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "100", "--json", "number,title,state,url"], 80_000);
|
||
}
|
||
|
||
function findBriefIssueInList(date: string): { number: number; url: string } | null {
|
||
const listed = listIssues();
|
||
if (!isCommandSuccess(listed)) return null;
|
||
const issues = commandData(listed).issues;
|
||
if (!Array.isArray(issues)) return null;
|
||
const title = dailyBriefTitle(date);
|
||
const match = issues.map((item) => record(item)).find((issue) => issue.title === title);
|
||
if (match === undefined) return null;
|
||
return { number: Number(match.number), url: String(match.url) };
|
||
}
|
||
|
||
function newDailyBriefBody(date: string): string {
|
||
return [
|
||
`# ${dailyBriefTitle(date)}`,
|
||
"",
|
||
"## 当日更新",
|
||
"",
|
||
"## 常驻观察与长期建议",
|
||
"",
|
||
"- G14 HWLAB source truth 固定为 G14 `/root/hwlab` 与 `origin/G14`；DEV rollout 只接受 G14 Tekton/GitOps/Argo 和公网 health 证据。",
|
||
"- GitHub issue/PR 写操作必须走 UniDesk `gh` 子命令；G14 k3s 操作必须走 UniDesk `ssh G14:k3s`。",
|
||
].join("\n");
|
||
}
|
||
|
||
function insertBriefIndexRow(indexBody: string, date: string, brief: { number: number; url: string }): string {
|
||
if (parseBriefIssueFromIndex(indexBody, date) !== null) return indexBody;
|
||
const row = `| ${date} | [#${brief.number}](${brief.url}) | 已创建当日 G14 DEV rollout 自动简报入口；后续每次 DEV 上线记录 CI/CD 耗时和上线 changelog。 | 继续监控 HWLAB base=G14 PR，ready 后自动合并并滚动到 G14 DEV。 |`;
|
||
const marker = "### 指挥简报索引";
|
||
const markerIndex = indexBody.indexOf(marker);
|
||
if (markerIndex === -1) return `${indexBody.trimEnd()}\n\n${marker}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n`;
|
||
const afterMarker = indexBody.slice(markerIndex);
|
||
const separator = "| --- | --- | --- | --- |";
|
||
const separatorInBody = afterMarker.indexOf(separator);
|
||
if (separatorInBody === -1) return `${indexBody.slice(0, markerIndex + marker.length)}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n${indexBody.slice(markerIndex + marker.length)}`;
|
||
const insertAt = markerIndex + separatorInBody + separator.length;
|
||
return `${indexBody.slice(0, insertAt)}\n${row}${indexBody.slice(insertAt)}`;
|
||
}
|
||
|
||
function ensureDailyBriefIssue(date: string, dryRun: boolean): Record<string, unknown> {
|
||
const indexRead = readIssue(G14_BRIEF_INDEX_ISSUE);
|
||
if (!isCommandSuccess(indexRead)) return { ok: false, phase: "read-index", indexRead };
|
||
const indexIssue = issueFromRead(indexRead);
|
||
const indexBody = String(indexIssue.body ?? "");
|
||
let brief = parseBriefIssueFromIndex(indexBody, date) ?? findBriefIssueInList(date);
|
||
const actions: Record<string, unknown>[] = [];
|
||
if (brief === null) {
|
||
const bodyPath = writeStateFile(`daily-brief-${date}.md`, `${newDailyBriefBody(date)}\n`);
|
||
if (dryRun) {
|
||
brief = { number: 0, url: `https://github.com/pikasTech/HWLAB/issues/new?title=${encodeURIComponent(dailyBriefTitle(date))}` };
|
||
actions.push({ action: "would-create-daily-brief", title: dailyBriefTitle(date), bodyPath });
|
||
} else {
|
||
const created = cliJson(["gh", "issue", "create", "--repo", HWLAB_REPO, "--title", dailyBriefTitle(date), "--body-file", bodyPath], 80_000);
|
||
if (!isCommandSuccess(created)) return { ok: false, phase: "create-daily-brief", created };
|
||
const createdIssue = record(commandData(created).issue);
|
||
brief = { number: Number(createdIssue.number), url: String(createdIssue.url) };
|
||
actions.push({ action: "created-daily-brief", issue: brief, bodyPath });
|
||
}
|
||
}
|
||
if (brief.number > 0 && parseBriefIssueFromIndex(indexBody, date) === null) {
|
||
const nextBody = insertBriefIndexRow(indexBody, date, brief);
|
||
const bodyPath = writeStateFile(`issue-${G14_BRIEF_INDEX_ISSUE}-brief-index-${date}.md`, `${nextBody.trimEnd()}\n`);
|
||
if (dryRun) {
|
||
actions.push({ action: "would-update-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||
} else {
|
||
const bodySha = String(indexIssue.bodySha ?? "");
|
||
const update = cliJson(["gh", "issue", "update", String(G14_BRIEF_INDEX_ISSUE), "--repo", HWLAB_REPO, "--mode", "replace", "--body-file", bodyPath, ...(bodySha.length > 0 ? ["--expect-body-sha", bodySha] : [])], 100_000);
|
||
if (!isCommandSuccess(update)) return { ok: false, phase: "update-brief-index", update, bodyPath };
|
||
actions.push({ action: "updated-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||
}
|
||
}
|
||
return { ok: true, date, brief, actions };
|
||
}
|
||
|
||
function readPullRequest(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "read", String(number), "--repo", HWLAB_REPO, "--json", "title,body,url,stateDetail,merged,mergedAt,mergeCommit,headRefName,baseRefName"], 80_000);
|
||
}
|
||
|
||
function readPullRequestFiles(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "files", String(number), "--repo", HWLAB_REPO, "--limit", "30"], 80_000);
|
||
}
|
||
|
||
function summarizePrFiles(filesResult: CommandJsonResult): Record<string, unknown> {
|
||
if (!isCommandSuccess(filesResult)) return { ok: false, filesResult };
|
||
const data = commandData(filesResult);
|
||
const files = Array.isArray(data.files) ? data.files.map((item) => record(item)) : [];
|
||
return {
|
||
ok: true,
|
||
summary: data.summary ?? null,
|
||
keyFiles: files.slice(0, 8).map((file) => `${file.status ?? "changed"} ${file.filename ?? ""}`),
|
||
};
|
||
}
|
||
|
||
function mergeCommitFromPr(prData: Record<string, unknown>): string | null {
|
||
const mergeCommit = record(record(prData.json).mergeCommit);
|
||
const fromJson = mergeCommit.oid;
|
||
if (typeof fromJson === "string") return fromJson;
|
||
const fromSummary = record(record(prData.pullRequest).mergeCommit).oid;
|
||
return typeof fromSummary === "string" ? fromSummary : null;
|
||
}
|
||
|
||
function currentGitopsRevision(): string | null {
|
||
const argo = getArgoStatus();
|
||
if (!isCommandSuccess(argo)) return null;
|
||
return statusText(argo).split(/\r?\n/u)[0] ?? null;
|
||
}
|
||
|
||
export function rolloutRecordBody(input: {
|
||
pr: OpenPullRequest;
|
||
prData: Record<string, unknown>;
|
||
fileSummary: Record<string, unknown>;
|
||
sourceCommit: string;
|
||
pipelineRun: string;
|
||
gitopsRevision: string | null;
|
||
mergedAt: string | null;
|
||
pipelineSucceededAt: string | null;
|
||
finishedAt: string | null;
|
||
rollout: Record<string, unknown>;
|
||
ciMetrics?: CiPipelineMetrics | null;
|
||
}): string {
|
||
const now = beijingParts();
|
||
const title = String(record(input.prData.json).title ?? input.pr.title ?? `PR #${input.pr.number}`);
|
||
const prBody = String(record(input.prData.json).body ?? record(input.prData.pullRequest).body ?? "");
|
||
const url = String(record(input.prData.json).url ?? input.pr.url ?? `https://github.com/pikasTech/HWLAB/pull/${input.pr.number}`);
|
||
const summary = record(input.fileSummary.summary);
|
||
const keyFiles = Array.isArray(input.fileSummary.keyFiles) ? input.fileSummary.keyFiles.map(String) : [];
|
||
const semanticBullets = semanticChangelogBullets(title, prBody, input.fileSummary);
|
||
const mergeToPipeline = durationSeconds(input.mergedAt, input.pipelineSucceededAt);
|
||
const pipelineToDev = durationSeconds(input.pipelineSucceededAt, input.finishedAt);
|
||
const mergeToDev = durationSeconds(input.mergedAt, input.finishedAt);
|
||
const metrics = input.ciMetrics ?? ciMetricsFromValue(input.rollout.ciMetrics);
|
||
return [
|
||
"",
|
||
"",
|
||
`## 更新 ${now.date} ${now.time.slice(0, 5)} 北京时间`,
|
||
"",
|
||
`### G14 DEV rollout：PR #${input.pr.number}`,
|
||
"",
|
||
`- PR: [#${input.pr.number} ${title}](${url})`,
|
||
`- 合并 commit: \`${input.sourceCommit}\``,
|
||
`- GitOps revision: \`${input.gitopsRevision ?? "unknown"}\``,
|
||
`- PipelineRun: \`${input.pipelineRun}\``,
|
||
"- CI/CD 耗时:",
|
||
` - merge -> pipeline succeeded: ${formatDuration(mergeToPipeline)}`,
|
||
` - pipeline succeeded -> DEV Healthy: ${formatDuration(pipelineToDev)}`,
|
||
` - merge -> DEV Healthy: ${formatDuration(mergeToDev)}`,
|
||
...ciMetricLines(metrics),
|
||
"- 上线 changelog（语义化）:",
|
||
...semanticBullets.map((bullet) => ` - ${bullet}`),
|
||
"- 自动 diff 摘要:",
|
||
` - changed files: ${String(summary.files ?? "n/a")}; +${String(summary.additions ?? "n/a")} / -${String(summary.deletions ?? "n/a")}; commits: ${String(summary.commits ?? "n/a")}`,
|
||
...keyFiles.map((file) => ` - ${file}`),
|
||
"- DEV 验证:",
|
||
` - Tekton: ${String(input.rollout.pipelineText ?? "Succeeded").split(/\r?\n/u).join(" / ")}`,
|
||
` - Argo: ${String(input.rollout.argoText ?? "Synced / Healthy").split(/\r?\n/u).slice(0, 5).join(" / ")}`,
|
||
` - health/live: ${input.rollout.healthOk === false ? "not-ok" : "ok"}`,
|
||
" - workload readiness: Deployment/StatefulSet ready；历史 Completed smoke/debug pod 不作为 DEV rollout blocker。",
|
||
].join("\n");
|
||
}
|
||
|
||
function appendRolloutBrief(options: G14RecordRolloutOptions, rollout: Record<string, unknown> = {}): Record<string, unknown> {
|
||
const now = beijingParts();
|
||
const ensured = ensureDailyBriefIssue(now.date, options.dryRun);
|
||
if (record(ensured).ok !== true) return ensured;
|
||
const brief = record(record(ensured).brief);
|
||
const prRead = readPullRequest(options.prNumber);
|
||
if (!isCommandSuccess(prRead)) return { ok: false, phase: "read-pr", prRead };
|
||
const prData = commandData(prRead);
|
||
const sourceCommit = options.sourceCommit ?? mergeCommitFromPr(prData);
|
||
if (sourceCommit === null || sourceCommit === undefined) return { ok: false, phase: "source-commit", message: "source commit unavailable", prData };
|
||
const pipelineRun = options.pipelineRun ?? `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||
const files = summarizePrFiles(readPullRequestFiles(options.prNumber));
|
||
const rolloutGitopsRevision = typeof rollout.gitopsRevision === "string" && rollout.gitopsRevision.length > 0 ? rollout.gitopsRevision : null;
|
||
const gitopsRevision = options.gitopsRevision ?? rolloutGitopsRevision ?? currentGitopsRevision();
|
||
const prJson = record(prData.json);
|
||
const prSummary = record(prData.pullRequest);
|
||
const prMergedAt = typeof prJson.mergedAt === "string" && prJson.mergedAt.length > 0
|
||
? prJson.mergedAt
|
||
: typeof prSummary.mergedAt === "string" && prSummary.mergedAt.length > 0
|
||
? prSummary.mergedAt
|
||
: null;
|
||
const rolloutPipelineSucceededAt = typeof rollout.pipelineSucceededAt === "string" && rollout.pipelineSucceededAt.length > 0 ? rollout.pipelineSucceededAt : null;
|
||
const rolloutFinishedAt = typeof rollout.finishedAt === "string" && rollout.finishedAt.length > 0 ? rollout.finishedAt : null;
|
||
const mergedAt = options.mergedAt ?? prMergedAt;
|
||
const pipelineSucceededAt = options.pipelineSucceededAt ?? rolloutPipelineSucceededAt;
|
||
const finishedAt = options.finishedAt ?? rolloutFinishedAt;
|
||
const ciMetrics = ciMetricsFromValue(rollout.ciMetrics) ?? collectPipelineMetrics(pipelineRun);
|
||
const pr: OpenPullRequest = {
|
||
number: options.prNumber,
|
||
title: String(record(prData.json).title ?? ""),
|
||
url: String(record(prData.json).url ?? ""),
|
||
baseRefName: String(record(prData.json).baseRefName ?? ""),
|
||
headRefName: String(record(prData.json).headRefName ?? ""),
|
||
};
|
||
const body = rolloutRecordBody({ pr, prData, fileSummary: files, sourceCommit, pipelineRun, gitopsRevision, mergedAt, pipelineSucceededAt, finishedAt, rollout, ciMetrics });
|
||
if (Number(brief.number) <= 0 || options.dryRun) {
|
||
return { ok: true, dryRun: true, date: now.date, brief, wouldAppend: { bodyPreview: body.slice(0, 1000), bodyChars: body.length }, ensured };
|
||
}
|
||
const currentBrief = readIssue(Number(brief.number));
|
||
if (!isCommandSuccess(currentBrief)) return { ok: false, phase: "read-daily-brief", currentBrief, brief };
|
||
const currentBody = String(issueFromRead(currentBrief).body ?? "");
|
||
if (currentBody.includes(`PR #${options.prNumber}`) && currentBody.includes(sourceCommit)) {
|
||
return { ok: true, skipped: true, reason: "rollout-brief-already-recorded", brief, sourceCommit, ensured };
|
||
}
|
||
const bodyPath = writeStateFile(`rollout-pr-${options.prNumber}-${shortSha(sourceCommit)}.md`, `${body}\n`);
|
||
const update = cliJson(["gh", "issue", "update", String(brief.number), "--repo", HWLAB_REPO, "--mode", "append", "--body-file", bodyPath, "--body-profile", "commander-brief"], 100_000);
|
||
if (!isCommandSuccess(update)) return { ok: false, phase: "append-rollout-brief", update, bodyPath, brief, ensured };
|
||
return { ok: true, date: now.date, brief, sourceCommit, pipelineRun, gitopsRevision, ciMetrics, bodyPath, update: commandData(update), ensured };
|
||
}
|
||
|
||
async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
const startedAt = new Date(started).toISOString();
|
||
let pipelineText = "";
|
||
let argoText = "";
|
||
let healthOk = false;
|
||
let workloadsReady: Record<string, unknown> = { ready: false };
|
||
let pipelineSucceededAt: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
const pipeline = getPipelineStatus(sourceCommit);
|
||
pipelineText = statusText(pipeline);
|
||
printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) });
|
||
if (!pipelineText.startsWith("True\nSucceeded")) {
|
||
if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText };
|
||
await sleep(30_000);
|
||
continue;
|
||
}
|
||
pipelineSucceededAt ??= new Date().toISOString();
|
||
refreshArgoDev();
|
||
const argo = getArgoStatus();
|
||
argoText = statusText(argo);
|
||
printEvent("g14.argo.status", { text: argoText.slice(0, 500) });
|
||
const argoLines = argoText.split(/\r?\n/u);
|
||
const synced = argoLines[1] === "Synced";
|
||
const healthy = argoLines[2] === "Healthy";
|
||
const gitopsRevision = argoLines[0] ?? null;
|
||
const workloads = getDevWorkloads();
|
||
const workloadsText = statusText(workloads);
|
||
workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads));
|
||
const health = getLiveHealth();
|
||
healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout);
|
||
if (synced && healthy && record(workloadsReady).ready === true && healthOk) {
|
||
const finishedAt = new Date().toISOString();
|
||
const pipelineRun = `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||
const ciMetrics = collectPipelineMetrics(pipelineRun);
|
||
return { ok: true, sourceCommit, gitopsRevision, pipelineRun, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk, ciMetrics };
|
||
}
|
||
await sleep(30_000);
|
||
}
|
||
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
|
||
}
|
||
|
||
async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||
const precheck = precheckWorkspace();
|
||
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
|
||
const listed = listOpenG14PullRequests();
|
||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||
const prs = extractPullRequests(listed);
|
||
printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||
if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] };
|
||
const merged: unknown[] = [];
|
||
for (const pr of prs) {
|
||
const preflight = preflightPullRequest(pr.number);
|
||
printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) });
|
||
if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) {
|
||
return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight };
|
||
}
|
||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||
printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||
if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge };
|
||
const sourceCommit = getG14Head();
|
||
const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds);
|
||
const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true
|
||
? { skipped: true, dryRun: options.dryRun }
|
||
: appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout);
|
||
printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null });
|
||
merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief });
|
||
if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged };
|
||
if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged };
|
||
}
|
||
return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged };
|
||
}
|
||
|
||
async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<string, unknown>> {
|
||
const maxCycles = options.once ? 1 : options.maxCycles;
|
||
let cycle = 0;
|
||
const results: unknown[] = [];
|
||
while (maxCycles === 0 || cycle < maxCycles) {
|
||
cycle += 1;
|
||
const result = await monitorCycle(options, cycle);
|
||
results.push(result);
|
||
printEvent("g14.monitor.cycle.done", { cycle, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
||
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
|
||
if (options.once || record(result).action !== "none") return { ok: true, cycles: cycle, lastResult: result, results };
|
||
printEvent("g14.monitor.sleep", { cycle, intervalSeconds: options.intervalSeconds });
|
||
await sleep(options.intervalSeconds * 1000);
|
||
}
|
||
return { ok: true, cycles: cycle, results };
|
||
}
|
||
|
||
export function hwlabG14Help(): Record<string, unknown> {
|
||
return {
|
||
command: "hwlab g14",
|
||
output: "json",
|
||
usage: [
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs",
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs --once --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 record-rollout --pr <number> [--source-commit sha]",
|
||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror status",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||
],
|
||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.",
|
||
defaults: {
|
||
repo: HWLAB_REPO,
|
||
base: G14_SOURCE_BRANCH,
|
||
provider: G14_PROVIDER,
|
||
workspace: G14_WORKSPACE,
|
||
v02Workspace: V02_WORKSPACE,
|
||
ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO,
|
||
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
|
||
devApplication: DEV_APP,
|
||
v02Application: V02_APP,
|
||
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
|
||
},
|
||
stateFiles: {
|
||
monitor: ".state/hwlab-g14/latest-monitor-job.json",
|
||
once: ".state/hwlab-g14/latest-once-job.json",
|
||
dryRun: ".state/hwlab-g14/latest-dry-run-job.json",
|
||
onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json",
|
||
},
|
||
};
|
||
}
|
||
|
||
export async function runHwlabG14Command(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
||
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return { ok: true, ...hwlabG14Help() };
|
||
const [action] = args;
|
||
if (action === "record-rollout") {
|
||
const options = parseRecordRolloutOptions(args.slice(1));
|
||
return appendRolloutBrief(options);
|
||
}
|
||
if (action === "control-plane") {
|
||
const options = parseControlPlaneOptions(args.slice(1));
|
||
if (options.action === "trigger-current" && options.confirm && !options.dryRun && !options.wait) {
|
||
return startControlPlaneTriggerJob(options);
|
||
}
|
||
return runV02ControlPlane(options);
|
||
}
|
||
if (action === "tools-image") {
|
||
const options = parseToolsImageOptions(args.slice(1));
|
||
return runG14ToolsImage(options);
|
||
}
|
||
if (action === "git-mirror") {
|
||
const options = parseGitMirrorOptions(args.slice(1));
|
||
if ((options.action === "sync" || options.action === "flush") && options.confirm && !options.dryRun && !options.wait) {
|
||
return startGitMirrorJob(options);
|
||
}
|
||
return runG14GitMirror(options);
|
||
}
|
||
if (action !== "monitor-prs") {
|
||
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 record-rollout, hwlab g14 control-plane, hwlab g14 git-mirror, hwlab g14 tools-image" };
|
||
}
|
||
const options = parseOptions(args.slice(1));
|
||
if (options.worker) return runMonitorWorker(options);
|
||
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
|
||
const job = startJob("hwlab_g14_pr_monitor", command, `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`);
|
||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
|
||
const stateDir = rootPath(".state", "hwlab-g14");
|
||
mkdirSync(stateDir, { recursive: true });
|
||
const stateFileName = hwlabG14MonitorStateFileName(options);
|
||
const stateFileRole = hwlabG14MonitorStateRole(options);
|
||
const latestPath = join(stateDir, stateFileName);
|
||
const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null;
|
||
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole }, null, 2)}\n`, "utf8");
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 monitor-prs",
|
||
mode: "async-job",
|
||
job,
|
||
statusCommand,
|
||
latestPath,
|
||
stateFileName,
|
||
stateFileRole,
|
||
previousLatest,
|
||
next: {
|
||
status: statusCommand,
|
||
tail: `tail -f ${job.stdoutFile}`,
|
||
},
|
||
};
|
||
}
|