Codex
3763 lines
162 KiB
TypeScript
3763 lines
162 KiB
TypeScript
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { createHash } from "node:crypto";
|
||
import { repoRoot, rootPath, type Config } from "./config";
|
||
import { runCommand } from "./command";
|
||
import { startJob } from "./jobs";
|
||
|
||
const HWLAB_REPO = "pikasTech/HWLAB";
|
||
const G14_SOURCE_BRANCH = "G14";
|
||
const G14_PROVIDER = "G14";
|
||
const G14_WORKSPACE = "/root/hwlab";
|
||
const V02_SOURCE_BRANCH = "v0.2";
|
||
const V02_WORKSPACE = "/root/hwlab-v02";
|
||
const V02_CICD_REPO = "/root/hwlab-v02-cicd.git";
|
||
const DEV_NAMESPACE = "hwlab-dev";
|
||
const CI_NAMESPACE = "hwlab-ci";
|
||
const ARGO_NAMESPACE = "argocd";
|
||
const DEV_APP = "hwlab-g14-dev";
|
||
const V02_APP = "hwlab-g14-v02";
|
||
const V02_PIPELINE = "hwlab-v02-ci-image-publish";
|
||
const V02_POLLER = "hwlab-v02-branch-poller";
|
||
const V02_RECONCILER = "hwlab-v02-control-plane-reconciler";
|
||
const V02_PIPELINERUN_PREFIX = "hwlab-v02-ci-poll";
|
||
const V02_CONTROL_PLANE_FIELD_MANAGER = "unidesk-hwlab-v02-control-plane";
|
||
const V02_GIT_URL = "git@github.com:pikasTech/HWLAB.git";
|
||
const V02_GIT_READ_URL = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git";
|
||
const V02_GIT_WRITE_URL = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git";
|
||
const V02_GITOPS_BRANCH = "v0.2-gitops";
|
||
const V02_CATALOG_PATH = "deploy/artifact-catalog.v02.json";
|
||
const V02_RUNTIME_PATH = "deploy/gitops/g14/runtime-v02";
|
||
const V02_REGISTRY_PREFIX = "127.0.0.1:5000/hwlab";
|
||
const V02_BASE_IMAGE = "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim";
|
||
const GIT_MIRROR_NAMESPACE = "devops-infra";
|
||
const GIT_MIRROR_MANIFEST_FIELD_MANAGER = "unidesk-hwlab-git-mirror";
|
||
const GIT_MIRROR_SYNC_JOB_PREFIX = "git-mirror-hwlab-sync-manual";
|
||
const GIT_MIRROR_LEGACY_CRONJOB = "git-mirror-hwlab-sync";
|
||
const V02_SERVICE_IDS = [
|
||
"hwlab-cloud-api",
|
||
"hwlab-cloud-web",
|
||
"hwlab-agent-mgr",
|
||
"hwlab-agent-worker",
|
||
"hwlab-device-pod",
|
||
"hwlab-gateway",
|
||
"hwlab-edge-proxy",
|
||
"hwlab-agent-skills",
|
||
];
|
||
const V02_CLOUD_WEB_URL = "http://74.48.78.17:19666";
|
||
const V02_CLOUD_API_URL = "http://74.48.78.17:19667";
|
||
|
||
export function v02PipelineServiceIds(): string[] {
|
||
return [...V02_SERVICE_IDS];
|
||
}
|
||
|
||
const G14_CI_TOOLS_IMAGE_REPO = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools";
|
||
const G14_CI_TOOLS_BASE_TAG = "node22-alpine-v1";
|
||
const DEFAULT_INTERVAL_SECONDS = 600;
|
||
const DEFAULT_MAX_CYCLES = 0;
|
||
const DEFAULT_TIMEOUT_SECONDS = 1800;
|
||
const G14_BRIEF_INDEX_ISSUE = 7;
|
||
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||
const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
|
||
const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
|
||
|
||
interface G14MonitorOptions {
|
||
intervalSeconds: number;
|
||
maxCycles: number;
|
||
once: boolean;
|
||
dryRun: boolean;
|
||
worker: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface G14RecordRolloutOptions {
|
||
prNumber: number;
|
||
sourceCommit?: string;
|
||
pipelineRun?: string;
|
||
gitopsRevision?: string;
|
||
mergedAt?: string;
|
||
pipelineSucceededAt?: string;
|
||
finishedAt?: string;
|
||
dryRun: boolean;
|
||
}
|
||
|
||
interface G14ControlPlaneOptions {
|
||
action: "status" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration";
|
||
lane: "v02" | "g14" | "all";
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
wait: boolean;
|
||
allowLiveDbRead: boolean;
|
||
timeoutSeconds: number;
|
||
minAgeMinutes: number;
|
||
limit: number;
|
||
sourceCommit?: string;
|
||
pipelineRun?: string;
|
||
}
|
||
|
||
type V02StatusTargetMode = "latest-source-head" | "source-commit" | "pipeline-run";
|
||
|
||
interface V02ControlPlaneStatusTarget {
|
||
sourceCommit?: string | null;
|
||
pipelineRun?: string | null;
|
||
mode?: V02StatusTargetMode;
|
||
}
|
||
|
||
interface G14ToolsImageOptions {
|
||
action: "status" | "build";
|
||
name: "ci-node-tools";
|
||
tag: string;
|
||
dockerfile: string;
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface G14GitMirrorOptions {
|
||
action: "status" | "apply" | "sync" | "flush";
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
wait: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface CommandJsonResult {
|
||
ok: boolean;
|
||
command: string[];
|
||
exitCode: number | null;
|
||
stdout: string;
|
||
stderr: string;
|
||
parsed: unknown | null;
|
||
}
|
||
|
||
interface ShellSection {
|
||
stdout: string;
|
||
exitCode: number | null;
|
||
}
|
||
|
||
interface OpenPullRequest {
|
||
number: number;
|
||
title?: string;
|
||
url?: string;
|
||
baseRefName?: string;
|
||
headRefName?: string;
|
||
mergeable?: string | null;
|
||
mergeStateStatus?: string | null;
|
||
draft?: boolean;
|
||
}
|
||
|
||
interface CiServiceMetric {
|
||
taskRun: string;
|
||
serviceId: string;
|
||
status: string;
|
||
durationSeconds: number | null;
|
||
buildBackend: string | null;
|
||
reusedFrom: string | null;
|
||
imageTag: string | null;
|
||
sourceCommitId: string | null;
|
||
}
|
||
|
||
interface CiPipelineMetrics {
|
||
ok: boolean;
|
||
pipelineRun: string;
|
||
serviceTotal: number;
|
||
reusedCount: number;
|
||
rebuildCount: number;
|
||
reusedServices: string[];
|
||
rebuildServices: string[];
|
||
services: CiServiceMetric[];
|
||
degradedReason?: string;
|
||
rawText?: string;
|
||
}
|
||
|
||
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun">): string {
|
||
if (options.once && options.dryRun) return "latest-once-dry-run-job.json";
|
||
if (options.once) return "latest-once-job.json";
|
||
if (options.dryRun) return "latest-dry-run-job.json";
|
||
return "latest-monitor-job.json";
|
||
}
|
||
|
||
function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun">): string {
|
||
if (options.once && options.dryRun) return "once-dry-run";
|
||
if (options.once) return "once";
|
||
if (options.dryRun) return "dry-run-monitor";
|
||
return "monitor";
|
||
}
|
||
|
||
function parseOptions(args: string[]): G14MonitorOptions {
|
||
return {
|
||
intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400),
|
||
maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000),
|
||
once: args.includes("--once"),
|
||
dryRun: args.includes("--dry-run"),
|
||
worker: args.includes("--worker"),
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", DEFAULT_TIMEOUT_SECONDS, 86400),
|
||
};
|
||
}
|
||
|
||
function optionValue(args: string[], name: string): string | undefined {
|
||
const index = args.indexOf(name);
|
||
if (index === -1) return undefined;
|
||
const value = args[index + 1];
|
||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||
return value;
|
||
}
|
||
|
||
function validateFullShaOption(value: string, name: string): string {
|
||
if (!/^[0-9a-f]{40}$/iu.test(value)) throw new Error(`${name} must be a full 40-character git SHA`);
|
||
return value.toLowerCase();
|
||
}
|
||
|
||
function validateV02PipelineRunOption(value: string): string {
|
||
if (!/^hwlab-v02-ci-poll-[0-9a-f]{7,40}$/iu.test(value)) throw new Error("--pipeline-run must be a hwlab-v02-ci-poll-<sha> PipelineRun name");
|
||
return value.toLowerCase();
|
||
}
|
||
|
||
function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||
const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number");
|
||
const prNumber = Number(prRaw);
|
||
if (!Number.isInteger(prNumber) || prNumber <= 0) throw new Error("record-rollout requires --pr <number>");
|
||
return {
|
||
prNumber,
|
||
sourceCommit: optionValue(args, "--source-commit"),
|
||
pipelineRun: optionValue(args, "--pipeline-run"),
|
||
gitopsRevision: optionValue(args, "--gitops-revision"),
|
||
mergedAt: optionValue(args, "--merged-at"),
|
||
pipelineSucceededAt: optionValue(args, "--pipeline-succeeded-at"),
|
||
finishedAt: optionValue(args, "--finished-at"),
|
||
dryRun: args.includes("--dry-run"),
|
||
};
|
||
}
|
||
|
||
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||
const [rawAction] = args;
|
||
const actionRaw = rawAction === "rerun-current" ? "trigger-current" : rawAction;
|
||
if (
|
||
actionRaw !== "status" &&
|
||
actionRaw !== "apply" &&
|
||
actionRaw !== "trigger-current" &&
|
||
actionRaw !== "cleanup-runs" &&
|
||
actionRaw !== "cleanup-released-pvs" &&
|
||
actionRaw !== "runtime-migration"
|
||
) {
|
||
throw new Error("control-plane usage: status|apply|trigger-current|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
|
||
}
|
||
const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
|
||
if (actionRaw === "cleanup-runs") {
|
||
if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
|
||
} else if (actionRaw === "cleanup-released-pvs") {
|
||
if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
|
||
} else if (lane !== "v02") {
|
||
throw new Error("control-plane status/apply/trigger-current/runtime-migration currently requires --lane v02");
|
||
}
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run");
|
||
const allowLiveDbRead = args.includes("--allow-live-db-read");
|
||
if (allowLiveDbRead && confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
|
||
if (allowLiveDbRead && actionRaw !== "runtime-migration") throw new Error("--allow-live-db-read is only valid for control-plane runtime-migration");
|
||
const sourceCommitRaw = optionValue(args, "--source-commit");
|
||
const pipelineRunRaw = optionValue(args, "--pipeline-run");
|
||
if ((sourceCommitRaw !== undefined || pipelineRunRaw !== undefined) && actionRaw !== "status") {
|
||
throw new Error("--source-commit and --pipeline-run are only valid for control-plane status");
|
||
}
|
||
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) throw new Error("control-plane status accepts only one of --source-commit or --pipeline-run");
|
||
return {
|
||
action: actionRaw,
|
||
lane,
|
||
confirm,
|
||
wait: args.includes("--wait"),
|
||
allowLiveDbRead,
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
||
limit: positiveIntegerOption(args, "--limit", 20, 200),
|
||
sourceCommit: sourceCommitRaw === undefined ? undefined : validateFullShaOption(sourceCommitRaw, "--source-commit"),
|
||
pipelineRun: pipelineRunRaw === undefined ? undefined : validateV02PipelineRunOption(pipelineRunRaw),
|
||
};
|
||
}
|
||
|
||
function parseToolsImageOptions(args: string[]): G14ToolsImageOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "status" && actionRaw !== "build") {
|
||
throw new Error("tools-image usage: status|build --name ci-node-tools --tag <tag> [--dockerfile path] [--dry-run|--confirm]");
|
||
}
|
||
const name = optionValue(args, "--name") ?? "ci-node-tools";
|
||
if (name !== "ci-node-tools") throw new Error("tools-image currently supports --name ci-node-tools");
|
||
const tag = optionValue(args, "--tag") ?? G14_CI_TOOLS_BASE_TAG;
|
||
if (!/^[A-Za-z0-9_.-]+$/u.test(tag)) throw new Error("--tag may only contain letters, numbers, dot, underscore, and dash");
|
||
const dockerfile = optionValue(args, "--dockerfile") ?? "deploy/ci/hwlab-ci-node-tools.Dockerfile";
|
||
if (dockerfile.startsWith("/") || dockerfile.includes("..")) throw new Error("--dockerfile must be a repo-relative path without '..'");
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run");
|
||
return {
|
||
action: actionRaw,
|
||
name,
|
||
tag,
|
||
dockerfile,
|
||
confirm,
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
|
||
};
|
||
}
|
||
|
||
function parseGitMirrorOptions(args: string[]): G14GitMirrorOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "sync" && actionRaw !== "flush") {
|
||
throw new Error("git-mirror usage: status|apply|sync|flush [--dry-run|--confirm]");
|
||
}
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("git-mirror accepts only one of --confirm or --dry-run");
|
||
return {
|
||
action: actionRaw,
|
||
confirm,
|
||
wait: args.includes("--wait"),
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "sync" || actionRaw === "flush" ? 300 : 120, 900),
|
||
};
|
||
}
|
||
|
||
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
||
const index = args.indexOf(name);
|
||
if (index === -1) return defaultValue;
|
||
const raw = args[index + 1];
|
||
const value = Number(raw);
|
||
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||
return Math.min(value, maxValue);
|
||
}
|
||
|
||
function shellQuote(value: string): string {
|
||
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
||
}
|
||
|
||
function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
const result = runCommand(command, repoRoot, { timeoutMs });
|
||
let parsed: unknown | null = null;
|
||
if (result.stdout.trim().length > 0) {
|
||
try {
|
||
parsed = JSON.parse(result.stdout) as unknown;
|
||
} catch {
|
||
parsed = null;
|
||
}
|
||
}
|
||
return {
|
||
ok: result.exitCode === 0,
|
||
command,
|
||
exitCode: result.exitCode,
|
||
stdout: result.stdout,
|
||
stderr: result.stderr,
|
||
parsed,
|
||
};
|
||
}
|
||
|
||
function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs);
|
||
}
|
||
|
||
function record(value: unknown): Record<string, unknown> {
|
||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||
}
|
||
|
||
function stringOrNull(value: unknown): string | null {
|
||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||
}
|
||
|
||
function nested(value: unknown, keys: string[]): unknown {
|
||
let current = value;
|
||
for (const key of keys) current = record(current)[key];
|
||
return current;
|
||
}
|
||
|
||
function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown> {
|
||
const dumpPath = nested(result.parsed, ["data", "dump", "path"]);
|
||
if (typeof dumpPath === "string" && existsSync(dumpPath)) {
|
||
try {
|
||
return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown);
|
||
} catch {
|
||
return record(result.parsed);
|
||
}
|
||
}
|
||
return record(result.parsed);
|
||
}
|
||
|
||
function commandData(result: CommandJsonResult): Record<string, unknown> {
|
||
return record(sanitizeCliOutput(record(expandedParsedRoot(result).data)));
|
||
}
|
||
|
||
function textHash(value: string): string {
|
||
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
||
}
|
||
|
||
function redactLargePayloads(value: string): string {
|
||
return value
|
||
.replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => {
|
||
return `manifest_b64='<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>'`;
|
||
})
|
||
.replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => {
|
||
return `manifest_b64="<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>"`;
|
||
})
|
||
.replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => {
|
||
return `manifest_b64=<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||
})
|
||
.replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => {
|
||
return `<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||
});
|
||
}
|
||
|
||
function compactInlineText(value: string, maxChars = 4000): string {
|
||
const redacted = redactLargePayloads(value);
|
||
if (redacted.length <= maxChars) return redacted;
|
||
const headChars = Math.floor(maxChars * 0.35);
|
||
const tailChars = maxChars - headChars;
|
||
const omittedChars = redacted.length - headChars - tailChars;
|
||
return [
|
||
redacted.slice(0, headChars),
|
||
`<truncated chars=${omittedChars} sha256=${textHash(redacted)}>`,
|
||
redacted.slice(-tailChars),
|
||
].join("\n");
|
||
}
|
||
|
||
function sanitizeCliOutput(value: unknown): unknown {
|
||
if (typeof value === "string") return compactInlineText(value);
|
||
if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item));
|
||
if (typeof value !== "object" || value === null) return value;
|
||
const output: Record<string, unknown> = {};
|
||
for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item);
|
||
return output;
|
||
}
|
||
|
||
function redactedCommand(command: string[]): string[] {
|
||
return command.map((arg) => compactInlineText(arg, 1200));
|
||
}
|
||
|
||
function compactCommandResult(result: CommandJsonResult): Record<string, unknown> {
|
||
const data = record(expandedParsedRoot(result).data);
|
||
const stdout = String(data.stdout ?? result.stdout ?? "");
|
||
const stderr = String(data.stderr ?? result.stderr ?? "");
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
exitCode: result.exitCode,
|
||
command: redactedCommand(result.command),
|
||
stdout: compactInlineText(stdout.trim(), 4000),
|
||
stderr: compactInlineText(stderr.trim(), 4000),
|
||
stdoutBytes: Buffer.byteLength(stdout),
|
||
stderrBytes: Buffer.byteLength(stderr),
|
||
};
|
||
}
|
||
|
||
function compactControlPlaneRefresh(value: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
ok: value.ok === true,
|
||
phase: value.phase ?? null,
|
||
sourceCommit: value.sourceCommit ?? null,
|
||
renderDir: value.renderDir ?? null,
|
||
applyOk: nested(value, ["apply", "ok"]) ?? null,
|
||
cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null,
|
||
applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200),
|
||
degradedReason: value.degradedReason ?? null,
|
||
};
|
||
}
|
||
|
||
function compactTriggerCommandResult(value: Record<string, unknown> | null): Record<string, unknown> | null {
|
||
if (value === null) return null;
|
||
return {
|
||
ok: value.ok ?? null,
|
||
exitCode: value.exitCode ?? null,
|
||
stdout: tailText(value.stdout, 1200),
|
||
stderr: tailText(value.stderr, 1200),
|
||
stdoutBytes: value.stdoutBytes ?? null,
|
||
stderrBytes: value.stderrBytes ?? null,
|
||
};
|
||
}
|
||
|
||
function isCommandSuccess(result: CommandJsonResult): boolean {
|
||
if (!result.ok) return false;
|
||
const topOk = record(result.parsed).ok;
|
||
if (topOk === false) return false;
|
||
const dataOk = nested(result.parsed, ["data", "ok"]);
|
||
return dataOk !== false;
|
||
}
|
||
|
||
function extractPullRequests(result: CommandJsonResult): OpenPullRequest[] {
|
||
const prs = nested(result.parsed, ["data", "pullRequests"]);
|
||
if (!Array.isArray(prs)) return [];
|
||
return prs
|
||
.map((item) => record(item))
|
||
.filter((item) => item.baseRefName === G14_SOURCE_BRANCH || record(item.base).ref === G14_SOURCE_BRANCH)
|
||
.map((item) => ({
|
||
number: Number(item.number),
|
||
title: typeof item.title === "string" ? item.title : undefined,
|
||
url: typeof item.url === "string" ? item.url : undefined,
|
||
baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined,
|
||
headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined,
|
||
mergeable: typeof item.mergeable === "string" ? item.mergeable : null,
|
||
mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null,
|
||
draft: item.draft === true,
|
||
}))
|
||
.filter((item) => Number.isInteger(item.number) && item.number > 0);
|
||
}
|
||
|
||
function sleep(ms: number): Promise<void> {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
function printEvent(event: string, data: Record<string, unknown> = {}): void {
|
||
process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||
}
|
||
|
||
function printProgressEvent(event: string, data: Record<string, unknown> = {}): void {
|
||
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||
}
|
||
|
||
function shortSha(sha: string): string {
|
||
return sha.slice(0, 12);
|
||
}
|
||
|
||
function precheckWorkspace(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
|
||
}
|
||
|
||
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
|
||
}
|
||
|
||
function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs);
|
||
}
|
||
|
||
function v02CicdRepoEnsureScript(): string {
|
||
return [
|
||
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
||
`cicd_url=${shellQuote(V02_GIT_URL)}`,
|
||
"mkdir -p \"$(dirname \"$cicd_repo\")\"",
|
||
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
||
" :",
|
||
"elif [ -e \"$cicd_repo\" ]; then",
|
||
" echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
|
||
" exit 41",
|
||
"else",
|
||
" git clone --bare \"$cicd_url\" \"$cicd_repo\"",
|
||
"fi",
|
||
"git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"",
|
||
"git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
|
||
"git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune",
|
||
].join("\n");
|
||
}
|
||
|
||
function getV02Head(): string | null {
|
||
const result = g14HostScript([
|
||
"set -eu",
|
||
v02CicdRepoEnsureScript(),
|
||
"git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2",
|
||
].join("\n"), 180_000);
|
||
if (!isCommandSuccess(result)) return null;
|
||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||
return match?.[0] ?? null;
|
||
}
|
||
|
||
function v02PipelineRunName(sourceCommit: string): string {
|
||
return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`;
|
||
}
|
||
|
||
function getPipelineRunCompact(name: string): Record<string, unknown> {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
name,
|
||
"-o",
|
||
"jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}",
|
||
], 60_000);
|
||
return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr);
|
||
}
|
||
|
||
function pipelineRunCompactFromText(
|
||
name: string,
|
||
text: string,
|
||
commandOk: boolean,
|
||
command: string[] | string,
|
||
exitCode: number | null,
|
||
stderr: string,
|
||
): Record<string, unknown> {
|
||
const [status = "", reason = "", message = ""] = text.split(/\r?\n/u);
|
||
const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`);
|
||
return {
|
||
ok: commandOk,
|
||
exists: commandOk || !notFound,
|
||
pipelineRun: name,
|
||
status: status || null,
|
||
reason: reason || null,
|
||
message: message || null,
|
||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
function timestampMs(value: unknown): number | null {
|
||
if (typeof value !== "string" || value.trim().length === 0) return null;
|
||
const parsed = Date.parse(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function secondsBetween(start: unknown, end: unknown): number | null {
|
||
const startMs = timestampMs(start);
|
||
const endMs = timestampMs(end);
|
||
if (startMs === null || endMs === null || endMs < startMs) return null;
|
||
return Math.round((endMs - startMs) / 1000);
|
||
}
|
||
|
||
function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record<string, unknown> | null {
|
||
const [
|
||
name = "",
|
||
sourceCommit = "",
|
||
createdAt = "",
|
||
startTime = "",
|
||
completionTime = "",
|
||
status = "",
|
||
reason = "",
|
||
] = line.split("\t");
|
||
if (!name.startsWith(V02_PIPELINERUN_PREFIX)) return null;
|
||
const startMs = timestampMs(startTime);
|
||
const completion = completionTime.trim().length > 0 ? completionTime : null;
|
||
const conditionStatus = status.trim().length > 0 ? status : null;
|
||
return {
|
||
name,
|
||
sourceCommit: sourceCommit || null,
|
||
status: conditionStatus,
|
||
reason: reason || null,
|
||
createdAt: createdAt || null,
|
||
startTime: startTime || null,
|
||
completionTime: completion,
|
||
durationSeconds: secondsBetween(startTime, completion),
|
||
elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)),
|
||
active: completion === null && conditionStatus !== "True" && conditionStatus !== "False",
|
||
};
|
||
}
|
||
|
||
function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now()): Record<string, unknown> {
|
||
const rows = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs))
|
||
.filter((item): item is Record<string, unknown> => item !== null)
|
||
.sort((left, right) => {
|
||
return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0);
|
||
});
|
||
const activeItems = rows.filter((item) => item.active === true);
|
||
return {
|
||
ok: true,
|
||
count: rows.length,
|
||
activeCount: activeItems.length,
|
||
items: rows.slice(0, limit),
|
||
activeItems: activeItems.slice(0, limit),
|
||
disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete",
|
||
};
|
||
}
|
||
|
||
function pipelineRunRowsJsonPath(): string {
|
||
return [
|
||
"jsonpath={range .items[*]}",
|
||
"{.metadata.name}",
|
||
"{\"\\t\"}",
|
||
"{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}",
|
||
"{\"\\t\"}",
|
||
"{.metadata.creationTimestamp}",
|
||
"{\"\\t\"}",
|
||
"{.status.startTime}",
|
||
"{\"\\t\"}",
|
||
"{.status.completionTime}",
|
||
"{\"\\t\"}",
|
||
"{.status.conditions[0].status}",
|
||
"{\"\\t\"}",
|
||
"{.status.conditions[0].reason}",
|
||
"{\"\\n\"}",
|
||
"{end}",
|
||
].join("");
|
||
}
|
||
|
||
function listV02PipelineRunsCompact(limit = 8): Record<string, unknown> {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-l",
|
||
"hwlab.pikastech.local/gitops-target=v02",
|
||
"-o",
|
||
pipelineRunRowsJsonPath(),
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
return {
|
||
ok: false,
|
||
command: redactedCommand(result.command),
|
||
exitCode: result.exitCode,
|
||
stderr: commandErrorSummary(result),
|
||
items: [],
|
||
activeItems: [],
|
||
};
|
||
}
|
||
return parsePipelineRunRows(statusText(result), limit);
|
||
}
|
||
|
||
function parseShellSections(output: string): Record<string, ShellSection> {
|
||
const sections: Record<string, ShellSection> = {};
|
||
let currentName: string | null = null;
|
||
let currentLines: string[] = [];
|
||
let currentExitCode: number | null = null;
|
||
const flush = (): void => {
|
||
if (currentName === null) return;
|
||
sections[currentName] = {
|
||
stdout: currentLines.join("\n").trim(),
|
||
exitCode: currentExitCode,
|
||
};
|
||
};
|
||
for (const line of output.split(/\r?\n/u)) {
|
||
const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line);
|
||
if (begin !== null) {
|
||
flush();
|
||
currentName = begin[1] ?? null;
|
||
currentLines = [];
|
||
currentExitCode = null;
|
||
continue;
|
||
}
|
||
const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line);
|
||
if (end !== null && currentName === end[1]) {
|
||
currentExitCode = Number(end[2]);
|
||
flush();
|
||
currentName = null;
|
||
currentLines = [];
|
||
currentExitCode = null;
|
||
continue;
|
||
}
|
||
if (currentName !== null) currentLines.push(line);
|
||
}
|
||
flush();
|
||
return sections;
|
||
}
|
||
|
||
function shellSectionOk(section: ShellSection | undefined): boolean {
|
||
return section?.exitCode === 0;
|
||
}
|
||
|
||
function v02ControlPlaneStatusBundle(target: V02ControlPlaneStatusTarget = {}): CommandJsonResult {
|
||
const targetMode: V02StatusTargetMode = target.mode
|
||
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
||
const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null
|
||
? [
|
||
`pipeline_run=${shellQuote(target.pipelineRun)}`,
|
||
`source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`,
|
||
`if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | cut -d- -f5-); fi`,
|
||
].join("\n")
|
||
: [
|
||
target.sourceCommit === undefined
|
||
? `source_commit=$(git --git-dir=${shellQuote(V02_CICD_REPO)} rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)`
|
||
: `source_commit=${shellQuote(target.sourceCommit ?? "")}`,
|
||
"pipeline_run=",
|
||
].join("\n");
|
||
const script = [
|
||
"set +e",
|
||
targetInit,
|
||
`target_mode=${shellQuote(targetMode)}`,
|
||
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${V02_PIPELINERUN_PREFIX}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`,
|
||
"section() {",
|
||
" name=\"$1\"",
|
||
" shift",
|
||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||
" \"$@\"",
|
||
" code=$?",
|
||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||
"}",
|
||
"section statusTarget printf 'mode\\t%s\\nsourceCommit\\t%s\\npipelineRun\\t%s\\n' \"$target_mode\" \"$source_commit\" \"$pipeline_run\"",
|
||
"section sourceCommit printf '%s\\n' \"$source_commit\"",
|
||
"section pipelineRunName printf '%s\\n' \"$pipeline_run\"",
|
||
`section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())}`,
|
||
"section queryNow date -u +%Y-%m-%dT%H:%M:%SZ",
|
||
`section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`,
|
||
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
|
||
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
||
`if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
|
||
`if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
|
||
`if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`,
|
||
`section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`,
|
||
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
|
||
`section gitMirrorCache kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(gitMirrorCacheProbeScript())}`,
|
||
`section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
|
||
].join("\n");
|
||
return g14K3s(["script", "--", script], 60_000);
|
||
}
|
||
|
||
function v02SourceHeadsProbeScript(): string {
|
||
return [
|
||
"set +e",
|
||
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
||
`workspace=${shellQuote(V02_WORKSPACE)}`,
|
||
"rev_cicd() { git --git-dir=\"$cicd_repo\" rev-parse \"$1\" 2>/dev/null || true; }",
|
||
"rev_workspace() { git -C \"$workspace\" rev-parse \"$1\" 2>/dev/null || true; }",
|
||
"printf 'cicdRepo\\t%s\\n' \"$cicd_repo\"",
|
||
"printf 'cicdRepoExists\\t%s\\n' \"$([ -d \"$cicd_repo/objects\" ] && printf yes || printf no)\"",
|
||
"printf 'cicdSourceHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"",
|
||
"printf 'originHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"",
|
||
"printf 'workspacePath\\t%s\\n' \"$workspace\"",
|
||
"printf 'workspaceBranch\\t%s\\n' \"$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)\"",
|
||
"printf 'workspaceHead\\t%s\\n' \"$(rev_workspace HEAD)\"",
|
||
"printf 'workspaceOriginHead\\t%s\\n' \"$(rev_workspace refs/remotes/origin/v0.2)\"",
|
||
"printf 'workspaceDirtyCount\\t%s\\n' \"$(git -C \"$workspace\" status --porcelain=v1 2>/dev/null | wc -l | tr -d ' ')\"",
|
||
].join("\n");
|
||
}
|
||
|
||
function v02PlanArtifactsLogScript(): string {
|
||
return [
|
||
"pipeline_run=\"$1\"",
|
||
`namespace=${shellQuote(CI_NAMESPACE)}`,
|
||
"[ -n \"$pipeline_run\" ] || exit 0",
|
||
"pods=$(kubectl -n \"$namespace\" get pods -l \"tekton.dev/pipelineRun=$pipeline_run,tekton.dev/pipelineTask=plan-artifacts\" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)",
|
||
"for pod in $pods; do",
|
||
" printf '__POD__\\t%s\\n' \"$pod\"",
|
||
" kubectl -n \"$namespace\" logs \"$pod\" --all-containers --tail=200 2>/dev/null || true",
|
||
"done",
|
||
].join("\n");
|
||
}
|
||
|
||
function v02RuntimeWorkloadsColumns(): string {
|
||
return [
|
||
"custom-columns=KIND:.kind",
|
||
"NAME:.metadata.name",
|
||
"SERVICE:.metadata.labels.hwlab\\.pikastech\\.local/service-id",
|
||
"ARTIFACT:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/artifact-source-commit",
|
||
"SOURCE:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/source-commit",
|
||
"IMAGE:.spec.template.spec.containers[0].image",
|
||
"READY:.status.readyReplicas",
|
||
"REPLICAS:.status.replicas",
|
||
].join(",");
|
||
}
|
||
|
||
function v02WebAssetsProbeScript(): string {
|
||
return [
|
||
"set +e",
|
||
`base=${shellQuote(V02_CLOUD_WEB_URL)}`,
|
||
`api=${shellQuote(V02_CLOUD_API_URL)}`,
|
||
"fetch_url() {",
|
||
" if command -v curl >/dev/null 2>&1; then",
|
||
" curl -fsS --connect-timeout 2 --max-time 5 \"$1\"",
|
||
" elif command -v wget >/dev/null 2>&1; then",
|
||
" wget -q -T 5 -O - \"$1\"",
|
||
" else",
|
||
" return 127",
|
||
" fi",
|
||
"}",
|
||
"printf 'baseUrl\\t%s\\n' \"$base\"",
|
||
"printf 'apiUrl\\t%s\\n' \"$api\"",
|
||
"html=$(fetch_url \"$base/\" 2>/dev/null)",
|
||
"html_code=$?",
|
||
"printf 'htmlOk\\t%s\\n' \"$html_code\"",
|
||
"printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"",
|
||
"css=$(fetch_url \"$base/styles.css\" 2>/dev/null)",
|
||
"css_code=$?",
|
||
"printf 'cssOk\\t%s\\n' \"$css_code\"",
|
||
"app_js=$(fetch_url \"$base/app.js\" 2>/dev/null)",
|
||
"app_js_code=$?",
|
||
"printf 'appJsOk\\t%s\\n' \"$app_js_code\"",
|
||
"printf 'appJsBytes\\t%s\\n' \"$(printf '%s' \"$app_js\" | wc -c | tr -d ' ')\"",
|
||
"printf 'sidebarFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||
"printf 'workspaceFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||
"printf 'eventPanelFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(132px,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||
"health=$(fetch_url \"$api/health/live\" 2>/dev/null)",
|
||
"health_code=$?",
|
||
"printf 'apiHealthOk\\t%s\\n' \"$health_code\"",
|
||
"printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"",
|
||
].join("\n");
|
||
}
|
||
|
||
function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
|
||
items: [],
|
||
performance: v02TaskRunPerformanceSummary([]),
|
||
};
|
||
}
|
||
const items = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
status: status || null,
|
||
reason: reason || null,
|
||
startTime: startTime || null,
|
||
completionTime: completionTime || null,
|
||
durationSeconds: secondsBetween(startTime, completionTime),
|
||
};
|
||
});
|
||
const counts = {
|
||
succeeded: items.filter((item) => item.status === "True").length,
|
||
failed: items.filter((item) => item.status === "False").length,
|
||
running: items.filter((item) => item.status === "Unknown").length,
|
||
unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
|
||
};
|
||
const performance = v02TaskRunPerformanceSummary(items);
|
||
const performanceWarning = performance.ok === false ? `; ${String(performance.summary ?? "")}` : "";
|
||
return {
|
||
ok: true,
|
||
pipelineRun,
|
||
counts,
|
||
items,
|
||
performance,
|
||
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}${performanceWarning}`,
|
||
disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
|
||
};
|
||
}
|
||
|
||
function v02TaskRunPipelineTaskName(name: string): string | null {
|
||
const buildIndex = name.lastIndexOf("-build-");
|
||
if (buildIndex >= 0) return name.slice(buildIndex + 1);
|
||
const knownSuffixes = [
|
||
"prepare-source",
|
||
"plan-artifacts",
|
||
"publish-artifact-catalog",
|
||
"gitops-render",
|
||
"gitops-promote",
|
||
"runtime-ready",
|
||
"collect-artifacts",
|
||
];
|
||
return knownSuffixes.find((suffix) => name.endsWith(`-${suffix}`)) ?? null;
|
||
}
|
||
|
||
export function v02TaskRunPerformanceSummary(taskRuns: unknown[]): Record<string, unknown> {
|
||
const slowTaskRuns: Record<string, unknown>[] = [];
|
||
for (const itemRaw of taskRuns) {
|
||
const item = record(itemRaw);
|
||
const name = stringOrNull(item.name) ?? "";
|
||
const pipelineTask = stringOrNull(item.pipelineTask) ?? v02TaskRunPipelineTaskName(name);
|
||
const durationSeconds = typeof item.durationSeconds === "number" ? item.durationSeconds : null;
|
||
if (!pipelineTask?.startsWith("build-") || durationSeconds === null || durationSeconds <= V02_BUILD_TASKRUN_WARNING_SECONDS) continue;
|
||
const serviceId = pipelineTask.slice("build-".length) || null;
|
||
slowTaskRuns.push({
|
||
name,
|
||
pipelineTask,
|
||
serviceId,
|
||
status: stringOrNull(item.status),
|
||
reason: stringOrNull(item.reason),
|
||
durationSeconds,
|
||
budgetSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
||
severity: durationSeconds >= V02_BUILD_TASKRUN_CRITICAL_SECONDS ? "critical" : "warning",
|
||
message: `${pipelineTask} took ${durationSeconds}s, above v0.2 build TaskRun warning budget ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
||
});
|
||
}
|
||
slowTaskRuns.sort((left, right) => Number(right.durationSeconds ?? 0) - Number(left.durationSeconds ?? 0));
|
||
const worst = slowTaskRuns[0];
|
||
return {
|
||
ok: slowTaskRuns.length === 0,
|
||
warningCount: slowTaskRuns.length,
|
||
thresholds: {
|
||
buildTaskRunWarningSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
||
buildTaskRunCriticalSeconds: V02_BUILD_TASKRUN_CRITICAL_SECONDS,
|
||
},
|
||
slowTaskRuns,
|
||
summary: worst
|
||
? `slow build taskruns=${slowTaskRuns.length}; worst=${String(worst.pipelineTask)} ${String(worst.durationSeconds)}s budget=${V02_BUILD_TASKRUN_WARNING_SECONDS}s`
|
||
: `no build taskrun over ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
||
};
|
||
}
|
||
|
||
function v02WebAssetsFromText(
|
||
text: string,
|
||
commandOk: boolean,
|
||
sourceCommit: string | null,
|
||
argoSyncRevision: string | null,
|
||
exitCode: number | null,
|
||
stderr: string,
|
||
activePipelineRuns: unknown[] = [],
|
||
): Record<string, unknown> {
|
||
const fields: Record<string, string> = {};
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const [key = "", ...rest] = line.split("\t");
|
||
if (key.length > 0) fields[key] = rest.join("\t");
|
||
}
|
||
const htmlOk = fields.htmlOk === "0";
|
||
const cssOk = fields.cssOk === "0";
|
||
const appJsOk = fields.appJsOk === "0";
|
||
const apiHealthOk = fields.apiHealthOk === "0";
|
||
const readonlyNoteAbsent = fields.readonlyNote === "1";
|
||
const sidebarFitCss = fields.sidebarFitCss === "0";
|
||
const workspaceFitCss = fields.workspaceFitCss === "0";
|
||
const eventPanelFitCss = fields.eventPanelFitCss === "0";
|
||
const apiRevision = fields.apiRevision || null;
|
||
const webChecksPass = htmlOk && cssOk && appJsOk && readonlyNoteAbsent && sidebarFitCss && workspaceFitCss && eventPanelFitCss && apiHealthOk;
|
||
const failedChecks = Object.entries({
|
||
htmlOk,
|
||
cssOk,
|
||
appJsOk,
|
||
readonlyNoteAbsent,
|
||
sidebarFitCss,
|
||
workspaceFitCss,
|
||
eventPanelFitCss,
|
||
apiHealthOk,
|
||
}).filter(([, ok]) => !ok).map(([name]) => name);
|
||
return {
|
||
ok: commandOk && webChecksPass,
|
||
summary: commandOk && webChecksPass ? "19666/19667 probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`,
|
||
baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL,
|
||
apiUrl: fields.apiUrl || V02_CLOUD_API_URL,
|
||
sourceCommit,
|
||
argoSyncRevision: argoSyncRevision || null,
|
||
checks: {
|
||
htmlOk,
|
||
cssOk,
|
||
appJsOk,
|
||
readonlyNoteAbsent,
|
||
sidebarFitCss,
|
||
workspaceFitCss,
|
||
eventPanelFitCss,
|
||
apiHealthOk,
|
||
},
|
||
probeExitCodes: {
|
||
html: numericField(fields.htmlOk),
|
||
css: numericField(fields.cssOk),
|
||
appJs: numericField(fields.appJsOk),
|
||
readonlyNoteGrep: numericField(fields.readonlyNote),
|
||
sidebarFitCssGrep: numericField(fields.sidebarFitCss),
|
||
workspaceFitCssGrep: numericField(fields.workspaceFitCss),
|
||
eventPanelFitCssGrep: numericField(fields.eventPanelFitCss),
|
||
apiHealth: numericField(fields.apiHealthOk),
|
||
},
|
||
assetBytes: {
|
||
appJs: numericField(fields.appJsBytes),
|
||
},
|
||
apiRevision,
|
||
note: webAssetsRevisionNote(apiRevision, sourceCommit, activePipelineRuns),
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
function v02PlanArtifactsFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
const events: Record<string, unknown>[] = [];
|
||
const pods: string[] = [];
|
||
for (const rawLine of text.split(/\r?\n/u)) {
|
||
const line = rawLine.trim();
|
||
if (line.startsWith("__POD__")) {
|
||
const [, pod = ""] = line.split("\t");
|
||
if (pod.length > 0) pods.push(pod);
|
||
continue;
|
||
}
|
||
if (!line.startsWith("{")) continue;
|
||
try {
|
||
const event = record(JSON.parse(line) as unknown);
|
||
if (event.event === "g14-ci-plan") events.push(event);
|
||
} catch {
|
||
// Ignore non-JSON log lines; the raw log remains visible through pod logs.
|
||
}
|
||
}
|
||
const latest = events.at(-1) ?? null;
|
||
if (!commandOk || latest === null) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
pods,
|
||
eventFound: latest !== null,
|
||
degradedReason: commandOk ? "g14-ci-plan-event-not-found" : "plan-artifacts-log-query-failed",
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
const audit = record(latest.artifactProvenanceAudit);
|
||
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
|
||
const provenanceRebuildServices = Array.isArray(audit.provenanceRebuildServices) ? audit.provenanceRebuildServices : [];
|
||
return {
|
||
ok: true,
|
||
pipelineRun,
|
||
pods,
|
||
sourceCommitId: stringOrNull(latest.sourceCommitId),
|
||
affectedServices: stringArray(latest.affectedServices),
|
||
rolloutServices: stringArray(latest.rolloutServices),
|
||
buildServices: stringArray(latest.buildServices),
|
||
reusedServices: stringArray(latest.reusedServices),
|
||
buildSkippedCount: numericValue(latest.buildSkippedCount),
|
||
artifactProvenanceAudit: Object.keys(audit).length > 0 ? audit : null,
|
||
summary: `build=${stringArray(latest.buildServices).length} reuse=${stringArray(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`,
|
||
disclosure: "parsed from plan-artifacts g14-ci-plan log event",
|
||
};
|
||
}
|
||
|
||
function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
items: [],
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
const items = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [kind = "", name = "", serviceId = "", artifactSourceCommit = "", sourceCommit = "", image = "", ready = "", replicas = ""] = line.split(/\s+/u);
|
||
return {
|
||
kind,
|
||
name,
|
||
serviceId: serviceId === "<none>" ? null : serviceId,
|
||
artifactSourceCommit: artifactSourceCommit === "<none>" ? null : artifactSourceCommit,
|
||
sourceCommit: sourceCommit === "<none>" ? null : sourceCommit,
|
||
image: image === "<none>" ? null : image,
|
||
readyReplicas: numericField(ready),
|
||
replicas: numericField(replicas),
|
||
};
|
||
});
|
||
return {
|
||
ok: true,
|
||
items,
|
||
summary: `runtime deployments=${items.length}`,
|
||
};
|
||
}
|
||
|
||
function keyValueLinesFromText(text: string): Record<string, string> {
|
||
const fields: Record<string, string> = {};
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const [key = "", ...rest] = line.split("\t");
|
||
if (key.trim().length > 0) fields[key.trim()] = rest.join("\t").trim();
|
||
}
|
||
return fields;
|
||
}
|
||
|
||
function v02SourceHeadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
const fields = keyValueLinesFromText(text);
|
||
const dirtyCount = numericField(fields.workspaceDirtyCount);
|
||
return {
|
||
ok: commandOk,
|
||
cicdRepo: fields.cicdRepo || V02_CICD_REPO,
|
||
cicdRepoExists: fields.cicdRepoExists === "yes",
|
||
cicdSourceHead: fields.cicdSourceHead || null,
|
||
originHead: fields.originHead || null,
|
||
workspace: {
|
||
path: fields.workspacePath || V02_WORKSPACE,
|
||
branch: fields.workspaceBranch || null,
|
||
head: fields.workspaceHead || null,
|
||
originHead: fields.workspaceOriginHead || null,
|
||
dirtyCount,
|
||
dirty: typeof dirtyCount === "number" ? dirtyCount > 0 : null,
|
||
isolatedFromCicd: true,
|
||
},
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
export function v02FalseGreenGuard(input: {
|
||
sourceCommit: string | null;
|
||
pipelineRun: Record<string, unknown> | null;
|
||
taskRuns: Record<string, unknown>;
|
||
planArtifacts: Record<string, unknown>;
|
||
runtimeWorkloads: Record<string, unknown>;
|
||
}): Record<string, unknown> {
|
||
const status = stringOrNull(input.pipelineRun?.status);
|
||
if (input.sourceCommit === null || input.pipelineRun === null || input.pipelineRun.exists === false) {
|
||
return { ok: null, state: "not-started", summary: "current source commit has no PipelineRun yet" };
|
||
}
|
||
if (status !== "True") {
|
||
return { ok: null, state: "pipeline-not-final", summary: `PipelineRun status=${status ?? "unknown"}` };
|
||
}
|
||
if (input.planArtifacts.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
state: "missing-plan-artifacts-evidence",
|
||
summary: "PipelineRun succeeded but plan-artifacts g14-ci-plan evidence was not visible",
|
||
planArtifacts: input.planArtifacts,
|
||
};
|
||
}
|
||
const sourceCommit = input.sourceCommit;
|
||
const buildServices = stringArray(input.planArtifacts.buildServices);
|
||
const reusedServices = stringArray(input.planArtifacts.reusedServices);
|
||
const taskItems = Array.isArray(input.taskRuns.items) ? input.taskRuns.items.map((item) => record(item)) : [];
|
||
const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : [];
|
||
const workloadByService = new Map(workloadItems.map((item) => [String(item.serviceId ?? ""), item]));
|
||
const missingBuildTaskRuns = buildServices.filter((serviceId) => !taskItems.some((item) => String(item.name ?? "").endsWith(`build-${serviceId}`) && item.status === "True"));
|
||
const runtimeMismatches = buildServices
|
||
.map((serviceId) => {
|
||
const workload = workloadByService.get(serviceId);
|
||
if (!workload) return { serviceId, reason: "runtime-workload-missing" };
|
||
if (workload.artifactSourceCommit !== sourceCommit) {
|
||
return {
|
||
serviceId,
|
||
reason: "artifact-source-commit-mismatch",
|
||
runtimeArtifactSourceCommit: workload.artifactSourceCommit ?? null,
|
||
expectedSourceCommit: sourceCommit,
|
||
image: workload.image ?? null,
|
||
};
|
||
}
|
||
return null;
|
||
})
|
||
.filter((item): item is Record<string, unknown> => item !== null);
|
||
const audit = record(input.planArtifacts.artifactProvenanceAudit);
|
||
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
|
||
const auditPresent = input.planArtifacts.artifactProvenanceAudit !== null;
|
||
const provenanceWarning = !auditPresent && reusedServices.length > 0
|
||
? { reason: "artifact-provenance-audit-missing-for-reuse", reusedServices }
|
||
: null;
|
||
const provenanceOk = auditPresent ? audit.ok !== false && unsafeReuseServices.length === 0 : true;
|
||
const failures = [
|
||
...(input.runtimeWorkloads.ok === true ? [] : [{ reason: "runtime-workload-query-failed" }]),
|
||
...(provenanceOk ? [] : [{ reason: "artifact-provenance-audit-failed", unsafeReuseServices }]),
|
||
...missingBuildTaskRuns.map((serviceId) => ({ serviceId, reason: "expected-build-taskrun-not-succeeded" })),
|
||
...runtimeMismatches,
|
||
];
|
||
return {
|
||
ok: failures.length === 0,
|
||
state: failures.length === 0 ? "passed" : "failed",
|
||
summary: failures.length === 0
|
||
? `artifact provenance and built runtime services align with ${shortSha(sourceCommit)}`
|
||
: `false-green risk: ${failures.length} invariant violation(s)`,
|
||
sourceCommit,
|
||
buildServices,
|
||
reusedServices,
|
||
provenanceOk,
|
||
provenanceAuditPresent: auditPresent,
|
||
provenanceWarnings: provenanceWarning ? [provenanceWarning] : [],
|
||
missingBuildTaskRuns,
|
||
runtimeMismatches,
|
||
failures,
|
||
};
|
||
}
|
||
|
||
export function v02CommitAlignment(input: {
|
||
expectedSourceHead: string | null;
|
||
sourceHeads: Record<string, unknown>;
|
||
gitMirrorSummary: Record<string, unknown>;
|
||
pipelineRun: Record<string, unknown> | null;
|
||
recentPipelineRuns: Record<string, unknown>;
|
||
planArtifacts?: Record<string, unknown>;
|
||
runtimeWorkloads: Record<string, unknown>;
|
||
webAssets: Record<string, unknown>;
|
||
}): Record<string, unknown> {
|
||
const expectedSourceHead = input.expectedSourceHead;
|
||
const cicdSourceHead = stringOrNull(input.sourceHeads.cicdSourceHead);
|
||
const originHead = stringOrNull(input.sourceHeads.originHead) ?? expectedSourceHead;
|
||
const workspace = record(input.sourceHeads.workspace);
|
||
const workspaceHead = stringOrNull(workspace.head);
|
||
const workspaceOriginHead = stringOrNull(workspace.originHead);
|
||
const mirrorSourceHead = stringOrNull(input.gitMirrorSummary.localV02);
|
||
const mirrorGithubSourceHead = stringOrNull(input.gitMirrorSummary.githubV02);
|
||
const recentItems = Array.isArray(input.recentPipelineRuns.items)
|
||
? input.recentPipelineRuns.items.map((item) => record(item))
|
||
: [];
|
||
const latestPipelineRun = recentItems[0] ?? null;
|
||
const latestPipelineSourceCommit = latestPipelineRun === null ? null : stringOrNull(latestPipelineRun.sourceCommit);
|
||
const currentPipelineStatus = stringOrNull(input.pipelineRun?.status);
|
||
const apiRevision = stringOrNull(input.webAssets.apiRevision);
|
||
const planArtifacts = record(input.planArtifacts);
|
||
const planSourceCommit = stringOrNull(planArtifacts.sourceCommitId);
|
||
const rolloutServices = stringArray(planArtifacts.rolloutServices);
|
||
const apiRevisionRequired = expectedSourceHead !== null && planSourceCommit === expectedSourceHead && rolloutServices.includes("hwlab-cloud-api");
|
||
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
|
||
? input.runtimeWorkloads.items.map((item) => record(item))
|
||
: [];
|
||
const serviceSourceCommits = Object.fromEntries(workloadItems
|
||
.filter((item) => typeof item.serviceId === "string" && item.serviceId.length > 0)
|
||
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
|
||
const staleReasons: string[] = [];
|
||
if (expectedSourceHead === null) staleReasons.push("expected-source-head-unresolved");
|
||
if (expectedSourceHead !== null && originHead !== null && originHead !== expectedSourceHead) staleReasons.push("origin-head-mismatch");
|
||
if (expectedSourceHead !== null && cicdSourceHead !== null && cicdSourceHead !== expectedSourceHead) staleReasons.push("cicd-source-repo-stale");
|
||
if (expectedSourceHead !== null && mirrorSourceHead !== expectedSourceHead) staleReasons.push("mirror-source-stale");
|
||
if (expectedSourceHead !== null && latestPipelineSourceCommit !== expectedSourceHead) staleReasons.push("latest-pipelinerun-not-current");
|
||
if (apiRevisionRequired && apiRevision !== expectedSourceHead) staleReasons.push("runtime-api-revision-stale");
|
||
const workspaceWarnings: string[] = [];
|
||
if (expectedSourceHead !== null && workspaceHead !== null && workspaceHead !== expectedSourceHead) workspaceWarnings.push("workspace-head-differs-from-latest-source-but-isolated");
|
||
if (expectedSourceHead !== null && workspaceOriginHead !== null && workspaceOriginHead !== expectedSourceHead) workspaceWarnings.push("workspace-origin-ref-stale-but-isolated");
|
||
if (workspace.dirty === true) workspaceWarnings.push("workspace-dirty-but-isolated-from-cicd");
|
||
const runtimeWarnings: string[] = [];
|
||
if (!apiRevisionRequired && expectedSourceHead !== null && apiRevision !== null && apiRevision !== expectedSourceHead) {
|
||
runtimeWarnings.push("api-revision-differs-without-current-cloud-api-rollout");
|
||
}
|
||
const latestPipelineSucceeded = latestPipelineRun !== null && latestPipelineRun.status === "True";
|
||
const aligned = staleReasons.length === 0;
|
||
const state = aligned
|
||
? "aligned"
|
||
: latestPipelineSucceeded
|
||
? "stale-success"
|
||
: currentPipelineStatus === "Unknown"
|
||
? "in-progress"
|
||
: "stale";
|
||
return {
|
||
aligned,
|
||
state,
|
||
expectedSourceHead,
|
||
originHead,
|
||
cicdSourceHead,
|
||
cicdRepo: input.sourceHeads.cicdRepo ?? V02_CICD_REPO,
|
||
mirrorSourceHead,
|
||
mirrorGithubSourceHead,
|
||
latestPipelineRun: latestPipelineRun === null
|
||
? null
|
||
: {
|
||
name: latestPipelineRun.name ?? null,
|
||
sourceCommit: latestPipelineSourceCommit,
|
||
status: latestPipelineRun.status ?? null,
|
||
reason: latestPipelineRun.reason ?? null,
|
||
createdAt: latestPipelineRun.createdAt ?? null,
|
||
durationSeconds: latestPipelineRun.durationSeconds ?? null,
|
||
},
|
||
latestPipelineSourceCommit,
|
||
currentPipelineRun: input.pipelineRun === null
|
||
? null
|
||
: {
|
||
name: input.pipelineRun.pipelineRun ?? null,
|
||
status: input.pipelineRun.status ?? null,
|
||
reason: input.pipelineRun.reason ?? null,
|
||
},
|
||
runtimeSourceCommit: apiRevision,
|
||
apiRevision,
|
||
apiRevisionRequired,
|
||
planSourceCommit,
|
||
rolloutServices,
|
||
serviceSourceCommits,
|
||
sourceInSync: input.gitMirrorSummary.sourceInSync ?? null,
|
||
gitopsInSync: input.gitMirrorSummary.gitopsInSync ?? null,
|
||
staleReasons,
|
||
workspace: {
|
||
path: workspace.path ?? V02_WORKSPACE,
|
||
branch: workspace.branch ?? null,
|
||
head: workspaceHead,
|
||
originHead: workspaceOriginHead,
|
||
dirty: workspace.dirty ?? null,
|
||
dirtyCount: workspace.dirtyCount ?? null,
|
||
isolatedFromCicd: true,
|
||
},
|
||
workspaceWarnings,
|
||
runtimeWarnings,
|
||
summary: aligned
|
||
? `v0.2 CI/CD source, mirror, PipelineRun, and runtime align with ${shortSha(expectedSourceHead ?? "")}`
|
||
: `v0.2 CI/CD alignment state=${state}; staleReasons=${staleReasons.join(",")}`,
|
||
};
|
||
}
|
||
|
||
function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null {
|
||
if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null;
|
||
const activeItems = activePipelineRuns.map((item) => record(item));
|
||
const sourceIsActive = activeItems.some((item) => item.sourceCommit === sourceCommit || String(item.name ?? "").includes(shortSha(sourceCommit)));
|
||
if (sourceIsActive) {
|
||
return `runtime rollout still in progress; cloud-api apiRevision=${apiRevision} has not reached sourceCommit=${sourceCommit} yet`;
|
||
}
|
||
if (activeItems.length > 0) {
|
||
return `runtime apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; active v02 PipelineRun observed, wait for rollout before judging runtime revision`;
|
||
}
|
||
return `cloud-api apiRevision=${apiRevision} is service-scoped and differs from sourceCommit=${sourceCommit}; when web asset probes pass, this is not proof that Cloud Web is stale`;
|
||
}
|
||
|
||
function numericField(value: string | undefined): number | null {
|
||
if (value === undefined || value.trim().length === 0) return null;
|
||
if (value === "<none>") return null;
|
||
const parsed = Number(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function numericValue(value: unknown): number | null {
|
||
const parsed = typeof value === "number" ? value : Number(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function stringArray(value: unknown): string[] {
|
||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : [];
|
||
}
|
||
|
||
function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 4000),
|
||
items: [],
|
||
activeItems: [],
|
||
};
|
||
}
|
||
return parsePipelineRunRows(text, limit, nowMs);
|
||
}
|
||
|
||
function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] {
|
||
if (lane === "v02") return ["hwlab-v02-ci-poll-"];
|
||
if (lane === "g14") return ["hwlab-g14-ci-poll-"];
|
||
return ["hwlab-v02-ci-poll-", "hwlab-g14-ci-poll-"];
|
||
}
|
||
|
||
function commandErrorSummary(result: CommandJsonResult): string {
|
||
const remoteStderr = String(nested(result.parsed, ["data", "stderr"]) ?? "").trim();
|
||
const remoteStdout = String(nested(result.parsed, ["data", "stdout"]) ?? "").trim();
|
||
const text = [result.stderr.trim(), remoteStderr, remoteStdout, result.stdout.trim()].find((item) => item.length > 0) ?? "";
|
||
return text.slice(0, 4000);
|
||
}
|
||
|
||
function tailText(value: unknown, maxBytes = 2000): string {
|
||
const text = String(value ?? "").trim();
|
||
return text.length <= maxBytes ? text : text.slice(-maxBytes);
|
||
}
|
||
|
||
function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
|
||
}
|
||
const prefixes = pipelinePrefixesForLane(options.lane);
|
||
const now = Date.now();
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", createdAt = "", status = "", reason = ""] = line.split("\t");
|
||
const createdMs = Date.parse(createdAt);
|
||
const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
||
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
|
||
})
|
||
.filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
|
||
.filter((item) => item.status === "True" || item.status === "False")
|
||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||
.slice(0, options.limit);
|
||
}
|
||
|
||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unknown>[] {
|
||
if (pipelineRunNames.length === 0) {
|
||
return [];
|
||
}
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pvc",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
|
||
}
|
||
const wanted = new Set(pipelineRunNames);
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
volume: volume || null,
|
||
phase: phase || null,
|
||
ownerKind: ownerKind || null,
|
||
owner: owner || null,
|
||
};
|
||
})
|
||
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
|
||
}
|
||
|
||
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||
if (names.length === 0) {
|
||
return {
|
||
ok: true,
|
||
command: [],
|
||
exitCode: 0,
|
||
stdout: "no candidates",
|
||
stderr: "",
|
||
parsed: null,
|
||
};
|
||
}
|
||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
|
||
}
|
||
|
||
function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pv",
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
|
||
}
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
createdAt,
|
||
phase,
|
||
storageClass,
|
||
reclaimPolicy,
|
||
claimNamespace,
|
||
claimName,
|
||
capacity,
|
||
hostPath: claimNamespace === CI_NAMESPACE && claimName.length > 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null,
|
||
};
|
||
})
|
||
.filter((item) => item.phase === "Released")
|
||
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
||
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||
.slice(0, options.limit);
|
||
}
|
||
|
||
function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
|
||
if (names.length === 0) {
|
||
return {
|
||
ok: true,
|
||
command: [],
|
||
exitCode: 0,
|
||
stdout: "no candidates",
|
||
stderr: "",
|
||
parsed: null,
|
||
};
|
||
}
|
||
return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs);
|
||
}
|
||
|
||
function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const candidates = listReleasedCiWorkspacePvs(options);
|
||
const candidateNames = candidates.map((item) => String(item.name));
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||
mode: "dry-run",
|
||
lane: options.lane,
|
||
limit: options.limit,
|
||
candidates,
|
||
candidateCount: candidates.length,
|
||
mutation: false,
|
||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
|
||
};
|
||
}
|
||
const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(deletion),
|
||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||
mode: "confirmed-cleanup",
|
||
lane: options.lane,
|
||
limit: options.limit,
|
||
deletedPersistentVolumes: candidateNames,
|
||
deletedPersistentVolumeCount: candidateNames.length,
|
||
candidatesBefore: candidates,
|
||
deletion,
|
||
followUp: {
|
||
diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||
storage: "bun scripts/cli.ts ssh G14 script -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runV02RuntimeMigration(options: G14ControlPlaneOptions, sourceCommit: string): Record<string, unknown> {
|
||
const reportPath = `/tmp/hwlab-v02-runtime-migration-${shortSha(sourceCommit)}.json`;
|
||
const migrationArgs = options.dryRun
|
||
? [
|
||
...(options.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
|
||
"--report",
|
||
reportPath,
|
||
]
|
||
: [
|
||
"--apply",
|
||
"--confirm-dev",
|
||
"--confirmed-non-production",
|
||
"--report",
|
||
reportPath,
|
||
];
|
||
const command = [
|
||
"exec",
|
||
"-n",
|
||
"hwlab-v02",
|
||
"deployment/hwlab-cloud-api",
|
||
"-c",
|
||
"hwlab-cloud-api",
|
||
"--",
|
||
"bun",
|
||
"cmd/hwlab-cloud-api/migrate.ts",
|
||
...migrationArgs,
|
||
];
|
||
const result = g14K3s(["kubectl", ...command], options.timeoutSeconds * 1000);
|
||
const ok = isCommandSuccess(result);
|
||
return {
|
||
ok,
|
||
command: "hwlab g14 control-plane runtime-migration --lane v02",
|
||
lane: "v02",
|
||
mode: options.dryRun ? options.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
|
||
sourceCommit,
|
||
runtimeNamespace: "hwlab-v02",
|
||
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
|
||
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
|
||
reportPath,
|
||
mutation: !options.dryRun,
|
||
result,
|
||
next: options.dryRun
|
||
? {
|
||
liveReadDryRun: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||
apply: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||
}
|
||
: {
|
||
health: "curl -fsS --max-time 20 http://74.48.78.17:19667/health/live",
|
||
triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const candidates = listCleanupPipelineRuns(options);
|
||
const candidateNames = candidates.map((item) => String(item.name));
|
||
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane cleanup-runs",
|
||
mode: "dry-run",
|
||
lane: options.lane,
|
||
minAgeMinutes: options.minAgeMinutes,
|
||
limit: options.limit,
|
||
candidates,
|
||
candidateCount: candidates.length,
|
||
ownedPvcs,
|
||
ownedPvcCount: ownedPvcs.length,
|
||
mutation: false,
|
||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane ${options.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
|
||
};
|
||
}
|
||
const deletion = deletePipelineRuns(candidateNames, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(deletion),
|
||
command: "hwlab g14 control-plane cleanup-runs",
|
||
mode: "confirmed-cleanup",
|
||
lane: options.lane,
|
||
minAgeMinutes: options.minAgeMinutes,
|
||
limit: options.limit,
|
||
deletedPipelineRuns: candidateNames,
|
||
deletedPipelineRunCount: candidateNames.length,
|
||
ownedPvcsBefore: ownedPvcs,
|
||
ownedPvcCountBefore: ownedPvcs.length,
|
||
deletion,
|
||
followUp: {
|
||
status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||
diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||
},
|
||
};
|
||
}
|
||
|
||
export function v02ControlPlaneRenderScript(sourceCommit: string): string {
|
||
const renderDir = v02RenderDir(sourceCommit);
|
||
const worktreeDir = v02RenderWorktreeDir(sourceCommit);
|
||
return [
|
||
"set -eu",
|
||
v02CicdRepoEnsureScript(),
|
||
`render_dir=${shellQuote(renderDir)}`,
|
||
`worktree_dir=${shellQuote(worktreeDir)}`,
|
||
"cleanup_render_worktree() { git --git-dir=\"$cicd_repo\" worktree remove --force \"$worktree_dir\" >/dev/null 2>&1 || rm -rf \"$worktree_dir\"; }",
|
||
"trap cleanup_render_worktree EXIT",
|
||
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/v0.2)" = ${shellQuote(sourceCommit)}`,
|
||
"cleanup_render_worktree",
|
||
"git --git-dir=\"$cicd_repo\" worktree prune >/dev/null 2>&1 || true",
|
||
"rm -rf \"$render_dir\"",
|
||
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
||
`git --git-dir="$cicd_repo" worktree add --detach "$worktree_dir" ${shellQuote(sourceCommit)}`,
|
||
"cd \"$worktree_dir\"",
|
||
`node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
||
].join("\n");
|
||
}
|
||
|
||
function runV02RenderToTemp(sourceCommit: string): CommandJsonResult {
|
||
return g14HostScript(v02ControlPlaneRenderScript(sourceCommit), 180_000);
|
||
}
|
||
|
||
function v02RenderDir(sourceCommit: string): string {
|
||
return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}`;
|
||
}
|
||
|
||
function v02RenderWorktreeDir(sourceCommit: string): string {
|
||
return `/tmp/hwlab-v02-control-plane-source-${shortSha(sourceCommit)}`;
|
||
}
|
||
|
||
function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||
const renderDir = v02RenderDir(sourceCommit);
|
||
return g14K3s([
|
||
"kubectl",
|
||
"apply",
|
||
"--server-side",
|
||
"--force-conflicts",
|
||
`--field-manager=${V02_CONTROL_PLANE_FIELD_MANAGER}`,
|
||
...(dryRun ? ["--dry-run=server"] : []),
|
||
"-f",
|
||
`${renderDir}/tekton-v02/rbac.yaml`,
|
||
"-f",
|
||
`${renderDir}/tekton-v02/pipeline.yaml`,
|
||
"-f",
|
||
`${renderDir}/argocd/project.yaml`,
|
||
"-f",
|
||
`${renderDir}/argocd/application-v02.yaml`,
|
||
], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render)) {
|
||
return {
|
||
ok: false,
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
renderDir: v02RenderDir(sourceCommit),
|
||
render,
|
||
degradedReason: "control-plane-render-failed",
|
||
};
|
||
}
|
||
const apply = applyV02ControlPlaneFiles(sourceCommit, false, timeoutSeconds);
|
||
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
|
||
return {
|
||
ok: isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs)),
|
||
phase: "control-plane-refresh",
|
||
sourceCommit,
|
||
renderDir: v02RenderDir(sourceCommit),
|
||
render: commandData(render),
|
||
apply: compactCommandResult(apply),
|
||
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
||
degradedReason: !isCommandSuccess(apply)
|
||
? "control-plane-apply-failed"
|
||
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
||
? "obsolete-cronjob-cleanup-failed"
|
||
: undefined,
|
||
};
|
||
}
|
||
|
||
function getV02ObsoleteCronJobs(): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"cronjob",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
V02_POLLER,
|
||
V02_RECONCILER,
|
||
"--ignore-not-found",
|
||
"-o",
|
||
"name",
|
||
], 60_000);
|
||
}
|
||
|
||
function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"delete",
|
||
"cronjob",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
V02_POLLER,
|
||
V02_RECONCILER,
|
||
"--ignore-not-found=true",
|
||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||
], 60_000);
|
||
}
|
||
|
||
function v02PipelineRunManifest(sourceCommit: string): Record<string, unknown> {
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
return {
|
||
apiVersion: "tekton.dev/v1",
|
||
kind: "PipelineRun",
|
||
metadata: {
|
||
name: pipelineRun,
|
||
namespace: CI_NAMESPACE,
|
||
labels: {
|
||
"app.kubernetes.io/part-of": "hwlab",
|
||
"hwlab.pikastech.local/gitops-target": "v02",
|
||
"hwlab.pikastech.local/source-commit": sourceCommit,
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
annotations: {
|
||
"hwlab.pikastech.local/source-branch": V02_SOURCE_BRANCH,
|
||
"hwlab.pikastech.local/gitops-branch": V02_GITOPS_BRANCH,
|
||
"hwlab.pikastech.local/triggered-by": "unidesk-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
pipelineRef: { name: V02_PIPELINE },
|
||
taskRunTemplate: {
|
||
serviceAccountName: "hwlab-v02-tekton-runner",
|
||
podTemplate: {
|
||
hostNetwork: true,
|
||
dnsPolicy: "ClusterFirstWithHostNet",
|
||
securityContext: { fsGroup: 1000 },
|
||
},
|
||
},
|
||
params: [
|
||
{ name: "git-url", value: V02_GIT_URL },
|
||
{ name: "git-read-url", value: V02_GIT_READ_URL },
|
||
{ name: "source-branch", value: V02_SOURCE_BRANCH },
|
||
{ name: "gitops-branch", value: V02_GITOPS_BRANCH },
|
||
{ name: "lane", value: "v02" },
|
||
{ name: "catalog-path", value: V02_CATALOG_PATH },
|
||
{ name: "image-tag-mode", value: "full" },
|
||
{ name: "runtime-path", value: V02_RUNTIME_PATH },
|
||
{ name: "revision", value: sourceCommit },
|
||
{ name: "registry-prefix", value: V02_REGISTRY_PREFIX },
|
||
{ name: "services", value: V02_SERVICE_IDS.join(",") },
|
||
{ name: "base-image", value: V02_BASE_IMAGE },
|
||
],
|
||
workspaces: [
|
||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
|
||
],
|
||
},
|
||
};
|
||
}
|
||
|
||
function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult {
|
||
const manifest = JSON.stringify(v02PipelineRunManifest(sourceCommit));
|
||
const manifestB64 = Buffer.from(manifest, "utf8").toString("base64");
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
const script = [
|
||
"set -eu",
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
`manifest_path=/tmp/${pipelineRun}.json`,
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`,
|
||
].join("\n");
|
||
return g14K3s(["script", "--", script], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function deleteV02PipelineRun(pipelineRun: string): CommandJsonResult {
|
||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, pipelineRun, "--ignore-not-found=true"], 60_000);
|
||
}
|
||
|
||
function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record<string, unknown> {
|
||
const targetMode: V02StatusTargetMode = target.mode
|
||
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
||
const strictHeadAlignment = targetMode === "latest-source-head";
|
||
const bundle = v02ControlPlaneStatusBundle({ ...target, mode: targetMode });
|
||
const sections = parseShellSections(statusText(bundle));
|
||
const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null;
|
||
const pipelineRun = stringOrNull(sections.pipelineRunName?.stdout) ?? (sourceCommit === null ? null : v02PipelineRunName(sourceCommit));
|
||
const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? "");
|
||
const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now();
|
||
const sourceHeadsSection = sections.sourceHeads;
|
||
const controlPlane = sections.controlPlane;
|
||
const obsoleteCronJobs = sections.obsoleteCronJobs;
|
||
const argo = sections.argo;
|
||
const pipelineRunSection = sections.pipelineRun;
|
||
const taskRunsSection = sections.taskRuns;
|
||
const planArtifactsSection = sections.planArtifacts;
|
||
const runtimeWorkloadsSection = sections.runtimeWorkloads;
|
||
const gitMirrorCacheSection = sections.gitMirrorCache;
|
||
const webAssetsSection = sections.webAssets;
|
||
const recentPipelineRuns = listV02PipelineRunsCompactFromText(
|
||
sections.recentPipelineRuns?.stdout ?? "",
|
||
shellSectionOk(sections.recentPipelineRuns),
|
||
"kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=<summary-fields>",
|
||
sections.recentPipelineRuns?.exitCode ?? null,
|
||
bundle.stderr,
|
||
8,
|
||
queryNowMs,
|
||
);
|
||
const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
|
||
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
|
||
const sourceHeads = v02SourceHeadsFromText(
|
||
sourceHeadsSection?.stdout ?? "",
|
||
shellSectionOk(sourceHeadsSection),
|
||
sourceHeadsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const pipelineRunInfo = pipelineRun === null
|
||
? null
|
||
: pipelineRunCompactFromText(
|
||
pipelineRun,
|
||
pipelineRunSection?.stdout ?? "",
|
||
shellSectionOk(pipelineRunSection),
|
||
`kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
|
||
pipelineRunSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const taskRuns = taskRunsCompactFromText(
|
||
taskRunsSection?.stdout ?? "",
|
||
shellSectionOk(taskRunsSection),
|
||
pipelineRun,
|
||
taskRunsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const planArtifacts = v02PlanArtifactsFromText(
|
||
planArtifactsSection?.stdout ?? "",
|
||
shellSectionOk(planArtifactsSection),
|
||
pipelineRun,
|
||
planArtifactsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const runtimeWorkloads = v02RuntimeWorkloadsFromText(
|
||
runtimeWorkloadsSection?.stdout ?? "",
|
||
shellSectionOk(runtimeWorkloadsSection),
|
||
runtimeWorkloadsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const webAssets = v02WebAssetsFromText(
|
||
webAssetsSection?.stdout ?? "",
|
||
shellSectionOk(webAssetsSection),
|
||
sourceCommit,
|
||
syncRevision,
|
||
webAssetsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
activePipelineRuns,
|
||
);
|
||
const gitMirror = {
|
||
ok: shellSectionOk(gitMirrorCacheSection),
|
||
summary: gitMirrorStatusSummary(String(gitMirrorCacheSection?.stdout ?? "").trim()),
|
||
raw: String(gitMirrorCacheSection?.stdout ?? "").trim(),
|
||
exitCode: gitMirrorCacheSection?.exitCode ?? null,
|
||
stderr: shellSectionOk(gitMirrorCacheSection) ? "" : bundle.stderr.trim().slice(0, 2000),
|
||
};
|
||
const commitAlignment = v02CommitAlignment({
|
||
expectedSourceHead: sourceCommit,
|
||
sourceHeads,
|
||
gitMirrorSummary: record(gitMirror.summary),
|
||
pipelineRun: pipelineRunInfo,
|
||
recentPipelineRuns,
|
||
planArtifacts,
|
||
runtimeWorkloads,
|
||
webAssets,
|
||
});
|
||
const baseOk = sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo);
|
||
const targetPipelineRunOk = strictHeadAlignment
|
||
? true
|
||
: pipelineRunInfo !== null && pipelineRunInfo.ok === true && pipelineRunInfo.exists !== false;
|
||
const targetDegradedReason = !strictHeadAlignment && !targetPipelineRunOk ? "target-pipelinerun-not-found-or-unreadable" : undefined;
|
||
const statusCommand = targetMode === "pipeline-run" && pipelineRun !== null
|
||
? `hwlab g14 control-plane status --lane v02 --pipeline-run ${pipelineRun}`
|
||
: targetMode === "source-commit" && sourceCommit !== null
|
||
? `hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}`
|
||
: "hwlab g14 control-plane status --lane v02";
|
||
const state = strictHeadAlignment ? commitAlignment.state : `target-${targetMode}`;
|
||
const sourceCommitSource = targetMode === "pipeline-run"
|
||
? "PipelineRun metadata label hwlab.pikastech.local/source-commit"
|
||
: targetMode === "source-commit"
|
||
? "explicit --source-commit"
|
||
: "G14 CI/CD dedicated bare repo refs/remotes/origin/v0.2; workspace checkout is observable but isolated";
|
||
return {
|
||
ok: baseOk && targetPipelineRunOk && (!strictHeadAlignment || commitAlignment.aligned !== false),
|
||
command: statusCommand,
|
||
lane: "v02",
|
||
state,
|
||
degradedReason: strictHeadAlignment && commitAlignment.aligned === false ? commitAlignment.state : targetDegradedReason,
|
||
statusTarget: {
|
||
mode: statusTargetFields.mode || targetMode,
|
||
sourceCommit,
|
||
pipelineRun,
|
||
strictHeadAlignment,
|
||
note: strictHeadAlignment
|
||
? "default status validates the latest v0.2 source head alignment"
|
||
: "targeted status inspects the requested PipelineRun/source commit without failing merely because origin/v0.2 advanced later",
|
||
},
|
||
sourceCommit,
|
||
sourceCommitSource,
|
||
expected: {
|
||
sourceRepo: V02_CICD_REPO,
|
||
workspace: V02_WORKSPACE,
|
||
workspaceIsolation: "workspace dirty/stale state must not select CI/CD source commits",
|
||
branch: V02_SOURCE_BRANCH,
|
||
namespace: CI_NAMESPACE,
|
||
runtimeNamespace: "hwlab-v02",
|
||
pipeline: V02_PIPELINE,
|
||
trigger: "manual UniDesk CLI PipelineRun creation",
|
||
argoApplication: V02_APP,
|
||
},
|
||
commitAlignment,
|
||
sourceHeads,
|
||
gitMirror,
|
||
controlPlane: {
|
||
ok: shellSectionOk(controlPlane),
|
||
names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
exitCode: controlPlane?.exitCode ?? null,
|
||
},
|
||
obsoleteCronJobs: {
|
||
ok: shellSectionOk(obsoleteCronJobs),
|
||
names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
exitCode: obsoleteCronJobs?.exitCode ?? null,
|
||
cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||
},
|
||
argo: {
|
||
ok: shellSectionOk(argo),
|
||
raw: argo?.stdout ?? "",
|
||
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
||
exitCode: argo?.exitCode ?? null,
|
||
},
|
||
pipelineRun: pipelineRunInfo,
|
||
taskRuns,
|
||
planArtifacts,
|
||
runtimeWorkloads,
|
||
falseGreenGuard: v02FalseGreenGuard({ sourceCommit, pipelineRun: pipelineRunInfo, taskRuns, planArtifacts, runtimeWorkloads }),
|
||
webAssets,
|
||
activePipelineRuns,
|
||
recentPipelineRuns,
|
||
query: {
|
||
ok: isCommandSuccess(bundle),
|
||
exitCode: bundle.exitCode,
|
||
stderr: bundle.stderr.trim().slice(0, 2000),
|
||
},
|
||
visibilityHint: activePipelineRuns.length > 0
|
||
? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger"
|
||
: "no active v02 PipelineRun observed",
|
||
};
|
||
}
|
||
|
||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
||
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
||
if (options.action === "status" && options.pipelineRun !== undefined) return v02ControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run" });
|
||
if (options.action === "status" && options.sourceCommit !== undefined) return v02ControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit" });
|
||
const sourceCommit = getV02Head();
|
||
if (sourceCommit === null) {
|
||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE };
|
||
}
|
||
if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit);
|
||
if (options.action === "status") return v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head" });
|
||
if (options.action === "apply") {
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render)) {
|
||
return {
|
||
ok: false,
|
||
command: `hwlab g14 control-plane ${options.action} --lane v02`,
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
render: compactCommandResult(render),
|
||
};
|
||
}
|
||
const apply = applyV02ControlPlaneFiles(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||
return {
|
||
ok: isCommandSuccess(apply),
|
||
command: "hwlab g14 control-plane apply --lane v02",
|
||
lane: "v02",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||
sourceCommit,
|
||
renderDir: v02RenderDir(sourceCommit),
|
||
render: commandData(render),
|
||
apply: compactCommandResult(apply),
|
||
cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false)),
|
||
status: v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head" }),
|
||
next: options.dryRun
|
||
? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" }
|
||
: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
const before = getPipelineRunCompact(v02PipelineRunName(sourceCommit));
|
||
if (options.dryRun) {
|
||
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "dry-run",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
gitMirrorPreSync,
|
||
controlPlaneRefresh: {
|
||
mode: "skipped-dry-run",
|
||
wouldRefreshBeforeCreate: true,
|
||
resources: ["tekton-v02/rbac.yaml", "tekton-v02/pipeline.yaml", "argocd/project.yaml", "argocd/application-v02.yaml"],
|
||
},
|
||
manifest: v02PipelineRunManifest(sourceCommit),
|
||
next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
if (before.status === "True" || before.status === "Unknown") {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
degradedReason: "refuse-active-or-successful-pipelinerun",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const controlPlaneRefresh = refreshV02ControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds);
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: controlPlaneRefresh.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), degradedReason: record(controlPlaneRefresh).degradedReason ?? null });
|
||
if (controlPlaneRefresh.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "control-plane-refresh",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh,
|
||
degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync",
|
||
status: "started",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
reason: "ensure local git mirror has source commit before creating PipelineRun",
|
||
});
|
||
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync",
|
||
status: gitMirrorPreSync.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
mode: record(gitMirrorPreSync).mode ?? null,
|
||
required: nested(gitMirrorPreSync, ["before", "required"]) ?? null,
|
||
localV02: nested(gitMirrorPreSync, ["before", "localV02"]) ?? null,
|
||
pendingFlush: nested(gitMirrorPreSync, ["before", "pendingFlush"]) ?? null,
|
||
reason: nested(gitMirrorPreSync, ["before", "reason"]) ?? null,
|
||
syncElapsedMs: nested(gitMirrorPreSync, ["sync", "elapsedMs"]) ?? null,
|
||
degradedReason: record(gitMirrorPreSync).degradedReason ?? null,
|
||
});
|
||
if (gitMirrorPreSync.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "git-mirror-pre-sync",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh,
|
||
gitMirrorPreSync,
|
||
degradedReason: "git-mirror-pre-sync-failed",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const deletePipelineRun = deleteV02PipelineRun(v02PipelineRunName(sourceCommit));
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: isCommandSuccess(deletePipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: deletePipelineRun.exitCode });
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const createPipelineRun = isCommandSuccess(deletePipelineRun)
|
||
? createV02PipelineRun(sourceCommit, options.timeoutSeconds)
|
||
: null;
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: createPipelineRun !== null && isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: createPipelineRun?.exitCode ?? null });
|
||
return {
|
||
ok: isCommandSuccess(deletePipelineRun) && createPipelineRun !== null && isCommandSuccess(createPipelineRun),
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh),
|
||
gitMirrorPreSync,
|
||
deletePipelineRun: compactTriggerCommandResult(compactCommandResult(deletePipelineRun)),
|
||
createPipelineRun: createPipelineRun === null ? null : compactTriggerCommandResult(compactCommandResult(createPipelineRun)),
|
||
after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)),
|
||
disclosure: {
|
||
fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.",
|
||
},
|
||
};
|
||
}
|
||
|
||
function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"delete",
|
||
"cronjob",
|
||
"-n",
|
||
GIT_MIRROR_NAMESPACE,
|
||
GIT_MIRROR_LEGACY_CRONJOB,
|
||
"--ignore-not-found=true",
|
||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||
], 60_000);
|
||
}
|
||
|
||
function applyGitMirrorManifestFile(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||
const renderDir = v02RenderDir(sourceCommit);
|
||
return g14K3s([
|
||
"kubectl",
|
||
"apply",
|
||
"--server-side",
|
||
"--force-conflicts",
|
||
`--field-manager=${GIT_MIRROR_MANIFEST_FIELD_MANAGER}`,
|
||
...(dryRun ? ["--dry-run=server"] : []),
|
||
"-f",
|
||
`${renderDir}/devops-infra/git-mirror.yaml`,
|
||
], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function gitMirrorSyncJobName(): string {
|
||
return `${GIT_MIRROR_SYNC_JOB_PREFIX}-${Date.now().toString(36)}`.slice(0, 63);
|
||
}
|
||
|
||
function gitMirrorFlushJobName(): string {
|
||
return `git-mirror-hwlab-flush-manual-${Date.now().toString(36)}`.slice(0, 63);
|
||
}
|
||
|
||
export function gitMirrorSyncJobManifest(name: string): Record<string, unknown> {
|
||
return {
|
||
apiVersion: "batch/v1",
|
||
kind: "Job",
|
||
metadata: {
|
||
name,
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
labels: {
|
||
"app.kubernetes.io/name": "git-mirror",
|
||
"app.kubernetes.io/part-of": "devops-infra",
|
||
"app.kubernetes.io/component": "sync-controller",
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
backoffLimit: 0,
|
||
activeDeadlineSeconds: 300,
|
||
ttlSecondsAfterFinished: 3600,
|
||
template: {
|
||
metadata: {
|
||
labels: {
|
||
"app.kubernetes.io/name": "git-mirror",
|
||
"app.kubernetes.io/part-of": "devops-infra",
|
||
"app.kubernetes.io/component": "sync-controller",
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
restartPolicy: "Never",
|
||
hostNetwork: true,
|
||
dnsPolicy: "ClusterFirstWithHostNet",
|
||
volumes: [
|
||
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
|
||
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
|
||
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } },
|
||
],
|
||
containers: [{
|
||
name: "sync",
|
||
image: "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1",
|
||
imagePullPolicy: "IfNotPresent",
|
||
command: ["/script/sync.sh"],
|
||
volumeMounts: [
|
||
{ name: "cache", mountPath: "/cache" },
|
||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||
{ name: "script", mountPath: "/script", readOnly: true },
|
||
],
|
||
}],
|
||
},
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
export function gitMirrorFlushJobManifest(name: string): Record<string, unknown> {
|
||
const manifest = gitMirrorSyncJobManifest(name);
|
||
record(record(manifest.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||
const template = record(record(manifest.spec).template);
|
||
record(record(template.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||
const podSpec = record(template.spec);
|
||
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
|
||
const first = record(containers[0]);
|
||
first.name = "flush";
|
||
first.command = ["/script/flush.sh"];
|
||
podSpec.containers = [first];
|
||
return manifest;
|
||
}
|
||
|
||
export function parseGitMirrorStatusRefs(raw: string): { refs: Record<string, string | null>; pendingFlush: boolean | null } {
|
||
const line = raw.split(/\r?\n/u).find((item) => item.startsWith("refs="));
|
||
if (line === undefined) return { refs: {}, pendingFlush: null };
|
||
try {
|
||
const parsed = record(JSON.parse(line.slice("refs=".length)) as unknown);
|
||
const refs = record(parsed.refs);
|
||
return {
|
||
refs: {
|
||
localV02: stringOrNull(refs.localV02),
|
||
githubV02: stringOrNull(refs.githubV02),
|
||
localG14: stringOrNull(refs.localG14),
|
||
githubG14: stringOrNull(refs.githubG14),
|
||
localGitops: stringOrNull(refs.localGitops),
|
||
githubGitops: stringOrNull(refs.githubGitops),
|
||
},
|
||
pendingFlush: typeof parsed.pendingFlush === "boolean" ? parsed.pendingFlush : null,
|
||
};
|
||
} catch {
|
||
return { refs: {}, pendingFlush: null };
|
||
}
|
||
}
|
||
|
||
function parseLabeledJsonLine(raw: string, label: string): Record<string, unknown> {
|
||
const line = raw.split(/\r?\n/u).find((item) => item.startsWith(`${label}=`));
|
||
if (line === undefined) return {};
|
||
const value = line.slice(label.length + 1).trim();
|
||
if (value.length === 0) return {};
|
||
try {
|
||
return record(JSON.parse(value) as unknown);
|
||
} catch {
|
||
return {};
|
||
}
|
||
}
|
||
|
||
export function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
|
||
const refs = parseGitMirrorStatusRefs(raw);
|
||
const lastSync = parseLabeledJsonLine(raw, "lastSync");
|
||
const lastWrite = parseLabeledJsonLine(raw, "lastWrite");
|
||
const lastFlush = parseLabeledJsonLine(raw, "lastFlush");
|
||
const localV02 = refs.refs.localV02 ?? null;
|
||
const githubV02 = refs.refs.githubV02 ?? null;
|
||
const localGitops = refs.refs.localGitops ?? null;
|
||
const githubGitops = refs.refs.githubGitops ?? null;
|
||
const sourceInSync = Boolean(localV02 && githubV02 && localV02 === githubV02);
|
||
const gitopsInSync = Boolean(localGitops && githubGitops && localGitops === githubGitops);
|
||
return {
|
||
localV02,
|
||
githubV02,
|
||
localG14: refs.refs.localG14 ?? null,
|
||
githubG14: refs.refs.githubG14 ?? null,
|
||
localGitops,
|
||
githubGitops,
|
||
pendingFlush: refs.pendingFlush,
|
||
lastSync: {
|
||
status: stringOrNull(lastSync.status) ?? null,
|
||
at: stringOrNull(lastSync.publishedAt) ?? stringOrNull(lastSync.syncedAt) ?? null,
|
||
pendingFlush: typeof lastSync.pendingFlush === "boolean" ? lastSync.pendingFlush : null,
|
||
},
|
||
lastWrite: {
|
||
status: stringOrNull(lastWrite.status) ?? null,
|
||
at: stringOrNull(lastWrite.writtenAt) ?? null,
|
||
pendingFlush: typeof lastWrite.pendingFlush === "boolean" ? lastWrite.pendingFlush : null,
|
||
},
|
||
lastFlush: {
|
||
status: stringOrNull(lastFlush.status) ?? null,
|
||
at: stringOrNull(lastFlush.flushedAt) ?? null,
|
||
pendingFlush: typeof lastFlush.pendingFlush === "boolean" ? lastFlush.pendingFlush : null,
|
||
},
|
||
flushNeeded: refs.pendingFlush === true,
|
||
flushCommand: refs.pendingFlush === true ? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" : null,
|
||
sourceInSync,
|
||
gitopsInSync,
|
||
githubInSync: sourceInSync && gitopsInSync,
|
||
};
|
||
}
|
||
|
||
export function gitMirrorV02SyncRequirement(sourceCommit: string, rawStatus: string): Record<string, unknown> {
|
||
const parsed = parseGitMirrorStatusRefs(rawStatus);
|
||
const localV02 = parsed.refs.localV02 ?? null;
|
||
return {
|
||
required: localV02 !== sourceCommit,
|
||
sourceCommit,
|
||
localV02,
|
||
pendingFlush: parsed.pendingFlush,
|
||
refs: parsed.refs,
|
||
reason: localV02 === sourceCommit ? "local-v02-current" : localV02 === null ? "local-v02-unresolved" : "local-v02-stale",
|
||
};
|
||
}
|
||
|
||
function gitMirrorStatusCacheRaw(status: Record<string, unknown>): string {
|
||
return String(nested(status, ["cache", "raw"]) ?? "");
|
||
}
|
||
|
||
function compactGitMirrorStatus(status: Record<string, unknown>, sourceCommit: string): Record<string, unknown> {
|
||
const requirement = gitMirrorV02SyncRequirement(sourceCommit, gitMirrorStatusCacheRaw(status));
|
||
return {
|
||
ok: status.ok === true,
|
||
readUrl: status.readUrl ?? V02_GIT_READ_URL,
|
||
writeUrl: status.writeUrl ?? V02_GIT_WRITE_URL,
|
||
elapsedMs: status.elapsedMs ?? null,
|
||
legacyCronJobExists: nested(status, ["legacyCronJob", "exists"]) === true,
|
||
required: requirement.required,
|
||
sourceCommit,
|
||
localV02: requirement.localV02 ?? null,
|
||
githubV02: requirement.refs.githubV02 ?? null,
|
||
pendingFlush: requirement.pendingFlush ?? null,
|
||
sourceInSync: Boolean(requirement.refs.localV02 && requirement.refs.githubV02 && requirement.refs.localV02 === requirement.refs.githubV02),
|
||
reason: requirement.reason,
|
||
cacheOk: nested(status, ["cache", "ok"]) === true,
|
||
cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000),
|
||
resourcesOk: nested(status, ["resources", "ok"]) === true,
|
||
};
|
||
}
|
||
|
||
function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
ok: sync.ok === true,
|
||
command: sync.command ?? "hwlab g14 git-mirror sync",
|
||
mode: sync.mode ?? null,
|
||
namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE,
|
||
jobName: sync.jobName ?? null,
|
||
elapsedMs: sync.elapsedMs ?? null,
|
||
exitCode: nested(sync, ["result", "exitCode"]) ?? null,
|
||
stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600),
|
||
stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200),
|
||
stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null,
|
||
stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null,
|
||
};
|
||
}
|
||
|
||
function gitMirrorCacheProbeScript(): string {
|
||
return [
|
||
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'refs='",
|
||
"node - <<'NODE' 2>/dev/null || true",
|
||
"const { execFileSync } = require('node:child_process');",
|
||
"const repo = '/cache/pikasTech/HWLAB.git';",
|
||
"function rev(ref) {",
|
||
" try {",
|
||
" return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim();",
|
||
" } catch {",
|
||
" return null;",
|
||
" }",
|
||
"}",
|
||
"const localGitops = rev('refs/heads/v0.2-gitops');",
|
||
"const githubGitops = rev('refs/mirror-stage/heads/v0.2-gitops');",
|
||
"console.log(JSON.stringify({",
|
||
" refs: {",
|
||
" localV02: rev('refs/heads/v0.2'),",
|
||
" githubV02: rev('refs/mirror-stage/heads/v0.2'),",
|
||
" localG14: rev('refs/heads/G14'),",
|
||
" githubG14: rev('refs/mirror-stage/heads/G14'),",
|
||
" localGitops,",
|
||
" githubGitops,",
|
||
" },",
|
||
" pendingFlush: Boolean(localGitops && githubGitops && localGitops !== githubGitops),",
|
||
"}));",
|
||
"NODE",
|
||
].join("\n");
|
||
}
|
||
|
||
function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
|
||
const statusStartMs = Date.now();
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
|
||
const before = runGitMirrorStatus();
|
||
const beforeSummary = compactGitMirrorStatus(before, sourceCommit);
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-status",
|
||
status: beforeSummary.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
durationMs: Date.now() - statusStartMs,
|
||
required: beforeSummary.required,
|
||
localV02: beforeSummary.localV02,
|
||
pendingFlush: beforeSummary.pendingFlush,
|
||
reason: beforeSummary.reason,
|
||
});
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
mode: "dry-run",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
action: beforeSummary.required === true ? "would-sync-before-trigger" : "already-current",
|
||
next: beforeSummary.required === true ? { syncAndTrigger: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" } : undefined,
|
||
};
|
||
}
|
||
if (beforeSummary.required !== true) {
|
||
return {
|
||
ok: true,
|
||
mode: "already-current",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-sync",
|
||
status: "started",
|
||
sourceCommit,
|
||
reason: beforeSummary.reason,
|
||
localV02: beforeSummary.localV02,
|
||
pendingFlush: beforeSummary.pendingFlush,
|
||
});
|
||
const sync = runGitMirrorSync({
|
||
action: "sync",
|
||
confirm: true,
|
||
dryRun: false,
|
||
timeoutSeconds: options.timeoutSeconds,
|
||
});
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-sync",
|
||
status: sync.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
durationMs: sync.elapsedMs ?? null,
|
||
jobName: sync.jobName ?? null,
|
||
});
|
||
const syncStatus = record(sync.status);
|
||
const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
|
||
const afterSummary = compactGitMirrorStatus(after, sourceCommit);
|
||
const ok = sync.ok === true && afterSummary.required === false;
|
||
return {
|
||
ok,
|
||
mode: "auto-sync-before-trigger",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
sync: compactGitMirrorSync(sync),
|
||
after: afterSummary,
|
||
degradedReason: ok ? undefined : "git-mirror-local-v02-not-current-after-sync",
|
||
};
|
||
}
|
||
|
||
function runGitMirrorStatus(): Record<string, unknown> {
|
||
const startedAtMs = Date.now();
|
||
const script = [
|
||
"set +e",
|
||
"section() {",
|
||
" name=\"$1\"",
|
||
" shift",
|
||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||
" \"$@\"",
|
||
" code=$?",
|
||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||
"}",
|
||
[
|
||
"section resources kubectl get deploy,svc,pvc,cm",
|
||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
"-l app.kubernetes.io/name=git-mirror",
|
||
"-o name",
|
||
].join(" "),
|
||
[
|
||
"section legacyCronJob kubectl get cronjob",
|
||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
shellQuote(GIT_MIRROR_LEGACY_CRONJOB),
|
||
"--ignore-not-found",
|
||
"-o name",
|
||
].join(" "),
|
||
[
|
||
`section jobs kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
"-l app.kubernetes.io/name=git-mirror",
|
||
"--sort-by=.metadata.creationTimestamp",
|
||
"-o custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime",
|
||
"--no-headers",
|
||
].join(" "),
|
||
[
|
||
"section cache kubectl exec",
|
||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
"deploy/git-mirror-http",
|
||
"-- sh -lc",
|
||
shellQuote(gitMirrorCacheProbeScript()),
|
||
].join(" "),
|
||
].join("\n");
|
||
const bundle = g14K3s(["script", "--", script], 60_000);
|
||
const sections = parseShellSections(statusText(bundle));
|
||
const resources = sections.resources;
|
||
const legacyCronJob = sections.legacyCronJob;
|
||
const jobs = sections.jobs;
|
||
const cache = sections.cache;
|
||
const bundleStderr = bundle.stderr.trim().slice(0, 2000);
|
||
return {
|
||
ok: isCommandSuccess(bundle) && shellSectionOk(resources),
|
||
command: "hwlab g14 git-mirror status",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
readUrl: V02_GIT_READ_URL,
|
||
writeUrl: V02_GIT_WRITE_URL,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
resources: {
|
||
ok: shellSectionOk(resources),
|
||
names: String(resources?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
stderr: shellSectionOk(resources) ? "" : bundleStderr,
|
||
},
|
||
legacyCronJob: {
|
||
ok: shellSectionOk(legacyCronJob),
|
||
exists: String(legacyCronJob?.stdout ?? "").trim().length > 0,
|
||
name: String(legacyCronJob?.stdout ?? "").trim() || null,
|
||
cleanupCommand: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
},
|
||
jobs: {
|
||
ok: shellSectionOk(jobs),
|
||
raw: String(jobs?.stdout ?? "").trim(),
|
||
stderr: shellSectionOk(jobs) ? "" : bundleStderr,
|
||
},
|
||
cache: {
|
||
ok: shellSectionOk(cache),
|
||
summary: gitMirrorStatusSummary(String(cache?.stdout ?? "").trim()),
|
||
raw: String(cache?.stdout ?? "").trim(),
|
||
stderr: shellSectionOk(cache) ? "" : bundleStderr,
|
||
},
|
||
next: {
|
||
apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
||
flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const sourceCommit = getV02Head();
|
||
if (sourceCommit === null) {
|
||
return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE };
|
||
}
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render)) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 git-mirror apply",
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
render: compactCommandResult(render),
|
||
};
|
||
}
|
||
const apply = applyGitMirrorManifestFile(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||
const cleanupLegacyCronJob = isCommandSuccess(apply)
|
||
? deleteLegacyGitMirrorCronJob(options.dryRun)
|
||
: {
|
||
ok: false,
|
||
command: [],
|
||
exitCode: null,
|
||
stdout: "",
|
||
stderr: "skipped because apply failed",
|
||
parsed: null,
|
||
};
|
||
return {
|
||
ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob),
|
||
command: "hwlab g14 git-mirror apply",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||
sourceCommit,
|
||
manifest: `${v02RenderDir(sourceCommit)}/devops-infra/git-mirror.yaml`,
|
||
render: commandData(render),
|
||
apply: compactCommandResult(apply),
|
||
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
|
||
status: runGitMirrorStatus(),
|
||
next: options.dryRun
|
||
? { apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm" }
|
||
: { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" },
|
||
};
|
||
}
|
||
|
||
function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const startedAtMs = Date.now();
|
||
const jobName = gitMirrorSyncJobName();
|
||
const manifest = gitMirrorSyncJobManifest(jobName);
|
||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||
const script = [
|
||
"set -eu",
|
||
`job=${shellQuote(jobName)}`,
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
"manifest_path=\"/tmp/$job.json\"",
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
options.dryRun
|
||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||
: [
|
||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||
"while :; do",
|
||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
" exit 44",
|
||
" fi",
|
||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||
" exit 45",
|
||
" fi",
|
||
" sleep 2",
|
||
"done",
|
||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2 refs/heads/v0.2-gitops refs/heads/G14 2>/dev/null || true'`,
|
||
].join("\n"),
|
||
].join("\n");
|
||
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 git-mirror sync",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-sync",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
jobName,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
manifest: options.dryRun ? manifest : undefined,
|
||
result: compactCommandResult(result),
|
||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||
next: options.dryRun ? { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
|
||
function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const jobName = gitMirrorFlushJobName();
|
||
const manifest = gitMirrorFlushJobManifest(jobName);
|
||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||
const script = [
|
||
"set -eu",
|
||
`job=${shellQuote(jobName)}`,
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
"manifest_path=\"/tmp/$job.json\"",
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
options.dryRun
|
||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||
: [
|
||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||
"while :; do",
|
||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||
" succeeded=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||
" failed=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
" exit 54",
|
||
" fi",
|
||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||
" exit 55",
|
||
" fi",
|
||
" sleep 2",
|
||
"done",
|
||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2-gitops refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true'`,
|
||
].join("\n"),
|
||
].join("\n");
|
||
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 git-mirror flush",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-flush",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
jobName,
|
||
manifest: options.dryRun ? manifest : undefined,
|
||
result: compactCommandResult(result),
|
||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||
next: options.dryRun ? { flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" } : { status: "bun scripts/cli.ts hwlab g14 git-mirror status" },
|
||
};
|
||
}
|
||
|
||
function runG14GitMirror(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
if (options.action === "status") return runGitMirrorStatus();
|
||
if (options.action === "apply") return runGitMirrorApply(options);
|
||
if (options.action === "flush") return runGitMirrorFlush(options);
|
||
return runGitMirrorSync(options);
|
||
}
|
||
|
||
function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record<string, unknown> {
|
||
const job = startJob(name, command, note);
|
||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
||
return {
|
||
ok: true,
|
||
mode: "async-job",
|
||
job,
|
||
statusCommand,
|
||
tailCommand: `tail -f ${job.stdoutFile}`,
|
||
next: {
|
||
status: statusCommand,
|
||
tail: `tail -f ${job.stdoutFile}`,
|
||
},
|
||
};
|
||
}
|
||
|
||
function startControlPlaneTriggerJob(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const command = [
|
||
"bun",
|
||
"scripts/cli.ts",
|
||
"hwlab",
|
||
"g14",
|
||
"control-plane",
|
||
"trigger-current",
|
||
"--lane",
|
||
options.lane,
|
||
"--confirm",
|
||
"--timeout-seconds",
|
||
String(options.timeoutSeconds),
|
||
"--wait",
|
||
];
|
||
return {
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: options.lane,
|
||
reason: "confirmed trigger can spend tens of seconds syncing git mirror and creating PipelineRun; default is fire-and-forget to avoid silent blocking",
|
||
waitCommand: command.join(" "),
|
||
...startAsyncHwlabG14Job(
|
||
"hwlab_g14_v02_trigger_current",
|
||
command,
|
||
"Trigger HWLAB v0.2 current commit PipelineRun with git mirror pre-sync through G14 control-plane",
|
||
),
|
||
};
|
||
}
|
||
|
||
function startGitMirrorJob(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const command = [
|
||
"bun",
|
||
"scripts/cli.ts",
|
||
"hwlab",
|
||
"g14",
|
||
"git-mirror",
|
||
options.action,
|
||
"--confirm",
|
||
"--timeout-seconds",
|
||
String(options.timeoutSeconds),
|
||
"--wait",
|
||
];
|
||
return {
|
||
command: `hwlab g14 git-mirror ${options.action}`,
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
reason: "manual git mirror sync/flush waits for a Kubernetes Job; default is fire-and-forget to keep CLI output immediately visible",
|
||
waitCommand: command.join(" "),
|
||
...startAsyncHwlabG14Job(
|
||
`hwlab_g14_git_mirror_${options.action}`,
|
||
command,
|
||
`Run HWLAB devops-infra git mirror ${options.action} through a bounded manual Kubernetes Job`,
|
||
),
|
||
};
|
||
}
|
||
|
||
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
|
||
}
|
||
|
||
function g14CiToolsImage(tag: string): string {
|
||
return `${G14_CI_TOOLS_IMAGE_REPO}:${tag}`;
|
||
}
|
||
|
||
function runG14ToolsImageStatus(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
const image = g14CiToolsImage(options.tag);
|
||
const script = [
|
||
"set -u",
|
||
`image=${shellQuote(image)}`,
|
||
"exists=false",
|
||
"image_id=",
|
||
"created=",
|
||
"size=",
|
||
"node_version=",
|
||
"npm_version=",
|
||
"bun_version=",
|
||
"git_version=",
|
||
"docker_version=",
|
||
"python_version=",
|
||
"registry_tags=",
|
||
"if docker image inspect \"$image\" >/tmp/hwlab-tools-image-inspect.json 2>/tmp/hwlab-tools-image-inspect.err; then",
|
||
" exists=true",
|
||
" image_id=$(docker image inspect \"$image\" --format '{{.Id}}' 2>/dev/null || true)",
|
||
" created=$(docker image inspect \"$image\" --format '{{.Created}}' 2>/dev/null || true)",
|
||
" size=$(docker image inspect \"$image\" --format '{{.Size}}' 2>/dev/null || true)",
|
||
" node_version=$(docker run --rm \"$image\" node --version 2>/dev/null || true)",
|
||
" npm_version=$(docker run --rm \"$image\" npm --version 2>/dev/null || true)",
|
||
" bun_version=$(docker run --rm \"$image\" bun --version 2>/dev/null || true)",
|
||
" git_version=$(docker run --rm \"$image\" git --version 2>/dev/null || true)",
|
||
" docker_version=$(docker run --rm \"$image\" docker --version 2>/dev/null || true)",
|
||
" python_version=$(docker run --rm \"$image\" python3 --version 2>/dev/null || true)",
|
||
"fi",
|
||
`registry_tags=$(curl -fsS --max-time 10 http://127.0.0.1:5000/v2/hwlab/hwlab-ci-node-tools/tags/list 2>/dev/null || true)`,
|
||
"export image exists image_id created size node_version npm_version bun_version git_version docker_version python_version registry_tags",
|
||
"node - <<'NODE'",
|
||
"const fs = require('node:fs');",
|
||
"const env = process.env;",
|
||
"const payload = {",
|
||
" image: env.image,",
|
||
" exists: env.exists === 'true',",
|
||
" imageId: env.image_id || null,",
|
||
" created: env.created || null,",
|
||
" sizeBytes: env.size ? Number(env.size) : null,",
|
||
" tools: {",
|
||
" node: env.node_version || null,",
|
||
" npm: env.npm_version || null,",
|
||
" bun: env.bun_version || null,",
|
||
" git: env.git_version || null,",
|
||
" docker: env.docker_version || null,",
|
||
" python: env.python_version || null",
|
||
" },",
|
||
" registryTagsRaw: env.registry_tags || null",
|
||
"};",
|
||
"console.log(JSON.stringify(payload, null, 2));",
|
||
"NODE",
|
||
].join("\n");
|
||
const result = g14HostScript(script, 180_000);
|
||
let parsedStatus: unknown = null;
|
||
const text = statusText(result);
|
||
if (text.length > 0) {
|
||
try {
|
||
parsedStatus = JSON.parse(text) as unknown;
|
||
} catch {
|
||
parsedStatus = null;
|
||
}
|
||
}
|
||
return {
|
||
ok: isCommandSuccess(result) && record(parsedStatus).exists === true,
|
||
command: "hwlab g14 tools-image status --name ci-node-tools",
|
||
name: options.name,
|
||
image,
|
||
status: parsedStatus ?? text,
|
||
result,
|
||
};
|
||
}
|
||
|
||
function runG14ToolsImageBuild(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
const image = g14CiToolsImage(options.tag);
|
||
const script = [
|
||
"set -eu",
|
||
`cd ${shellQuote(V02_WORKSPACE)}`,
|
||
"git fetch origin v0.2 --prune",
|
||
"git checkout v0.2 >/dev/null 2>&1 || true",
|
||
"git merge --ff-only origin/v0.2",
|
||
`test -f ${shellQuote(options.dockerfile)}`,
|
||
`image=${shellQuote(image)}`,
|
||
`dockerfile=${shellQuote(options.dockerfile)}`,
|
||
"echo \"{\\\"phase\\\":\\\"preflight\\\",\\\"image\\\":\\\"$image\\\",\\\"dockerfile\\\":\\\"$dockerfile\\\",\\\"commit\\\":\\\"$(git rev-parse HEAD)\\\"}\"",
|
||
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
|
||
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
|
||
"export no_proxy=$NO_PROXY",
|
||
"docker build --pull --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg http_proxy --build-arg https_proxy --build-arg NO_PROXY --build-arg no_proxy -f \"$dockerfile\" -t \"$image\" .",
|
||
"docker run --rm \"$image\" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V'",
|
||
"docker push \"$image\"",
|
||
"digest=$(docker image inspect \"$image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
|
||
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"image\\\":\\\"$image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
|
||
].join("\n");
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||
mode: "dry-run",
|
||
image,
|
||
workspace: V02_WORKSPACE,
|
||
dockerfile: options.dockerfile,
|
||
buildScriptPreview: script.split(/\n/u),
|
||
next: { build: `bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag ${options.tag} --confirm` },
|
||
};
|
||
}
|
||
const result = g14HostScript(script, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||
mode: "confirmed-build",
|
||
image,
|
||
workspace: V02_WORKSPACE,
|
||
dockerfile: options.dockerfile,
|
||
result,
|
||
status: runG14ToolsImageStatus({ ...options, action: "status", dryRun: true, confirm: false }),
|
||
};
|
||
}
|
||
|
||
function runG14ToolsImage(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
if (options.action === "status") return runG14ToolsImageStatus(options);
|
||
return runG14ToolsImageBuild(options);
|
||
}
|
||
|
||
function listOpenG14PullRequests(): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "30", "--json", "number,title,state,url,head,base,draft,headRefName,baseRefName"], 60_000);
|
||
}
|
||
|
||
function preflightPullRequest(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "preflight", String(number), "--repo", HWLAB_REPO], 80_000);
|
||
}
|
||
|
||
function mergePullRequest(number: number, dryRun: boolean): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "merge", String(number), "--repo", HWLAB_REPO, "--merge", "--delete-branch", ...(dryRun ? ["--dry-run"] : [])], 100_000);
|
||
}
|
||
|
||
function getG14Head(): string | null {
|
||
const result = cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "git fetch origin G14 --prune >/dev/null 2>&1; git rev-parse origin/G14"], 120_000);
|
||
if (!isCommandSuccess(result)) return null;
|
||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||
return match?.[0] ?? null;
|
||
}
|
||
|
||
function refreshArgoDev(): void {
|
||
cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "annotate", "application", "-n", ARGO_NAMESPACE, DEV_APP, "argocd.argoproj.io/refresh=hard", "--overwrite"], 60_000);
|
||
}
|
||
|
||
function getPipelineStatus(sourceCommit: string): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}"], 60_000);
|
||
}
|
||
|
||
function getArgoStatus(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "application", "-n", ARGO_NAMESPACE, DEV_APP, "-o", "jsonpath={.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.operationState.phase}{\"\\n\"}{.status.operationState.syncResult.revision}{\"\\n\"}"], 60_000);
|
||
}
|
||
|
||
function getDevWorkloads(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "deploy,statefulset,pod", "-n", DEV_NAMESPACE, "-o", "wide"], 60_000);
|
||
}
|
||
|
||
function getLiveHealth(): CommandJsonResult {
|
||
return commandJson(["curl", "-fsS", "--max-time", "20", "http://74.48.78.17:17667/health/live"], 30_000);
|
||
}
|
||
|
||
function statusText(result: CommandJsonResult): string {
|
||
return String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
}
|
||
|
||
function workloadReadiness(workloadsText: string, commandOk: boolean): Record<string, unknown> {
|
||
const blockers: string[] = [];
|
||
const ignoredPods: string[] = [];
|
||
const readyWorkloads: string[] = [];
|
||
const allowedZeroDeployments = new Set(["deployment.apps/hwlab-gateway", "deployment.apps/hwlab-tunnel-client"]);
|
||
for (const line of workloadsText.split(/\r?\n/u)) {
|
||
const trimmed = line.trim();
|
||
if (trimmed.length === 0 || trimmed.startsWith("NAME ")) continue;
|
||
const fields = trimmed.split(/\s+/u);
|
||
const name = fields[0] ?? "";
|
||
const ready = fields[1] ?? "";
|
||
if (name.startsWith("deployment.apps/") || name.startsWith("statefulset.apps/")) {
|
||
if (ready === "0/0" && allowedZeroDeployments.has(name)) {
|
||
readyWorkloads.push(`${name}:${ready}:scaled-zero`);
|
||
continue;
|
||
}
|
||
const match = /^(\d+)\/(\d+)$/u.exec(ready);
|
||
if (match === null) {
|
||
blockers.push(`${name}:unparseable-ready:${ready}`);
|
||
continue;
|
||
}
|
||
const readyCount = Number(match[1]);
|
||
const desiredCount = Number(match[2]);
|
||
if (desiredCount > 0 && readyCount < desiredCount) blockers.push(`${name}:not-ready:${ready}`);
|
||
else readyWorkloads.push(`${name}:${ready}`);
|
||
continue;
|
||
}
|
||
if (name.startsWith("pod/")) {
|
||
const status = fields[2] ?? "";
|
||
if (status === "Completed" || status === "Succeeded") ignoredPods.push(`${name}:${status}`);
|
||
if (/^(CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError|RunContainerError)$/u.test(status)) {
|
||
blockers.push(`${name}:pod-status:${status}`);
|
||
}
|
||
}
|
||
}
|
||
return {
|
||
ready: commandOk && blockers.length === 0,
|
||
blockers,
|
||
readyWorkloads: readyWorkloads.slice(0, 20),
|
||
ignoredPods: ignoredPods.slice(0, 20),
|
||
};
|
||
}
|
||
|
||
function beijingParts(date = new Date()): { date: string; time: string; iso: string } {
|
||
const shifted = new Date(date.getTime() + BEIJING_OFFSET_MS).toISOString();
|
||
return { date: shifted.slice(0, 10), time: shifted.slice(11, 19), iso: shifted };
|
||
}
|
||
|
||
function durationSeconds(start?: string | null, end?: string | null): number | null {
|
||
if (start === undefined || start === null || end === undefined || end === null) return null;
|
||
const startMs = Date.parse(start);
|
||
const endMs = Date.parse(end);
|
||
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) return null;
|
||
return Math.round((endMs - startMs) / 1000);
|
||
}
|
||
|
||
function formatDuration(seconds: number | null): string {
|
||
if (seconds === null) return "n/a";
|
||
const minutes = Math.floor(seconds / 60);
|
||
const rest = seconds % 60;
|
||
if (minutes === 0) return `${rest}s`;
|
||
return `${minutes}m${String(rest).padStart(2, "0")}s`;
|
||
}
|
||
|
||
function pipelineTaskRunResultsJsonPath(): string {
|
||
return [
|
||
"jsonpath=",
|
||
"{range .items[*]}",
|
||
"{.metadata.name}{\"\\t\"}",
|
||
"{.metadata.labels.tekton\\.dev/pipelineTask}{\"\\t\"}",
|
||
"{.status.startTime}{\"\\t\"}",
|
||
"{.status.completionTime}{\"\\t\"}",
|
||
"{range .status.results[*]}{.name}{\"=\"}{.value}{\";\"}{end}",
|
||
"{\"\\n\"}",
|
||
"{end}",
|
||
].join("");
|
||
}
|
||
|
||
function getPipelineTaskRunResults(pipelineRun: string): CommandJsonResult {
|
||
return cliJson([
|
||
"ssh",
|
||
`${G14_PROVIDER}:k3s`,
|
||
"kubectl",
|
||
"get",
|
||
"taskrun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-l",
|
||
`tekton.dev/pipelineRun=${pipelineRun}`,
|
||
"-o",
|
||
pipelineTaskRunResultsJsonPath(),
|
||
], 80_000);
|
||
}
|
||
|
||
function parseResultMap(raw: string): Record<string, string> {
|
||
const values: Record<string, string> = {};
|
||
for (const item of raw.split(";")) {
|
||
if (item.length === 0) continue;
|
||
const separator = item.indexOf("=");
|
||
if (separator <= 0) continue;
|
||
values[item.slice(0, separator)] = item.slice(separator + 1);
|
||
}
|
||
return values;
|
||
}
|
||
|
||
function serviceMetricSort(left: CiServiceMetric, right: CiServiceMetric): number {
|
||
return left.serviceId.localeCompare(right.serviceId);
|
||
}
|
||
|
||
export function parsePipelineTaskRunMetrics(pipelineRun: string, text: string): CiPipelineMetrics {
|
||
const services: CiServiceMetric[] = [];
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const trimmed = line.trimEnd();
|
||
if (trimmed.length === 0) continue;
|
||
const [taskRunName = "", pipelineTask = "", startTime = "", completionTime = "", resultsRaw = ""] = trimmed.split("\t");
|
||
if (!pipelineTask.startsWith("build-")) continue;
|
||
const results = parseResultMap(resultsRaw);
|
||
const serviceId = results["service-id"] ?? pipelineTask.replace(/^build-/u, "");
|
||
if (serviceId.length === 0) continue;
|
||
services.push({
|
||
taskRun: taskRunName,
|
||
serviceId,
|
||
status: results.status ?? "unknown",
|
||
durationSeconds: durationSeconds(startTime, completionTime),
|
||
buildBackend: results["build-backend"] ?? null,
|
||
reusedFrom: results["reused-from"] ?? null,
|
||
imageTag: results["image-tag"] ?? null,
|
||
sourceCommitId: results["source-commit-id"] ?? null,
|
||
});
|
||
}
|
||
services.sort(serviceMetricSort);
|
||
const reusedServices = services.filter((service) => service.status === "reused").map((service) => service.serviceId);
|
||
const rebuildServices = services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
|
||
return {
|
||
ok: services.length > 0,
|
||
pipelineRun,
|
||
serviceTotal: services.length,
|
||
reusedCount: reusedServices.length,
|
||
rebuildCount: rebuildServices.length,
|
||
reusedServices,
|
||
rebuildServices,
|
||
services,
|
||
...(services.length === 0 ? { degradedReason: "no-build-task-results", rawText: text.slice(0, 2000) } : {}),
|
||
};
|
||
}
|
||
|
||
function ciMetricsFromValue(value: unknown): CiPipelineMetrics | null {
|
||
const data = record(value);
|
||
if (data.ok !== true || typeof data.pipelineRun !== "string") return null;
|
||
const services = Array.isArray(data.services) ? data.services.map((item) => {
|
||
const service = record(item);
|
||
return {
|
||
taskRun: String(service.taskRun ?? ""),
|
||
serviceId: String(service.serviceId ?? ""),
|
||
status: String(service.status ?? "unknown"),
|
||
durationSeconds: typeof service.durationSeconds === "number" ? service.durationSeconds : null,
|
||
buildBackend: typeof service.buildBackend === "string" ? service.buildBackend : null,
|
||
reusedFrom: typeof service.reusedFrom === "string" ? service.reusedFrom : null,
|
||
imageTag: typeof service.imageTag === "string" ? service.imageTag : null,
|
||
sourceCommitId: typeof service.sourceCommitId === "string" ? service.sourceCommitId : null,
|
||
};
|
||
}).filter((service) => service.serviceId.length > 0).sort(serviceMetricSort) : [];
|
||
const reusedServices = Array.isArray(data.reusedServices) ? data.reusedServices.map(String) : services.filter((service) => service.status === "reused").map((service) => service.serviceId);
|
||
const rebuildServices = Array.isArray(data.rebuildServices) ? data.rebuildServices.map(String) : services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
|
||
return {
|
||
ok: true,
|
||
pipelineRun: data.pipelineRun,
|
||
serviceTotal: typeof data.serviceTotal === "number" ? data.serviceTotal : services.length,
|
||
reusedCount: typeof data.reusedCount === "number" ? data.reusedCount : reusedServices.length,
|
||
rebuildCount: typeof data.rebuildCount === "number" ? data.rebuildCount : rebuildServices.length,
|
||
reusedServices,
|
||
rebuildServices,
|
||
services,
|
||
};
|
||
}
|
||
|
||
function collectPipelineMetrics(pipelineRun: string): CiPipelineMetrics {
|
||
const result = getPipelineTaskRunResults(pipelineRun);
|
||
if (!isCommandSuccess(result)) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
serviceTotal: 0,
|
||
reusedCount: 0,
|
||
rebuildCount: 0,
|
||
reusedServices: [],
|
||
rebuildServices: [],
|
||
services: [],
|
||
degradedReason: "taskrun-query-failed",
|
||
rawText: `${result.stderr}\n${result.stdout}`.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
return parsePipelineTaskRunMetrics(pipelineRun, statusText(result));
|
||
}
|
||
|
||
function ciMetricLines(metrics: CiPipelineMetrics | null): string[] {
|
||
if (metrics === null) {
|
||
return [
|
||
"- CI/CD 关键指标:",
|
||
" - lazy build reused: n/a",
|
||
" - reused services: n/a",
|
||
" - rebuild services: n/a",
|
||
" - service durations: n/a",
|
||
];
|
||
}
|
||
if (metrics.ok !== true) {
|
||
return [
|
||
"- CI/CD 关键指标:",
|
||
` - lazy build reused: unavailable (${metrics.degradedReason ?? "unknown"})`,
|
||
" - reused services: n/a",
|
||
" - rebuild services: n/a",
|
||
" - service durations: n/a",
|
||
];
|
||
}
|
||
const lines = [
|
||
"- CI/CD 关键指标:",
|
||
` - lazy build reused: ${metrics.reusedCount}/${metrics.serviceTotal}`,
|
||
` - reused services: ${metrics.reusedServices.length > 0 ? metrics.reusedServices.join(", ") : "none"}`,
|
||
` - rebuild services: ${metrics.rebuildServices.length > 0 ? metrics.rebuildServices.join(", ") : "none"}`,
|
||
" - service durations:",
|
||
];
|
||
for (const service of metrics.services) {
|
||
const backend = service.buildBackend === null ? "" : `, backend=${service.buildBackend}`;
|
||
lines.push(` - ${service.serviceId}: ${service.status}, ${formatDuration(service.durationSeconds)}${backend}`);
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
function stripMarkdownInline(value: string): string {
|
||
return value
|
||
.replace(/\[([^\]]+)\]\([^)]+\)/gu, "$1")
|
||
.replace(/\*\*([^*]+)\*\*/gu, "$1")
|
||
.replace(/\*([^*]+)\*/gu, "$1")
|
||
.replace(/__([^_]+)__/gu, "$1")
|
||
.replace(/_([^_]+)_/gu, "$1")
|
||
.replace(/\s+/gu, " ")
|
||
.trim();
|
||
}
|
||
|
||
function markdownSection(body: string, headings: string[]): string {
|
||
const normalizedHeadings = new Set(headings.map((heading) => heading.toLowerCase()));
|
||
const lines = body.split(/\r?\n/u);
|
||
let capturing = false;
|
||
const captured: string[] = [];
|
||
for (const line of lines) {
|
||
const heading = /^(#{1,6})\s+(.+?)\s*$/u.exec(line);
|
||
if (heading !== null) {
|
||
const title = stripMarkdownInline(heading[2] ?? "").replace(/[：:]\s*$/u, "").toLowerCase();
|
||
if (capturing) break;
|
||
capturing = normalizedHeadings.has(title);
|
||
continue;
|
||
}
|
||
if (capturing) captured.push(line);
|
||
}
|
||
return captured.join("\n").trim();
|
||
}
|
||
|
||
function firstParagraph(section: string): string | null {
|
||
for (const paragraph of section.split(/\n\s*\n/u)) {
|
||
const trimmed = stripMarkdownInline(paragraph.replace(/^[-*]\s+/gmu, ""));
|
||
if (trimmed.length > 0) return trimmed;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function topLevelBullets(section: string, limit = 8): string[] {
|
||
const bullets: string[] = [];
|
||
for (const line of section.split(/\r?\n/u)) {
|
||
const match = /^[-*]\s+(.+)$/u.exec(line);
|
||
if (match === null) continue;
|
||
const text = stripMarkdownInline(match[1] ?? "");
|
||
if (text.length > 0) bullets.push(text);
|
||
if (bullets.length >= limit) break;
|
||
}
|
||
return bullets;
|
||
}
|
||
|
||
function conventionalTitleSummary(title: string): string {
|
||
const match = /^(feat|fix|docs|test|refactor|perf|ci|build|chore)(?:\([^)]+\))?:\s*(.+)$/iu.exec(title.trim());
|
||
if (match === null) return stripMarkdownInline(title);
|
||
const kind = String(match[1] ?? "").toLowerCase();
|
||
const subject = stripMarkdownInline(match[2] ?? title);
|
||
const labels: Record<string, string> = {
|
||
feat: "功能",
|
||
fix: "修复",
|
||
docs: "文档",
|
||
test: "测试",
|
||
refactor: "重构",
|
||
perf: "性能",
|
||
ci: "CI/CD",
|
||
build: "构建",
|
||
chore: "维护",
|
||
};
|
||
return `${labels[kind] ?? "变更"}：${subject}`;
|
||
}
|
||
|
||
function fileHeuristicBullets(fileSummary: Record<string, unknown>): string[] {
|
||
const keyFiles = Array.isArray(fileSummary.keyFiles) ? fileSummary.keyFiles.map(String) : [];
|
||
const bullets: string[] = [];
|
||
if (keyFiles.some((file) => file.includes("web/hwlab-cloud-web/"))) {
|
||
bullets.push("前端：更新 HWLAB Cloud Web / Workbench 交互行为。");
|
||
}
|
||
if (keyFiles.some((file) => /\.(test|spec)\./u.test(file) || file.includes("/test"))) {
|
||
bullets.push("验证：补充或更新回归测试，覆盖本次变更路径。");
|
||
}
|
||
if (keyFiles.some((file) => file.includes("deploy/") || file.includes("k8s") || file.includes("gitops"))) {
|
||
bullets.push("部署：更新 G14 Kubernetes/GitOps 发布声明或发布计划。");
|
||
}
|
||
if (keyFiles.some((file) => file.includes("internal/cloud/") || file.includes("code-agent"))) {
|
||
bullets.push("运行时：调整 Cloud API / Code Agent 运行链路。");
|
||
}
|
||
return bullets;
|
||
}
|
||
|
||
export function semanticChangelogBullets(title: string, body: string, fileSummary: Record<string, unknown>): string[] {
|
||
const bullets: string[] = [];
|
||
const background = firstParagraph(markdownSection(body, ["背景", "问题", "problem", "background", "context"]));
|
||
if (background !== null && /^(fix|perf)(?:\([^)]+\))?:/iu.test(title)) {
|
||
bullets.push(`修复目标：${background}`);
|
||
}
|
||
const changeSection = markdownSection(body, ["上线 changelog", "changelog", "摘要", "概述", "总结", "修改", "变更", "改动", "changes", "what changed", "summary"]);
|
||
bullets.push(...topLevelBullets(changeSection));
|
||
if (bullets.length === 0) bullets.push(conventionalTitleSummary(title));
|
||
for (const bullet of fileHeuristicBullets(fileSummary)) {
|
||
if (bullets.length >= 8) break;
|
||
if (!bullets.includes(bullet)) bullets.push(bullet);
|
||
}
|
||
return bullets.slice(0, 8);
|
||
}
|
||
|
||
function readIssue(issueNumber: number): CommandJsonResult {
|
||
return cliJson(["gh", "issue", "read", String(issueNumber), "--repo", HWLAB_REPO, "--json", "title,body,state,updatedAt", "--raw"], 80_000);
|
||
}
|
||
|
||
function issueFromRead(result: CommandJsonResult): Record<string, unknown> {
|
||
return record(commandData(result).issue);
|
||
}
|
||
|
||
function parseBriefIssueFromIndex(indexBody: string, date: string): { number: number; url: string } | null {
|
||
const escapedDate = date.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||
const linePattern = new RegExp(`^\\|\\s*${escapedDate}\\s*\\|([^\\n]+)$`, "mu");
|
||
const line = linePattern.exec(indexBody)?.[0];
|
||
if (line === undefined) return null;
|
||
const match = /\[#(\d+)\]\((https:\/\/github\.com\/pikasTech\/HWLAB\/issues\/\d+)\)/u.exec(line);
|
||
if (match === null) return null;
|
||
return { number: Number(match[1]), url: match[2] };
|
||
}
|
||
|
||
function dailyBriefTitle(date: string): string {
|
||
return `${date} 指挥简报（北京时间）`;
|
||
}
|
||
|
||
function writeStateFile(name: string, content: string): string {
|
||
const dir = rootPath(".state", "hwlab-g14");
|
||
mkdirSync(dir, { recursive: true });
|
||
const path = join(dir, name);
|
||
writeFileSync(path, content, "utf8");
|
||
return path;
|
||
}
|
||
|
||
function listIssues(): CommandJsonResult {
|
||
return cliJson(["gh", "issue", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "100", "--json", "number,title,state,url"], 80_000);
|
||
}
|
||
|
||
function findBriefIssueInList(date: string): { number: number; url: string } | null {
|
||
const listed = listIssues();
|
||
if (!isCommandSuccess(listed)) return null;
|
||
const issues = commandData(listed).issues;
|
||
if (!Array.isArray(issues)) return null;
|
||
const title = dailyBriefTitle(date);
|
||
const match = issues.map((item) => record(item)).find((issue) => issue.title === title);
|
||
if (match === undefined) return null;
|
||
return { number: Number(match.number), url: String(match.url) };
|
||
}
|
||
|
||
function newDailyBriefBody(date: string): string {
|
||
return [
|
||
`# ${dailyBriefTitle(date)}`,
|
||
"",
|
||
"## 当日更新",
|
||
"",
|
||
"## 常驻观察与长期建议",
|
||
"",
|
||
"- G14 HWLAB source truth 固定为 G14 `/root/hwlab` 与 `origin/G14`；DEV rollout 只接受 G14 Tekton/GitOps/Argo 和公网 health 证据。",
|
||
"- GitHub issue/PR 写操作必须走 UniDesk `gh` 子命令；G14 k3s 操作必须走 UniDesk `ssh G14:k3s`。",
|
||
].join("\n");
|
||
}
|
||
|
||
function insertBriefIndexRow(indexBody: string, date: string, brief: { number: number; url: string }): string {
|
||
if (parseBriefIssueFromIndex(indexBody, date) !== null) return indexBody;
|
||
const row = `| ${date} | [#${brief.number}](${brief.url}) | 已创建当日 G14 DEV rollout 自动简报入口；后续每次 DEV 上线记录 CI/CD 耗时和上线 changelog。 | 继续监控 HWLAB base=G14 PR，ready 后自动合并并滚动到 G14 DEV。 |`;
|
||
const marker = "### 指挥简报索引";
|
||
const markerIndex = indexBody.indexOf(marker);
|
||
if (markerIndex === -1) return `${indexBody.trimEnd()}\n\n${marker}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n`;
|
||
const afterMarker = indexBody.slice(markerIndex);
|
||
const separator = "| --- | --- | --- | --- |";
|
||
const separatorInBody = afterMarker.indexOf(separator);
|
||
if (separatorInBody === -1) return `${indexBody.slice(0, markerIndex + marker.length)}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n${indexBody.slice(markerIndex + marker.length)}`;
|
||
const insertAt = markerIndex + separatorInBody + separator.length;
|
||
return `${indexBody.slice(0, insertAt)}\n${row}${indexBody.slice(insertAt)}`;
|
||
}
|
||
|
||
function ensureDailyBriefIssue(date: string, dryRun: boolean): Record<string, unknown> {
|
||
const indexRead = readIssue(G14_BRIEF_INDEX_ISSUE);
|
||
if (!isCommandSuccess(indexRead)) return { ok: false, phase: "read-index", indexRead };
|
||
const indexIssue = issueFromRead(indexRead);
|
||
const indexBody = String(indexIssue.body ?? "");
|
||
let brief = parseBriefIssueFromIndex(indexBody, date) ?? findBriefIssueInList(date);
|
||
const actions: Record<string, unknown>[] = [];
|
||
if (brief === null) {
|
||
const bodyPath = writeStateFile(`daily-brief-${date}.md`, `${newDailyBriefBody(date)}\n`);
|
||
if (dryRun) {
|
||
brief = { number: 0, url: `https://github.com/pikasTech/HWLAB/issues/new?title=${encodeURIComponent(dailyBriefTitle(date))}` };
|
||
actions.push({ action: "would-create-daily-brief", title: dailyBriefTitle(date), bodyPath });
|
||
} else {
|
||
const created = cliJson(["gh", "issue", "create", "--repo", HWLAB_REPO, "--title", dailyBriefTitle(date), "--body-file", bodyPath], 80_000);
|
||
if (!isCommandSuccess(created)) return { ok: false, phase: "create-daily-brief", created };
|
||
const createdIssue = record(commandData(created).issue);
|
||
brief = { number: Number(createdIssue.number), url: String(createdIssue.url) };
|
||
actions.push({ action: "created-daily-brief", issue: brief, bodyPath });
|
||
}
|
||
}
|
||
if (brief.number > 0 && parseBriefIssueFromIndex(indexBody, date) === null) {
|
||
const nextBody = insertBriefIndexRow(indexBody, date, brief);
|
||
const bodyPath = writeStateFile(`issue-${G14_BRIEF_INDEX_ISSUE}-brief-index-${date}.md`, `${nextBody.trimEnd()}\n`);
|
||
if (dryRun) {
|
||
actions.push({ action: "would-update-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||
} else {
|
||
const bodySha = String(indexIssue.bodySha ?? "");
|
||
const update = cliJson(["gh", "issue", "update", String(G14_BRIEF_INDEX_ISSUE), "--repo", HWLAB_REPO, "--mode", "replace", "--body-file", bodyPath, ...(bodySha.length > 0 ? ["--expect-body-sha", bodySha] : [])], 100_000);
|
||
if (!isCommandSuccess(update)) return { ok: false, phase: "update-brief-index", update, bodyPath };
|
||
actions.push({ action: "updated-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||
}
|
||
}
|
||
return { ok: true, date, brief, actions };
|
||
}
|
||
|
||
function readPullRequest(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "read", String(number), "--repo", HWLAB_REPO, "--json", "title,body,url,stateDetail,merged,mergedAt,mergeCommit,headRefName,baseRefName"], 80_000);
|
||
}
|
||
|
||
function readPullRequestFiles(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "files", String(number), "--repo", HWLAB_REPO, "--limit", "30"], 80_000);
|
||
}
|
||
|
||
function summarizePrFiles(filesResult: CommandJsonResult): Record<string, unknown> {
|
||
if (!isCommandSuccess(filesResult)) return { ok: false, filesResult };
|
||
const data = commandData(filesResult);
|
||
const files = Array.isArray(data.files) ? data.files.map((item) => record(item)) : [];
|
||
return {
|
||
ok: true,
|
||
summary: data.summary ?? null,
|
||
keyFiles: files.slice(0, 8).map((file) => `${file.status ?? "changed"} ${file.filename ?? ""}`),
|
||
};
|
||
}
|
||
|
||
function mergeCommitFromPr(prData: Record<string, unknown>): string | null {
|
||
const mergeCommit = record(record(prData.json).mergeCommit);
|
||
const fromJson = mergeCommit.oid;
|
||
if (typeof fromJson === "string") return fromJson;
|
||
const fromSummary = record(record(prData.pullRequest).mergeCommit).oid;
|
||
return typeof fromSummary === "string" ? fromSummary : null;
|
||
}
|
||
|
||
function currentGitopsRevision(): string | null {
|
||
const argo = getArgoStatus();
|
||
if (!isCommandSuccess(argo)) return null;
|
||
return statusText(argo).split(/\r?\n/u)[0] ?? null;
|
||
}
|
||
|
||
export function rolloutRecordBody(input: {
|
||
pr: OpenPullRequest;
|
||
prData: Record<string, unknown>;
|
||
fileSummary: Record<string, unknown>;
|
||
sourceCommit: string;
|
||
pipelineRun: string;
|
||
gitopsRevision: string | null;
|
||
mergedAt: string | null;
|
||
pipelineSucceededAt: string | null;
|
||
finishedAt: string | null;
|
||
rollout: Record<string, unknown>;
|
||
ciMetrics?: CiPipelineMetrics | null;
|
||
}): string {
|
||
const now = beijingParts();
|
||
const title = String(record(input.prData.json).title ?? input.pr.title ?? `PR #${input.pr.number}`);
|
||
const prBody = String(record(input.prData.json).body ?? record(input.prData.pullRequest).body ?? "");
|
||
const url = String(record(input.prData.json).url ?? input.pr.url ?? `https://github.com/pikasTech/HWLAB/pull/${input.pr.number}`);
|
||
const summary = record(input.fileSummary.summary);
|
||
const keyFiles = Array.isArray(input.fileSummary.keyFiles) ? input.fileSummary.keyFiles.map(String) : [];
|
||
const semanticBullets = semanticChangelogBullets(title, prBody, input.fileSummary);
|
||
const mergeToPipeline = durationSeconds(input.mergedAt, input.pipelineSucceededAt);
|
||
const pipelineToDev = durationSeconds(input.pipelineSucceededAt, input.finishedAt);
|
||
const mergeToDev = durationSeconds(input.mergedAt, input.finishedAt);
|
||
const metrics = input.ciMetrics ?? ciMetricsFromValue(input.rollout.ciMetrics);
|
||
return [
|
||
"",
|
||
"",
|
||
`## 更新 ${now.date} ${now.time.slice(0, 5)} 北京时间`,
|
||
"",
|
||
`### G14 DEV rollout：PR #${input.pr.number}`,
|
||
"",
|
||
`- PR: [#${input.pr.number} ${title}](${url})`,
|
||
`- 合并 commit: \`${input.sourceCommit}\``,
|
||
`- GitOps revision: \`${input.gitopsRevision ?? "unknown"}\``,
|
||
`- PipelineRun: \`${input.pipelineRun}\``,
|
||
"- CI/CD 耗时:",
|
||
` - merge -> pipeline succeeded: ${formatDuration(mergeToPipeline)}`,
|
||
` - pipeline succeeded -> DEV Healthy: ${formatDuration(pipelineToDev)}`,
|
||
` - merge -> DEV Healthy: ${formatDuration(mergeToDev)}`,
|
||
...ciMetricLines(metrics),
|
||
"- 上线 changelog（语义化）:",
|
||
...semanticBullets.map((bullet) => ` - ${bullet}`),
|
||
"- 自动 diff 摘要:",
|
||
` - changed files: ${String(summary.files ?? "n/a")}; +${String(summary.additions ?? "n/a")} / -${String(summary.deletions ?? "n/a")}; commits: ${String(summary.commits ?? "n/a")}`,
|
||
...keyFiles.map((file) => ` - ${file}`),
|
||
"- DEV 验证:",
|
||
` - Tekton: ${String(input.rollout.pipelineText ?? "Succeeded").split(/\r?\n/u).join(" / ")}`,
|
||
` - Argo: ${String(input.rollout.argoText ?? "Synced / Healthy").split(/\r?\n/u).slice(0, 5).join(" / ")}`,
|
||
` - health/live: ${input.rollout.healthOk === false ? "not-ok" : "ok"}`,
|
||
" - workload readiness: Deployment/StatefulSet ready；历史 Completed smoke/debug pod 不作为 DEV rollout blocker。",
|
||
].join("\n");
|
||
}
|
||
|
||
function appendRolloutBrief(options: G14RecordRolloutOptions, rollout: Record<string, unknown> = {}): Record<string, unknown> {
|
||
const now = beijingParts();
|
||
const ensured = ensureDailyBriefIssue(now.date, options.dryRun);
|
||
if (record(ensured).ok !== true) return ensured;
|
||
const brief = record(record(ensured).brief);
|
||
const prRead = readPullRequest(options.prNumber);
|
||
if (!isCommandSuccess(prRead)) return { ok: false, phase: "read-pr", prRead };
|
||
const prData = commandData(prRead);
|
||
const sourceCommit = options.sourceCommit ?? mergeCommitFromPr(prData);
|
||
if (sourceCommit === null || sourceCommit === undefined) return { ok: false, phase: "source-commit", message: "source commit unavailable", prData };
|
||
const pipelineRun = options.pipelineRun ?? `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||
const files = summarizePrFiles(readPullRequestFiles(options.prNumber));
|
||
const rolloutGitopsRevision = typeof rollout.gitopsRevision === "string" && rollout.gitopsRevision.length > 0 ? rollout.gitopsRevision : null;
|
||
const gitopsRevision = options.gitopsRevision ?? rolloutGitopsRevision ?? currentGitopsRevision();
|
||
const prJson = record(prData.json);
|
||
const prSummary = record(prData.pullRequest);
|
||
const prMergedAt = typeof prJson.mergedAt === "string" && prJson.mergedAt.length > 0
|
||
? prJson.mergedAt
|
||
: typeof prSummary.mergedAt === "string" && prSummary.mergedAt.length > 0
|
||
? prSummary.mergedAt
|
||
: null;
|
||
const rolloutPipelineSucceededAt = typeof rollout.pipelineSucceededAt === "string" && rollout.pipelineSucceededAt.length > 0 ? rollout.pipelineSucceededAt : null;
|
||
const rolloutFinishedAt = typeof rollout.finishedAt === "string" && rollout.finishedAt.length > 0 ? rollout.finishedAt : null;
|
||
const mergedAt = options.mergedAt ?? prMergedAt;
|
||
const pipelineSucceededAt = options.pipelineSucceededAt ?? rolloutPipelineSucceededAt;
|
||
const finishedAt = options.finishedAt ?? rolloutFinishedAt;
|
||
const ciMetrics = ciMetricsFromValue(rollout.ciMetrics) ?? collectPipelineMetrics(pipelineRun);
|
||
const pr: OpenPullRequest = {
|
||
number: options.prNumber,
|
||
title: String(record(prData.json).title ?? ""),
|
||
url: String(record(prData.json).url ?? ""),
|
||
baseRefName: String(record(prData.json).baseRefName ?? ""),
|
||
headRefName: String(record(prData.json).headRefName ?? ""),
|
||
};
|
||
const body = rolloutRecordBody({ pr, prData, fileSummary: files, sourceCommit, pipelineRun, gitopsRevision, mergedAt, pipelineSucceededAt, finishedAt, rollout, ciMetrics });
|
||
if (Number(brief.number) <= 0 || options.dryRun) {
|
||
return { ok: true, dryRun: true, date: now.date, brief, wouldAppend: { bodyPreview: body.slice(0, 1000), bodyChars: body.length }, ensured };
|
||
}
|
||
const currentBrief = readIssue(Number(brief.number));
|
||
if (!isCommandSuccess(currentBrief)) return { ok: false, phase: "read-daily-brief", currentBrief, brief };
|
||
const currentBody = String(issueFromRead(currentBrief).body ?? "");
|
||
if (currentBody.includes(`PR #${options.prNumber}`) && currentBody.includes(sourceCommit)) {
|
||
return { ok: true, skipped: true, reason: "rollout-brief-already-recorded", brief, sourceCommit, ensured };
|
||
}
|
||
const bodyPath = writeStateFile(`rollout-pr-${options.prNumber}-${shortSha(sourceCommit)}.md`, `${body}\n`);
|
||
const update = cliJson(["gh", "issue", "update", String(brief.number), "--repo", HWLAB_REPO, "--mode", "append", "--body-file", bodyPath, "--body-profile", "commander-brief"], 100_000);
|
||
if (!isCommandSuccess(update)) return { ok: false, phase: "append-rollout-brief", update, bodyPath, brief, ensured };
|
||
return { ok: true, date: now.date, brief, sourceCommit, pipelineRun, gitopsRevision, ciMetrics, bodyPath, update: commandData(update), ensured };
|
||
}
|
||
|
||
async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
const startedAt = new Date(started).toISOString();
|
||
let pipelineText = "";
|
||
let argoText = "";
|
||
let healthOk = false;
|
||
let workloadsReady: Record<string, unknown> = { ready: false };
|
||
let pipelineSucceededAt: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
const pipeline = getPipelineStatus(sourceCommit);
|
||
pipelineText = statusText(pipeline);
|
||
printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) });
|
||
if (!pipelineText.startsWith("True\nSucceeded")) {
|
||
if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText };
|
||
await sleep(30_000);
|
||
continue;
|
||
}
|
||
pipelineSucceededAt ??= new Date().toISOString();
|
||
refreshArgoDev();
|
||
const argo = getArgoStatus();
|
||
argoText = statusText(argo);
|
||
printEvent("g14.argo.status", { text: argoText.slice(0, 500) });
|
||
const argoLines = argoText.split(/\r?\n/u);
|
||
const synced = argoLines[1] === "Synced";
|
||
const healthy = argoLines[2] === "Healthy";
|
||
const gitopsRevision = argoLines[0] ?? null;
|
||
const workloads = getDevWorkloads();
|
||
const workloadsText = statusText(workloads);
|
||
workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads));
|
||
const health = getLiveHealth();
|
||
healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout);
|
||
if (synced && healthy && record(workloadsReady).ready === true && healthOk) {
|
||
const finishedAt = new Date().toISOString();
|
||
const pipelineRun = `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||
const ciMetrics = collectPipelineMetrics(pipelineRun);
|
||
return { ok: true, sourceCommit, gitopsRevision, pipelineRun, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk, ciMetrics };
|
||
}
|
||
await sleep(30_000);
|
||
}
|
||
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
|
||
}
|
||
|
||
async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||
const precheck = precheckWorkspace();
|
||
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
|
||
const listed = listOpenG14PullRequests();
|
||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||
const prs = extractPullRequests(listed);
|
||
printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||
if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] };
|
||
const merged: unknown[] = [];
|
||
for (const pr of prs) {
|
||
const preflight = preflightPullRequest(pr.number);
|
||
printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) });
|
||
if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) {
|
||
return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight };
|
||
}
|
||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||
printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||
if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge };
|
||
const sourceCommit = getG14Head();
|
||
const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds);
|
||
const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true
|
||
? { skipped: true, dryRun: options.dryRun }
|
||
: appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout);
|
||
printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null });
|
||
merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief });
|
||
if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged };
|
||
if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged };
|
||
}
|
||
return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged };
|
||
}
|
||
|
||
async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<string, unknown>> {
|
||
const maxCycles = options.once ? 1 : options.maxCycles;
|
||
let cycle = 0;
|
||
const results: unknown[] = [];
|
||
while (maxCycles === 0 || cycle < maxCycles) {
|
||
cycle += 1;
|
||
const result = await monitorCycle(options, cycle);
|
||
results.push(result);
|
||
printEvent("g14.monitor.cycle.done", { cycle, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
||
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
|
||
if (options.once || record(result).action !== "none") return { ok: true, cycles: cycle, lastResult: result, results };
|
||
printEvent("g14.monitor.sleep", { cycle, intervalSeconds: options.intervalSeconds });
|
||
await sleep(options.intervalSeconds * 1000);
|
||
}
|
||
return { ok: true, cycles: cycle, results };
|
||
}
|
||
|
||
export function hwlabG14Help(): Record<string, unknown> {
|
||
return {
|
||
command: "hwlab g14",
|
||
output: "json",
|
||
usage: [
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs",
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs --once --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 record-rollout --pr <number> [--source-commit sha]",
|
||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
|
||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit <full-sha>",
|
||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror status",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||
],
|
||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.",
|
||
defaults: {
|
||
repo: HWLAB_REPO,
|
||
base: G14_SOURCE_BRANCH,
|
||
provider: G14_PROVIDER,
|
||
workspace: G14_WORKSPACE,
|
||
v02Workspace: V02_WORKSPACE,
|
||
ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO,
|
||
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
|
||
devApplication: DEV_APP,
|
||
v02Application: V02_APP,
|
||
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
|
||
},
|
||
stateFiles: {
|
||
monitor: ".state/hwlab-g14/latest-monitor-job.json",
|
||
once: ".state/hwlab-g14/latest-once-job.json",
|
||
dryRun: ".state/hwlab-g14/latest-dry-run-job.json",
|
||
onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json",
|
||
},
|
||
};
|
||
}
|
||
|
||
export async function runHwlabG14Command(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
||
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return { ok: true, ...hwlabG14Help() };
|
||
const [action] = args;
|
||
if (action === "record-rollout") {
|
||
const options = parseRecordRolloutOptions(args.slice(1));
|
||
return appendRolloutBrief(options);
|
||
}
|
||
if (action === "control-plane") {
|
||
const options = parseControlPlaneOptions(args.slice(1));
|
||
if (options.action === "trigger-current" && options.confirm && !options.dryRun && !options.wait) {
|
||
return startControlPlaneTriggerJob(options);
|
||
}
|
||
return runV02ControlPlane(options);
|
||
}
|
||
if (action === "tools-image") {
|
||
const options = parseToolsImageOptions(args.slice(1));
|
||
return runG14ToolsImage(options);
|
||
}
|
||
if (action === "git-mirror") {
|
||
const options = parseGitMirrorOptions(args.slice(1));
|
||
if ((options.action === "sync" || options.action === "flush") && options.confirm && !options.dryRun && !options.wait) {
|
||
return startGitMirrorJob(options);
|
||
}
|
||
return runG14GitMirror(options);
|
||
}
|
||
if (action !== "monitor-prs") {
|
||
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 record-rollout, hwlab g14 control-plane, hwlab g14 git-mirror, hwlab g14 tools-image" };
|
||
}
|
||
const options = parseOptions(args.slice(1));
|
||
if (options.worker) return runMonitorWorker(options);
|
||
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
|
||
const job = startJob("hwlab_g14_pr_monitor", command, `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`);
|
||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
|
||
const stateDir = rootPath(".state", "hwlab-g14");
|
||
mkdirSync(stateDir, { recursive: true });
|
||
const stateFileName = hwlabG14MonitorStateFileName(options);
|
||
const stateFileRole = hwlabG14MonitorStateRole(options);
|
||
const latestPath = join(stateDir, stateFileName);
|
||
const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null;
|
||
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole }, null, 2)}\n`, "utf8");
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 monitor-prs",
|
||
mode: "async-job",
|
||
job,
|
||
statusCommand,
|
||
latestPath,
|
||
stateFileName,
|
||
stateFileRole,
|
||
previousLatest,
|
||
next: {
|
||
status: statusCommand,
|
||
tail: `tail -f ${job.stdoutFile}`,
|
||
},
|
||
};
|
||
}
|