Codex
5474 lines
239 KiB
TypeScript
5474 lines
239 KiB
TypeScript
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||
import { dirname, join } from "node:path";
|
||
import { createHash } from "node:crypto";
|
||
import { repoRoot, rootPath, type Config } from "./config";
|
||
import { runCommand } from "./command";
|
||
import { startJob } from "./jobs";
|
||
|
||
const HWLAB_REPO = "pikasTech/HWLAB";
|
||
const G14_SOURCE_BRANCH = "G14";
|
||
const G14_PROVIDER = "G14";
|
||
const G14_WORKSPACE = "/root/hwlab";
|
||
const V02_SOURCE_BRANCH = "v0.2";
|
||
const V02_WORKSPACE = "/root/hwlab-v02";
|
||
const V02_CICD_REPO = "/root/hwlab-v02-cicd.git";
|
||
const DEV_NAMESPACE = "hwlab-dev";
|
||
const CI_NAMESPACE = "hwlab-ci";
|
||
const ARGO_NAMESPACE = "argocd";
|
||
const DEV_APP = "hwlab-g14-dev";
|
||
const V02_APP = "hwlab-g14-v02";
|
||
const V02_PIPELINE = "hwlab-v02-ci-image-publish";
|
||
const V02_POLLER = "hwlab-v02-branch-poller";
|
||
const V02_RECONCILER = "hwlab-v02-control-plane-reconciler";
|
||
const V02_PIPELINERUN_PREFIX = "hwlab-v02-ci-poll";
|
||
const V02_CONTROL_PLANE_FIELD_MANAGER = "unidesk-hwlab-v02-control-plane";
|
||
const V02_SECRET_FIELD_MANAGER = "unidesk-hwlab-v02-secret";
|
||
const V02_GIT_URL = "git@github.com:pikasTech/HWLAB.git";
|
||
const V02_GIT_READ_URL = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git";
|
||
const V02_GIT_WRITE_URL = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git";
|
||
const V02_GITOPS_BRANCH = "v0.2-gitops";
|
||
const V02_CATALOG_PATH = "deploy/artifact-catalog.v02.json";
|
||
const V02_RUNTIME_PATH = "deploy/gitops/g14/runtime-v02";
|
||
const V02_RUNTIME_NAMESPACE = "hwlab-v02";
|
||
const V02_DEVICE_POD_API_KEY_SECRET = "hwlab-v02-device-pod-api-key";
|
||
const V02_DEVICE_POD_API_KEY_SECRET_KEY = "api-key";
|
||
const V02_REGISTRY_PREFIX = "127.0.0.1:5000/hwlab";
|
||
const V02_BASE_IMAGE = "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim";
|
||
const GIT_MIRROR_NAMESPACE = "devops-infra";
|
||
const GIT_MIRROR_MANIFEST_FIELD_MANAGER = "unidesk-hwlab-git-mirror";
|
||
const GIT_MIRROR_SYNC_JOB_PREFIX = "git-mirror-hwlab-sync-manual";
|
||
const GIT_MIRROR_LEGACY_CRONJOB = "git-mirror-hwlab-sync";
|
||
const V02_SERVICE_IDS = [
|
||
"hwlab-cloud-api",
|
||
"hwlab-cloud-web",
|
||
"hwlab-agent-mgr",
|
||
"hwlab-agent-worker",
|
||
"hwlab-device-pod",
|
||
"hwlab-gateway",
|
||
"hwlab-edge-proxy",
|
||
"hwlab-agent-skills",
|
||
];
|
||
const V02_CLOUD_WEB_URL = "http://74.48.78.17:19666";
|
||
const V02_CLOUD_API_URL = "http://74.48.78.17:19667";
|
||
|
||
export function v02PipelineServiceIds(): string[] {
|
||
return [...V02_SERVICE_IDS];
|
||
}
|
||
|
||
const G14_CI_TOOLS_IMAGE_REPO = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools";
|
||
const G14_CI_TOOLS_BASE_TAG = "node22-alpine-v1";
|
||
const DEFAULT_INTERVAL_SECONDS = 600;
|
||
const DEFAULT_MAX_CYCLES = 0;
|
||
const DEFAULT_TIMEOUT_SECONDS = 1800;
|
||
const G14_BRIEF_INDEX_ISSUE = 7;
|
||
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||
const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
|
||
const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
|
||
const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000;
|
||
const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000;
|
||
const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000;
|
||
const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000;
|
||
const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000;
|
||
|
||
export type G14MonitorLane = "g14" | "v02";
|
||
|
||
interface G14MonitorOptions {
|
||
lane: G14MonitorLane;
|
||
intervalSeconds: number;
|
||
maxCycles: number;
|
||
once: boolean;
|
||
dryRun: boolean;
|
||
worker: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface G14RecordRolloutOptions {
|
||
prNumber: number;
|
||
sourceCommit?: string;
|
||
pipelineRun?: string;
|
||
gitopsRevision?: string;
|
||
mergedAt?: string;
|
||
pipelineSucceededAt?: string;
|
||
finishedAt?: string;
|
||
dryRun: boolean;
|
||
}
|
||
|
||
interface G14ControlPlaneOptions {
|
||
action: "status" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration";
|
||
lane: "v02" | "g14" | "all";
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
wait: boolean;
|
||
allowLiveDbRead: boolean;
|
||
timeoutSeconds: number;
|
||
minAgeMinutes: number;
|
||
limit: number;
|
||
sourceCommit?: string;
|
||
pipelineRun?: string;
|
||
}
|
||
|
||
type V02StatusTargetMode = "latest-source-head" | "source-commit" | "pipeline-run";
|
||
|
||
interface V02ControlPlaneStatusTarget {
|
||
sourceCommit?: string | null;
|
||
pipelineRun?: string | null;
|
||
mode?: V02StatusTargetMode;
|
||
}
|
||
|
||
interface G14ToolsImageOptions {
|
||
action: "status" | "build";
|
||
name: "ci-node-tools";
|
||
tag: string;
|
||
dockerfile: string;
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface G14GitMirrorOptions {
|
||
action: "status" | "apply" | "sync" | "flush";
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
wait: boolean;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface G14SecretOptions {
|
||
action: "status" | "ensure";
|
||
lane: "v02";
|
||
dryRun: boolean;
|
||
confirm: boolean;
|
||
name: typeof V02_DEVICE_POD_API_KEY_SECRET;
|
||
key: typeof V02_DEVICE_POD_API_KEY_SECRET_KEY;
|
||
timeoutSeconds: number;
|
||
}
|
||
|
||
interface CommandJsonResult {
|
||
ok: boolean;
|
||
command: string[];
|
||
exitCode: number | null;
|
||
stdout: string;
|
||
stderr: string;
|
||
parsed: unknown | null;
|
||
}
|
||
|
||
interface ShellSection {
|
||
stdout: string;
|
||
exitCode: number | null;
|
||
}
|
||
|
||
export interface OpenPullRequest {
|
||
number: number;
|
||
title?: string;
|
||
url?: string;
|
||
baseRefName?: string;
|
||
headRefName?: string;
|
||
mergeable?: string | null;
|
||
mergeStateStatus?: string | null;
|
||
draft?: boolean;
|
||
}
|
||
|
||
interface CiServiceMetric {
|
||
taskRun: string;
|
||
serviceId: string;
|
||
status: string;
|
||
durationSeconds: number | null;
|
||
buildBackend: string | null;
|
||
reusedFrom: string | null;
|
||
imageTag: string | null;
|
||
sourceCommitId: string | null;
|
||
}
|
||
|
||
interface CiPipelineMetrics {
|
||
ok: boolean;
|
||
pipelineRun: string;
|
||
serviceTotal: number;
|
||
reusedCount: number;
|
||
rebuildCount: number;
|
||
reusedServices: string[];
|
||
rebuildServices: string[];
|
||
services: CiServiceMetric[];
|
||
degradedReason?: string;
|
||
rawText?: string;
|
||
}
|
||
|
||
export interface V02PrCommentInput {
|
||
pr: OpenPullRequest;
|
||
phase: string;
|
||
state: "waiting-ci" | "blocked" | "merge-failed" | "cd-started" | "cd-succeeded" | "cd-failed" | "cd-timeout" | "cd-blocked";
|
||
startedAt: string;
|
||
observedAt: string;
|
||
elapsedSeconds: number | null;
|
||
preflight?: Record<string, unknown>;
|
||
merge?: Record<string, unknown>;
|
||
sourceCommit?: string | null;
|
||
pipelineRun?: string | null;
|
||
cd?: Record<string, unknown>;
|
||
flush?: Record<string, unknown>;
|
||
dryRun?: boolean;
|
||
message?: string;
|
||
}
|
||
|
||
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||
const prefix = options.lane === "v02" ? "latest-v02-" : "latest-";
|
||
if (options.once && options.dryRun) return options.lane === "v02" ? `${prefix}once-dry-run-job.json` : "latest-once-dry-run-job.json";
|
||
if (options.once && options.lane === "v02") return `${prefix}once-job.json`;
|
||
if (options.once) return "latest-once-job.json";
|
||
if (options.dryRun) return `${prefix}dry-run-job.json`;
|
||
return `${prefix}monitor-job.json`;
|
||
}
|
||
|
||
function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||
const lanePrefix = options.lane === "v02" ? "v02-" : "";
|
||
if (options.once && options.dryRun) return `${lanePrefix}once-dry-run`;
|
||
if (options.once) return `${lanePrefix}once`;
|
||
if (options.dryRun) return `${lanePrefix}dry-run-monitor`;
|
||
return `${lanePrefix}monitor`;
|
||
}
|
||
|
||
function parseMonitorLane(args: string[]): G14MonitorLane {
|
||
const lane = optionValue(args, "--lane") ?? "g14";
|
||
if (lane !== "g14" && lane !== "v02") throw new Error("monitor-prs --lane must be g14 or v02");
|
||
return lane;
|
||
}
|
||
|
||
function parseOptions(args: string[]): G14MonitorOptions {
|
||
return {
|
||
lane: parseMonitorLane(args),
|
||
intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400),
|
||
maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000),
|
||
once: args.includes("--once"),
|
||
dryRun: args.includes("--dry-run"),
|
||
worker: args.includes("--worker"),
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", DEFAULT_TIMEOUT_SECONDS, 86400),
|
||
};
|
||
}
|
||
|
||
function optionValue(args: string[], name: string): string | undefined {
|
||
const index = args.indexOf(name);
|
||
if (index === -1) return undefined;
|
||
const value = args[index + 1];
|
||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||
return value;
|
||
}
|
||
|
||
function validateFullShaOption(value: string, name: string): string {
|
||
if (!/^[0-9a-f]{40}$/iu.test(value)) throw new Error(`${name} must be a full 40-character git SHA`);
|
||
return value.toLowerCase();
|
||
}
|
||
|
||
function validateV02PipelineRunOption(value: string): string {
|
||
if (!/^hwlab-v02-ci-poll-[0-9a-f]{7,40}$/iu.test(value)) throw new Error("--pipeline-run must be a hwlab-v02-ci-poll-<sha> PipelineRun name");
|
||
return value.toLowerCase();
|
||
}
|
||
|
||
function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||
const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number");
|
||
const prNumber = Number(prRaw);
|
||
if (!Number.isInteger(prNumber) || prNumber <= 0) throw new Error("record-rollout requires --pr <number>");
|
||
return {
|
||
prNumber,
|
||
sourceCommit: optionValue(args, "--source-commit"),
|
||
pipelineRun: optionValue(args, "--pipeline-run"),
|
||
gitopsRevision: optionValue(args, "--gitops-revision"),
|
||
mergedAt: optionValue(args, "--merged-at"),
|
||
pipelineSucceededAt: optionValue(args, "--pipeline-succeeded-at"),
|
||
finishedAt: optionValue(args, "--finished-at"),
|
||
dryRun: args.includes("--dry-run"),
|
||
};
|
||
}
|
||
|
||
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||
const [rawAction] = args;
|
||
const actionRaw = rawAction === "rerun-current" ? "trigger-current" : rawAction;
|
||
if (
|
||
actionRaw !== "status" &&
|
||
actionRaw !== "apply" &&
|
||
actionRaw !== "trigger-current" &&
|
||
actionRaw !== "cleanup-runs" &&
|
||
actionRaw !== "cleanup-released-pvs" &&
|
||
actionRaw !== "runtime-migration"
|
||
) {
|
||
throw new Error("control-plane usage: status|apply|trigger-current|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
|
||
}
|
||
const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
|
||
if (actionRaw === "cleanup-runs") {
|
||
if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
|
||
} else if (actionRaw === "cleanup-released-pvs") {
|
||
if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
|
||
} else if (lane !== "v02") {
|
||
throw new Error("control-plane status/apply/trigger-current/runtime-migration currently requires --lane v02");
|
||
}
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run");
|
||
const allowLiveDbRead = args.includes("--allow-live-db-read");
|
||
if (allowLiveDbRead && confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
|
||
if (allowLiveDbRead && actionRaw !== "runtime-migration") throw new Error("--allow-live-db-read is only valid for control-plane runtime-migration");
|
||
const sourceCommitRaw = optionValue(args, "--source-commit");
|
||
const pipelineRunRaw = optionValue(args, "--pipeline-run");
|
||
if ((sourceCommitRaw !== undefined || pipelineRunRaw !== undefined) && actionRaw !== "status") {
|
||
throw new Error("--source-commit and --pipeline-run are only valid for control-plane status");
|
||
}
|
||
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) throw new Error("control-plane status accepts only one of --source-commit or --pipeline-run");
|
||
return {
|
||
action: actionRaw,
|
||
lane,
|
||
confirm,
|
||
wait: args.includes("--wait"),
|
||
allowLiveDbRead,
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
||
limit: positiveIntegerOption(args, "--limit", 20, 200),
|
||
sourceCommit: sourceCommitRaw === undefined ? undefined : validateFullShaOption(sourceCommitRaw, "--source-commit"),
|
||
pipelineRun: pipelineRunRaw === undefined ? undefined : validateV02PipelineRunOption(pipelineRunRaw),
|
||
};
|
||
}
|
||
|
||
function parseToolsImageOptions(args: string[]): G14ToolsImageOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "status" && actionRaw !== "build") {
|
||
throw new Error("tools-image usage: status|build --name ci-node-tools --tag <tag> [--dockerfile path] [--dry-run|--confirm]");
|
||
}
|
||
const name = optionValue(args, "--name") ?? "ci-node-tools";
|
||
if (name !== "ci-node-tools") throw new Error("tools-image currently supports --name ci-node-tools");
|
||
const tag = optionValue(args, "--tag") ?? G14_CI_TOOLS_BASE_TAG;
|
||
if (!/^[A-Za-z0-9_.-]+$/u.test(tag)) throw new Error("--tag may only contain letters, numbers, dot, underscore, and dash");
|
||
const dockerfile = optionValue(args, "--dockerfile") ?? "deploy/ci/hwlab-ci-node-tools.Dockerfile";
|
||
if (dockerfile.startsWith("/") || dockerfile.includes("..")) throw new Error("--dockerfile must be a repo-relative path without '..'");
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run");
|
||
return {
|
||
action: actionRaw,
|
||
name,
|
||
tag,
|
||
dockerfile,
|
||
confirm,
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
|
||
};
|
||
}
|
||
|
||
function parseGitMirrorOptions(args: string[]): G14GitMirrorOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "sync" && actionRaw !== "flush") {
|
||
throw new Error("git-mirror usage: status|apply|sync|flush [--dry-run|--confirm]");
|
||
}
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("git-mirror accepts only one of --confirm or --dry-run");
|
||
return {
|
||
action: actionRaw,
|
||
confirm,
|
||
wait: args.includes("--wait"),
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "sync" || actionRaw === "flush" ? 300 : 120, 900),
|
||
};
|
||
}
|
||
|
||
function parseSecretOptions(args: string[]): G14SecretOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "status" && actionRaw !== "ensure") {
|
||
throw new Error("secret usage: status|ensure --lane v02 --name hwlab-v02-device-pod-api-key --key api-key [--dry-run|--confirm]");
|
||
}
|
||
const lane = optionValue(args, "--lane") ?? "v02";
|
||
if (lane !== "v02") throw new Error("secret currently supports --lane v02");
|
||
const name = optionValue(args, "--name") ?? V02_DEVICE_POD_API_KEY_SECRET;
|
||
if (name !== V02_DEVICE_POD_API_KEY_SECRET) throw new Error(`secret currently supports --name ${V02_DEVICE_POD_API_KEY_SECRET}`);
|
||
const key = optionValue(args, "--key") ?? V02_DEVICE_POD_API_KEY_SECRET_KEY;
|
||
if (key !== V02_DEVICE_POD_API_KEY_SECRET_KEY) throw new Error(`secret currently supports --key ${V02_DEVICE_POD_API_KEY_SECRET_KEY}`);
|
||
const confirm = args.includes("--confirm");
|
||
const explicitDryRun = args.includes("--dry-run");
|
||
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
||
return {
|
||
action: actionRaw,
|
||
lane,
|
||
confirm,
|
||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||
name,
|
||
key,
|
||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||
};
|
||
}
|
||
|
||
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
||
const index = args.indexOf(name);
|
||
if (index === -1) return defaultValue;
|
||
const raw = args[index + 1];
|
||
const value = Number(raw);
|
||
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||
return Math.min(value, maxValue);
|
||
}
|
||
|
||
function shellQuote(value: string): string {
|
||
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
||
}
|
||
|
||
function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
const result = runCommand(command, repoRoot, { timeoutMs });
|
||
let parsed: unknown | null = null;
|
||
if (result.stdout.trim().length > 0) {
|
||
try {
|
||
parsed = JSON.parse(result.stdout) as unknown;
|
||
} catch {
|
||
parsed = null;
|
||
}
|
||
}
|
||
return {
|
||
ok: result.exitCode === 0,
|
||
command,
|
||
exitCode: result.exitCode,
|
||
stdout: result.stdout,
|
||
stderr: result.stderr,
|
||
parsed,
|
||
};
|
||
}
|
||
|
||
function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs);
|
||
}
|
||
|
||
function record(value: unknown): Record<string, unknown> {
|
||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||
}
|
||
|
||
function stringOrNull(value: unknown): string | null {
|
||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||
}
|
||
|
||
function nested(value: unknown, keys: string[]): unknown {
|
||
let current = value;
|
||
for (const key of keys) current = record(current)[key];
|
||
return current;
|
||
}
|
||
|
||
function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown> {
|
||
const dumpPath = nested(result.parsed, ["data", "dump", "path"]);
|
||
if (typeof dumpPath === "string" && existsSync(dumpPath)) {
|
||
try {
|
||
return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown);
|
||
} catch {
|
||
return record(result.parsed);
|
||
}
|
||
}
|
||
return record(result.parsed);
|
||
}
|
||
|
||
function commandData(result: CommandJsonResult): Record<string, unknown> {
|
||
return record(sanitizeCliOutput(record(expandedParsedRoot(result).data)));
|
||
}
|
||
|
||
function textHash(value: string): string {
|
||
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
||
}
|
||
|
||
function redactLargePayloads(value: string): string {
|
||
return value
|
||
.replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => {
|
||
return `manifest_b64='<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>'`;
|
||
})
|
||
.replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => {
|
||
return `manifest_b64="<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>"`;
|
||
})
|
||
.replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => {
|
||
return `manifest_b64=<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||
})
|
||
.replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => {
|
||
return `<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
||
});
|
||
}
|
||
|
||
function compactInlineText(value: string, maxChars = 4000): string {
|
||
const redacted = redactLargePayloads(value);
|
||
if (redacted.length <= maxChars) return redacted;
|
||
const headChars = Math.floor(maxChars * 0.35);
|
||
const tailChars = maxChars - headChars;
|
||
const omittedChars = redacted.length - headChars - tailChars;
|
||
return [
|
||
redacted.slice(0, headChars),
|
||
`<truncated chars=${omittedChars} sha256=${textHash(redacted)}>`,
|
||
redacted.slice(-tailChars),
|
||
].join("\n");
|
||
}
|
||
|
||
function sanitizeCliOutput(value: unknown): unknown {
|
||
if (typeof value === "string") return compactInlineText(value);
|
||
if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item));
|
||
if (typeof value !== "object" || value === null) return value;
|
||
const output: Record<string, unknown> = {};
|
||
for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item);
|
||
return output;
|
||
}
|
||
|
||
function redactedCommand(command: string[]): string[] {
|
||
return command.map((arg) => compactInlineText(arg, 1200));
|
||
}
|
||
|
||
function compactCommandResult(result: CommandJsonResult): Record<string, unknown> {
|
||
const data = record(expandedParsedRoot(result).data);
|
||
const stdout = String(data.stdout ?? result.stdout ?? "");
|
||
const stderr = String(data.stderr ?? result.stderr ?? "");
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
exitCode: result.exitCode,
|
||
command: redactedCommand(result.command),
|
||
stdout: compactInlineText(stdout.trim(), 4000),
|
||
stderr: compactInlineText(stderr.trim(), 4000),
|
||
stdoutBytes: Buffer.byteLength(stdout),
|
||
stderrBytes: Buffer.byteLength(stderr),
|
||
};
|
||
}
|
||
|
||
function compactControlPlaneRefresh(value: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
ok: value.ok === true,
|
||
phase: value.phase ?? null,
|
||
sourceCommit: value.sourceCommit ?? null,
|
||
renderDir: value.renderDir ?? null,
|
||
applyOk: nested(value, ["apply", "ok"]) ?? null,
|
||
cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null,
|
||
applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200),
|
||
degradedReason: value.degradedReason ?? null,
|
||
};
|
||
}
|
||
|
||
function compactTriggerCommandResult(value: Record<string, unknown> | null): Record<string, unknown> | null {
|
||
if (value === null) return null;
|
||
return {
|
||
ok: value.ok ?? null,
|
||
exitCode: value.exitCode ?? null,
|
||
stdout: tailText(value.stdout, 1200),
|
||
stderr: tailText(value.stderr, 1200),
|
||
stdoutBytes: value.stdoutBytes ?? null,
|
||
stderrBytes: value.stderrBytes ?? null,
|
||
};
|
||
}
|
||
|
||
function isCommandSuccess(result: CommandJsonResult): boolean {
|
||
if (!result.ok) return false;
|
||
const topOk = record(result.parsed).ok;
|
||
if (topOk === false) return false;
|
||
const dataOk = nested(result.parsed, ["data", "ok"]);
|
||
return dataOk !== false;
|
||
}
|
||
|
||
function monitorBaseBranch(lane: G14MonitorLane): string {
|
||
return lane === "v02" ? V02_SOURCE_BRANCH : G14_SOURCE_BRANCH;
|
||
}
|
||
|
||
function extractPullRequests(result: CommandJsonResult, baseBranch = G14_SOURCE_BRANCH): OpenPullRequest[] {
|
||
const prs = nested(result.parsed, ["data", "pullRequests"]);
|
||
if (!Array.isArray(prs)) return [];
|
||
return prs
|
||
.map((item) => record(item))
|
||
.filter((item) => item.baseRefName === baseBranch || record(item.base).ref === baseBranch)
|
||
.map((item) => ({
|
||
number: Number(item.number),
|
||
title: typeof item.title === "string" ? item.title : undefined,
|
||
url: typeof item.url === "string" ? item.url : undefined,
|
||
baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined,
|
||
headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined,
|
||
mergeable: typeof item.mergeable === "string" ? item.mergeable : null,
|
||
mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null,
|
||
draft: item.draft === true,
|
||
}))
|
||
.filter((item) => Number.isInteger(item.number) && item.number > 0);
|
||
}
|
||
|
||
function sleep(ms: number): Promise<void> {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
function printEvent(event: string, data: Record<string, unknown> = {}): void {
|
||
process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||
}
|
||
|
||
function printProgressEvent(event: string, data: Record<string, unknown> = {}): void {
|
||
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
||
}
|
||
|
||
function printV02PrMonitorProgress(data: Record<string, unknown> = {}): void {
|
||
printProgressEvent("hwlab.v02.pr-monitor.progress", data);
|
||
}
|
||
|
||
function shortSha(sha: string): string {
|
||
return sha.slice(0, 12);
|
||
}
|
||
|
||
function precheckWorkspace(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
|
||
}
|
||
|
||
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
|
||
}
|
||
|
||
function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs);
|
||
}
|
||
|
||
function v02CicdRepoEnsureScript(): string {
|
||
return [
|
||
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
||
`cicd_url=${shellQuote(V02_GIT_URL)}`,
|
||
"cicd_repo_lock=/tmp/hwlab-v02-cicd-repo.lock",
|
||
"cicd_repo_ensure_fetch() {",
|
||
" mkdir -p \"$(dirname \"$cicd_repo\")\" || return $?",
|
||
" if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
||
" :",
|
||
" elif [ -e \"$cicd_repo\" ]; then",
|
||
" echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
|
||
" return 41",
|
||
" else",
|
||
" git clone --bare \"$cicd_url\" \"$cicd_repo\" || return $?",
|
||
" fi",
|
||
" git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\" || return $?",
|
||
" git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' || return $?",
|
||
" git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune || return $?",
|
||
"}",
|
||
"cicd_repo_lock_mode=none",
|
||
"if command -v flock >/dev/null 2>&1; then",
|
||
" exec 9>\"$cicd_repo_lock\"",
|
||
" flock -w 120 9 || { echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock\" >&2; exit 42; }",
|
||
" cicd_repo_lock_mode=flock",
|
||
"else",
|
||
" cicd_repo_lock_dir=\"$cicd_repo_lock.d\"",
|
||
" cicd_repo_lock_attempt=0",
|
||
" while ! mkdir \"$cicd_repo_lock_dir\" 2>/dev/null; do",
|
||
" cicd_repo_lock_attempt=$((cicd_repo_lock_attempt + 1))",
|
||
" if [ \"$cicd_repo_lock_attempt\" -ge 120 ]; then echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock_dir\" >&2; exit 42; fi",
|
||
" sleep 1",
|
||
" done",
|
||
" cicd_repo_lock_mode=dir",
|
||
"fi",
|
||
"cicd_repo_status=0",
|
||
"cicd_repo_ensure_fetch || cicd_repo_status=$?",
|
||
"if [ \"$cicd_repo_lock_mode\" = flock ]; then flock -u 9 >/dev/null 2>&1 || true; fi",
|
||
"if [ \"$cicd_repo_lock_mode\" = dir ]; then rmdir \"$cicd_repo_lock_dir\" >/dev/null 2>&1 || true; fi",
|
||
"if [ \"$cicd_repo_status\" -ne 0 ]; then echo \"v0.2 CI/CD repo refresh failed status=$cicd_repo_status repo=$cicd_repo\" >&2; exit \"$cicd_repo_status\"; fi",
|
||
].join("\n");
|
||
}
|
||
|
||
function resolveV02Head(): { sourceCommit: string | null; result: CommandJsonResult } {
|
||
const result = g14HostScript([
|
||
"set -eu",
|
||
v02CicdRepoEnsureScript(),
|
||
"git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2",
|
||
].join("\n"), 180_000);
|
||
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||
return { sourceCommit: match?.[0] ?? null, result };
|
||
}
|
||
|
||
function getV02Head(): string | null {
|
||
return resolveV02Head().sourceCommit;
|
||
}
|
||
|
||
function resolveV02LatestRemoteHead(): { sourceCommit: string | null; result: CommandJsonResult } {
|
||
const result = commandJson(["git", "ls-remote", V02_GIT_URL, "refs/heads/v0.2"], 30_000);
|
||
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
||
const output = statusText(result);
|
||
const match = /^[0-9a-f]{40}/imu.exec(output);
|
||
return { sourceCommit: match?.[0] ?? null, result };
|
||
}
|
||
|
||
function v02PipelineRunName(sourceCommit: string): string {
|
||
return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`;
|
||
}
|
||
|
||
interface V02TriggerSnapshot {
|
||
sourceCommit: string | null;
|
||
pipelineRun: string | null;
|
||
before: Record<string, unknown> | null;
|
||
result: CommandJsonResult;
|
||
observedAtMs: number;
|
||
}
|
||
|
||
export function parseV02TriggerSnapshot(result: CommandJsonResult, observedAtMs: number): V02TriggerSnapshot {
|
||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout);
|
||
const fields = keyValueLinesFromText(output);
|
||
const sourceCommit = stringOrNull(fields.sourceCommit?.match(/^[0-9a-f]{40}$/iu)?.[0]);
|
||
const pipelineRun = stringOrNull(fields.pipelineRun);
|
||
const pipelineRunExitCode = numericField(fields.pipelineRunExitCode);
|
||
const before = pipelineRun !== null && pipelineRunExitCode !== null
|
||
? pipelineRunCompactFromText(
|
||
pipelineRun,
|
||
[fields.status ?? "", fields.reason ?? "", fields.message ?? ""].join("\n"),
|
||
pipelineRunExitCode === 0,
|
||
"hwlab-v02-trigger-snapshot:pipelinerun",
|
||
pipelineRunExitCode,
|
||
fields.pipelineRunStderr ?? "",
|
||
)
|
||
: null;
|
||
return {
|
||
sourceCommit,
|
||
pipelineRun,
|
||
before,
|
||
result,
|
||
observedAtMs,
|
||
};
|
||
}
|
||
|
||
function getV02TriggerSnapshot(): V02TriggerSnapshot {
|
||
const script = [
|
||
"set +e",
|
||
"one_line() { tr '\\r\\n\\t' ' ' | sed 's/[[:space:]][[:space:]]*/ /g' | cut -c1-2000; }",
|
||
"tmp_dir=$(mktemp -d /tmp/hwlab-v02-trigger-snapshot.XXXXXX)",
|
||
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
||
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
||
"source_commit=",
|
||
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
||
" source_commit=$(git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)",
|
||
"elif [ -e \"$cicd_repo\" ]; then",
|
||
" printf 'snapshotError\\t%s\\n' \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\"",
|
||
"fi",
|
||
"pipeline_run=",
|
||
`if [ -n "$source_commit" ]; then pipeline_run=${shellQuote(V02_PIPELINERUN_PREFIX)}-$(printf '%s' "$source_commit" | cut -c1-12); fi`,
|
||
"if [ -n \"$pipeline_run\" ]; then",
|
||
` (kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}' >\"$tmp_dir/pipelinerun.out\" 2>\"$tmp_dir/pipelinerun.err\"; printf '%s' "$?" >\"$tmp_dir/pipelinerun.status\") &`,
|
||
" pipeline_pid=$!",
|
||
"else",
|
||
" printf '' >\"$tmp_dir/pipelinerun.out\"",
|
||
" printf '' >\"$tmp_dir/pipelinerun.err\"",
|
||
" printf '' >\"$tmp_dir/pipelinerun.status\"",
|
||
" pipeline_pid=",
|
||
"fi",
|
||
"if [ -n \"$pipeline_pid\" ]; then wait \"$pipeline_pid\" 2>/dev/null || true; pipeline_pid=; fi",
|
||
"status_text=$(cat \"$tmp_dir/pipelinerun.out\" 2>/dev/null)",
|
||
"pipeline_status=$(cat \"$tmp_dir/pipelinerun.status\" 2>/dev/null)",
|
||
"printf 'sourceCommit\\t%s\\n' \"$source_commit\"",
|
||
"printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"",
|
||
"printf 'pipelineRunExitCode\\t%s\\n' \"$pipeline_status\"",
|
||
"printf 'status\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '1p')\"",
|
||
"printf 'reason\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '2p')\"",
|
||
"printf 'message\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '3p')\"",
|
||
"printf 'pipelineRunStderr\\t%s\\n' \"$(one_line < \"$tmp_dir/pipelinerun.err\")\"",
|
||
].join("\n");
|
||
const result = g14K3s(["script", "--", script], 180_000);
|
||
return parseV02TriggerSnapshot(result, Date.now());
|
||
}
|
||
|
||
export function v02ExistingPipelineRunReuseDecision(input: {
|
||
sourceCommit: string;
|
||
before: Record<string, unknown>;
|
||
latestSourceCommit: string | null;
|
||
}): Record<string, unknown> {
|
||
const status = stringOrNull(input.before.status);
|
||
const exists = input.before.exists === true;
|
||
const latestSourceCommit = input.latestSourceCommit;
|
||
const alreadyUsable = exists && (status === "True" || status === "Unknown");
|
||
if (latestSourceCommit === null) {
|
||
return {
|
||
reusable: false,
|
||
alreadyUsable,
|
||
reason: "source-head-recheck-unresolved-before-existing-pipelinerun-reuse",
|
||
sourceCommit: input.sourceCommit,
|
||
latestSourceCommit,
|
||
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
|
||
};
|
||
}
|
||
if (latestSourceCommit !== null && latestSourceCommit !== input.sourceCommit) {
|
||
return {
|
||
reusable: false,
|
||
alreadyUsable,
|
||
reason: "source-head-advanced-before-existing-pipelinerun-reuse",
|
||
sourceCommit: input.sourceCommit,
|
||
latestSourceCommit,
|
||
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
|
||
latestPipelineRun: v02PipelineRunName(latestSourceCommit),
|
||
};
|
||
}
|
||
return {
|
||
reusable: exists && status !== null,
|
||
alreadyUsable,
|
||
reason: alreadyUsable ? "existing-pipelinerun-reused" : "existing-pipelinerun-terminal-failed",
|
||
sourceCommit: input.sourceCommit,
|
||
latestSourceCommit,
|
||
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
|
||
};
|
||
}
|
||
|
||
function getPipelineRunCompact(name: string): Record<string, unknown> {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
name,
|
||
"-o",
|
||
"jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}",
|
||
], 60_000);
|
||
return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr);
|
||
}
|
||
|
||
function pipelineRunCompactFromText(
|
||
name: string,
|
||
text: string,
|
||
commandOk: boolean,
|
||
command: string[] | string,
|
||
exitCode: number | null,
|
||
stderr: string,
|
||
): Record<string, unknown> {
|
||
const [status = "", reason = "", message = ""] = text.split(/\r?\n/u);
|
||
const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`);
|
||
return {
|
||
ok: commandOk,
|
||
exists: commandOk || !notFound,
|
||
pipelineRun: name,
|
||
status: status || null,
|
||
reason: reason || null,
|
||
message: message || null,
|
||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
function timestampMs(value: unknown): number | null {
|
||
if (typeof value !== "string" || value.trim().length === 0) return null;
|
||
const parsed = Date.parse(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function secondsBetween(start: unknown, end: unknown): number | null {
|
||
const startMs = timestampMs(start);
|
||
const endMs = timestampMs(end);
|
||
if (startMs === null || endMs === null || endMs < startMs) return null;
|
||
return Math.round((endMs - startMs) / 1000);
|
||
}
|
||
|
||
function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record<string, unknown> | null {
|
||
const [
|
||
name = "",
|
||
sourceCommit = "",
|
||
createdAt = "",
|
||
startTime = "",
|
||
completionTime = "",
|
||
status = "",
|
||
reason = "",
|
||
] = line.split("\t");
|
||
if (!name.startsWith(V02_PIPELINERUN_PREFIX)) return null;
|
||
const startMs = timestampMs(startTime);
|
||
const completion = completionTime.trim().length > 0 ? completionTime : null;
|
||
const conditionStatus = status.trim().length > 0 ? status : null;
|
||
return {
|
||
name,
|
||
sourceCommit: sourceCommit || null,
|
||
status: conditionStatus,
|
||
reason: reason || null,
|
||
createdAt: createdAt || null,
|
||
startTime: startTime || null,
|
||
completionTime: completion,
|
||
durationSeconds: secondsBetween(startTime, completion),
|
||
elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)),
|
||
active: completion === null && conditionStatus !== "True" && conditionStatus !== "False",
|
||
};
|
||
}
|
||
|
||
function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now()): Record<string, unknown> {
|
||
const rows = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs))
|
||
.filter((item): item is Record<string, unknown> => item !== null)
|
||
.sort((left, right) => {
|
||
return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0);
|
||
});
|
||
const activeItems = rows.filter((item) => item.active === true);
|
||
return {
|
||
ok: true,
|
||
count: rows.length,
|
||
activeCount: activeItems.length,
|
||
items: rows.slice(0, limit),
|
||
activeItems: activeItems.slice(0, limit),
|
||
disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete",
|
||
};
|
||
}
|
||
|
||
function pipelineRunRowsJsonPath(): string {
|
||
return [
|
||
"jsonpath={range .items[*]}",
|
||
"{.metadata.name}",
|
||
"{\"\\t\"}",
|
||
"{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}",
|
||
"{\"\\t\"}",
|
||
"{.metadata.creationTimestamp}",
|
||
"{\"\\t\"}",
|
||
"{.status.startTime}",
|
||
"{\"\\t\"}",
|
||
"{.status.completionTime}",
|
||
"{\"\\t\"}",
|
||
"{.status.conditions[0].status}",
|
||
"{\"\\t\"}",
|
||
"{.status.conditions[0].reason}",
|
||
"{\"\\n\"}",
|
||
"{end}",
|
||
].join("");
|
||
}
|
||
|
||
function listV02PipelineRunsCompact(limit = 8): Record<string, unknown> {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-l",
|
||
"hwlab.pikastech.local/gitops-target=v02",
|
||
"-o",
|
||
pipelineRunRowsJsonPath(),
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
return {
|
||
ok: false,
|
||
command: redactedCommand(result.command),
|
||
exitCode: result.exitCode,
|
||
stderr: commandErrorSummary(result),
|
||
items: [],
|
||
activeItems: [],
|
||
};
|
||
}
|
||
return parsePipelineRunRows(statusText(result), limit);
|
||
}
|
||
|
||
function parseShellSections(output: string): Record<string, ShellSection> {
|
||
const sections: Record<string, ShellSection> = {};
|
||
let currentName: string | null = null;
|
||
let currentLines: string[] = [];
|
||
let currentExitCode: number | null = null;
|
||
const flush = (): void => {
|
||
if (currentName === null) return;
|
||
sections[currentName] = {
|
||
stdout: currentLines.join("\n").trim(),
|
||
exitCode: currentExitCode,
|
||
};
|
||
};
|
||
for (const line of output.split(/\r?\n/u)) {
|
||
const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line);
|
||
if (begin !== null) {
|
||
flush();
|
||
currentName = begin[1] ?? null;
|
||
currentLines = [];
|
||
currentExitCode = null;
|
||
continue;
|
||
}
|
||
const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line);
|
||
if (end !== null && currentName === end[1]) {
|
||
currentExitCode = Number(end[2]);
|
||
flush();
|
||
currentName = null;
|
||
currentLines = [];
|
||
currentExitCode = null;
|
||
continue;
|
||
}
|
||
if (currentName !== null) currentLines.push(line);
|
||
}
|
||
flush();
|
||
return sections;
|
||
}
|
||
|
||
function shellSectionOk(section: ShellSection | undefined): boolean {
|
||
return section?.exitCode === 0;
|
||
}
|
||
|
||
function v02ControlPlaneStatusBundle(target: V02ControlPlaneStatusTarget = {}): CommandJsonResult {
|
||
const targetMode: V02StatusTargetMode = target.mode
|
||
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
||
const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null
|
||
? [
|
||
`pipeline_run=${shellQuote(target.pipelineRun)}`,
|
||
`source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`,
|
||
`if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | cut -d- -f5-); fi`,
|
||
].join("\n")
|
||
: [
|
||
target.sourceCommit === undefined
|
||
? `source_commit=$(git --git-dir=${shellQuote(V02_CICD_REPO)} rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)`
|
||
: `source_commit=${shellQuote(target.sourceCommit ?? "")}`,
|
||
"pipeline_run=",
|
||
].join("\n");
|
||
const script = [
|
||
"set +e",
|
||
targetInit,
|
||
`target_mode=${shellQuote(targetMode)}`,
|
||
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${V02_PIPELINERUN_PREFIX}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`,
|
||
"section() {",
|
||
" name=\"$1\"",
|
||
" shift",
|
||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||
" \"$@\"",
|
||
" code=$?",
|
||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||
"}",
|
||
"section statusTarget printf 'mode\\t%s\\nsourceCommit\\t%s\\npipelineRun\\t%s\\n' \"$target_mode\" \"$source_commit\" \"$pipeline_run\"",
|
||
"section sourceCommit printf '%s\\n' \"$source_commit\"",
|
||
"section pipelineRunName printf '%s\\n' \"$pipeline_run\"",
|
||
`section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())}`,
|
||
"section queryNow date -u +%Y-%m-%dT%H:%M:%SZ",
|
||
`section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`,
|
||
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
|
||
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
||
`if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
|
||
`if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
|
||
`if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`,
|
||
`section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`,
|
||
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
|
||
`section gitMirrorCache kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(gitMirrorCacheProbeScript())}`,
|
||
`section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
|
||
].join("\n");
|
||
return g14K3s(["script", "--", script], 60_000);
|
||
}
|
||
|
||
function v02SourceHeadsProbeScript(): string {
|
||
return [
|
||
"set +e",
|
||
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
||
`workspace=${shellQuote(V02_WORKSPACE)}`,
|
||
"rev_cicd() { git --git-dir=\"$cicd_repo\" rev-parse \"$1\" 2>/dev/null || true; }",
|
||
"rev_workspace() { git -C \"$workspace\" rev-parse \"$1\" 2>/dev/null || true; }",
|
||
"printf 'cicdRepo\\t%s\\n' \"$cicd_repo\"",
|
||
"printf 'cicdRepoExists\\t%s\\n' \"$([ -d \"$cicd_repo/objects\" ] && printf yes || printf no)\"",
|
||
"printf 'cicdSourceHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"",
|
||
"printf 'originHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"",
|
||
"printf 'workspacePath\\t%s\\n' \"$workspace\"",
|
||
"printf 'workspaceBranch\\t%s\\n' \"$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)\"",
|
||
"printf 'workspaceHead\\t%s\\n' \"$(rev_workspace HEAD)\"",
|
||
"printf 'workspaceOriginHead\\t%s\\n' \"$(rev_workspace refs/remotes/origin/v0.2)\"",
|
||
"printf 'workspaceDirtyCount\\t%s\\n' \"$(git -C \"$workspace\" status --porcelain=v1 2>/dev/null | wc -l | tr -d ' ')\"",
|
||
].join("\n");
|
||
}
|
||
|
||
function v02PlanArtifactsLogScript(): string {
|
||
return [
|
||
"pipeline_run=\"$1\"",
|
||
`namespace=${shellQuote(CI_NAMESPACE)}`,
|
||
"[ -n \"$pipeline_run\" ] || exit 0",
|
||
"pods=$(kubectl -n \"$namespace\" get pods -l \"tekton.dev/pipelineRun=$pipeline_run,tekton.dev/pipelineTask=plan-artifacts\" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)",
|
||
"for pod in $pods; do",
|
||
" printf '__POD__\\t%s\\n' \"$pod\"",
|
||
" kubectl -n \"$namespace\" logs \"$pod\" --all-containers --tail=200 2>/dev/null || true",
|
||
"done",
|
||
].join("\n");
|
||
}
|
||
|
||
function v02RuntimeWorkloadsColumns(): string {
|
||
return [
|
||
"custom-columns=KIND:.kind",
|
||
"NAME:.metadata.name",
|
||
"SERVICE:.metadata.labels.hwlab\\.pikastech\\.local/service-id",
|
||
"ARTIFACT:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/artifact-source-commit",
|
||
"SOURCE:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/source-commit",
|
||
"IMAGE:.spec.template.spec.containers[0].image",
|
||
"READY:.status.readyReplicas",
|
||
"REPLICAS:.status.replicas",
|
||
].join(",");
|
||
}
|
||
|
||
function v02WebAssetsProbeScript(): string {
|
||
return [
|
||
"set +e",
|
||
`base=${shellQuote(V02_CLOUD_WEB_URL)}`,
|
||
`api=${shellQuote(V02_CLOUD_API_URL)}`,
|
||
"fetch_url() {",
|
||
" if command -v curl >/dev/null 2>&1; then",
|
||
" curl -fsS --connect-timeout 2 --max-time 10 \"$1\"",
|
||
" elif command -v wget >/dev/null 2>&1; then",
|
||
" wget -q -T 10 -O - \"$1\"",
|
||
" else",
|
||
" return 127",
|
||
" fi",
|
||
"}",
|
||
"printf 'baseUrl\\t%s\\n' \"$base\"",
|
||
"printf 'apiUrl\\t%s\\n' \"$api\"",
|
||
"probe_start_s=$(date +%s 2>/dev/null || printf '0')",
|
||
"html=$(fetch_url \"$base/\" 2>/dev/null)",
|
||
"html_code=$?",
|
||
"printf 'htmlOk\\t%s\\n' \"$html_code\"",
|
||
"css_path=$(printf '%s' \"$html\" | sed -n 's/.*href=\"\\([^\"]*\\.css\\)\".*/\\1/p' | head -1)",
|
||
"printf 'cssPath\\t%s\\n' \"$css_path\"",
|
||
"case \"$css_path\" in",
|
||
" http://*|https://*) css_url=\"$css_path\" ;;",
|
||
" /*) css_url=\"$base$css_path\" ;;",
|
||
" \"\") css_url=\"\" ;;",
|
||
" *) css_url=\"$base/$css_path\" ;;",
|
||
"esac",
|
||
"printf 'htmlAppScript\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eq 'src=\"/app\\.js\"|src=\"app\\.js\"'; printf '%s' \"$?\")\"",
|
||
"printf 'htmlCssLink\\t%s\\n' \"$(test -n \"$css_path\"; printf '%s' \"$?\")\"",
|
||
"printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"",
|
||
"if [ -n \"$css_url\" ]; then css=$(fetch_url \"$css_url\" 2>/dev/null); else css=\"\"; false; fi",
|
||
"css_code=$?",
|
||
"printf 'cssOk\\t%s\\n' \"$css_code\"",
|
||
"printf 'cssBytes\\t%s\\n' \"$(printf '%s' \"$css\" | wc -c | tr -d ' ')\"",
|
||
"app_js=$(fetch_url \"$base/app.js\" 2>/dev/null)",
|
||
"app_js_code=$?",
|
||
"probe_end_s=$(date +%s 2>/dev/null || printf '0')",
|
||
"printf 'appJsOk\\t%s\\n' \"$app_js_code\"",
|
||
"printf 'appJsBytes\\t%s\\n' \"$(printf '%s' \"$app_js\" | wc -c | tr -d ' ')\"",
|
||
"printf 'probeElapsedMs\\t%s\\n' \"$(((probe_end_s - probe_start_s) * 1000))\"",
|
||
"fetch_url \"$base/styles.css\" >/dev/null 2>&1",
|
||
"legacy_styles_code=$?",
|
||
"if [ \"$legacy_styles_code\" = \"0\" ]; then legacy_styles_absent=1; else legacy_styles_absent=0; fi",
|
||
"printf 'legacyStylesAbsent\\t%s\\n' \"$legacy_styles_absent\"",
|
||
"printf 'reactWorkbenchCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'workbench-shell|activity-rail|session-sidebar|conversation-panel'; printf '%s' \"$?\")\"",
|
||
"health=$(fetch_url \"$api/health/live\" 2>/dev/null)",
|
||
"health_code=$?",
|
||
"printf 'apiHealthOk\\t%s\\n' \"$health_code\"",
|
||
"printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"",
|
||
].join("\n");
|
||
}
|
||
|
||
function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
|
||
items: [],
|
||
performance: v02TaskRunPerformanceSummary([]),
|
||
};
|
||
}
|
||
const items = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
status: status || null,
|
||
reason: reason || null,
|
||
startTime: startTime || null,
|
||
completionTime: completionTime || null,
|
||
durationSeconds: secondsBetween(startTime, completionTime),
|
||
};
|
||
});
|
||
const counts = {
|
||
succeeded: items.filter((item) => item.status === "True").length,
|
||
failed: items.filter((item) => item.status === "False").length,
|
||
running: items.filter((item) => item.status === "Unknown").length,
|
||
unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
|
||
};
|
||
const performance = v02TaskRunPerformanceSummary(items);
|
||
const performanceWarning = performance.ok === false ? `; ${String(performance.summary ?? "")}` : "";
|
||
return {
|
||
ok: true,
|
||
pipelineRun,
|
||
counts,
|
||
items,
|
||
performance,
|
||
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}${performanceWarning}`,
|
||
disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
|
||
};
|
||
}
|
||
|
||
function v02TaskRunPipelineTaskName(name: string): string | null {
|
||
const buildIndex = name.lastIndexOf("-build-");
|
||
if (buildIndex >= 0) return name.slice(buildIndex + 1);
|
||
const knownSuffixes = [
|
||
"prepare-source",
|
||
"plan-artifacts",
|
||
"publish-artifact-catalog",
|
||
"gitops-render",
|
||
"gitops-promote",
|
||
"runtime-ready",
|
||
"collect-artifacts",
|
||
];
|
||
return knownSuffixes.find((suffix) => name.endsWith(`-${suffix}`)) ?? null;
|
||
}
|
||
|
||
export function v02TaskRunPerformanceSummary(taskRuns: unknown[]): Record<string, unknown> {
|
||
const slowTaskRuns: Record<string, unknown>[] = [];
|
||
for (const itemRaw of taskRuns) {
|
||
const item = record(itemRaw);
|
||
const name = stringOrNull(item.name) ?? "";
|
||
const pipelineTask = stringOrNull(item.pipelineTask) ?? v02TaskRunPipelineTaskName(name);
|
||
const durationSeconds = typeof item.durationSeconds === "number" ? item.durationSeconds : null;
|
||
if (!pipelineTask?.startsWith("build-") || durationSeconds === null || durationSeconds <= V02_BUILD_TASKRUN_WARNING_SECONDS) continue;
|
||
const serviceId = pipelineTask.slice("build-".length) || null;
|
||
slowTaskRuns.push({
|
||
name,
|
||
pipelineTask,
|
||
serviceId,
|
||
status: stringOrNull(item.status),
|
||
reason: stringOrNull(item.reason),
|
||
durationSeconds,
|
||
budgetSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
||
severity: durationSeconds >= V02_BUILD_TASKRUN_CRITICAL_SECONDS ? "critical" : "warning",
|
||
message: `${pipelineTask} took ${durationSeconds}s, above v0.2 build TaskRun warning budget ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
||
});
|
||
}
|
||
slowTaskRuns.sort((left, right) => Number(right.durationSeconds ?? 0) - Number(left.durationSeconds ?? 0));
|
||
const worst = slowTaskRuns[0];
|
||
return {
|
||
ok: slowTaskRuns.length === 0,
|
||
warningCount: slowTaskRuns.length,
|
||
thresholds: {
|
||
buildTaskRunWarningSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
||
buildTaskRunCriticalSeconds: V02_BUILD_TASKRUN_CRITICAL_SECONDS,
|
||
},
|
||
slowTaskRuns,
|
||
summary: worst
|
||
? `slow build taskruns=${slowTaskRuns.length}; worst=${String(worst.pipelineTask)} ${String(worst.durationSeconds)}s budget=${V02_BUILD_TASKRUN_WARNING_SECONDS}s`
|
||
: `no build taskrun over ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
||
};
|
||
}
|
||
|
||
function v02WebAssetsFromText(
|
||
text: string,
|
||
commandOk: boolean,
|
||
sourceCommit: string | null,
|
||
argoSyncRevision: string | null,
|
||
exitCode: number | null,
|
||
stderr: string,
|
||
activePipelineRuns: unknown[] = [],
|
||
): Record<string, unknown> {
|
||
const fields: Record<string, string> = {};
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const [key = "", ...rest] = line.split("\t");
|
||
if (key.length > 0) fields[key] = rest.join("\t");
|
||
}
|
||
const htmlOk = fields.htmlOk === "0";
|
||
const htmlAppScript = fields.htmlAppScript === "0";
|
||
const htmlCssLink = fields.htmlCssLink === "0";
|
||
const cssOk = fields.cssOk === "0";
|
||
const appJsOk = fields.appJsOk === "0";
|
||
const apiHealthOk = fields.apiHealthOk === "0";
|
||
const readonlyNoteAbsent = fields.readonlyNote === "1";
|
||
const legacyStylesAbsent = fields.legacyStylesAbsent === "0";
|
||
const reactWorkbenchCss = fields.reactWorkbenchCss === "0";
|
||
const apiRevision = fields.apiRevision || null;
|
||
const webChecksPass = htmlOk && htmlAppScript && htmlCssLink && cssOk && appJsOk && readonlyNoteAbsent && legacyStylesAbsent && reactWorkbenchCss && apiHealthOk;
|
||
const failedChecks = Object.entries({
|
||
htmlOk,
|
||
htmlAppScript,
|
||
htmlCssLink,
|
||
cssOk,
|
||
appJsOk,
|
||
readonlyNoteAbsent,
|
||
legacyStylesAbsent,
|
||
reactWorkbenchCss,
|
||
apiHealthOk,
|
||
}).filter(([, ok]) => !ok).map(([name]) => name);
|
||
return {
|
||
ok: commandOk && webChecksPass,
|
||
summary: commandOk && webChecksPass ? "19666/19667 React/Vite asset probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`,
|
||
baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL,
|
||
apiUrl: fields.apiUrl || V02_CLOUD_API_URL,
|
||
sourceCommit,
|
||
argoSyncRevision: argoSyncRevision || null,
|
||
cssPath: fields.cssPath || null,
|
||
checks: {
|
||
htmlOk,
|
||
htmlAppScript,
|
||
htmlCssLink,
|
||
cssOk,
|
||
appJsOk,
|
||
readonlyNoteAbsent,
|
||
legacyStylesAbsent,
|
||
reactWorkbenchCss,
|
||
apiHealthOk,
|
||
},
|
||
probeExitCodes: {
|
||
html: numericField(fields.htmlOk),
|
||
htmlAppScript: numericField(fields.htmlAppScript),
|
||
htmlCssLink: numericField(fields.htmlCssLink),
|
||
css: numericField(fields.cssOk),
|
||
appJs: numericField(fields.appJsOk),
|
||
readonlyNoteGrep: numericField(fields.readonlyNote),
|
||
legacyStylesAbsent,
|
||
reactWorkbenchCssGrep: numericField(fields.reactWorkbenchCss),
|
||
apiHealth: numericField(fields.apiHealthOk),
|
||
},
|
||
assetBytes: {
|
||
css: numericField(fields.cssBytes),
|
||
appJs: numericField(fields.appJsBytes),
|
||
},
|
||
probeElapsedMs: numericField(fields.probeElapsedMs),
|
||
apiRevision,
|
||
note: webAssetsRevisionNote(apiRevision, sourceCommit, activePipelineRuns),
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
function v02PlanArtifactsFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
const events: Record<string, unknown>[] = [];
|
||
const pods: string[] = [];
|
||
for (const rawLine of text.split(/\r?\n/u)) {
|
||
const line = rawLine.trim();
|
||
if (line.startsWith("__POD__")) {
|
||
const [, pod = ""] = line.split("\t");
|
||
if (pod.length > 0) pods.push(pod);
|
||
continue;
|
||
}
|
||
if (!line.startsWith("{")) continue;
|
||
try {
|
||
const event = record(JSON.parse(line) as unknown);
|
||
if (event.event === "g14-ci-plan") events.push(event);
|
||
} catch {
|
||
// Ignore non-JSON log lines; the raw log remains visible through pod logs.
|
||
}
|
||
}
|
||
const latest = events.at(-1) ?? null;
|
||
if (!commandOk || latest === null) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
pods,
|
||
eventFound: latest !== null,
|
||
degradedReason: commandOk ? "g14-ci-plan-event-not-found" : "plan-artifacts-log-query-failed",
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
const audit = record(latest.artifactProvenanceAudit);
|
||
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
|
||
const provenanceRebuildServices = Array.isArray(audit.provenanceRebuildServices) ? audit.provenanceRebuildServices : [];
|
||
return {
|
||
ok: true,
|
||
pipelineRun,
|
||
pods,
|
||
sourceCommitId: stringOrNull(latest.sourceCommitId),
|
||
affectedServices: stringArray(latest.affectedServices),
|
||
rolloutServices: stringArray(latest.rolloutServices),
|
||
buildServices: stringArray(latest.buildServices),
|
||
reusedServices: stringArray(latest.reusedServices),
|
||
buildSkippedCount: numericValue(latest.buildSkippedCount),
|
||
artifactProvenanceAudit: Object.keys(audit).length > 0 ? audit : null,
|
||
summary: `build=${stringArray(latest.buildServices).length} reuse=${stringArray(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`,
|
||
disclosure: "parsed from plan-artifacts g14-ci-plan log event",
|
||
};
|
||
}
|
||
|
||
function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
items: [],
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
const items = text
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [kind = "", name = "", serviceId = "", artifactSourceCommit = "", sourceCommit = "", image = "", ready = "", replicas = ""] = line.split(/\s+/u);
|
||
return {
|
||
kind,
|
||
name,
|
||
serviceId: serviceId === "<none>" ? null : serviceId,
|
||
artifactSourceCommit: artifactSourceCommit === "<none>" ? null : artifactSourceCommit,
|
||
sourceCommit: sourceCommit === "<none>" ? null : sourceCommit,
|
||
image: image === "<none>" ? null : image,
|
||
readyReplicas: numericField(ready),
|
||
replicas: numericField(replicas),
|
||
};
|
||
});
|
||
return {
|
||
ok: true,
|
||
items,
|
||
summary: `runtime deployments=${items.length}`,
|
||
};
|
||
}
|
||
|
||
function keyValueLinesFromText(text: string): Record<string, string> {
|
||
const fields: Record<string, string> = {};
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const [key = "", ...rest] = line.split("\t");
|
||
if (key.trim().length > 0) fields[key.trim()] = rest.join("\t").trim();
|
||
}
|
||
return fields;
|
||
}
|
||
|
||
function v02SourceHeadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
const fields = keyValueLinesFromText(text);
|
||
const dirtyCount = numericField(fields.workspaceDirtyCount);
|
||
return {
|
||
ok: commandOk,
|
||
cicdRepo: fields.cicdRepo || V02_CICD_REPO,
|
||
cicdRepoExists: fields.cicdRepoExists === "yes",
|
||
cicdSourceHead: fields.cicdSourceHead || null,
|
||
originHead: fields.originHead || null,
|
||
workspace: {
|
||
path: fields.workspacePath || V02_WORKSPACE,
|
||
branch: fields.workspaceBranch || null,
|
||
head: fields.workspaceHead || null,
|
||
originHead: fields.workspaceOriginHead || null,
|
||
dirtyCount,
|
||
dirty: typeof dirtyCount === "number" ? dirtyCount > 0 : null,
|
||
isolatedFromCicd: true,
|
||
},
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
|
||
export function v02FalseGreenGuard(input: {
|
||
sourceCommit: string | null;
|
||
pipelineRun: Record<string, unknown> | null;
|
||
taskRuns: Record<string, unknown>;
|
||
planArtifacts: Record<string, unknown>;
|
||
runtimeWorkloads: Record<string, unknown>;
|
||
}): Record<string, unknown> {
|
||
const status = stringOrNull(input.pipelineRun?.status);
|
||
if (input.sourceCommit === null || input.pipelineRun === null || input.pipelineRun.exists === false) {
|
||
return { ok: null, state: "not-started", summary: "current source commit has no PipelineRun yet" };
|
||
}
|
||
if (status !== "True") {
|
||
return { ok: null, state: "pipeline-not-final", summary: `PipelineRun status=${status ?? "unknown"}` };
|
||
}
|
||
if (input.planArtifacts.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
state: "missing-plan-artifacts-evidence",
|
||
summary: "PipelineRun succeeded but plan-artifacts g14-ci-plan evidence was not visible",
|
||
planArtifacts: input.planArtifacts,
|
||
};
|
||
}
|
||
const sourceCommit = input.sourceCommit;
|
||
const buildServices = stringArray(input.planArtifacts.buildServices);
|
||
const reusedServices = stringArray(input.planArtifacts.reusedServices);
|
||
const taskItems = Array.isArray(input.taskRuns.items) ? input.taskRuns.items.map((item) => record(item)) : [];
|
||
const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : [];
|
||
const workloadByService = new Map(workloadItems.map((item) => [String(item.serviceId ?? ""), item]));
|
||
const missingBuildTaskRuns = buildServices.filter((serviceId) => !taskItems.some((item) => String(item.name ?? "").endsWith(`build-${serviceId}`) && item.status === "True"));
|
||
const runtimeMismatches = buildServices
|
||
.map((serviceId) => {
|
||
const workload = workloadByService.get(serviceId);
|
||
if (!workload) return { serviceId, reason: "runtime-workload-missing" };
|
||
if (workload.artifactSourceCommit !== sourceCommit) {
|
||
return {
|
||
serviceId,
|
||
reason: "artifact-source-commit-mismatch",
|
||
runtimeArtifactSourceCommit: workload.artifactSourceCommit ?? null,
|
||
expectedSourceCommit: sourceCommit,
|
||
image: workload.image ?? null,
|
||
};
|
||
}
|
||
return null;
|
||
})
|
||
.filter((item): item is Record<string, unknown> => item !== null);
|
||
const audit = record(input.planArtifacts.artifactProvenanceAudit);
|
||
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
|
||
const auditPresent = input.planArtifacts.artifactProvenanceAudit !== null;
|
||
const provenanceWarning = !auditPresent && reusedServices.length > 0
|
||
? { reason: "artifact-provenance-audit-missing-for-reuse", reusedServices }
|
||
: null;
|
||
const provenanceOk = auditPresent ? audit.ok !== false && unsafeReuseServices.length === 0 : true;
|
||
const failures = [
|
||
...(input.runtimeWorkloads.ok === true ? [] : [{ reason: "runtime-workload-query-failed" }]),
|
||
...(provenanceOk ? [] : [{ reason: "artifact-provenance-audit-failed", unsafeReuseServices }]),
|
||
...missingBuildTaskRuns.map((serviceId) => ({ serviceId, reason: "expected-build-taskrun-not-succeeded" })),
|
||
...runtimeMismatches,
|
||
];
|
||
return {
|
||
ok: failures.length === 0,
|
||
state: failures.length === 0 ? "passed" : "failed",
|
||
summary: failures.length === 0
|
||
? `artifact provenance and built runtime services align with ${shortSha(sourceCommit)}`
|
||
: `false-green risk: ${failures.length} invariant violation(s)`,
|
||
sourceCommit,
|
||
buildServices,
|
||
reusedServices,
|
||
provenanceOk,
|
||
provenanceAuditPresent: auditPresent,
|
||
provenanceWarnings: provenanceWarning ? [provenanceWarning] : [],
|
||
missingBuildTaskRuns,
|
||
runtimeMismatches,
|
||
failures,
|
||
};
|
||
}
|
||
|
||
function v02TargetValidation(input: {
|
||
targetMode: V02StatusTargetMode;
|
||
sourceCommit: string | null;
|
||
pipelineRun: Record<string, unknown> | null;
|
||
argo: Record<string, unknown>;
|
||
planArtifacts: Record<string, unknown>;
|
||
runtimeWorkloads: Record<string, unknown>;
|
||
webAssets: Record<string, unknown>;
|
||
gitMirror: Record<string, unknown>;
|
||
recentPipelineRuns: Record<string, unknown>;
|
||
}): Record<string, unknown> {
|
||
if (input.targetMode === "latest-source-head") {
|
||
return {
|
||
applicable: false,
|
||
state: "latest-source-head",
|
||
summary: "default status uses strict latest source head alignment; use --pipeline-run or --source-commit for historical run validation",
|
||
};
|
||
}
|
||
const sourceCommit = input.sourceCommit;
|
||
const argoFields = record(input.argo.fields);
|
||
const gitMirrorSummary = record(input.gitMirror.summary);
|
||
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
|
||
? input.runtimeWorkloads.items.map((item) => record(item))
|
||
: [];
|
||
const serviceSourceCommits = Object.fromEntries(workloadItems
|
||
.filter((item) => item.serviceId === "hwlab-cloud-api" || item.serviceId === "hwlab-cloud-web")
|
||
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
|
||
const rolloutServices = stringArray(input.planArtifacts.rolloutServices)
|
||
.filter((serviceId) => serviceId === "hwlab-cloud-api" || serviceId === "hwlab-cloud-web");
|
||
const requiredRuntimeServices = rolloutServices;
|
||
const recentItems = Array.isArray(input.recentPipelineRuns.items)
|
||
? input.recentPipelineRuns.items.map((item) => record(item))
|
||
: [];
|
||
const targetPipelineRunName = stringOrNull(input.pipelineRun?.pipelineRun) ?? stringOrNull(input.pipelineRun?.name);
|
||
const targetRecentIndex = recentItems.findIndex((item) => (
|
||
(targetPipelineRunName !== null && item.name === targetPipelineRunName)
|
||
|| (sourceCommit !== null && item.sourceCommit === sourceCommit)
|
||
));
|
||
const newerCandidates = targetRecentIndex >= 0 ? recentItems.slice(0, targetRecentIndex) : recentItems;
|
||
const newerSucceededRuns = sourceCommit === null
|
||
? []
|
||
: newerCandidates.filter((item) => item.sourceCommit !== sourceCommit && item.status === "True");
|
||
const failures: Record<string, unknown>[] = [];
|
||
const supersededRuntimeServices: Record<string, unknown>[] = [];
|
||
if (sourceCommit === null) failures.push({ reason: "source-commit-unresolved" });
|
||
if (input.pipelineRun === null || input.pipelineRun.exists === false) {
|
||
failures.push({ reason: "pipeline-run-missing" });
|
||
} else if (input.pipelineRun.status !== "True") {
|
||
failures.push({ reason: "pipeline-run-not-succeeded", status: input.pipelineRun.status ?? null, reasonDetail: input.pipelineRun.reason ?? null });
|
||
}
|
||
if (input.argo.ok !== true || argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") {
|
||
failures.push({ reason: "argo-not-synced-healthy", syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null });
|
||
}
|
||
if (input.webAssets.ok !== true) failures.push({ reason: "web-assets-probe-failed", summary: input.webAssets.summary ?? null });
|
||
if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) {
|
||
failures.push({
|
||
reason: "git-mirror-not-flushed",
|
||
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
|
||
githubInSync: gitMirrorSummary.githubInSync ?? null,
|
||
});
|
||
}
|
||
for (const serviceId of requiredRuntimeServices) {
|
||
const serviceCommit = serviceSourceCommits[serviceId];
|
||
if (sourceCommit !== null && serviceCommit !== sourceCommit) {
|
||
const mismatch = { reason: "runtime-service-source-mismatch", serviceId, expectedSourceCommit: sourceCommit, actualSourceCommit: serviceCommit ?? null };
|
||
if (newerSucceededRuns.length > 0) supersededRuntimeServices.push(mismatch);
|
||
else failures.push(mismatch);
|
||
}
|
||
}
|
||
const superseded = failures.length === 0 && supersededRuntimeServices.length > 0;
|
||
return {
|
||
applicable: true,
|
||
ok: failures.length === 0,
|
||
state: failures.length === 0 ? (superseded ? "superseded" : "passed") : "failed",
|
||
summary: failures.length === 0
|
||
? superseded
|
||
? `target ${input.targetMode} completed for ${shortSha(sourceCommit ?? "")}; runtime now reflects newer v0.2 PipelineRun`
|
||
: requiredRuntimeServices.length === 0
|
||
? `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}; no runtime service rollout required`
|
||
: `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}`
|
||
: `target ${input.targetMode} validation failed with ${failures.length} issue(s)`,
|
||
sourceCommit,
|
||
pipelineRun: input.pipelineRun === null
|
||
? null
|
||
: { name: input.pipelineRun.pipelineRun ?? null, status: input.pipelineRun.status ?? null, reason: input.pipelineRun.reason ?? null },
|
||
argo: { syncRevision: argoFields.syncRevision ?? null, syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null },
|
||
runtimeServices: serviceSourceCommits,
|
||
requiredRuntimeServices,
|
||
rolloutServices: stringArray(input.planArtifacts.rolloutServices),
|
||
reusedServices: stringArray(input.planArtifacts.reusedServices),
|
||
superseded,
|
||
supersededRuntimeServices,
|
||
newerSucceededRuns: newerSucceededRuns.slice(0, 3).map((item) => ({
|
||
name: item.name ?? null,
|
||
sourceCommit: item.sourceCommit ?? null,
|
||
createdAt: item.createdAt ?? null,
|
||
durationSeconds: item.durationSeconds ?? null,
|
||
})),
|
||
webAssets: { ok: input.webAssets.ok ?? null, summary: input.webAssets.summary ?? null, appJsBytes: record(input.webAssets.assetBytes).appJs ?? null },
|
||
gitMirror: { pendingFlush: gitMirrorSummary.pendingFlush ?? null, githubInSync: gitMirrorSummary.githubInSync ?? null },
|
||
failures,
|
||
};
|
||
}
|
||
|
||
export function v02LatestOnlyTargetValidation(input: {
|
||
targetMode: string;
|
||
sourceCommit: string | null;
|
||
pipelineRun: Record<string, unknown> | null;
|
||
commitAlignment: Record<string, unknown>;
|
||
targetValidation: Record<string, unknown>;
|
||
}): Record<string, unknown> {
|
||
if (input.targetMode === "latest-source-head") return input.targetValidation;
|
||
if (input.sourceCommit === null) return input.targetValidation;
|
||
if (input.pipelineRun === null || input.pipelineRun.status !== "True") return input.targetValidation;
|
||
const validation = record(input.targetValidation);
|
||
const staleReasons = stringArray(input.commitAlignment.staleReasons);
|
||
const sourceHeadAdvancedReasons = staleReasons.filter((reason) => (
|
||
reason === "origin-head-mismatch"
|
||
|| reason === "cicd-source-repo-stale"
|
||
|| reason === "latest-pipelinerun-not-current"
|
||
));
|
||
if (sourceHeadAdvancedReasons.length === 0) return input.targetValidation;
|
||
const failures = Array.isArray(validation.failures) ? validation.failures : [];
|
||
return {
|
||
...validation,
|
||
ok: true,
|
||
state: "superseded",
|
||
superseded: true,
|
||
latestOnlySuperseded: true,
|
||
latestOnlyReasons: sourceHeadAdvancedReasons,
|
||
originalState: validation.state ?? null,
|
||
summary: validation.ok === true || validation.state === "passed"
|
||
? `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and is superseded by the newer v0.2 source head`
|
||
: `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`,
|
||
supersededFailures: failures.slice(0, 10),
|
||
failures: [],
|
||
};
|
||
}
|
||
|
||
export function v02CommitAlignment(input: {
|
||
expectedSourceHead: string | null;
|
||
sourceHeads: Record<string, unknown>;
|
||
gitMirrorSummary: Record<string, unknown>;
|
||
pipelineRun: Record<string, unknown> | null;
|
||
recentPipelineRuns: Record<string, unknown>;
|
||
planArtifacts?: Record<string, unknown>;
|
||
runtimeWorkloads: Record<string, unknown>;
|
||
webAssets: Record<string, unknown>;
|
||
}): Record<string, unknown> {
|
||
const expectedSourceHead = input.expectedSourceHead;
|
||
const cicdSourceHead = stringOrNull(input.sourceHeads.cicdSourceHead);
|
||
const originHead = stringOrNull(input.sourceHeads.originHead) ?? expectedSourceHead;
|
||
const workspace = record(input.sourceHeads.workspace);
|
||
const workspaceHead = stringOrNull(workspace.head);
|
||
const workspaceOriginHead = stringOrNull(workspace.originHead);
|
||
const mirrorSourceHead = stringOrNull(input.gitMirrorSummary.localV02);
|
||
const mirrorGithubSourceHead = stringOrNull(input.gitMirrorSummary.githubV02);
|
||
const recentItems = Array.isArray(input.recentPipelineRuns.items)
|
||
? input.recentPipelineRuns.items.map((item) => record(item))
|
||
: [];
|
||
const latestPipelineRun = recentItems[0] ?? null;
|
||
const latestPipelineSourceCommit = latestPipelineRun === null ? null : stringOrNull(latestPipelineRun.sourceCommit);
|
||
const currentPipelineStatus = stringOrNull(input.pipelineRun?.status);
|
||
const apiRevision = stringOrNull(input.webAssets.apiRevision);
|
||
const planArtifacts = record(input.planArtifacts);
|
||
const planSourceCommit = stringOrNull(planArtifacts.sourceCommitId);
|
||
const rolloutServices = stringArray(planArtifacts.rolloutServices);
|
||
const apiRevisionRequired = expectedSourceHead !== null && planSourceCommit === expectedSourceHead && rolloutServices.includes("hwlab-cloud-api");
|
||
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
|
||
? input.runtimeWorkloads.items.map((item) => record(item))
|
||
: [];
|
||
const serviceSourceCommits = Object.fromEntries(workloadItems
|
||
.filter((item) => typeof item.serviceId === "string" && item.serviceId.length > 0)
|
||
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
|
||
const staleReasons: string[] = [];
|
||
if (expectedSourceHead === null) staleReasons.push("expected-source-head-unresolved");
|
||
if (expectedSourceHead !== null && originHead !== null && originHead !== expectedSourceHead) staleReasons.push("origin-head-mismatch");
|
||
if (expectedSourceHead !== null && cicdSourceHead !== null && cicdSourceHead !== expectedSourceHead) staleReasons.push("cicd-source-repo-stale");
|
||
if (expectedSourceHead !== null && mirrorSourceHead !== expectedSourceHead) staleReasons.push("mirror-source-stale");
|
||
if (expectedSourceHead !== null && latestPipelineSourceCommit !== expectedSourceHead) staleReasons.push("latest-pipelinerun-not-current");
|
||
if (apiRevisionRequired && apiRevision !== expectedSourceHead) staleReasons.push("runtime-api-revision-stale");
|
||
const workspaceWarnings: string[] = [];
|
||
if (expectedSourceHead !== null && workspaceHead !== null && workspaceHead !== expectedSourceHead) workspaceWarnings.push("workspace-head-differs-from-latest-source-but-isolated");
|
||
if (expectedSourceHead !== null && workspaceOriginHead !== null && workspaceOriginHead !== expectedSourceHead) workspaceWarnings.push("workspace-origin-ref-stale-but-isolated");
|
||
if (workspace.dirty === true) workspaceWarnings.push("workspace-dirty-but-isolated-from-cicd");
|
||
const runtimeWarnings: string[] = [];
|
||
if (!apiRevisionRequired && expectedSourceHead !== null && apiRevision !== null && apiRevision !== expectedSourceHead) {
|
||
runtimeWarnings.push("api-revision-differs-without-current-cloud-api-rollout");
|
||
}
|
||
const latestPipelineSucceeded = latestPipelineRun !== null && latestPipelineRun.status === "True";
|
||
const aligned = staleReasons.length === 0;
|
||
const state = aligned
|
||
? "aligned"
|
||
: latestPipelineSucceeded
|
||
? "stale-success"
|
||
: currentPipelineStatus === "Unknown"
|
||
? "in-progress"
|
||
: "stale";
|
||
return {
|
||
aligned,
|
||
state,
|
||
expectedSourceHead,
|
||
originHead,
|
||
cicdSourceHead,
|
||
cicdRepo: input.sourceHeads.cicdRepo ?? V02_CICD_REPO,
|
||
mirrorSourceHead,
|
||
mirrorGithubSourceHead,
|
||
latestPipelineRun: latestPipelineRun === null
|
||
? null
|
||
: {
|
||
name: latestPipelineRun.name ?? null,
|
||
sourceCommit: latestPipelineSourceCommit,
|
||
status: latestPipelineRun.status ?? null,
|
||
reason: latestPipelineRun.reason ?? null,
|
||
createdAt: latestPipelineRun.createdAt ?? null,
|
||
durationSeconds: latestPipelineRun.durationSeconds ?? null,
|
||
},
|
||
latestPipelineSourceCommit,
|
||
currentPipelineRun: input.pipelineRun === null
|
||
? null
|
||
: {
|
||
name: input.pipelineRun.pipelineRun ?? null,
|
||
status: input.pipelineRun.status ?? null,
|
||
reason: input.pipelineRun.reason ?? null,
|
||
},
|
||
runtimeSourceCommit: apiRevision,
|
||
apiRevision,
|
||
apiRevisionRequired,
|
||
planSourceCommit,
|
||
rolloutServices,
|
||
serviceSourceCommits,
|
||
sourceInSync: input.gitMirrorSummary.sourceInSync ?? null,
|
||
gitopsInSync: input.gitMirrorSummary.gitopsInSync ?? null,
|
||
staleReasons,
|
||
workspace: {
|
||
path: workspace.path ?? V02_WORKSPACE,
|
||
branch: workspace.branch ?? null,
|
||
head: workspaceHead,
|
||
originHead: workspaceOriginHead,
|
||
dirty: workspace.dirty ?? null,
|
||
dirtyCount: workspace.dirtyCount ?? null,
|
||
isolatedFromCicd: true,
|
||
},
|
||
workspaceWarnings,
|
||
runtimeWarnings,
|
||
summary: aligned
|
||
? apiRevisionRequired
|
||
? `v0.2 CI/CD source, mirror, PipelineRun, and cloud-api runtime align with ${shortSha(expectedSourceHead ?? "")}`
|
||
: `v0.2 CI/CD source, mirror, and PipelineRun align with ${shortSha(expectedSourceHead ?? "")}; cloud-api runtime rollout not required`
|
||
: `v0.2 CI/CD alignment state=${state}; staleReasons=${staleReasons.join(",")}`,
|
||
};
|
||
}
|
||
|
||
function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null {
|
||
if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null;
|
||
const activeItems = activePipelineRuns.map((item) => record(item));
|
||
const sourceIsActive = activeItems.some((item) => item.sourceCommit === sourceCommit || String(item.name ?? "").includes(shortSha(sourceCommit)));
|
||
if (sourceIsActive) {
|
||
return `runtime rollout still in progress; cloud-api apiRevision=${apiRevision} has not reached sourceCommit=${sourceCommit} yet`;
|
||
}
|
||
if (activeItems.length > 0) {
|
||
return `runtime apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; active v02 PipelineRun observed, wait for rollout before judging runtime revision`;
|
||
}
|
||
return `cloud-api apiRevision=${apiRevision} is service-scoped and differs from sourceCommit=${sourceCommit}; when web asset probes pass, this is not proof that Cloud Web is stale`;
|
||
}
|
||
|
||
function numericField(value: string | undefined): number | null {
|
||
if (value === undefined || value.trim().length === 0) return null;
|
||
if (value === "<none>") return null;
|
||
const parsed = Number(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function numericValue(value: unknown): number | null {
|
||
const parsed = typeof value === "number" ? value : Number(value);
|
||
return Number.isFinite(parsed) ? parsed : null;
|
||
}
|
||
|
||
function stringArray(value: unknown): string[] {
|
||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : [];
|
||
}
|
||
|
||
function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
|
||
if (!commandOk) {
|
||
return {
|
||
ok: false,
|
||
command: Array.isArray(command) ? redactedCommand(command) : command,
|
||
exitCode,
|
||
stderr: stderr.trim().slice(0, 4000),
|
||
items: [],
|
||
activeItems: [],
|
||
};
|
||
}
|
||
return parsePipelineRunRows(text, limit, nowMs);
|
||
}
|
||
|
||
function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] {
|
||
if (lane === "v02") return ["hwlab-v02-ci-poll-"];
|
||
if (lane === "g14") return ["hwlab-g14-ci-poll-"];
|
||
return ["hwlab-v02-ci-poll-", "hwlab-g14-ci-poll-"];
|
||
}
|
||
|
||
function commandErrorSummary(result: CommandJsonResult): string {
|
||
const remoteStderr = String(nested(result.parsed, ["data", "stderr"]) ?? "").trim();
|
||
const remoteStdout = String(nested(result.parsed, ["data", "stdout"]) ?? "").trim();
|
||
const text = [result.stderr.trim(), remoteStderr, remoteStdout, result.stdout.trim()].find((item) => item.length > 0) ?? "";
|
||
return text.slice(0, 4000);
|
||
}
|
||
|
||
function tailText(value: unknown, maxBytes = 2000): string {
|
||
const text = String(value ?? "").trim();
|
||
return text.length <= maxBytes ? text : text.slice(-maxBytes);
|
||
}
|
||
|
||
function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pipelinerun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
|
||
}
|
||
const prefixes = pipelinePrefixesForLane(options.lane);
|
||
const now = Date.now();
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", createdAt = "", status = "", reason = ""] = line.split("\t");
|
||
const createdMs = Date.parse(createdAt);
|
||
const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
||
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
|
||
})
|
||
.filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
|
||
.filter((item) => item.status === "True" || item.status === "False")
|
||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||
.slice(0, options.limit);
|
||
}
|
||
|
||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unknown>[] {
|
||
if (pipelineRunNames.length === 0) {
|
||
return [];
|
||
}
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pvc",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
|
||
}
|
||
const wanted = new Set(pipelineRunNames);
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
volume: volume || null,
|
||
phase: phase || null,
|
||
ownerKind: ownerKind || null,
|
||
owner: owner || null,
|
||
};
|
||
})
|
||
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
|
||
}
|
||
|
||
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||
if (names.length === 0) {
|
||
return {
|
||
ok: true,
|
||
command: [],
|
||
exitCode: 0,
|
||
stdout: "no candidates",
|
||
stderr: "",
|
||
parsed: null,
|
||
};
|
||
}
|
||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
|
||
}
|
||
|
||
function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||
const result = g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"pv",
|
||
"-o",
|
||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\n"}{end}',
|
||
], 60_000);
|
||
if (!isCommandSuccess(result)) {
|
||
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
|
||
}
|
||
return statusText(result)
|
||
.split(/\r?\n/u)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t");
|
||
return {
|
||
name,
|
||
createdAt,
|
||
phase,
|
||
storageClass,
|
||
reclaimPolicy,
|
||
claimNamespace,
|
||
claimName,
|
||
capacity,
|
||
hostPath: claimNamespace === CI_NAMESPACE && claimName.length > 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null,
|
||
};
|
||
})
|
||
.filter((item) => item.phase === "Released")
|
||
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
||
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||
.slice(0, options.limit);
|
||
}
|
||
|
||
function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
|
||
if (names.length === 0) {
|
||
return {
|
||
ok: true,
|
||
command: [],
|
||
exitCode: 0,
|
||
stdout: "no candidates",
|
||
stderr: "",
|
||
parsed: null,
|
||
};
|
||
}
|
||
return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs);
|
||
}
|
||
|
||
function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const candidates = listReleasedCiWorkspacePvs(options);
|
||
const candidateNames = candidates.map((item) => String(item.name));
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||
mode: "dry-run",
|
||
lane: options.lane,
|
||
limit: options.limit,
|
||
candidates,
|
||
candidateCount: candidates.length,
|
||
mutation: false,
|
||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
|
||
};
|
||
}
|
||
const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(deletion),
|
||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||
mode: "confirmed-cleanup",
|
||
lane: options.lane,
|
||
limit: options.limit,
|
||
deletedPersistentVolumes: candidateNames,
|
||
deletedPersistentVolumeCount: candidateNames.length,
|
||
candidatesBefore: candidates,
|
||
deletion,
|
||
followUp: {
|
||
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||
storage: "trans G14 script -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runV02RuntimeMigration(options: G14ControlPlaneOptions, sourceCommit: string): Record<string, unknown> {
|
||
const reportPath = `/tmp/hwlab-v02-runtime-migration-${shortSha(sourceCommit)}.json`;
|
||
const migrationArgs = options.dryRun
|
||
? [
|
||
...(options.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
|
||
"--report",
|
||
reportPath,
|
||
]
|
||
: [
|
||
"--apply",
|
||
"--confirm-dev",
|
||
"--confirmed-non-production",
|
||
"--report",
|
||
reportPath,
|
||
];
|
||
const command = [
|
||
"exec",
|
||
"-n",
|
||
"hwlab-v02",
|
||
"deployment/hwlab-cloud-api",
|
||
"-c",
|
||
"hwlab-cloud-api",
|
||
"--",
|
||
"bun",
|
||
"cmd/hwlab-cloud-api/migrate.ts",
|
||
...migrationArgs,
|
||
];
|
||
const result = g14K3s(["kubectl", ...command], options.timeoutSeconds * 1000);
|
||
const ok = isCommandSuccess(result);
|
||
return {
|
||
ok,
|
||
command: "hwlab g14 control-plane runtime-migration --lane v02",
|
||
lane: "v02",
|
||
mode: options.dryRun ? options.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
|
||
sourceCommit,
|
||
runtimeNamespace: "hwlab-v02",
|
||
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
|
||
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
|
||
reportPath,
|
||
mutation: !options.dryRun,
|
||
result,
|
||
next: options.dryRun
|
||
? {
|
||
liveReadDryRun: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||
apply: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||
}
|
||
: {
|
||
health: "curl -fsS --max-time 20 http://74.48.78.17:19667/health/live",
|
||
triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const candidates = listCleanupPipelineRuns(options);
|
||
const candidateNames = candidates.map((item) => String(item.name));
|
||
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane cleanup-runs",
|
||
mode: "dry-run",
|
||
lane: options.lane,
|
||
minAgeMinutes: options.minAgeMinutes,
|
||
limit: options.limit,
|
||
candidates,
|
||
candidateCount: candidates.length,
|
||
ownedPvcs,
|
||
ownedPvcCount: ownedPvcs.length,
|
||
mutation: false,
|
||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane ${options.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
|
||
};
|
||
}
|
||
const deletion = deletePipelineRuns(candidateNames, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(deletion),
|
||
command: "hwlab g14 control-plane cleanup-runs",
|
||
mode: "confirmed-cleanup",
|
||
lane: options.lane,
|
||
minAgeMinutes: options.minAgeMinutes,
|
||
limit: options.limit,
|
||
deletedPipelineRuns: candidateNames,
|
||
deletedPipelineRunCount: candidateNames.length,
|
||
ownedPvcsBefore: ownedPvcs,
|
||
ownedPvcCountBefore: ownedPvcs.length,
|
||
deletion,
|
||
followUp: {
|
||
status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||
},
|
||
};
|
||
}
|
||
|
||
export function v02ControlPlaneRenderScript(sourceCommit: string, token = v02RenderToken()): string {
|
||
const renderDir = v02RenderDir(sourceCommit, token);
|
||
const worktreeDir = v02RenderWorktreeDir(sourceCommit, token);
|
||
return [
|
||
"set -eu",
|
||
v02CicdRepoEnsureScript(),
|
||
`source_commit=${shellQuote(sourceCommit)}`,
|
||
`render_dir=${shellQuote(renderDir)}`,
|
||
`worktree_dir=${shellQuote(worktreeDir)}`,
|
||
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
|
||
"trap cleanup_render_worktree EXIT",
|
||
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/v0.2)" = ${shellQuote(sourceCommit)}`,
|
||
"cleanup_render_worktree",
|
||
"rm -rf \"$render_dir\"",
|
||
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
||
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
||
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
||
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
||
"cd \"$worktree_dir\"",
|
||
`node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
||
].join("\n");
|
||
}
|
||
|
||
interface V02RenderTempResult {
|
||
result: CommandJsonResult;
|
||
renderDir: string;
|
||
worktreeDir: string;
|
||
token: string;
|
||
}
|
||
|
||
function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult {
|
||
const token = v02RenderToken();
|
||
return {
|
||
result: g14HostScript(v02ControlPlaneRenderScript(sourceCommit, token), 180_000),
|
||
renderDir: v02RenderDir(sourceCommit, token),
|
||
worktreeDir: v02RenderWorktreeDir(sourceCommit, token),
|
||
token,
|
||
};
|
||
}
|
||
|
||
function v02RenderToken(): string {
|
||
const random = Math.random().toString(16).slice(2, 10);
|
||
return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-");
|
||
}
|
||
|
||
function v02RenderDir(sourceCommit: string, token?: string): string {
|
||
return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
||
}
|
||
|
||
function v02RenderWorktreeDir(sourceCommit: string, token?: string): string {
|
||
return `/tmp/hwlab-v02-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
||
}
|
||
|
||
function cleanupV02RenderDir(renderDir: string): CommandJsonResult {
|
||
return g14HostScript([
|
||
"set -eu",
|
||
`render_dir=${shellQuote(renderDir)}`,
|
||
"case \"$render_dir\" in",
|
||
" /tmp/hwlab-v02-control-plane-*) rm -rf \"$render_dir\" ;;",
|
||
" *) echo \"refusing to cleanup unexpected v0.2 render dir: $render_dir\" >&2; exit 44 ;;",
|
||
"esac",
|
||
].join("\n"), 60_000);
|
||
}
|
||
|
||
function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"apply",
|
||
"--server-side",
|
||
"--force-conflicts",
|
||
`--field-manager=${V02_CONTROL_PLANE_FIELD_MANAGER}`,
|
||
...(dryRun ? ["--dry-run=server"] : []),
|
||
"-f",
|
||
`${renderDir}/tekton-v02/rbac.yaml`,
|
||
"-f",
|
||
`${renderDir}/tekton-v02/pipeline.yaml`,
|
||
"-f",
|
||
`${renderDir}/argocd/project.yaml`,
|
||
"-f",
|
||
`${renderDir}/argocd/application-v02.yaml`,
|
||
], timeoutSeconds * 1000);
|
||
}
|
||
|
||
interface V02RefreshMarker {
|
||
ok: boolean;
|
||
sourceCommit: string;
|
||
refreshedAt: string;
|
||
renderScriptHash: string;
|
||
renderDir?: string | null;
|
||
}
|
||
|
||
function v02ControlPlaneRefreshStateDir(): string {
|
||
return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh");
|
||
}
|
||
|
||
export function v02ControlPlaneRefreshScriptHash(sourceCommit: string): string {
|
||
return textHash(v02ControlPlaneRenderScript(sourceCommit, "contract-token"));
|
||
}
|
||
|
||
function v02ControlPlaneRefreshMarkerPath(sourceCommit: string): string {
|
||
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`);
|
||
}
|
||
|
||
function v02ControlPlaneRefreshLockDir(sourceCommit: string): string {
|
||
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`);
|
||
}
|
||
|
||
function readV02RefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
||
const path = v02ControlPlaneRefreshMarkerPath(sourceCommit);
|
||
if (!existsSync(path)) return null;
|
||
try {
|
||
const marker = JSON.parse(readFileSync(path, "utf8")) as V02RefreshMarker;
|
||
return v02ReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs);
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
export function v02ReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
||
const candidate = record(marker) as V02RefreshMarker;
|
||
const refreshedAtMs = timestampMs(candidate.refreshedAt);
|
||
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null;
|
||
if (refreshedAtMs === null || nowMs - refreshedAtMs > V02_CONTROL_PLANE_REFRESH_TTL_MS) return null;
|
||
return candidate;
|
||
}
|
||
|
||
function writeV02RefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): V02RefreshMarker {
|
||
const marker = {
|
||
ok: true,
|
||
sourceCommit,
|
||
refreshedAt: new Date().toISOString(),
|
||
renderScriptHash: scriptHash,
|
||
renderDir,
|
||
};
|
||
const dir = v02ControlPlaneRefreshStateDir();
|
||
mkdirSync(dir, { recursive: true });
|
||
writeFileSync(v02ControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
||
return marker;
|
||
}
|
||
|
||
function acquireV02RefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } {
|
||
const stateDir = v02ControlPlaneRefreshStateDir();
|
||
mkdirSync(stateDir, { recursive: true });
|
||
const lockDir = v02ControlPlaneRefreshLockDir(sourceCommit);
|
||
const startedAtMs = Date.now();
|
||
for (;;) {
|
||
try {
|
||
mkdirSync(lockDir);
|
||
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
||
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
} catch {
|
||
const ageMs = (() => {
|
||
try {
|
||
return Date.now() - statSync(lockDir).mtimeMs;
|
||
} catch {
|
||
return 0;
|
||
}
|
||
})();
|
||
if (ageMs > V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) {
|
||
try {
|
||
rmSync(lockDir, { recursive: true, force: true });
|
||
continue;
|
||
} catch {
|
||
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
}
|
||
}
|
||
if (Date.now() - startedAtMs >= V02_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
||
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
}
|
||
}
|
||
}
|
||
|
||
function releaseV02RefreshLock(lockDir: string): void {
|
||
try {
|
||
rmSync(lockDir, { recursive: true, force: true });
|
||
} catch {
|
||
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
||
}
|
||
}
|
||
|
||
function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
|
||
const scriptHash = v02ControlPlaneRefreshScriptHash(sourceCommit);
|
||
const existingMarker = readV02RefreshMarker(sourceCommit, scriptHash);
|
||
if (existingMarker !== null) {
|
||
return {
|
||
ok: true,
|
||
phase: "control-plane-refresh",
|
||
mode: "reused-recent-refresh-marker",
|
||
sourceCommit,
|
||
renderDir: existingMarker.renderDir ?? null,
|
||
marker: existingMarker,
|
||
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
||
};
|
||
}
|
||
const lock = acquireV02RefreshLock(sourceCommit);
|
||
if (!lock.acquired) {
|
||
return {
|
||
ok: false,
|
||
phase: "control-plane-refresh-lock",
|
||
mode: "local-refresh-lock",
|
||
sourceCommit,
|
||
lockDir: lock.lockDir,
|
||
waitedMs: lock.waitedMs,
|
||
degradedReason: "control-plane-refresh-lock-timeout",
|
||
};
|
||
}
|
||
try {
|
||
const markerAfterWait = readV02RefreshMarker(sourceCommit, scriptHash);
|
||
if (markerAfterWait !== null) {
|
||
return {
|
||
ok: true,
|
||
phase: "control-plane-refresh",
|
||
mode: "waited-for-recent-refresh-marker",
|
||
sourceCommit,
|
||
renderDir: markerAfterWait.renderDir ?? null,
|
||
marker: markerAfterWait,
|
||
waitedMs: lock.waitedMs,
|
||
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
||
};
|
||
}
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render.result)) {
|
||
return {
|
||
ok: false,
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
renderDir: render.renderDir,
|
||
render: compactCommandResult(render.result),
|
||
degradedReason: "control-plane-render-failed",
|
||
};
|
||
}
|
||
const apply = applyV02ControlPlaneFiles(render.renderDir, false, timeoutSeconds);
|
||
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
|
||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
||
const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs));
|
||
const marker = ok ? writeV02RefreshMarker(sourceCommit, scriptHash, render.renderDir) : null;
|
||
return {
|
||
ok,
|
||
phase: "control-plane-refresh",
|
||
mode: "refreshed",
|
||
sourceCommit,
|
||
renderDir: render.renderDir,
|
||
render: compactCommandResult(render.result),
|
||
apply: compactCommandResult(apply),
|
||
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||
marker,
|
||
waitedMs: lock.waitedMs,
|
||
degradedReason: !isCommandSuccess(apply)
|
||
? "control-plane-apply-failed"
|
||
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
||
? "obsolete-cronjob-cleanup-failed"
|
||
: undefined,
|
||
};
|
||
} finally {
|
||
releaseV02RefreshLock(lock.lockDir);
|
||
}
|
||
}
|
||
|
||
function getV02ObsoleteCronJobs(): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"get",
|
||
"cronjob",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
V02_POLLER,
|
||
V02_RECONCILER,
|
||
"--ignore-not-found",
|
||
"-o",
|
||
"name",
|
||
], 60_000);
|
||
}
|
||
|
||
function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"delete",
|
||
"cronjob",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
V02_POLLER,
|
||
V02_RECONCILER,
|
||
"--ignore-not-found=true",
|
||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||
], 60_000);
|
||
}
|
||
|
||
function v02PipelineRunManifest(sourceCommit: string): Record<string, unknown> {
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
return {
|
||
apiVersion: "tekton.dev/v1",
|
||
kind: "PipelineRun",
|
||
metadata: {
|
||
name: pipelineRun,
|
||
namespace: CI_NAMESPACE,
|
||
labels: {
|
||
"app.kubernetes.io/part-of": "hwlab",
|
||
"hwlab.pikastech.local/gitops-target": "v02",
|
||
"hwlab.pikastech.local/source-commit": sourceCommit,
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
annotations: {
|
||
"hwlab.pikastech.local/source-branch": V02_SOURCE_BRANCH,
|
||
"hwlab.pikastech.local/gitops-branch": V02_GITOPS_BRANCH,
|
||
"hwlab.pikastech.local/triggered-by": "unidesk-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
pipelineRef: { name: V02_PIPELINE },
|
||
taskRunTemplate: {
|
||
serviceAccountName: "hwlab-v02-tekton-runner",
|
||
podTemplate: {
|
||
hostNetwork: true,
|
||
dnsPolicy: "ClusterFirstWithHostNet",
|
||
securityContext: { fsGroup: 1000 },
|
||
},
|
||
},
|
||
params: [
|
||
{ name: "git-url", value: V02_GIT_URL },
|
||
{ name: "git-read-url", value: V02_GIT_READ_URL },
|
||
{ name: "source-branch", value: V02_SOURCE_BRANCH },
|
||
{ name: "gitops-branch", value: V02_GITOPS_BRANCH },
|
||
{ name: "lane", value: "v02" },
|
||
{ name: "catalog-path", value: V02_CATALOG_PATH },
|
||
{ name: "image-tag-mode", value: "full" },
|
||
{ name: "runtime-path", value: V02_RUNTIME_PATH },
|
||
{ name: "revision", value: sourceCommit },
|
||
{ name: "registry-prefix", value: V02_REGISTRY_PREFIX },
|
||
{ name: "services", value: V02_SERVICE_IDS.join(",") },
|
||
{ name: "base-image", value: V02_BASE_IMAGE },
|
||
],
|
||
workspaces: [
|
||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
|
||
],
|
||
},
|
||
};
|
||
}
|
||
|
||
function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult {
|
||
const manifest = JSON.stringify(v02PipelineRunManifest(sourceCommit));
|
||
const manifestB64 = Buffer.from(manifest, "utf8").toString("base64");
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
const script = [
|
||
"set -eu",
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
`manifest_path=/tmp/${pipelineRun}.json`,
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
"if kubectl create -f \"$manifest_path\"; then",
|
||
" :",
|
||
"else",
|
||
" code=$?",
|
||
` if kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} >/dev/null 2>&1; then`,
|
||
` printf 'PipelineRun %s already exists; reusing existing object\\n' ${shellQuote(pipelineRun)} >&2`,
|
||
" else",
|
||
" exit \"$code\"",
|
||
" fi",
|
||
"fi",
|
||
`kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`,
|
||
].join("\n");
|
||
return g14K3s(["script", "--", script], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record<string, unknown> {
|
||
const targetMode: V02StatusTargetMode = target.mode
|
||
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
||
const strictHeadAlignment = targetMode === "latest-source-head";
|
||
const bundle = v02ControlPlaneStatusBundle({ ...target, mode: targetMode });
|
||
const sections = parseShellSections(statusText(bundle));
|
||
const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null;
|
||
const pipelineRun = stringOrNull(sections.pipelineRunName?.stdout) ?? (sourceCommit === null ? null : v02PipelineRunName(sourceCommit));
|
||
const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? "");
|
||
const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now();
|
||
const sourceHeadsSection = sections.sourceHeads;
|
||
const controlPlane = sections.controlPlane;
|
||
const obsoleteCronJobs = sections.obsoleteCronJobs;
|
||
const argo = sections.argo;
|
||
const pipelineRunSection = sections.pipelineRun;
|
||
const taskRunsSection = sections.taskRuns;
|
||
const planArtifactsSection = sections.planArtifacts;
|
||
const runtimeWorkloadsSection = sections.runtimeWorkloads;
|
||
const gitMirrorCacheSection = sections.gitMirrorCache;
|
||
const webAssetsSection = sections.webAssets;
|
||
const recentPipelineRuns = listV02PipelineRunsCompactFromText(
|
||
sections.recentPipelineRuns?.stdout ?? "",
|
||
shellSectionOk(sections.recentPipelineRuns),
|
||
"kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=<summary-fields>",
|
||
sections.recentPipelineRuns?.exitCode ?? null,
|
||
bundle.stderr,
|
||
8,
|
||
queryNowMs,
|
||
);
|
||
const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
|
||
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
|
||
const sourceHeads = v02SourceHeadsFromText(
|
||
sourceHeadsSection?.stdout ?? "",
|
||
shellSectionOk(sourceHeadsSection),
|
||
sourceHeadsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const pipelineRunInfo = pipelineRun === null
|
||
? null
|
||
: pipelineRunCompactFromText(
|
||
pipelineRun,
|
||
pipelineRunSection?.stdout ?? "",
|
||
shellSectionOk(pipelineRunSection),
|
||
`kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
|
||
pipelineRunSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const taskRuns = taskRunsCompactFromText(
|
||
taskRunsSection?.stdout ?? "",
|
||
shellSectionOk(taskRunsSection),
|
||
pipelineRun,
|
||
taskRunsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const planArtifacts = v02PlanArtifactsFromText(
|
||
planArtifactsSection?.stdout ?? "",
|
||
shellSectionOk(planArtifactsSection),
|
||
pipelineRun,
|
||
planArtifactsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const runtimeWorkloads = v02RuntimeWorkloadsFromText(
|
||
runtimeWorkloadsSection?.stdout ?? "",
|
||
shellSectionOk(runtimeWorkloadsSection),
|
||
runtimeWorkloadsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
);
|
||
const webAssets = v02WebAssetsFromText(
|
||
webAssetsSection?.stdout ?? "",
|
||
shellSectionOk(webAssetsSection),
|
||
sourceCommit,
|
||
syncRevision,
|
||
webAssetsSection?.exitCode ?? null,
|
||
bundle.stderr,
|
||
activePipelineRuns,
|
||
);
|
||
const gitMirror = {
|
||
ok: shellSectionOk(gitMirrorCacheSection),
|
||
summary: gitMirrorStatusSummary(String(gitMirrorCacheSection?.stdout ?? "").trim()),
|
||
raw: String(gitMirrorCacheSection?.stdout ?? "").trim(),
|
||
exitCode: gitMirrorCacheSection?.exitCode ?? null,
|
||
stderr: shellSectionOk(gitMirrorCacheSection) ? "" : bundle.stderr.trim().slice(0, 2000),
|
||
};
|
||
const commitAlignment = v02CommitAlignment({
|
||
expectedSourceHead: sourceCommit,
|
||
sourceHeads,
|
||
gitMirrorSummary: record(gitMirror.summary),
|
||
pipelineRun: pipelineRunInfo,
|
||
recentPipelineRuns,
|
||
planArtifacts,
|
||
runtimeWorkloads,
|
||
webAssets,
|
||
});
|
||
const targetValidationBase = v02TargetValidation({
|
||
targetMode,
|
||
sourceCommit,
|
||
pipelineRun: pipelineRunInfo,
|
||
argo: {
|
||
ok: shellSectionOk(argo),
|
||
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
||
},
|
||
planArtifacts,
|
||
runtimeWorkloads,
|
||
webAssets,
|
||
gitMirror,
|
||
recentPipelineRuns,
|
||
});
|
||
const targetValidation = v02LatestOnlyTargetValidation({
|
||
targetMode,
|
||
sourceCommit,
|
||
pipelineRun: pipelineRunInfo,
|
||
commitAlignment,
|
||
targetValidation: targetValidationBase,
|
||
});
|
||
const falseGreenGuard = targetValidation.state === "superseded"
|
||
? {
|
||
ok: null,
|
||
state: "superseded",
|
||
summary: "target PipelineRun succeeded but runtime now reflects a newer v0.2 PipelineRun; current-runtime false-green guard is not applicable to this historical target",
|
||
sourceCommit,
|
||
targetValidationState: targetValidation.state,
|
||
newerSucceededRuns: targetValidation.newerSucceededRuns ?? [],
|
||
supersededRuntimeServices: targetValidation.supersededRuntimeServices ?? [],
|
||
}
|
||
: v02FalseGreenGuard({ sourceCommit, pipelineRun: pipelineRunInfo, taskRuns, planArtifacts, runtimeWorkloads });
|
||
const baseOk = sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo);
|
||
const targetPipelineRunOk = strictHeadAlignment
|
||
? true
|
||
: pipelineRunInfo !== null && pipelineRunInfo.ok === true && pipelineRunInfo.exists !== false;
|
||
const targetDegradedReason = !strictHeadAlignment && !targetPipelineRunOk ? "target-pipelinerun-not-found-or-unreadable" : undefined;
|
||
const statusCommand = targetMode === "pipeline-run" && pipelineRun !== null
|
||
? `hwlab g14 control-plane status --lane v02 --pipeline-run ${pipelineRun}`
|
||
: targetMode === "source-commit" && sourceCommit !== null
|
||
? `hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}`
|
||
: "hwlab g14 control-plane status --lane v02";
|
||
const state = strictHeadAlignment ? commitAlignment.state : `target-${targetMode}`;
|
||
const sourceCommitSource = targetMode === "pipeline-run"
|
||
? "PipelineRun metadata label hwlab.pikastech.local/source-commit"
|
||
: targetMode === "source-commit"
|
||
? "explicit --source-commit"
|
||
: "G14 CI/CD dedicated bare repo refs/remotes/origin/v0.2; workspace checkout is observable but isolated";
|
||
return {
|
||
ok: baseOk && targetPipelineRunOk && (!strictHeadAlignment || commitAlignment.aligned !== false),
|
||
command: statusCommand,
|
||
lane: "v02",
|
||
state,
|
||
degradedReason: strictHeadAlignment && commitAlignment.aligned === false ? commitAlignment.state : targetDegradedReason,
|
||
statusTarget: {
|
||
mode: statusTargetFields.mode || targetMode,
|
||
sourceCommit,
|
||
pipelineRun,
|
||
strictHeadAlignment,
|
||
note: strictHeadAlignment
|
||
? "default status validates the latest v0.2 source head alignment"
|
||
: "targeted status inspects the requested PipelineRun/source commit without failing merely because origin/v0.2 advanced later",
|
||
},
|
||
sourceCommit,
|
||
sourceCommitSource,
|
||
expected: {
|
||
sourceRepo: V02_CICD_REPO,
|
||
workspace: V02_WORKSPACE,
|
||
workspaceIsolation: "workspace dirty/stale state must not select CI/CD source commits",
|
||
branch: V02_SOURCE_BRANCH,
|
||
namespace: CI_NAMESPACE,
|
||
runtimeNamespace: "hwlab-v02",
|
||
pipeline: V02_PIPELINE,
|
||
trigger: "manual UniDesk CLI PipelineRun creation",
|
||
argoApplication: V02_APP,
|
||
},
|
||
commitAlignment,
|
||
targetValidation,
|
||
sourceHeads,
|
||
gitMirror,
|
||
controlPlane: {
|
||
ok: shellSectionOk(controlPlane),
|
||
names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
exitCode: controlPlane?.exitCode ?? null,
|
||
},
|
||
obsoleteCronJobs: {
|
||
ok: shellSectionOk(obsoleteCronJobs),
|
||
names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
exitCode: obsoleteCronJobs?.exitCode ?? null,
|
||
cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||
},
|
||
argo: {
|
||
ok: shellSectionOk(argo),
|
||
raw: argo?.stdout ?? "",
|
||
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
||
exitCode: argo?.exitCode ?? null,
|
||
},
|
||
pipelineRun: pipelineRunInfo,
|
||
taskRuns,
|
||
planArtifacts,
|
||
runtimeWorkloads,
|
||
falseGreenGuard,
|
||
webAssets,
|
||
activePipelineRuns,
|
||
recentPipelineRuns,
|
||
query: {
|
||
ok: isCommandSuccess(bundle),
|
||
exitCode: bundle.exitCode,
|
||
stderr: bundle.stderr.trim().slice(0, 2000),
|
||
},
|
||
visibilityHint: activePipelineRuns.length > 0
|
||
? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger"
|
||
: "no active v02 PipelineRun observed",
|
||
};
|
||
}
|
||
|
||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
||
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
||
if (options.action === "status" && options.pipelineRun !== undefined) return v02ControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run" });
|
||
if (options.action === "status" && options.sourceCommit !== undefined) return v02ControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit" });
|
||
const snapshot = options.action === "trigger-current" ? getV02TriggerSnapshot() : null;
|
||
const latestHead = options.action === "trigger-current" ? resolveV02LatestRemoteHead() : null;
|
||
const head = snapshot === null
|
||
? resolveV02Head()
|
||
: snapshot.sourceCommit === null
|
||
? resolveV02Head()
|
||
: { sourceCommit: snapshot.sourceCommit, result: snapshot.result };
|
||
let sourceCommit = head.sourceCommit;
|
||
if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) {
|
||
const refreshedHead = resolveV02Head();
|
||
sourceCommit = refreshedHead.sourceCommit;
|
||
if (sourceCommit !== latestHead.sourceCommit) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "source-head-refresh",
|
||
sourceCommit,
|
||
latestSourceHeadProbe: {
|
||
sourceCommit: latestHead.sourceCommit,
|
||
result: compactCommandResult(latestHead.result),
|
||
},
|
||
refreshedHeadProbe: compactCommandResult(refreshedHead.result),
|
||
degradedReason: "v02-head-refresh-did-not-reach-latest-source",
|
||
};
|
||
}
|
||
}
|
||
if (sourceCommit === null) {
|
||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) };
|
||
}
|
||
if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit);
|
||
if (options.action === "status") return v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head" });
|
||
if (options.action === "apply") {
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render.result)) {
|
||
return {
|
||
ok: false,
|
||
command: `hwlab g14 control-plane ${options.action} --lane v02`,
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
renderDir: render.renderDir,
|
||
render: compactCommandResult(render.result),
|
||
};
|
||
}
|
||
const apply = applyV02ControlPlaneFiles(render.renderDir, options.dryRun, options.timeoutSeconds);
|
||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
||
return {
|
||
ok: isCommandSuccess(apply),
|
||
command: "hwlab g14 control-plane apply --lane v02",
|
||
lane: "v02",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||
sourceCommit,
|
||
renderDir: render.renderDir,
|
||
render: compactCommandResult(render.result),
|
||
apply: compactCommandResult(apply),
|
||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||
cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false)),
|
||
status: v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head" }),
|
||
next: options.dryRun
|
||
? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" }
|
||
: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
let before = snapshot !== null && snapshot.sourceCommit === sourceCommit && snapshot.before !== null
|
||
? snapshot.before
|
||
: getPipelineRunCompact(v02PipelineRunName(sourceCommit));
|
||
if (options.dryRun) {
|
||
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "dry-run",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
gitMirrorPreSync,
|
||
controlPlaneRefresh: {
|
||
mode: "skipped-dry-run",
|
||
wouldRefreshBeforeCreate: true,
|
||
resources: ["tekton-v02/rbac.yaml", "tekton-v02/pipeline.yaml", "argocd/project.yaml", "argocd/application-v02.yaml"],
|
||
},
|
||
manifest: v02PipelineRunManifest(sourceCommit),
|
||
next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
if (before.exists === true && before.status !== null) {
|
||
const reuseHead = latestHead ?? resolveV02LatestRemoteHead();
|
||
const reuseDecision = v02ExistingPipelineRunReuseDecision({ sourceCommit, before, latestSourceCommit: reuseHead.sourceCommit });
|
||
if (reuseDecision.reusable === true) {
|
||
const alreadyUsable = reuseDecision.alreadyUsable === true;
|
||
return {
|
||
ok: alreadyUsable,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
skipped: true,
|
||
reuseDecision,
|
||
reason: reuseDecision.reason,
|
||
degradedReason: alreadyUsable ? undefined : "existing-pipelinerun-terminal-failed",
|
||
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse",
|
||
};
|
||
}
|
||
if (reuseHead.sourceCommit !== null && reuseHead.sourceCommit !== sourceCommit) {
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "source-head-recheck",
|
||
status: "advanced",
|
||
sourceCommit,
|
||
latestSourceCommit: reuseHead.sourceCommit,
|
||
previousPipelineRun: v02PipelineRunName(sourceCommit),
|
||
latestPipelineRun: v02PipelineRunName(reuseHead.sourceCommit),
|
||
});
|
||
sourceCommit = reuseHead.sourceCommit;
|
||
before = getPipelineRunCompact(v02PipelineRunName(sourceCommit));
|
||
if (before.exists === true && before.status !== null) {
|
||
const advancedAlreadyUsable = before.status === "True" || before.status === "Unknown";
|
||
return {
|
||
ok: advancedAlreadyUsable,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
skipped: true,
|
||
reuseDecision: {
|
||
...reuseDecision,
|
||
advancedSourceCommit: sourceCommit,
|
||
advancedPipelineRun: v02PipelineRunName(sourceCommit),
|
||
advancedPipelineRunStatus: before.status ?? null,
|
||
},
|
||
reason: advancedAlreadyUsable ? "advanced-existing-pipelinerun-reused" : "advanced-existing-pipelinerun-terminal-failed",
|
||
degradedReason: advancedAlreadyUsable ? undefined : "advanced-existing-pipelinerun-terminal-failed",
|
||
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse",
|
||
};
|
||
}
|
||
} else {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
reuseDecision,
|
||
degradedReason: "existing-pipelinerun-reuse-rejected",
|
||
latestSourceHeadProbe: {
|
||
sourceCommit: reuseHead.sourceCommit,
|
||
exitCode: reuseHead.result.exitCode,
|
||
stderr: reuseHead.result.stderr.trim().slice(0, 2000),
|
||
},
|
||
};
|
||
}
|
||
}
|
||
if (latestHead !== null && latestHead.sourceCommit !== sourceCommit) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "source-head-recheck",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
latestSourceHeadProbe: {
|
||
sourceCommit: latestHead.sourceCommit,
|
||
exitCode: latestHead.result.exitCode,
|
||
stderr: latestHead.result.stderr.trim().slice(0, 2000),
|
||
},
|
||
degradedReason: latestHead.sourceCommit === null
|
||
? "source-head-recheck-unresolved-before-pipelinerun-create"
|
||
: "source-head-advanced-before-pipelinerun-create",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const controlPlaneRefresh = refreshV02ControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds);
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "control-plane-refresh",
|
||
status: controlPlaneRefresh.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
mode: record(controlPlaneRefresh).mode ?? null,
|
||
waitedMs: record(controlPlaneRefresh).waitedMs ?? null,
|
||
renderDir: record(controlPlaneRefresh).renderDir ?? null,
|
||
degradedReason: record(controlPlaneRefresh).degradedReason ?? null,
|
||
});
|
||
if (controlPlaneRefresh.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "control-plane-refresh",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh,
|
||
degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync",
|
||
status: "started",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
reason: "ensure local git mirror has source commit before creating PipelineRun",
|
||
});
|
||
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync",
|
||
status: gitMirrorPreSync.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
mode: record(gitMirrorPreSync).mode ?? null,
|
||
required: nested(gitMirrorPreSync, ["before", "required"]) ?? null,
|
||
localV02: nested(gitMirrorPreSync, ["before", "localV02"]) ?? null,
|
||
finalLocalV02: nested(gitMirrorPreSync, ["final", "localV02"]) ?? nested(gitMirrorPreSync, ["marker", "localV02"]) ?? nested(gitMirrorPreSync, ["after", "localV02"]) ?? null,
|
||
pendingFlush: nested(gitMirrorPreSync, ["before", "pendingFlush"]) ?? null,
|
||
reason: nested(gitMirrorPreSync, ["before", "reason"]) ?? null,
|
||
syncElapsedMs: nested(gitMirrorPreSync, ["sync", "elapsedMs"]) ?? null,
|
||
waitElapsedMs: nested(gitMirrorPreSync, ["wait", "elapsedMs"]) ?? null,
|
||
waitAttempts: nested(gitMirrorPreSync, ["wait", "attempts"]) ?? null,
|
||
lockWaitedMs: record(gitMirrorPreSync).waitedMs ?? null,
|
||
degradedReason: record(gitMirrorPreSync).degradedReason ?? null,
|
||
});
|
||
if (gitMirrorPreSync.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
phase: "git-mirror-pre-sync",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh,
|
||
gitMirrorPreSync,
|
||
degradedReason: "git-mirror-pre-sync-failed",
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
|
||
const createPipelineRun = createV02PipelineRun(sourceCommit, options.timeoutSeconds);
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: createPipelineRun.exitCode });
|
||
return {
|
||
ok: isCommandSuccess(createPipelineRun),
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: "v02",
|
||
mode: "confirmed-trigger",
|
||
sourceCommit,
|
||
pipelineRun: v02PipelineRunName(sourceCommit),
|
||
before,
|
||
controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh),
|
||
gitMirrorPreSync,
|
||
createPipelineRun: compactTriggerCommandResult(compactCommandResult(createPipelineRun)),
|
||
after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)),
|
||
latestOnlyPolicy: "no old PipelineRun is canceled; stale commits self-supersede before GitOps writeback",
|
||
disclosure: {
|
||
fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.",
|
||
},
|
||
};
|
||
}
|
||
|
||
function v02SecretScript(options: G14SecretOptions): string {
|
||
const emitAfterStatus = [
|
||
"after_exists=$(secret_exists_flag)",
|
||
"after_b64=$(secret_b64)",
|
||
"after_key_present=$([ -n \"$after_b64\" ] && printf yes || printf no)",
|
||
"after_value_bytes=$(decoded_length \"$after_b64\")",
|
||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||
"printf 'secret\\t%s\\n' \"$name\"",
|
||
"printf 'key\\t%s\\n' \"$key\"",
|
||
"printf 'action\\t%s\\n' \"$action\"",
|
||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
||
"printf 'beforeKeyPresent\\t%s\\n' \"$before_key_present\"",
|
||
"printf 'beforeValueBytes\\t%s\\n' \"$before_value_bytes\"",
|
||
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
||
"printf 'afterKeyPresent\\t%s\\n' \"$after_key_present\"",
|
||
"printf 'afterValueBytes\\t%s\\n' \"$after_value_bytes\"",
|
||
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
||
].join("\n");
|
||
return [
|
||
"set +e",
|
||
`namespace=${shellQuote(V02_RUNTIME_NAMESPACE)}`,
|
||
`name=${shellQuote(options.name)}`,
|
||
`key=${shellQuote(options.key)}`,
|
||
`action_request=${shellQuote(options.action)}`,
|
||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||
`field_manager=${shellQuote(V02_SECRET_FIELD_MANAGER)}`,
|
||
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||
"secret_b64() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$key\\\" }}\" 2>/dev/null || true; }",
|
||
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
||
"before_exists=$(secret_exists_flag)",
|
||
"before_b64=$(secret_b64)",
|
||
"before_key_present=$([ -n \"$before_b64\" ] && printf yes || printf no)",
|
||
"before_value_bytes=$(decoded_length \"$before_b64\")",
|
||
"action=observed",
|
||
"mutation=false",
|
||
"apply_exit=",
|
||
"if [ \"$action_request\" = ensure ]; then",
|
||
" if [ \"$dry_run\" = true ]; then",
|
||
" if [ \"$before_key_present\" = yes ] && [ \"$before_value_bytes\" -gt 0 ]; then action=kept; else action=would-create; fi",
|
||
" elif [ \"$before_key_present\" = yes ] && [ \"$before_value_bytes\" -gt 0 ]; then",
|
||
" action=kept",
|
||
" else",
|
||
" if ! command -v openssl >/dev/null 2>&1; then",
|
||
" action=openssl-missing",
|
||
" apply_exit=127",
|
||
" else",
|
||
" generated_key=$(openssl rand -base64 48)",
|
||
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$key=$generated_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
||
" apply_exit=$?",
|
||
" generated_key=",
|
||
" if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
|
||
" fi",
|
||
" fi",
|
||
"fi",
|
||
emitAfterStatus,
|
||
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
||
].join("\n");
|
||
}
|
||
|
||
function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||
const fields = keyValueLinesFromText(text);
|
||
const afterExists = fields.afterExists === "yes";
|
||
const afterKeyPresent = fields.afterKeyPresent === "yes";
|
||
const afterValueBytes = numericField(fields.afterValueBytes);
|
||
const healthy = afterExists && afterKeyPresent && typeof afterValueBytes === "number" && afterValueBytes > 0;
|
||
return {
|
||
ok: commandOk && healthy,
|
||
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
|
||
secret: fields.secret || V02_DEVICE_POD_API_KEY_SECRET,
|
||
key: fields.key || V02_DEVICE_POD_API_KEY_SECRET_KEY,
|
||
action: fields.action || null,
|
||
dryRun: fields.dryRun === "true",
|
||
mutation: fields.mutation === "true",
|
||
before: {
|
||
exists: fields.beforeExists === "yes",
|
||
keyPresent: fields.beforeKeyPresent === "yes",
|
||
valueBytes: numericField(fields.beforeValueBytes),
|
||
},
|
||
after: {
|
||
exists: afterExists,
|
||
keyPresent: afterKeyPresent,
|
||
valueBytes: afterValueBytes,
|
||
},
|
||
applyExitCode: numericField(fields.applyExitCode),
|
||
exitCode,
|
||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||
valuesRedacted: true,
|
||
summary: healthy
|
||
? `${fields.secret || V02_DEVICE_POD_API_KEY_SECRET}/${fields.key || V02_DEVICE_POD_API_KEY_SECRET_KEY} exists`
|
||
: `${fields.secret || V02_DEVICE_POD_API_KEY_SECRET}/${fields.key || V02_DEVICE_POD_API_KEY_SECRET_KEY} missing or empty`,
|
||
};
|
||
}
|
||
|
||
function runG14Secret(options: G14SecretOptions): Record<string, unknown> {
|
||
const script = v02SecretScript(options);
|
||
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000);
|
||
const status = v02SecretStatusFromText(statusText(result), isCommandSuccess(result), result.exitCode, result.stderr);
|
||
const dryRunOk = options.action === "ensure" && options.dryRun && isCommandSuccess(result);
|
||
const ok = dryRunOk ? true : status.ok === true;
|
||
return {
|
||
ok,
|
||
command: `hwlab g14 secret ${options.action} --lane v02`,
|
||
lane: options.lane,
|
||
namespace: V02_RUNTIME_NAMESPACE,
|
||
secret: options.name,
|
||
key: options.key,
|
||
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : "confirmed-ensure",
|
||
status,
|
||
mutation: status.mutation === true,
|
||
result: compactCommandResult(result),
|
||
valuesRedacted: true,
|
||
next: ok && options.action === "status"
|
||
? undefined
|
||
: { ensure: `bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name ${options.name} --key ${options.key} --confirm` },
|
||
};
|
||
}
|
||
|
||
function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"delete",
|
||
"cronjob",
|
||
"-n",
|
||
GIT_MIRROR_NAMESPACE,
|
||
GIT_MIRROR_LEGACY_CRONJOB,
|
||
"--ignore-not-found=true",
|
||
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
||
], 60_000);
|
||
}
|
||
|
||
function applyGitMirrorManifestFile(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||
return g14K3s([
|
||
"kubectl",
|
||
"apply",
|
||
"--server-side",
|
||
"--force-conflicts",
|
||
`--field-manager=${GIT_MIRROR_MANIFEST_FIELD_MANAGER}`,
|
||
...(dryRun ? ["--dry-run=server"] : []),
|
||
"-f",
|
||
`${renderDir}/devops-infra/git-mirror.yaml`,
|
||
], timeoutSeconds * 1000);
|
||
}
|
||
|
||
function gitMirrorSyncJobName(): string {
|
||
return `${GIT_MIRROR_SYNC_JOB_PREFIX}-${Date.now().toString(36)}`.slice(0, 63);
|
||
}
|
||
|
||
function gitMirrorFlushJobName(): string {
|
||
return `git-mirror-hwlab-flush-manual-${Date.now().toString(36)}`.slice(0, 63);
|
||
}
|
||
|
||
export function gitMirrorSyncJobManifest(name: string): Record<string, unknown> {
|
||
return {
|
||
apiVersion: "batch/v1",
|
||
kind: "Job",
|
||
metadata: {
|
||
name,
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
labels: {
|
||
"app.kubernetes.io/name": "git-mirror",
|
||
"app.kubernetes.io/part-of": "devops-infra",
|
||
"app.kubernetes.io/component": "sync-controller",
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
backoffLimit: 0,
|
||
activeDeadlineSeconds: 300,
|
||
ttlSecondsAfterFinished: 3600,
|
||
template: {
|
||
metadata: {
|
||
labels: {
|
||
"app.kubernetes.io/name": "git-mirror",
|
||
"app.kubernetes.io/part-of": "devops-infra",
|
||
"app.kubernetes.io/component": "sync-controller",
|
||
"hwlab.pikastech.local/trigger": "manual-cli",
|
||
},
|
||
},
|
||
spec: {
|
||
restartPolicy: "Never",
|
||
hostNetwork: true,
|
||
dnsPolicy: "ClusterFirstWithHostNet",
|
||
volumes: [
|
||
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
|
||
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
|
||
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } },
|
||
],
|
||
containers: [{
|
||
name: "sync",
|
||
image: "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1",
|
||
imagePullPolicy: "IfNotPresent",
|
||
command: ["/script/sync.sh"],
|
||
volumeMounts: [
|
||
{ name: "cache", mountPath: "/cache" },
|
||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||
{ name: "script", mountPath: "/script", readOnly: true },
|
||
],
|
||
}],
|
||
},
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
export function gitMirrorFlushJobManifest(name: string): Record<string, unknown> {
|
||
const manifest = gitMirrorSyncJobManifest(name);
|
||
record(record(manifest.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||
const template = record(record(manifest.spec).template);
|
||
record(record(template.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
||
const podSpec = record(template.spec);
|
||
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
|
||
const first = record(containers[0]);
|
||
first.name = "flush";
|
||
first.command = ["/script/flush.sh"];
|
||
podSpec.containers = [first];
|
||
return manifest;
|
||
}
|
||
|
||
export function parseGitMirrorStatusRefs(raw: string): { refs: Record<string, string | null>; pendingFlush: boolean | null } {
|
||
const line = raw.split(/\r?\n/u).find((item) => item.startsWith("refs="));
|
||
if (line === undefined) return { refs: {}, pendingFlush: null };
|
||
try {
|
||
const parsed = record(JSON.parse(line.slice("refs=".length)) as unknown);
|
||
const refs = record(parsed.refs);
|
||
return {
|
||
refs: {
|
||
localV02: stringOrNull(refs.localV02),
|
||
githubV02: stringOrNull(refs.githubV02),
|
||
localG14: stringOrNull(refs.localG14),
|
||
githubG14: stringOrNull(refs.githubG14),
|
||
localGitops: stringOrNull(refs.localGitops),
|
||
githubGitops: stringOrNull(refs.githubGitops),
|
||
},
|
||
pendingFlush: typeof parsed.pendingFlush === "boolean" ? parsed.pendingFlush : null,
|
||
};
|
||
} catch {
|
||
return { refs: {}, pendingFlush: null };
|
||
}
|
||
}
|
||
|
||
function parseLabeledJsonLine(raw: string, label: string): Record<string, unknown> {
|
||
const line = raw.split(/\r?\n/u).find((item) => item.startsWith(`${label}=`));
|
||
if (line === undefined) return {};
|
||
const value = line.slice(label.length + 1).trim();
|
||
if (value.length === 0) return {};
|
||
try {
|
||
return record(JSON.parse(value) as unknown);
|
||
} catch {
|
||
return {};
|
||
}
|
||
}
|
||
|
||
export function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
|
||
const refs = parseGitMirrorStatusRefs(raw);
|
||
const lastSync = parseLabeledJsonLine(raw, "lastSync");
|
||
const lastWrite = parseLabeledJsonLine(raw, "lastWrite");
|
||
const lastFlush = parseLabeledJsonLine(raw, "lastFlush");
|
||
const localV02 = refs.refs.localV02 ?? null;
|
||
const githubV02 = refs.refs.githubV02 ?? null;
|
||
const localGitops = refs.refs.localGitops ?? null;
|
||
const githubGitops = refs.refs.githubGitops ?? null;
|
||
const sourceInSync = Boolean(localV02 && githubV02 && localV02 === githubV02);
|
||
const gitopsInSync = Boolean(localGitops && githubGitops && localGitops === githubGitops);
|
||
return {
|
||
localV02,
|
||
githubV02,
|
||
localG14: refs.refs.localG14 ?? null,
|
||
githubG14: refs.refs.githubG14 ?? null,
|
||
localGitops,
|
||
githubGitops,
|
||
pendingFlush: refs.pendingFlush,
|
||
lastSync: {
|
||
status: stringOrNull(lastSync.status) ?? null,
|
||
at: stringOrNull(lastSync.publishedAt) ?? stringOrNull(lastSync.syncedAt) ?? null,
|
||
pendingFlush: typeof lastSync.pendingFlush === "boolean" ? lastSync.pendingFlush : null,
|
||
},
|
||
lastWrite: {
|
||
status: stringOrNull(lastWrite.status) ?? null,
|
||
at: stringOrNull(lastWrite.writtenAt) ?? null,
|
||
pendingFlush: typeof lastWrite.pendingFlush === "boolean" ? lastWrite.pendingFlush : null,
|
||
},
|
||
lastFlush: {
|
||
status: stringOrNull(lastFlush.status) ?? null,
|
||
at: stringOrNull(lastFlush.flushedAt) ?? null,
|
||
pendingFlush: typeof lastFlush.pendingFlush === "boolean" ? lastFlush.pendingFlush : null,
|
||
},
|
||
flushNeeded: refs.pendingFlush === true,
|
||
flushCommand: refs.pendingFlush === true ? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" : null,
|
||
sourceInSync,
|
||
gitopsInSync,
|
||
githubInSync: sourceInSync && gitopsInSync,
|
||
};
|
||
}
|
||
|
||
export function gitMirrorV02SyncRequirement(sourceCommit: string, rawStatus: string): Record<string, unknown> {
|
||
const parsed = parseGitMirrorStatusRefs(rawStatus);
|
||
const localV02 = parsed.refs.localV02 ?? null;
|
||
return {
|
||
required: localV02 !== sourceCommit,
|
||
sourceCommit,
|
||
localV02,
|
||
pendingFlush: parsed.pendingFlush,
|
||
refs: parsed.refs,
|
||
reason: localV02 === sourceCommit ? "local-v02-current" : localV02 === null ? "local-v02-unresolved" : "local-v02-stale",
|
||
};
|
||
}
|
||
|
||
function gitMirrorStatusCacheRaw(status: Record<string, unknown>): string {
|
||
return String(nested(status, ["cache", "raw"]) ?? "");
|
||
}
|
||
|
||
function compactGitMirrorStatus(status: Record<string, unknown>, sourceCommit: string): Record<string, unknown> {
|
||
const requirement = gitMirrorV02SyncRequirement(sourceCommit, gitMirrorStatusCacheRaw(status));
|
||
return {
|
||
ok: status.ok === true,
|
||
readUrl: status.readUrl ?? V02_GIT_READ_URL,
|
||
writeUrl: status.writeUrl ?? V02_GIT_WRITE_URL,
|
||
elapsedMs: status.elapsedMs ?? null,
|
||
legacyCronJobExists: nested(status, ["legacyCronJob", "exists"]) === true,
|
||
required: requirement.required,
|
||
sourceCommit,
|
||
localV02: requirement.localV02 ?? null,
|
||
githubV02: requirement.refs.githubV02 ?? null,
|
||
pendingFlush: requirement.pendingFlush ?? null,
|
||
sourceInSync: Boolean(requirement.refs.localV02 && requirement.refs.githubV02 && requirement.refs.localV02 === requirement.refs.githubV02),
|
||
reason: requirement.reason,
|
||
cacheOk: nested(status, ["cache", "ok"]) === true,
|
||
cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000),
|
||
resourcesOk: nested(status, ["resources", "ok"]) === true,
|
||
};
|
||
}
|
||
|
||
function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
ok: sync.ok === true,
|
||
command: sync.command ?? "hwlab g14 git-mirror sync",
|
||
mode: sync.mode ?? null,
|
||
namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE,
|
||
jobName: sync.jobName ?? null,
|
||
elapsedMs: sync.elapsedMs ?? null,
|
||
exitCode: nested(sync, ["result", "exitCode"]) ?? null,
|
||
stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600),
|
||
stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200),
|
||
stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null,
|
||
stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null,
|
||
};
|
||
}
|
||
|
||
interface V02GitMirrorPreSyncMarker {
|
||
ok: boolean;
|
||
sourceCommit: string;
|
||
syncedAt: string;
|
||
localV02?: string | null;
|
||
githubV02?: string | null;
|
||
reason?: string | null;
|
||
}
|
||
|
||
function v02GitMirrorPreSyncStateDir(): string {
|
||
return rootPath(".state", "hwlab-g14", "v02-git-mirror-presync");
|
||
}
|
||
|
||
function v02GitMirrorPreSyncMarkerPath(sourceCommit: string): string {
|
||
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.json`);
|
||
}
|
||
|
||
function v02GitMirrorPreSyncLockDir(sourceCommit: string): string {
|
||
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.lock`);
|
||
}
|
||
|
||
export function v02GitMirrorPreSyncWaitMs(timeoutSeconds: number): number {
|
||
const requestedMs = Math.max(0, Math.floor(timeoutSeconds * 1000));
|
||
if (requestedMs === 0) return V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS;
|
||
return Math.min(V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, requestedMs);
|
||
}
|
||
|
||
export function v02ReusableGitMirrorPreSyncMarker(marker: unknown, sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
|
||
const candidate = record(marker) as V02GitMirrorPreSyncMarker;
|
||
const syncedAtMs = timestampMs(candidate.syncedAt);
|
||
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit) return null;
|
||
if (syncedAtMs === null || nowMs - syncedAtMs > V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS) return null;
|
||
if (syncedAtMs < minSyncedAtMs) return null;
|
||
return candidate;
|
||
}
|
||
|
||
function readV02GitMirrorPreSyncMarker(sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
|
||
const path = v02GitMirrorPreSyncMarkerPath(sourceCommit);
|
||
if (!existsSync(path)) return null;
|
||
try {
|
||
return v02ReusableGitMirrorPreSyncMarker(JSON.parse(readFileSync(path, "utf8")) as unknown, sourceCommit, nowMs, minSyncedAtMs);
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function writeV02GitMirrorPreSyncMarker(sourceCommit: string, summary: Record<string, unknown>): V02GitMirrorPreSyncMarker {
|
||
const marker = {
|
||
ok: true,
|
||
sourceCommit,
|
||
syncedAt: new Date().toISOString(),
|
||
localV02: stringOrNull(summary.localV02),
|
||
githubV02: stringOrNull(summary.githubV02),
|
||
reason: stringOrNull(summary.reason),
|
||
};
|
||
const dir = v02GitMirrorPreSyncStateDir();
|
||
mkdirSync(dir, { recursive: true });
|
||
writeFileSync(v02GitMirrorPreSyncMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
||
return marker;
|
||
}
|
||
|
||
function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number, minSyncedAtMs = 0): { acquired: boolean; lockDir: string; waitedMs: number; marker?: V02GitMirrorPreSyncMarker } {
|
||
const stateDir = v02GitMirrorPreSyncStateDir();
|
||
mkdirSync(stateDir, { recursive: true });
|
||
const lockDir = v02GitMirrorPreSyncLockDir(sourceCommit);
|
||
const startedAtMs = Date.now();
|
||
for (;;) {
|
||
const marker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), minSyncedAtMs);
|
||
if (marker !== null) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs, marker };
|
||
try {
|
||
mkdirSync(lockDir);
|
||
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
||
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
} catch {
|
||
const ageMs = (() => {
|
||
try {
|
||
return Date.now() - statSync(lockDir).mtimeMs;
|
||
} catch {
|
||
return 0;
|
||
}
|
||
})();
|
||
if (ageMs > V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS) {
|
||
try {
|
||
rmSync(lockDir, { recursive: true, force: true });
|
||
continue;
|
||
} catch {
|
||
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
}
|
||
}
|
||
if (Date.now() - startedAtMs >= waitMs) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
||
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||
}
|
||
}
|
||
}
|
||
|
||
function releaseV02GitMirrorPreSyncLock(lockDir: string): void {
|
||
try {
|
||
rmSync(lockDir, { recursive: true, force: true });
|
||
} catch {
|
||
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
||
}
|
||
}
|
||
|
||
function waitForV02GitMirrorPreSync(sourceCommit: string, waitMs: number): Record<string, unknown> {
|
||
const startedAtMs = Date.now();
|
||
const observations: Record<string, unknown>[] = [];
|
||
let attempt = 0;
|
||
for (;;) {
|
||
attempt += 1;
|
||
const statusStartMs = Date.now();
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-wait",
|
||
status: "polling",
|
||
sourceCommit,
|
||
attempt,
|
||
elapsedMs: statusStartMs - startedAtMs,
|
||
});
|
||
const status = runGitMirrorStatus();
|
||
const summary = compactGitMirrorStatus(status, sourceCommit);
|
||
observations.push({
|
||
attempt,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
statusElapsedMs: Date.now() - statusStartMs,
|
||
ok: summary.ok,
|
||
required: summary.required,
|
||
localV02: summary.localV02,
|
||
githubV02: summary.githubV02,
|
||
pendingFlush: summary.pendingFlush,
|
||
reason: summary.reason,
|
||
});
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-wait",
|
||
status: summary.required === false ? "succeeded" : "waiting",
|
||
sourceCommit,
|
||
attempt,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
statusElapsedMs: Date.now() - statusStartMs,
|
||
required: summary.required,
|
||
localV02: summary.localV02,
|
||
githubV02: summary.githubV02,
|
||
pendingFlush: summary.pendingFlush,
|
||
reason: summary.reason,
|
||
});
|
||
if (summary.ok === true && summary.required === false) {
|
||
return {
|
||
ok: true,
|
||
sourceCommit,
|
||
attempts: attempt,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
final: summary,
|
||
observations: observations.slice(-6),
|
||
};
|
||
}
|
||
if (Date.now() - startedAtMs >= waitMs) {
|
||
return {
|
||
ok: false,
|
||
sourceCommit,
|
||
attempts: attempt,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
final: summary,
|
||
observations: observations.slice(-6),
|
||
degradedReason: "git-mirror-local-v02-not-current-after-wait",
|
||
};
|
||
}
|
||
const remainingMs = waitMs - (Date.now() - startedAtMs);
|
||
const sleepMs = Math.max(500, Math.min(V02_GIT_MIRROR_PRESYNC_POLL_MS, remainingMs));
|
||
const wait = runCommand(["sleep", String(Math.ceil(sleepMs / 1000))], repoRoot, { timeoutMs: sleepMs + 2_000 });
|
||
if (wait.exitCode !== 0 && wait.timedOut) {
|
||
return {
|
||
ok: false,
|
||
sourceCommit,
|
||
attempts: attempt,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
final: summary,
|
||
observations: observations.slice(-6),
|
||
degradedReason: "git-mirror-pre-sync-wait-sleep-timeout",
|
||
};
|
||
}
|
||
}
|
||
}
|
||
|
||
function gitMirrorCacheProbeScript(): string {
|
||
return [
|
||
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
|
||
"printf 'refs='",
|
||
"node - <<'NODE' 2>/dev/null || true",
|
||
"const { execFileSync } = require('node:child_process');",
|
||
"const repo = '/cache/pikasTech/HWLAB.git';",
|
||
"function rev(ref) {",
|
||
" try {",
|
||
" return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim();",
|
||
" } catch {",
|
||
" return null;",
|
||
" }",
|
||
"}",
|
||
"const localGitops = rev('refs/heads/v0.2-gitops');",
|
||
"const githubGitops = rev('refs/mirror-stage/heads/v0.2-gitops');",
|
||
"console.log(JSON.stringify({",
|
||
" refs: {",
|
||
" localV02: rev('refs/heads/v0.2'),",
|
||
" githubV02: rev('refs/mirror-stage/heads/v0.2'),",
|
||
" localG14: rev('refs/heads/G14'),",
|
||
" githubG14: rev('refs/mirror-stage/heads/G14'),",
|
||
" localGitops,",
|
||
" githubGitops,",
|
||
" },",
|
||
" pendingFlush: Boolean(localGitops && githubGitops && localGitops !== githubGitops),",
|
||
"}));",
|
||
"NODE",
|
||
].join("\n");
|
||
}
|
||
|
||
function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
|
||
const waitMs = v02GitMirrorPreSyncWaitMs(options.timeoutSeconds);
|
||
const statusStartMs = Date.now();
|
||
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
|
||
const before = runGitMirrorStatus();
|
||
const beforeSummary = compactGitMirrorStatus(before, sourceCommit);
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-status",
|
||
status: beforeSummary.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
durationMs: Date.now() - statusStartMs,
|
||
required: beforeSummary.required,
|
||
localV02: beforeSummary.localV02,
|
||
pendingFlush: beforeSummary.pendingFlush,
|
||
reason: beforeSummary.reason,
|
||
});
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
mode: "dry-run",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
action: beforeSummary.required === true ? "would-sync-before-trigger" : "already-current",
|
||
next: beforeSummary.required === true ? { syncAndTrigger: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" } : undefined,
|
||
};
|
||
}
|
||
if (beforeSummary.required !== true) {
|
||
return {
|
||
ok: true,
|
||
mode: "already-current",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
};
|
||
}
|
||
const existingMarker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
||
if (existingMarker !== null) {
|
||
return {
|
||
ok: true,
|
||
mode: "reused-recent-git-mirror-presync-marker",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
marker: existingMarker,
|
||
waitMs,
|
||
};
|
||
}
|
||
const lock = acquireV02GitMirrorPreSyncLock(sourceCommit, waitMs, statusStartMs);
|
||
if (!lock.acquired) {
|
||
const markerAfterLockTimeout = lock.marker ?? readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
||
if (markerAfterLockTimeout !== null) {
|
||
return {
|
||
ok: true,
|
||
mode: "waited-for-recent-git-mirror-presync-marker",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
marker: markerAfterLockTimeout,
|
||
waitedMs: lock.waitedMs,
|
||
waitMs,
|
||
};
|
||
}
|
||
return {
|
||
ok: false,
|
||
mode: "local-git-mirror-presync-lock",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
lockDir: lock.lockDir,
|
||
waitedMs: lock.waitedMs,
|
||
waitMs,
|
||
degradedReason: "git-mirror-pre-sync-lock-timeout",
|
||
};
|
||
}
|
||
try {
|
||
const markerAfterWait = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
||
if (markerAfterWait !== null) {
|
||
return {
|
||
ok: true,
|
||
mode: "waited-for-recent-git-mirror-presync-marker",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
marker: markerAfterWait,
|
||
waitedMs: lock.waitedMs,
|
||
waitMs,
|
||
};
|
||
}
|
||
const recheckStartMs = Date.now();
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-recheck",
|
||
status: "started",
|
||
sourceCommit,
|
||
waitedMs: lock.waitedMs,
|
||
});
|
||
const recheck = runGitMirrorStatus();
|
||
const recheckSummary = compactGitMirrorStatus(recheck, sourceCommit);
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-recheck",
|
||
status: recheckSummary.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
durationMs: Date.now() - recheckStartMs,
|
||
required: recheckSummary.required,
|
||
localV02: recheckSummary.localV02,
|
||
githubV02: recheckSummary.githubV02,
|
||
pendingFlush: recheckSummary.pendingFlush,
|
||
reason: recheckSummary.reason,
|
||
});
|
||
if (recheckSummary.ok === true && recheckSummary.required === false) {
|
||
const marker = writeV02GitMirrorPreSyncMarker(sourceCommit, recheckSummary);
|
||
return {
|
||
ok: true,
|
||
mode: "became-current-after-lock-wait",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
recheck: recheckSummary,
|
||
marker,
|
||
waitedMs: lock.waitedMs,
|
||
waitMs,
|
||
};
|
||
}
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-sync",
|
||
status: "started",
|
||
sourceCommit,
|
||
reason: recheckSummary.reason,
|
||
localV02: recheckSummary.localV02,
|
||
pendingFlush: recheckSummary.pendingFlush,
|
||
});
|
||
const sync = runGitMirrorSync({
|
||
action: "sync",
|
||
confirm: true,
|
||
dryRun: false,
|
||
timeoutSeconds: options.timeoutSeconds,
|
||
});
|
||
printProgressEvent("hwlab.v02.trigger.progress", {
|
||
stage: "git-mirror-pre-sync-sync",
|
||
status: sync.ok === true ? "succeeded" : "failed",
|
||
sourceCommit,
|
||
durationMs: sync.elapsedMs ?? null,
|
||
jobName: sync.jobName ?? null,
|
||
});
|
||
const syncStatus = record(sync.status);
|
||
const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
|
||
const afterSummary = compactGitMirrorStatus(after, sourceCommit);
|
||
const wait = sync.ok === true && afterSummary.required === true ? waitForV02GitMirrorPreSync(sourceCommit, waitMs) : null;
|
||
const waitFinal = wait === null ? null : record(record(wait).final);
|
||
const finalSummary = wait !== null && wait.ok === true ? waitFinal : afterSummary;
|
||
const ok = sync.ok === true && finalSummary.required === false;
|
||
const marker = ok ? writeV02GitMirrorPreSyncMarker(sourceCommit, finalSummary) : null;
|
||
return {
|
||
ok,
|
||
mode: "auto-sync-before-trigger",
|
||
sourceCommit,
|
||
before: beforeSummary,
|
||
recheck: recheckSummary,
|
||
sync: compactGitMirrorSync(sync),
|
||
after: afterSummary,
|
||
wait,
|
||
final: finalSummary,
|
||
marker,
|
||
waitedMs: lock.waitedMs,
|
||
waitMs,
|
||
degradedReason: ok ? undefined : stringOrNull(record(wait).degradedReason) ?? "git-mirror-local-v02-not-current-after-sync",
|
||
};
|
||
} finally {
|
||
releaseV02GitMirrorPreSyncLock(lock.lockDir);
|
||
}
|
||
}
|
||
|
||
function runGitMirrorStatus(): Record<string, unknown> {
|
||
const startedAtMs = Date.now();
|
||
const script = [
|
||
"set +e",
|
||
"section() {",
|
||
" name=\"$1\"",
|
||
" shift",
|
||
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
||
" \"$@\"",
|
||
" code=$?",
|
||
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
||
"}",
|
||
[
|
||
"section resources kubectl get deploy,svc,pvc,cm",
|
||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
"-l app.kubernetes.io/name=git-mirror",
|
||
"-o name",
|
||
].join(" "),
|
||
[
|
||
"section legacyCronJob kubectl get cronjob",
|
||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
shellQuote(GIT_MIRROR_LEGACY_CRONJOB),
|
||
"--ignore-not-found",
|
||
"-o name",
|
||
].join(" "),
|
||
[
|
||
`section jobs kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
"-l app.kubernetes.io/name=git-mirror",
|
||
"--sort-by=.metadata.creationTimestamp",
|
||
"-o custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime",
|
||
"--no-headers",
|
||
].join(" "),
|
||
[
|
||
"section cache kubectl exec",
|
||
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
||
"deploy/git-mirror-http",
|
||
"-- sh -lc",
|
||
shellQuote(gitMirrorCacheProbeScript()),
|
||
].join(" "),
|
||
].join("\n");
|
||
const bundle = g14K3s(["script", "--", script], 60_000);
|
||
const sections = parseShellSections(statusText(bundle));
|
||
const resources = sections.resources;
|
||
const legacyCronJob = sections.legacyCronJob;
|
||
const jobs = sections.jobs;
|
||
const cache = sections.cache;
|
||
const bundleStderr = bundle.stderr.trim().slice(0, 2000);
|
||
return {
|
||
ok: isCommandSuccess(bundle) && shellSectionOk(resources),
|
||
command: "hwlab g14 git-mirror status",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
readUrl: V02_GIT_READ_URL,
|
||
writeUrl: V02_GIT_WRITE_URL,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
resources: {
|
||
ok: shellSectionOk(resources),
|
||
names: String(resources?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
||
stderr: shellSectionOk(resources) ? "" : bundleStderr,
|
||
},
|
||
legacyCronJob: {
|
||
ok: shellSectionOk(legacyCronJob),
|
||
exists: String(legacyCronJob?.stdout ?? "").trim().length > 0,
|
||
name: String(legacyCronJob?.stdout ?? "").trim() || null,
|
||
cleanupCommand: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
},
|
||
jobs: {
|
||
ok: shellSectionOk(jobs),
|
||
raw: String(jobs?.stdout ?? "").trim(),
|
||
stderr: shellSectionOk(jobs) ? "" : bundleStderr,
|
||
},
|
||
cache: {
|
||
ok: shellSectionOk(cache),
|
||
summary: gitMirrorStatusSummary(String(cache?.stdout ?? "").trim()),
|
||
raw: String(cache?.stdout ?? "").trim(),
|
||
stderr: shellSectionOk(cache) ? "" : bundleStderr,
|
||
},
|
||
next: {
|
||
apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
||
flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
||
},
|
||
};
|
||
}
|
||
|
||
function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const head = resolveV02Head();
|
||
const sourceCommit = head.sourceCommit;
|
||
if (sourceCommit === null) {
|
||
return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) };
|
||
}
|
||
const render = runV02RenderToTemp(sourceCommit);
|
||
if (!isCommandSuccess(render.result)) {
|
||
return {
|
||
ok: false,
|
||
command: "hwlab g14 git-mirror apply",
|
||
phase: "source-render",
|
||
sourceCommit,
|
||
renderDir: render.renderDir,
|
||
render: compactCommandResult(render.result),
|
||
};
|
||
}
|
||
const apply = applyGitMirrorManifestFile(render.renderDir, options.dryRun, options.timeoutSeconds);
|
||
const cleanupLegacyCronJob = isCommandSuccess(apply)
|
||
? deleteLegacyGitMirrorCronJob(options.dryRun)
|
||
: {
|
||
ok: false,
|
||
command: [],
|
||
exitCode: null,
|
||
stdout: "",
|
||
stderr: "skipped because apply failed",
|
||
parsed: null,
|
||
};
|
||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
||
return {
|
||
ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob),
|
||
command: "hwlab g14 git-mirror apply",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||
sourceCommit,
|
||
manifest: `${render.renderDir}/devops-infra/git-mirror.yaml`,
|
||
render: compactCommandResult(render.result),
|
||
apply: compactCommandResult(apply),
|
||
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
|
||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||
status: runGitMirrorStatus(),
|
||
next: options.dryRun
|
||
? { apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm" }
|
||
: { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" },
|
||
};
|
||
}
|
||
|
||
function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const startedAtMs = Date.now();
|
||
const jobName = gitMirrorSyncJobName();
|
||
const manifest = gitMirrorSyncJobManifest(jobName);
|
||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||
const script = [
|
||
"set -eu",
|
||
`job=${shellQuote(jobName)}`,
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
"manifest_path=\"/tmp/$job.json\"",
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
options.dryRun
|
||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||
: [
|
||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||
"while :; do",
|
||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
" exit 44",
|
||
" fi",
|
||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||
" exit 45",
|
||
" fi",
|
||
" sleep 2",
|
||
"done",
|
||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2 refs/heads/v0.2-gitops refs/heads/G14 2>/dev/null || true'`,
|
||
].join("\n"),
|
||
].join("\n");
|
||
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 git-mirror sync",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-sync",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
jobName,
|
||
elapsedMs: Date.now() - startedAtMs,
|
||
manifest: options.dryRun ? manifest : undefined,
|
||
result: compactCommandResult(result),
|
||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||
next: options.dryRun ? { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
||
};
|
||
}
|
||
|
||
function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const jobName = gitMirrorFlushJobName();
|
||
const manifest = gitMirrorFlushJobManifest(jobName);
|
||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||
const script = [
|
||
"set -eu",
|
||
`job=${shellQuote(jobName)}`,
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
"manifest_path=\"/tmp/$job.json\"",
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
options.dryRun
|
||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||
: [
|
||
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
||
"kubectl create -f \"$manifest_path\"",
|
||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||
"while :; do",
|
||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||
" succeeded=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||
" failed=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
" exit 54",
|
||
" fi",
|
||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
||
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
||
" exit 55",
|
||
" fi",
|
||
" sleep 2",
|
||
"done",
|
||
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2-gitops refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true'`,
|
||
].join("\n"),
|
||
].join("\n");
|
||
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 git-mirror flush",
|
||
mode: options.dryRun ? "dry-run" : "confirmed-flush",
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
jobName,
|
||
manifest: options.dryRun ? manifest : undefined,
|
||
result: compactCommandResult(result),
|
||
status: options.dryRun ? undefined : runGitMirrorStatus(),
|
||
next: options.dryRun ? { flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" } : { status: "bun scripts/cli.ts hwlab g14 git-mirror status" },
|
||
};
|
||
}
|
||
|
||
function runG14GitMirror(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
if (options.action === "status") return runGitMirrorStatus();
|
||
if (options.action === "apply") return runGitMirrorApply(options);
|
||
if (options.action === "flush") return runGitMirrorFlush(options);
|
||
return runGitMirrorSync(options);
|
||
}
|
||
|
||
function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record<string, unknown> {
|
||
const job = startJob(name, command, note);
|
||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
||
return {
|
||
ok: true,
|
||
mode: "async-job",
|
||
job,
|
||
statusCommand,
|
||
tailCommand: `tail -f ${job.stdoutFile}`,
|
||
next: {
|
||
status: statusCommand,
|
||
tail: `tail -f ${job.stdoutFile}`,
|
||
},
|
||
};
|
||
}
|
||
|
||
function startControlPlaneTriggerJob(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||
const command = [
|
||
"bun",
|
||
"scripts/cli.ts",
|
||
"hwlab",
|
||
"g14",
|
||
"control-plane",
|
||
"trigger-current",
|
||
"--lane",
|
||
options.lane,
|
||
"--confirm",
|
||
"--timeout-seconds",
|
||
String(options.timeoutSeconds),
|
||
"--wait",
|
||
];
|
||
return {
|
||
command: "hwlab g14 control-plane trigger-current --lane v02",
|
||
lane: options.lane,
|
||
reason: "confirmed trigger can spend tens of seconds syncing git mirror and creating PipelineRun; default is fire-and-forget to avoid silent blocking",
|
||
waitCommand: command.join(" "),
|
||
...startAsyncHwlabG14Job(
|
||
"hwlab_g14_v02_trigger_current",
|
||
command,
|
||
"Trigger HWLAB v0.2 current commit PipelineRun with git mirror pre-sync through G14 control-plane",
|
||
),
|
||
};
|
||
}
|
||
|
||
function startGitMirrorJob(options: G14GitMirrorOptions): Record<string, unknown> {
|
||
const command = [
|
||
"bun",
|
||
"scripts/cli.ts",
|
||
"hwlab",
|
||
"g14",
|
||
"git-mirror",
|
||
options.action,
|
||
"--confirm",
|
||
"--timeout-seconds",
|
||
String(options.timeoutSeconds),
|
||
"--wait",
|
||
];
|
||
return {
|
||
command: `hwlab g14 git-mirror ${options.action}`,
|
||
namespace: GIT_MIRROR_NAMESPACE,
|
||
reason: "manual git mirror sync/flush waits for a Kubernetes Job; default is fire-and-forget to keep CLI output immediately visible",
|
||
waitCommand: command.join(" "),
|
||
...startAsyncHwlabG14Job(
|
||
`hwlab_g14_git_mirror_${options.action}`,
|
||
command,
|
||
`Run HWLAB devops-infra git mirror ${options.action} through a bounded manual Kubernetes Job`,
|
||
),
|
||
};
|
||
}
|
||
|
||
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
||
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
|
||
}
|
||
|
||
function g14CiToolsImage(tag: string): string {
|
||
return `${G14_CI_TOOLS_IMAGE_REPO}:${tag}`;
|
||
}
|
||
|
||
function runG14ToolsImageStatus(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
const image = g14CiToolsImage(options.tag);
|
||
const script = [
|
||
"set -u",
|
||
`image=${shellQuote(image)}`,
|
||
"exists=false",
|
||
"image_id=",
|
||
"created=",
|
||
"size=",
|
||
"node_version=",
|
||
"npm_version=",
|
||
"bun_version=",
|
||
"git_version=",
|
||
"docker_version=",
|
||
"python_version=",
|
||
"registry_tags=",
|
||
"if docker image inspect \"$image\" >/tmp/hwlab-tools-image-inspect.json 2>/tmp/hwlab-tools-image-inspect.err; then",
|
||
" exists=true",
|
||
" image_id=$(docker image inspect \"$image\" --format '{{.Id}}' 2>/dev/null || true)",
|
||
" created=$(docker image inspect \"$image\" --format '{{.Created}}' 2>/dev/null || true)",
|
||
" size=$(docker image inspect \"$image\" --format '{{.Size}}' 2>/dev/null || true)",
|
||
" node_version=$(docker run --rm \"$image\" node --version 2>/dev/null || true)",
|
||
" npm_version=$(docker run --rm \"$image\" npm --version 2>/dev/null || true)",
|
||
" bun_version=$(docker run --rm \"$image\" bun --version 2>/dev/null || true)",
|
||
" git_version=$(docker run --rm \"$image\" git --version 2>/dev/null || true)",
|
||
" docker_version=$(docker run --rm \"$image\" docker --version 2>/dev/null || true)",
|
||
" python_version=$(docker run --rm \"$image\" python3 --version 2>/dev/null || true)",
|
||
"fi",
|
||
`registry_tags=$(curl -fsS --max-time 10 http://127.0.0.1:5000/v2/hwlab/hwlab-ci-node-tools/tags/list 2>/dev/null || true)`,
|
||
"export image exists image_id created size node_version npm_version bun_version git_version docker_version python_version registry_tags",
|
||
"node - <<'NODE'",
|
||
"const fs = require('node:fs');",
|
||
"const env = process.env;",
|
||
"const payload = {",
|
||
" image: env.image,",
|
||
" exists: env.exists === 'true',",
|
||
" imageId: env.image_id || null,",
|
||
" created: env.created || null,",
|
||
" sizeBytes: env.size ? Number(env.size) : null,",
|
||
" tools: {",
|
||
" node: env.node_version || null,",
|
||
" npm: env.npm_version || null,",
|
||
" bun: env.bun_version || null,",
|
||
" git: env.git_version || null,",
|
||
" docker: env.docker_version || null,",
|
||
" python: env.python_version || null",
|
||
" },",
|
||
" registryTagsRaw: env.registry_tags || null",
|
||
"};",
|
||
"console.log(JSON.stringify(payload, null, 2));",
|
||
"NODE",
|
||
].join("\n");
|
||
const result = g14HostScript(script, 180_000);
|
||
let parsedStatus: unknown = null;
|
||
const text = statusText(result);
|
||
if (text.length > 0) {
|
||
try {
|
||
parsedStatus = JSON.parse(text) as unknown;
|
||
} catch {
|
||
parsedStatus = null;
|
||
}
|
||
}
|
||
return {
|
||
ok: isCommandSuccess(result) && record(parsedStatus).exists === true,
|
||
command: "hwlab g14 tools-image status --name ci-node-tools",
|
||
name: options.name,
|
||
image,
|
||
status: parsedStatus ?? text,
|
||
result,
|
||
};
|
||
}
|
||
|
||
function runG14ToolsImageBuild(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
const image = g14CiToolsImage(options.tag);
|
||
const script = [
|
||
"set -eu",
|
||
`cd ${shellQuote(V02_WORKSPACE)}`,
|
||
"git fetch origin v0.2 --prune",
|
||
"git checkout v0.2 >/dev/null 2>&1 || true",
|
||
"git merge --ff-only origin/v0.2",
|
||
`test -f ${shellQuote(options.dockerfile)}`,
|
||
`image=${shellQuote(image)}`,
|
||
`dockerfile=${shellQuote(options.dockerfile)}`,
|
||
"echo \"{\\\"phase\\\":\\\"preflight\\\",\\\"image\\\":\\\"$image\\\",\\\"dockerfile\\\":\\\"$dockerfile\\\",\\\"commit\\\":\\\"$(git rev-parse HEAD)\\\"}\"",
|
||
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
|
||
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
|
||
"export no_proxy=$NO_PROXY",
|
||
"docker build --pull --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg http_proxy --build-arg https_proxy --build-arg NO_PROXY --build-arg no_proxy -f \"$dockerfile\" -t \"$image\" .",
|
||
"docker run --rm \"$image\" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V'",
|
||
"docker push \"$image\"",
|
||
"digest=$(docker image inspect \"$image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
|
||
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"image\\\":\\\"$image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
|
||
].join("\n");
|
||
if (options.dryRun) {
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||
mode: "dry-run",
|
||
image,
|
||
workspace: V02_WORKSPACE,
|
||
dockerfile: options.dockerfile,
|
||
buildScriptPreview: script.split(/\n/u),
|
||
next: { build: `bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag ${options.tag} --confirm` },
|
||
};
|
||
}
|
||
const result = g14HostScript(script, options.timeoutSeconds * 1000);
|
||
return {
|
||
ok: isCommandSuccess(result),
|
||
command: "hwlab g14 tools-image build --name ci-node-tools",
|
||
mode: "confirmed-build",
|
||
image,
|
||
workspace: V02_WORKSPACE,
|
||
dockerfile: options.dockerfile,
|
||
result,
|
||
status: runG14ToolsImageStatus({ ...options, action: "status", dryRun: true, confirm: false }),
|
||
};
|
||
}
|
||
|
||
function runG14ToolsImage(options: G14ToolsImageOptions): Record<string, unknown> {
|
||
if (options.action === "status") return runG14ToolsImageStatus(options);
|
||
return runG14ToolsImageBuild(options);
|
||
}
|
||
|
||
function listOpenG14PullRequests(): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "30", "--json", "number,title,state,url,head,base,draft,headRefName,baseRefName"], 60_000);
|
||
}
|
||
|
||
function preflightPullRequest(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "preflight", String(number), "--repo", HWLAB_REPO], 80_000);
|
||
}
|
||
|
||
function mergePullRequest(number: number, dryRun: boolean): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "merge", String(number), "--repo", HWLAB_REPO, "--merge", "--delete-branch", ...(dryRun ? ["--dry-run"] : [])], 100_000);
|
||
}
|
||
|
||
function getG14Head(): string | null {
|
||
const result = cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "git fetch origin G14 --prune >/dev/null 2>&1; git rev-parse origin/G14"], 120_000);
|
||
if (!isCommandSuccess(result)) return null;
|
||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||
return match?.[0] ?? null;
|
||
}
|
||
|
||
function refreshArgoDev(): void {
|
||
cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "annotate", "application", "-n", ARGO_NAMESPACE, DEV_APP, "argocd.argoproj.io/refresh=hard", "--overwrite"], 60_000);
|
||
}
|
||
|
||
function getPipelineStatus(sourceCommit: string): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}"], 60_000);
|
||
}
|
||
|
||
function getArgoStatus(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "application", "-n", ARGO_NAMESPACE, DEV_APP, "-o", "jsonpath={.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.operationState.phase}{\"\\n\"}{.status.operationState.syncResult.revision}{\"\\n\"}"], 60_000);
|
||
}
|
||
|
||
function getDevWorkloads(): CommandJsonResult {
|
||
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "deploy,statefulset,pod", "-n", DEV_NAMESPACE, "-o", "wide"], 60_000);
|
||
}
|
||
|
||
function getLiveHealth(): CommandJsonResult {
|
||
return commandJson(["curl", "-fsS", "--max-time", "20", "http://74.48.78.17:17667/health/live"], 30_000);
|
||
}
|
||
|
||
function statusText(result: CommandJsonResult): string {
|
||
return String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||
}
|
||
|
||
function workloadReadiness(workloadsText: string, commandOk: boolean): Record<string, unknown> {
|
||
const blockers: string[] = [];
|
||
const ignoredPods: string[] = [];
|
||
const readyWorkloads: string[] = [];
|
||
const allowedZeroDeployments = new Set(["deployment.apps/hwlab-gateway", "deployment.apps/hwlab-tunnel-client"]);
|
||
for (const line of workloadsText.split(/\r?\n/u)) {
|
||
const trimmed = line.trim();
|
||
if (trimmed.length === 0 || trimmed.startsWith("NAME ")) continue;
|
||
const fields = trimmed.split(/\s+/u);
|
||
const name = fields[0] ?? "";
|
||
const ready = fields[1] ?? "";
|
||
if (name.startsWith("deployment.apps/") || name.startsWith("statefulset.apps/")) {
|
||
if (ready === "0/0" && allowedZeroDeployments.has(name)) {
|
||
readyWorkloads.push(`${name}:${ready}:scaled-zero`);
|
||
continue;
|
||
}
|
||
const match = /^(\d+)\/(\d+)$/u.exec(ready);
|
||
if (match === null) {
|
||
blockers.push(`${name}:unparseable-ready:${ready}`);
|
||
continue;
|
||
}
|
||
const readyCount = Number(match[1]);
|
||
const desiredCount = Number(match[2]);
|
||
if (desiredCount > 0 && readyCount < desiredCount) blockers.push(`${name}:not-ready:${ready}`);
|
||
else readyWorkloads.push(`${name}:${ready}`);
|
||
continue;
|
||
}
|
||
if (name.startsWith("pod/")) {
|
||
const status = fields[2] ?? "";
|
||
if (status === "Completed" || status === "Succeeded") ignoredPods.push(`${name}:${status}`);
|
||
if (/^(CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError|RunContainerError)$/u.test(status)) {
|
||
blockers.push(`${name}:pod-status:${status}`);
|
||
}
|
||
}
|
||
}
|
||
return {
|
||
ready: commandOk && blockers.length === 0,
|
||
blockers,
|
||
readyWorkloads: readyWorkloads.slice(0, 20),
|
||
ignoredPods: ignoredPods.slice(0, 20),
|
||
};
|
||
}
|
||
|
||
function beijingParts(date = new Date()): { date: string; time: string; iso: string } {
|
||
const shifted = new Date(date.getTime() + BEIJING_OFFSET_MS).toISOString();
|
||
return { date: shifted.slice(0, 10), time: shifted.slice(11, 19), iso: shifted };
|
||
}
|
||
|
||
function durationSeconds(start?: string | null, end?: string | null): number | null {
|
||
if (start === undefined || start === null || end === undefined || end === null) return null;
|
||
const startMs = Date.parse(start);
|
||
const endMs = Date.parse(end);
|
||
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) return null;
|
||
return Math.round((endMs - startMs) / 1000);
|
||
}
|
||
|
||
function formatDuration(seconds: number | null): string {
|
||
if (seconds === null) return "n/a";
|
||
const minutes = Math.floor(seconds / 60);
|
||
const rest = seconds % 60;
|
||
if (minutes === 0) return `${rest}s`;
|
||
return `${minutes}m${String(rest).padStart(2, "0")}s`;
|
||
}
|
||
|
||
function pipelineTaskRunResultsJsonPath(): string {
|
||
return [
|
||
"jsonpath=",
|
||
"{range .items[*]}",
|
||
"{.metadata.name}{\"\\t\"}",
|
||
"{.metadata.labels.tekton\\.dev/pipelineTask}{\"\\t\"}",
|
||
"{.status.startTime}{\"\\t\"}",
|
||
"{.status.completionTime}{\"\\t\"}",
|
||
"{range .status.results[*]}{.name}{\"=\"}{.value}{\";\"}{end}",
|
||
"{\"\\n\"}",
|
||
"{end}",
|
||
].join("");
|
||
}
|
||
|
||
function getPipelineTaskRunResults(pipelineRun: string): CommandJsonResult {
|
||
return cliJson([
|
||
"ssh",
|
||
`${G14_PROVIDER}:k3s`,
|
||
"kubectl",
|
||
"get",
|
||
"taskrun",
|
||
"-n",
|
||
CI_NAMESPACE,
|
||
"-l",
|
||
`tekton.dev/pipelineRun=${pipelineRun}`,
|
||
"-o",
|
||
pipelineTaskRunResultsJsonPath(),
|
||
], 80_000);
|
||
}
|
||
|
||
function parseResultMap(raw: string): Record<string, string> {
|
||
const values: Record<string, string> = {};
|
||
for (const item of raw.split(";")) {
|
||
if (item.length === 0) continue;
|
||
const separator = item.indexOf("=");
|
||
if (separator <= 0) continue;
|
||
values[item.slice(0, separator)] = item.slice(separator + 1);
|
||
}
|
||
return values;
|
||
}
|
||
|
||
function serviceMetricSort(left: CiServiceMetric, right: CiServiceMetric): number {
|
||
return left.serviceId.localeCompare(right.serviceId);
|
||
}
|
||
|
||
export function parsePipelineTaskRunMetrics(pipelineRun: string, text: string): CiPipelineMetrics {
|
||
const services: CiServiceMetric[] = [];
|
||
for (const line of text.split(/\r?\n/u)) {
|
||
const trimmed = line.trimEnd();
|
||
if (trimmed.length === 0) continue;
|
||
const [taskRunName = "", pipelineTask = "", startTime = "", completionTime = "", resultsRaw = ""] = trimmed.split("\t");
|
||
if (!pipelineTask.startsWith("build-")) continue;
|
||
const results = parseResultMap(resultsRaw);
|
||
const serviceId = results["service-id"] ?? pipelineTask.replace(/^build-/u, "");
|
||
if (serviceId.length === 0) continue;
|
||
services.push({
|
||
taskRun: taskRunName,
|
||
serviceId,
|
||
status: results.status ?? "unknown",
|
||
durationSeconds: durationSeconds(startTime, completionTime),
|
||
buildBackend: results["build-backend"] ?? null,
|
||
reusedFrom: results["reused-from"] ?? null,
|
||
imageTag: results["image-tag"] ?? null,
|
||
sourceCommitId: results["source-commit-id"] ?? null,
|
||
});
|
||
}
|
||
services.sort(serviceMetricSort);
|
||
const reusedServices = services.filter((service) => service.status === "reused").map((service) => service.serviceId);
|
||
const rebuildServices = services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
|
||
return {
|
||
ok: services.length > 0,
|
||
pipelineRun,
|
||
serviceTotal: services.length,
|
||
reusedCount: reusedServices.length,
|
||
rebuildCount: rebuildServices.length,
|
||
reusedServices,
|
||
rebuildServices,
|
||
services,
|
||
...(services.length === 0 ? { degradedReason: "no-build-task-results", rawText: text.slice(0, 2000) } : {}),
|
||
};
|
||
}
|
||
|
||
function ciMetricsFromValue(value: unknown): CiPipelineMetrics | null {
|
||
const data = record(value);
|
||
if (data.ok !== true || typeof data.pipelineRun !== "string") return null;
|
||
const services = Array.isArray(data.services) ? data.services.map((item) => {
|
||
const service = record(item);
|
||
return {
|
||
taskRun: String(service.taskRun ?? ""),
|
||
serviceId: String(service.serviceId ?? ""),
|
||
status: String(service.status ?? "unknown"),
|
||
durationSeconds: typeof service.durationSeconds === "number" ? service.durationSeconds : null,
|
||
buildBackend: typeof service.buildBackend === "string" ? service.buildBackend : null,
|
||
reusedFrom: typeof service.reusedFrom === "string" ? service.reusedFrom : null,
|
||
imageTag: typeof service.imageTag === "string" ? service.imageTag : null,
|
||
sourceCommitId: typeof service.sourceCommitId === "string" ? service.sourceCommitId : null,
|
||
};
|
||
}).filter((service) => service.serviceId.length > 0).sort(serviceMetricSort) : [];
|
||
const reusedServices = Array.isArray(data.reusedServices) ? data.reusedServices.map(String) : services.filter((service) => service.status === "reused").map((service) => service.serviceId);
|
||
const rebuildServices = Array.isArray(data.rebuildServices) ? data.rebuildServices.map(String) : services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
|
||
return {
|
||
ok: true,
|
||
pipelineRun: data.pipelineRun,
|
||
serviceTotal: typeof data.serviceTotal === "number" ? data.serviceTotal : services.length,
|
||
reusedCount: typeof data.reusedCount === "number" ? data.reusedCount : reusedServices.length,
|
||
rebuildCount: typeof data.rebuildCount === "number" ? data.rebuildCount : rebuildServices.length,
|
||
reusedServices,
|
||
rebuildServices,
|
||
services,
|
||
};
|
||
}
|
||
|
||
function collectPipelineMetrics(pipelineRun: string): CiPipelineMetrics {
|
||
const result = getPipelineTaskRunResults(pipelineRun);
|
||
if (!isCommandSuccess(result)) {
|
||
return {
|
||
ok: false,
|
||
pipelineRun,
|
||
serviceTotal: 0,
|
||
reusedCount: 0,
|
||
rebuildCount: 0,
|
||
reusedServices: [],
|
||
rebuildServices: [],
|
||
services: [],
|
||
degradedReason: "taskrun-query-failed",
|
||
rawText: `${result.stderr}\n${result.stdout}`.trim().slice(0, 2000),
|
||
};
|
||
}
|
||
return parsePipelineTaskRunMetrics(pipelineRun, statusText(result));
|
||
}
|
||
|
||
function ciMetricLines(metrics: CiPipelineMetrics | null): string[] {
|
||
if (metrics === null) {
|
||
return [
|
||
"- CI/CD 关键指标:",
|
||
" - lazy build reused: n/a",
|
||
" - reused services: n/a",
|
||
" - rebuild services: n/a",
|
||
" - service durations: n/a",
|
||
];
|
||
}
|
||
if (metrics.ok !== true) {
|
||
return [
|
||
"- CI/CD 关键指标:",
|
||
` - lazy build reused: unavailable (${metrics.degradedReason ?? "unknown"})`,
|
||
" - reused services: n/a",
|
||
" - rebuild services: n/a",
|
||
" - service durations: n/a",
|
||
];
|
||
}
|
||
const lines = [
|
||
"- CI/CD 关键指标:",
|
||
` - lazy build reused: ${metrics.reusedCount}/${metrics.serviceTotal}`,
|
||
` - reused services: ${metrics.reusedServices.length > 0 ? metrics.reusedServices.join(", ") : "none"}`,
|
||
` - rebuild services: ${metrics.rebuildServices.length > 0 ? metrics.rebuildServices.join(", ") : "none"}`,
|
||
" - service durations:",
|
||
];
|
||
for (const service of metrics.services) {
|
||
const backend = service.buildBackend === null ? "" : `, backend=${service.buildBackend}`;
|
||
lines.push(` - ${service.serviceId}: ${service.status}, ${formatDuration(service.durationSeconds)}${backend}`);
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
function stripMarkdownInline(value: string): string {
|
||
return value
|
||
.replace(/\[([^\]]+)\]\([^)]+\)/gu, "$1")
|
||
.replace(/\*\*([^*]+)\*\*/gu, "$1")
|
||
.replace(/\*([^*]+)\*/gu, "$1")
|
||
.replace(/__([^_]+)__/gu, "$1")
|
||
.replace(/_([^_]+)_/gu, "$1")
|
||
.replace(/\s+/gu, " ")
|
||
.trim();
|
||
}
|
||
|
||
function markdownSection(body: string, headings: string[]): string {
|
||
const normalizedHeadings = new Set(headings.map((heading) => heading.toLowerCase()));
|
||
const lines = body.split(/\r?\n/u);
|
||
let capturing = false;
|
||
const captured: string[] = [];
|
||
for (const line of lines) {
|
||
const heading = /^(#{1,6})\s+(.+?)\s*$/u.exec(line);
|
||
if (heading !== null) {
|
||
const title = stripMarkdownInline(heading[2] ?? "").replace(/[：:]\s*$/u, "").toLowerCase();
|
||
if (capturing) break;
|
||
capturing = normalizedHeadings.has(title);
|
||
continue;
|
||
}
|
||
if (capturing) captured.push(line);
|
||
}
|
||
return captured.join("\n").trim();
|
||
}
|
||
|
||
function firstParagraph(section: string): string | null {
|
||
for (const paragraph of section.split(/\n\s*\n/u)) {
|
||
const trimmed = stripMarkdownInline(paragraph.replace(/^[-*]\s+/gmu, ""));
|
||
if (trimmed.length > 0) return trimmed;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function topLevelBullets(section: string, limit = 8): string[] {
|
||
const bullets: string[] = [];
|
||
for (const line of section.split(/\r?\n/u)) {
|
||
const match = /^[-*]\s+(.+)$/u.exec(line);
|
||
if (match === null) continue;
|
||
const text = stripMarkdownInline(match[1] ?? "");
|
||
if (text.length > 0) bullets.push(text);
|
||
if (bullets.length >= limit) break;
|
||
}
|
||
return bullets;
|
||
}
|
||
|
||
function conventionalTitleSummary(title: string): string {
|
||
const match = /^(feat|fix|docs|test|refactor|perf|ci|build|chore)(?:\([^)]+\))?:\s*(.+)$/iu.exec(title.trim());
|
||
if (match === null) return stripMarkdownInline(title);
|
||
const kind = String(match[1] ?? "").toLowerCase();
|
||
const subject = stripMarkdownInline(match[2] ?? title);
|
||
const labels: Record<string, string> = {
|
||
feat: "功能",
|
||
fix: "修复",
|
||
docs: "文档",
|
||
test: "测试",
|
||
refactor: "重构",
|
||
perf: "性能",
|
||
ci: "CI/CD",
|
||
build: "构建",
|
||
chore: "维护",
|
||
};
|
||
return `${labels[kind] ?? "变更"}：${subject}`;
|
||
}
|
||
|
||
function fileHeuristicBullets(fileSummary: Record<string, unknown>): string[] {
|
||
const keyFiles = Array.isArray(fileSummary.keyFiles) ? fileSummary.keyFiles.map(String) : [];
|
||
const bullets: string[] = [];
|
||
if (keyFiles.some((file) => file.includes("web/hwlab-cloud-web/"))) {
|
||
bullets.push("前端：更新 HWLAB Cloud Web / Workbench 交互行为。");
|
||
}
|
||
if (keyFiles.some((file) => /\.(test|spec)\./u.test(file) || file.includes("/test"))) {
|
||
bullets.push("验证：补充或更新回归测试，覆盖本次变更路径。");
|
||
}
|
||
if (keyFiles.some((file) => file.includes("deploy/") || file.includes("k8s") || file.includes("gitops"))) {
|
||
bullets.push("部署：更新 G14 Kubernetes/GitOps 发布声明或发布计划。");
|
||
}
|
||
if (keyFiles.some((file) => file.includes("internal/cloud/") || file.includes("code-agent"))) {
|
||
bullets.push("运行时：调整 Cloud API / Code Agent 运行链路。");
|
||
}
|
||
return bullets;
|
||
}
|
||
|
||
export function semanticChangelogBullets(title: string, body: string, fileSummary: Record<string, unknown>): string[] {
|
||
const bullets: string[] = [];
|
||
const background = firstParagraph(markdownSection(body, ["背景", "问题", "problem", "background", "context"]));
|
||
if (background !== null && /^(fix|perf)(?:\([^)]+\))?:/iu.test(title)) {
|
||
bullets.push(`修复目标：${background}`);
|
||
}
|
||
const changeSection = markdownSection(body, ["上线 changelog", "changelog", "摘要", "概述", "总结", "修改", "变更", "改动", "changes", "what changed", "summary"]);
|
||
bullets.push(...topLevelBullets(changeSection));
|
||
if (bullets.length === 0) bullets.push(conventionalTitleSummary(title));
|
||
for (const bullet of fileHeuristicBullets(fileSummary)) {
|
||
if (bullets.length >= 8) break;
|
||
if (!bullets.includes(bullet)) bullets.push(bullet);
|
||
}
|
||
return bullets.slice(0, 8);
|
||
}
|
||
|
||
function readIssue(issueNumber: number): CommandJsonResult {
|
||
return cliJson(["gh", "issue", "read", String(issueNumber), "--repo", HWLAB_REPO, "--json", "title,body,state,updatedAt", "--raw"], 80_000);
|
||
}
|
||
|
||
function issueFromRead(result: CommandJsonResult): Record<string, unknown> {
|
||
return record(commandData(result).issue);
|
||
}
|
||
|
||
function parseBriefIssueFromIndex(indexBody: string, date: string): { number: number; url: string } | null {
|
||
const escapedDate = date.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||
const linePattern = new RegExp(`^\\|\\s*${escapedDate}\\s*\\|([^\\n]+)$`, "mu");
|
||
const line = linePattern.exec(indexBody)?.[0];
|
||
if (line === undefined) return null;
|
||
const match = /\[#(\d+)\]\((https:\/\/github\.com\/pikasTech\/HWLAB\/issues\/\d+)\)/u.exec(line);
|
||
if (match === null) return null;
|
||
return { number: Number(match[1]), url: match[2] };
|
||
}
|
||
|
||
function dailyBriefTitle(date: string): string {
|
||
return `${date} 指挥简报（北京时间）`;
|
||
}
|
||
|
||
function writeStateFile(name: string, content: string): string {
|
||
const dir = rootPath(".state", "hwlab-g14");
|
||
mkdirSync(dir, { recursive: true });
|
||
const path = join(dir, name);
|
||
writeFileSync(path, content, "utf8");
|
||
return path;
|
||
}
|
||
|
||
function listIssues(): CommandJsonResult {
|
||
return cliJson(["gh", "issue", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "100", "--json", "number,title,state,url"], 80_000);
|
||
}
|
||
|
||
function findBriefIssueInList(date: string): { number: number; url: string } | null {
|
||
const listed = listIssues();
|
||
if (!isCommandSuccess(listed)) return null;
|
||
const issues = commandData(listed).issues;
|
||
if (!Array.isArray(issues)) return null;
|
||
const title = dailyBriefTitle(date);
|
||
const match = issues.map((item) => record(item)).find((issue) => issue.title === title);
|
||
if (match === undefined) return null;
|
||
return { number: Number(match.number), url: String(match.url) };
|
||
}
|
||
|
||
function newDailyBriefBody(date: string): string {
|
||
return [
|
||
`# ${dailyBriefTitle(date)}`,
|
||
"",
|
||
"## 当日更新",
|
||
"",
|
||
"## 常驻观察与长期建议",
|
||
"",
|
||
"- G14 HWLAB source truth 固定为 G14 `/root/hwlab` 与 `origin/G14`；DEV rollout 只接受 G14 Tekton/GitOps/Argo 和公网 health 证据。",
|
||
"- GitHub issue/PR 写操作必须走 UniDesk `gh` 子命令；G14 k3s 操作必须走 UniDesk `trans G14:k3s`。",
|
||
].join("\n");
|
||
}
|
||
|
||
function insertBriefIndexRow(indexBody: string, date: string, brief: { number: number; url: string }): string {
|
||
if (parseBriefIssueFromIndex(indexBody, date) !== null) return indexBody;
|
||
const row = `| ${date} | [#${brief.number}](${brief.url}) | 已创建当日 G14 DEV rollout 自动简报入口；后续每次 DEV 上线记录 CI/CD 耗时和上线 changelog。 | 继续监控 HWLAB base=G14 PR，ready 后自动合并并滚动到 G14 DEV。 |`;
|
||
const marker = "### 指挥简报索引";
|
||
const markerIndex = indexBody.indexOf(marker);
|
||
if (markerIndex === -1) return `${indexBody.trimEnd()}\n\n${marker}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n`;
|
||
const afterMarker = indexBody.slice(markerIndex);
|
||
const separator = "| --- | --- | --- | --- |";
|
||
const separatorInBody = afterMarker.indexOf(separator);
|
||
if (separatorInBody === -1) return `${indexBody.slice(0, markerIndex + marker.length)}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n${indexBody.slice(markerIndex + marker.length)}`;
|
||
const insertAt = markerIndex + separatorInBody + separator.length;
|
||
return `${indexBody.slice(0, insertAt)}\n${row}${indexBody.slice(insertAt)}`;
|
||
}
|
||
|
||
function ensureDailyBriefIssue(date: string, dryRun: boolean): Record<string, unknown> {
|
||
const indexRead = readIssue(G14_BRIEF_INDEX_ISSUE);
|
||
if (!isCommandSuccess(indexRead)) return { ok: false, phase: "read-index", indexRead };
|
||
const indexIssue = issueFromRead(indexRead);
|
||
const indexBody = String(indexIssue.body ?? "");
|
||
let brief = parseBriefIssueFromIndex(indexBody, date) ?? findBriefIssueInList(date);
|
||
const actions: Record<string, unknown>[] = [];
|
||
if (brief === null) {
|
||
const bodyPath = writeStateFile(`daily-brief-${date}.md`, `${newDailyBriefBody(date)}\n`);
|
||
if (dryRun) {
|
||
brief = { number: 0, url: `https://github.com/pikasTech/HWLAB/issues/new?title=${encodeURIComponent(dailyBriefTitle(date))}` };
|
||
actions.push({ action: "would-create-daily-brief", title: dailyBriefTitle(date), bodyPath });
|
||
} else {
|
||
const created = cliJson(["gh", "issue", "create", "--repo", HWLAB_REPO, "--title", dailyBriefTitle(date), "--body-file", bodyPath], 80_000);
|
||
if (!isCommandSuccess(created)) return { ok: false, phase: "create-daily-brief", created };
|
||
const createdIssue = record(commandData(created).issue);
|
||
brief = { number: Number(createdIssue.number), url: String(createdIssue.url) };
|
||
actions.push({ action: "created-daily-brief", issue: brief, bodyPath });
|
||
}
|
||
}
|
||
if (brief.number > 0 && parseBriefIssueFromIndex(indexBody, date) === null) {
|
||
const nextBody = insertBriefIndexRow(indexBody, date, brief);
|
||
const bodyPath = writeStateFile(`issue-${G14_BRIEF_INDEX_ISSUE}-brief-index-${date}.md`, `${nextBody.trimEnd()}\n`);
|
||
if (dryRun) {
|
||
actions.push({ action: "would-update-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||
} else {
|
||
const bodySha = String(indexIssue.bodySha ?? "");
|
||
const update = cliJson(["gh", "issue", "update", String(G14_BRIEF_INDEX_ISSUE), "--repo", HWLAB_REPO, "--mode", "replace", "--body-file", bodyPath, ...(bodySha.length > 0 ? ["--expect-body-sha", bodySha] : [])], 100_000);
|
||
if (!isCommandSuccess(update)) return { ok: false, phase: "update-brief-index", update, bodyPath };
|
||
actions.push({ action: "updated-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
|
||
}
|
||
}
|
||
return { ok: true, date, brief, actions };
|
||
}
|
||
|
||
function readPullRequest(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "read", String(number), "--repo", HWLAB_REPO, "--json", "title,body,url,stateDetail,merged,mergedAt,mergeCommit,headRefName,baseRefName"], 80_000);
|
||
}
|
||
|
||
function readPullRequestFiles(number: number): CommandJsonResult {
|
||
return cliJson(["gh", "pr", "files", String(number), "--repo", HWLAB_REPO, "--limit", "30"], 80_000);
|
||
}
|
||
|
||
function summarizePrFiles(filesResult: CommandJsonResult): Record<string, unknown> {
|
||
if (!isCommandSuccess(filesResult)) return { ok: false, filesResult };
|
||
const data = commandData(filesResult);
|
||
const files = Array.isArray(data.files) ? data.files.map((item) => record(item)) : [];
|
||
return {
|
||
ok: true,
|
||
summary: data.summary ?? null,
|
||
keyFiles: files.slice(0, 8).map((file) => `${file.status ?? "changed"} ${file.filename ?? ""}`),
|
||
};
|
||
}
|
||
|
||
function mergeCommitFromPr(prData: Record<string, unknown>): string | null {
|
||
const mergeCommit = record(record(prData.json).mergeCommit);
|
||
const fromJson = mergeCommit.oid ?? mergeCommit.sha;
|
||
if (typeof fromJson === "string") return fromJson;
|
||
const fromSummary = record(record(prData.pullRequest).mergeCommit).oid ?? record(record(prData.pullRequest).mergeCommit).sha;
|
||
return typeof fromSummary === "string" ? fromSummary : null;
|
||
}
|
||
|
||
function sourceCommitFromMergeResult(merge: CommandJsonResult, prNumber: number): string | null {
|
||
const fromMerge = mergeCommitFromPr(commandData(merge));
|
||
if (fromMerge !== null) return fromMerge;
|
||
const prRead = readPullRequest(prNumber);
|
||
if (!isCommandSuccess(prRead)) return null;
|
||
return mergeCommitFromPr(commandData(prRead));
|
||
}
|
||
|
||
async function waitForV02Head(expectedCommit: string | null, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
let head: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
head = getV02Head();
|
||
if (expectedCommit === null) {
|
||
if (head !== null) return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||
} else if (head === expectedCommit) {
|
||
return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
|
||
}
|
||
printEvent("v02.source.wait", { expectedCommit, head, waitedSeconds: Math.round((Date.now() - started) / 1000) });
|
||
await sleep(5_000);
|
||
}
|
||
return { ok: false, sourceCommit: head, expectedCommit, matched: false, waitedSeconds: Math.round((Date.now() - started) / 1000), degradedReason: "v02-source-head-not-aligned-after-merge" };
|
||
}
|
||
|
||
function currentGitopsRevision(): string | null {
|
||
const argo = getArgoStatus();
|
||
if (!isCommandSuccess(argo)) return null;
|
||
return statusText(argo).split(/\r?\n/u)[0] ?? null;
|
||
}
|
||
|
||
export function rolloutRecordBody(input: {
|
||
pr: OpenPullRequest;
|
||
prData: Record<string, unknown>;
|
||
fileSummary: Record<string, unknown>;
|
||
sourceCommit: string;
|
||
pipelineRun: string;
|
||
gitopsRevision: string | null;
|
||
mergedAt: string | null;
|
||
pipelineSucceededAt: string | null;
|
||
finishedAt: string | null;
|
||
rollout: Record<string, unknown>;
|
||
ciMetrics?: CiPipelineMetrics | null;
|
||
}): string {
|
||
const now = beijingParts();
|
||
const title = String(record(input.prData.json).title ?? input.pr.title ?? `PR #${input.pr.number}`);
|
||
const prBody = String(record(input.prData.json).body ?? record(input.prData.pullRequest).body ?? "");
|
||
const url = String(record(input.prData.json).url ?? input.pr.url ?? `https://github.com/pikasTech/HWLAB/pull/${input.pr.number}`);
|
||
const summary = record(input.fileSummary.summary);
|
||
const keyFiles = Array.isArray(input.fileSummary.keyFiles) ? input.fileSummary.keyFiles.map(String) : [];
|
||
const semanticBullets = semanticChangelogBullets(title, prBody, input.fileSummary);
|
||
const mergeToPipeline = durationSeconds(input.mergedAt, input.pipelineSucceededAt);
|
||
const pipelineToDev = durationSeconds(input.pipelineSucceededAt, input.finishedAt);
|
||
const mergeToDev = durationSeconds(input.mergedAt, input.finishedAt);
|
||
const metrics = input.ciMetrics ?? ciMetricsFromValue(input.rollout.ciMetrics);
|
||
return [
|
||
"",
|
||
"",
|
||
`## 更新 ${now.date} ${now.time.slice(0, 5)} 北京时间`,
|
||
"",
|
||
`### G14 DEV rollout：PR #${input.pr.number}`,
|
||
"",
|
||
`- PR: [#${input.pr.number} ${title}](${url})`,
|
||
`- 合并 commit: \`${input.sourceCommit}\``,
|
||
`- GitOps revision: \`${input.gitopsRevision ?? "unknown"}\``,
|
||
`- PipelineRun: \`${input.pipelineRun}\``,
|
||
"- CI/CD 耗时:",
|
||
` - merge -> pipeline succeeded: ${formatDuration(mergeToPipeline)}`,
|
||
` - pipeline succeeded -> DEV Healthy: ${formatDuration(pipelineToDev)}`,
|
||
` - merge -> DEV Healthy: ${formatDuration(mergeToDev)}`,
|
||
...ciMetricLines(metrics),
|
||
"- 上线 changelog（语义化）:",
|
||
...semanticBullets.map((bullet) => ` - ${bullet}`),
|
||
"- 自动 diff 摘要:",
|
||
` - changed files: ${String(summary.files ?? "n/a")}; +${String(summary.additions ?? "n/a")} / -${String(summary.deletions ?? "n/a")}; commits: ${String(summary.commits ?? "n/a")}`,
|
||
...keyFiles.map((file) => ` - ${file}`),
|
||
"- DEV 验证:",
|
||
` - Tekton: ${String(input.rollout.pipelineText ?? "Succeeded").split(/\r?\n/u).join(" / ")}`,
|
||
` - Argo: ${String(input.rollout.argoText ?? "Synced / Healthy").split(/\r?\n/u).slice(0, 5).join(" / ")}`,
|
||
` - health/live: ${input.rollout.healthOk === false ? "not-ok" : "ok"}`,
|
||
" - workload readiness: Deployment/StatefulSet ready；历史 Completed smoke/debug pod 不作为 DEV rollout blocker。",
|
||
].join("\n");
|
||
}
|
||
|
||
function appendRolloutBrief(options: G14RecordRolloutOptions, rollout: Record<string, unknown> = {}): Record<string, unknown> {
|
||
const now = beijingParts();
|
||
const ensured = ensureDailyBriefIssue(now.date, options.dryRun);
|
||
if (record(ensured).ok !== true) return ensured;
|
||
const brief = record(record(ensured).brief);
|
||
const prRead = readPullRequest(options.prNumber);
|
||
if (!isCommandSuccess(prRead)) return { ok: false, phase: "read-pr", prRead };
|
||
const prData = commandData(prRead);
|
||
const sourceCommit = options.sourceCommit ?? mergeCommitFromPr(prData);
|
||
if (sourceCommit === null || sourceCommit === undefined) return { ok: false, phase: "source-commit", message: "source commit unavailable", prData };
|
||
const pipelineRun = options.pipelineRun ?? `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||
const files = summarizePrFiles(readPullRequestFiles(options.prNumber));
|
||
const rolloutGitopsRevision = typeof rollout.gitopsRevision === "string" && rollout.gitopsRevision.length > 0 ? rollout.gitopsRevision : null;
|
||
const gitopsRevision = options.gitopsRevision ?? rolloutGitopsRevision ?? currentGitopsRevision();
|
||
const prJson = record(prData.json);
|
||
const prSummary = record(prData.pullRequest);
|
||
const prMergedAt = typeof prJson.mergedAt === "string" && prJson.mergedAt.length > 0
|
||
? prJson.mergedAt
|
||
: typeof prSummary.mergedAt === "string" && prSummary.mergedAt.length > 0
|
||
? prSummary.mergedAt
|
||
: null;
|
||
const rolloutPipelineSucceededAt = typeof rollout.pipelineSucceededAt === "string" && rollout.pipelineSucceededAt.length > 0 ? rollout.pipelineSucceededAt : null;
|
||
const rolloutFinishedAt = typeof rollout.finishedAt === "string" && rollout.finishedAt.length > 0 ? rollout.finishedAt : null;
|
||
const mergedAt = options.mergedAt ?? prMergedAt;
|
||
const pipelineSucceededAt = options.pipelineSucceededAt ?? rolloutPipelineSucceededAt;
|
||
const finishedAt = options.finishedAt ?? rolloutFinishedAt;
|
||
const ciMetrics = ciMetricsFromValue(rollout.ciMetrics) ?? collectPipelineMetrics(pipelineRun);
|
||
const pr: OpenPullRequest = {
|
||
number: options.prNumber,
|
||
title: String(record(prData.json).title ?? ""),
|
||
url: String(record(prData.json).url ?? ""),
|
||
baseRefName: String(record(prData.json).baseRefName ?? ""),
|
||
headRefName: String(record(prData.json).headRefName ?? ""),
|
||
};
|
||
const body = rolloutRecordBody({ pr, prData, fileSummary: files, sourceCommit, pipelineRun, gitopsRevision, mergedAt, pipelineSucceededAt, finishedAt, rollout, ciMetrics });
|
||
if (Number(brief.number) <= 0 || options.dryRun) {
|
||
return { ok: true, dryRun: true, date: now.date, brief, wouldAppend: { bodyPreview: body.slice(0, 1000), bodyChars: body.length }, ensured };
|
||
}
|
||
const currentBrief = readIssue(Number(brief.number));
|
||
if (!isCommandSuccess(currentBrief)) return { ok: false, phase: "read-daily-brief", currentBrief, brief };
|
||
const currentBody = String(issueFromRead(currentBrief).body ?? "");
|
||
if (currentBody.includes(`PR #${options.prNumber}`) && currentBody.includes(sourceCommit)) {
|
||
return { ok: true, skipped: true, reason: "rollout-brief-already-recorded", brief, sourceCommit, ensured };
|
||
}
|
||
const bodyPath = writeStateFile(`rollout-pr-${options.prNumber}-${shortSha(sourceCommit)}.md`, `${body}\n`);
|
||
const update = cliJson(["gh", "issue", "update", String(brief.number), "--repo", HWLAB_REPO, "--mode", "append", "--body-file", bodyPath, "--body-profile", "commander-brief"], 100_000);
|
||
if (!isCommandSuccess(update)) return { ok: false, phase: "append-rollout-brief", update, bodyPath, brief, ensured };
|
||
return { ok: true, date: now.date, brief, sourceCommit, pipelineRun, gitopsRevision, ciMetrics, bodyPath, update: commandData(update), ensured };
|
||
}
|
||
|
||
function v02PrCommentStatePath(): string {
|
||
return rootPath(".state", "hwlab-g14", "v02-pr-comment-signatures.json");
|
||
}
|
||
|
||
function readV02PrCommentState(): Record<string, string> {
|
||
const path = v02PrCommentStatePath();
|
||
if (!existsSync(path)) return {};
|
||
try {
|
||
const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
||
const state: Record<string, string> = {};
|
||
for (const [key, value] of Object.entries(record(parsed))) {
|
||
if (typeof value === "string") state[key] = value;
|
||
}
|
||
return state;
|
||
} catch {
|
||
return {};
|
||
}
|
||
}
|
||
|
||
function writeV02PrCommentState(state: Record<string, string>): string {
|
||
const path = v02PrCommentStatePath();
|
||
mkdirSync(dirname(path), { recursive: true });
|
||
writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
||
return path;
|
||
}
|
||
|
||
function preflightSummary(preflight: CommandJsonResult): Record<string, unknown> {
|
||
const data = commandData(preflight);
|
||
const mergeability = record(data.mergeability);
|
||
return {
|
||
ok: isCommandSuccess(preflight),
|
||
conclusion: stringOrNull(mergeability.conclusion) ?? (isCommandSuccess(preflight) ? "unknown" : "preflight-command-failed"),
|
||
readyForCommanderMerge: mergeability.readyForCommanderMerge === true,
|
||
blockers: Array.isArray(mergeability.blockers) ? mergeability.blockers.map(String).slice(0, 20) : [],
|
||
pending: Array.isArray(mergeability.pending) ? mergeability.pending.map(String).slice(0, 20) : [],
|
||
mergeable: mergeability.mergeable ?? nested(data, ["pullRequest", "mergeable"]) ?? null,
|
||
mergeStateStatus: mergeability.mergeStateStatus ?? nested(data, ["pullRequest", "mergeStateStatus"]) ?? null,
|
||
headRefName: nested(data, ["pullRequest", "headRefName"]) ?? null,
|
||
baseRefName: nested(data, ["pullRequest", "baseRefName"]) ?? null,
|
||
statusChecks: record(data.statusChecks).summary ?? data.statusChecks ?? null,
|
||
};
|
||
}
|
||
|
||
function v02PrConflictState(summary: Record<string, unknown>): string {
|
||
const mergeable = String(summary.mergeable ?? "").toUpperCase();
|
||
const mergeStateStatus = String(summary.mergeStateStatus ?? "").toUpperCase();
|
||
const blockers = Array.isArray(summary.blockers) ? summary.blockers.map(String).join("\n") : "";
|
||
if (mergeable.includes("CONFLICT") || mergeStateStatus.includes("DIRTY") || /conflict/i.test(blockers)) return "conflict";
|
||
return "clear-or-unknown";
|
||
}
|
||
|
||
function summarizeV02CdStatus(status: Record<string, unknown>): Record<string, unknown> {
|
||
const pipelineRun = record(status.pipelineRun);
|
||
const targetValidation = record(status.targetValidation);
|
||
const gitMirror = record(status.gitMirror);
|
||
const gitMirrorSummary = record(gitMirror.summary);
|
||
const planArtifacts = record(status.planArtifacts);
|
||
const planSummary = record(planArtifacts.summary);
|
||
const argo = record(status.argo);
|
||
const webAssets = record(status.webAssets);
|
||
return {
|
||
ok: status.ok === true,
|
||
sourceCommit: status.sourceCommit ?? nested(status, ["statusTarget", "sourceCommit"]) ?? null,
|
||
pipelineRun: pipelineRun.pipelineRun ?? pipelineRun.name ?? nested(status, ["statusTarget", "pipelineRun"]) ?? null,
|
||
pipelineStatus: pipelineRun.status ?? null,
|
||
pipelineReason: pipelineRun.reason ?? null,
|
||
targetValidationState: targetValidation.state ?? null,
|
||
targetValidationOk: targetValidation.ok ?? null,
|
||
targetValidationFailures: Array.isArray(targetValidation.failures) ? targetValidation.failures.slice(0, 10) : [],
|
||
targetValidationWarnings: Array.isArray(targetValidation.warnings) ? targetValidation.warnings.slice(0, 10) : [],
|
||
argoSync: argo.syncStatus ?? nested(argo, ["summary", "syncStatus"]) ?? null,
|
||
argoHealth: argo.healthStatus ?? nested(argo, ["summary", "healthStatus"]) ?? null,
|
||
webAssetsOk: webAssets.ok ?? null,
|
||
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
|
||
githubInSync: gitMirrorSummary.githubInSync ?? null,
|
||
buildServices: planSummary.buildServices ?? planArtifacts.buildServices ?? null,
|
||
reusedServices: planSummary.reusedServices ?? planArtifacts.reusedServices ?? null,
|
||
rolloutServices: planSummary.rolloutServices ?? planArtifacts.rolloutServices ?? null,
|
||
};
|
||
}
|
||
|
||
function v02CdPassed(status: Record<string, unknown>): boolean {
|
||
const state = String(nested(status, ["targetValidation", "state"]) ?? "");
|
||
return state === "passed" || state === "superseded";
|
||
}
|
||
|
||
function v02PipelineSucceeded(status: Record<string, unknown>): boolean {
|
||
return String(nested(status, ["pipelineRun", "status"]) ?? "") === "True";
|
||
}
|
||
|
||
function v02CdFailed(status: Record<string, unknown>): boolean {
|
||
const pipelineStatus = String(nested(status, ["pipelineRun", "status"]) ?? "");
|
||
return pipelineStatus === "False";
|
||
}
|
||
|
||
function activeV02PipelineRuns(status: Record<string, unknown>): Record<string, unknown>[] {
|
||
const active = record(status.activePipelineRuns);
|
||
const items = Array.isArray(active.items) ? active.items : [];
|
||
return items.map((item) => record(item)).filter((item) => String(item.status ?? "") === "Unknown");
|
||
}
|
||
|
||
function v02PrCommentSignature(input: V02PrCommentInput): string {
|
||
const summary = input.preflight ?? {};
|
||
const cd = input.cd ?? {};
|
||
const flush = input.flush ?? {};
|
||
return textHash(JSON.stringify({
|
||
pr: input.pr.number,
|
||
state: input.state,
|
||
phase: input.phase,
|
||
conclusion: summary.conclusion ?? null,
|
||
readyForCommanderMerge: summary.readyForCommanderMerge ?? null,
|
||
conflict: v02PrConflictState(summary),
|
||
blockers: summary.blockers ?? [],
|
||
pending: summary.pending ?? [],
|
||
sourceCommit: input.sourceCommit ?? null,
|
||
pipelineRun: input.pipelineRun ?? null,
|
||
pipelineStatus: cd.pipelineStatus ?? null,
|
||
pipelineReason: cd.pipelineReason ?? null,
|
||
targetValidationState: cd.targetValidationState ?? null,
|
||
pendingFlush: cd.pendingFlush ?? null,
|
||
flushOk: flush.ok ?? null,
|
||
dryRun: input.dryRun === true,
|
||
}));
|
||
}
|
||
|
||
function listValue(value: unknown): string {
|
||
if (!Array.isArray(value) || value.length === 0) return "none";
|
||
return value.map(String).slice(0, 8).join("; ");
|
||
}
|
||
|
||
function scalarValue(value: unknown): string {
|
||
if (value === null || value === undefined || value === "") return "n/a";
|
||
if (typeof value === "object") return JSON.stringify(value).slice(0, 240);
|
||
return String(value);
|
||
}
|
||
|
||
export function v02PrAutomationCommentBody(input: V02PrCommentInput): string {
|
||
const preflight = input.preflight ?? {};
|
||
const cd = input.cd ?? {};
|
||
const flush = input.flush ?? {};
|
||
const title = input.pr.title ?? `PR #${input.pr.number}`;
|
||
const url = input.pr.url ?? `https://github.com/${HWLAB_REPO}/pull/${input.pr.number}`;
|
||
const elapsed = formatDuration(input.elapsedSeconds);
|
||
const dryRunPrefix = input.dryRun === true ? "[dry-run] " : "";
|
||
return [
|
||
`### ${dryRunPrefix}v0.2 自动 CI/CD 状态`,
|
||
"",
|
||
`${input.message ?? "UniDesk monitor 已记录本轮 PR 自动化状态。"}`,
|
||
"",
|
||
"- PR:",
|
||
` - [#${input.pr.number} ${title}](${url})`,
|
||
` - base: \`${input.pr.baseRefName ?? V02_SOURCE_BRANCH}\`; head: \`${input.pr.headRefName ?? "unknown"}\``,
|
||
"- 自动化状态:",
|
||
` - state: \`${input.state}\`; phase: \`${input.phase}\``,
|
||
` - startedAt: \`${input.startedAt}\`; observedAt: \`${input.observedAt}\`; elapsed: ${elapsed}`,
|
||
"- CI / mergeability:",
|
||
` - conclusion: \`${scalarValue(preflight.conclusion)}\`; readyForCommanderMerge: \`${String(preflight.readyForCommanderMerge ?? "n/a")}\`; conflict: \`${v02PrConflictState(preflight)}\``,
|
||
` - blockers: ${listValue(preflight.blockers)}`,
|
||
` - pending: ${listValue(preflight.pending)}`,
|
||
"- CD / runtime:",
|
||
` - sourceCommit: \`${input.sourceCommit ?? "n/a"}\``,
|
||
` - PipelineRun: \`${input.pipelineRun ?? scalarValue(cd.pipelineRun)}\``,
|
||
` - pipeline: \`${scalarValue(cd.pipelineStatus)} / ${scalarValue(cd.pipelineReason)}\`; targetValidation: \`${scalarValue(cd.targetValidationState)}\``,
|
||
` - Argo: \`${scalarValue(cd.argoSync)} / ${scalarValue(cd.argoHealth)}\`; webAssets.ok: \`${String(cd.webAssetsOk ?? "n/a")}\``,
|
||
` - Git mirror: pendingFlush=\`${String(cd.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(cd.githubInSync ?? "n/a")}\`, flush.ok=\`${String(flush.ok ?? "n/a")}\``,
|
||
` - rolloutServices: ${listValue(cd.rolloutServices)}; buildServices: ${listValue(cd.buildServices)}; reusedServices: ${listValue(cd.reusedServices)}`,
|
||
"",
|
||
"后续动作由 `bun scripts/cli.ts hwlab g14 monitor-prs --lane v02` 继续驱动；同一状态签名不会重复刷评论。",
|
||
].join("\n");
|
||
}
|
||
|
||
function commentV02PullRequest(input: V02PrCommentInput): Record<string, unknown> {
|
||
const body = v02PrAutomationCommentBody(input);
|
||
const signature = v02PrCommentSignature(input);
|
||
const cacheKey = `pr-${input.pr.number}`;
|
||
if (input.dryRun === true) {
|
||
return { ok: true, dryRun: true, skipped: true, signature, wouldComment: { bodyPreview: body.slice(0, 1200), bodyChars: body.length } };
|
||
}
|
||
const state = readV02PrCommentState();
|
||
if (state[cacheKey] === signature) {
|
||
return { ok: true, skipped: true, reason: "same-pr-comment-signature", signature, stateFile: v02PrCommentStatePath() };
|
||
}
|
||
const bodyPath = writeStateFile(`v02-pr-${input.pr.number}-${input.state}-${signature}.md`, `${body}\n`);
|
||
const create = cliJson(["gh", "pr", "comment", "create", String(input.pr.number), "--repo", HWLAB_REPO, "--body-file", bodyPath], 100_000);
|
||
if (!isCommandSuccess(create)) return { ok: false, phase: "pr-comment", signature, bodyPath, create };
|
||
state[cacheKey] = signature;
|
||
const stateFile = writeV02PrCommentState(state);
|
||
return { ok: true, signature, bodyPath, stateFile, create: commandData(create) };
|
||
}
|
||
|
||
function triggerV02Current(timeoutSeconds: number): Record<string, unknown> {
|
||
return runV02ControlPlane({
|
||
action: "trigger-current",
|
||
lane: "v02",
|
||
dryRun: false,
|
||
confirm: true,
|
||
wait: true,
|
||
allowLiveDbRead: false,
|
||
timeoutSeconds,
|
||
minAgeMinutes: 30,
|
||
limit: 20,
|
||
});
|
||
}
|
||
|
||
function flushV02GitMirrorIfNeeded(status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
|
||
const pendingFlush = nested(status, ["gitMirror", "summary", "pendingFlush"]);
|
||
if (pendingFlush !== true) return { ok: true, skipped: true, reason: "git-mirror-already-flushed" };
|
||
return runG14GitMirror({
|
||
action: "flush",
|
||
confirm: true,
|
||
dryRun: false,
|
||
wait: true,
|
||
timeoutSeconds,
|
||
});
|
||
}
|
||
|
||
function pullRequestFromMergeCommand(merge: CommandJsonResult): Record<string, unknown> {
|
||
const data = commandData(merge);
|
||
const direct = record(data.pullRequest);
|
||
if (Object.keys(direct).length > 0) return direct;
|
||
return record(nested(data, ["details", "details", "pullRequest"]));
|
||
}
|
||
|
||
function mergeCommandRaceState(merge: CommandJsonResult): "merged" | "closed" | null {
|
||
const pullRequest = pullRequestFromMergeCommand(merge);
|
||
if (pullRequest.merged === true || pullRequest.stateDetail === "merged") return "merged";
|
||
if (pullRequest.closed === true || pullRequest.state === "closed") return "closed";
|
||
return null;
|
||
}
|
||
|
||
async function waitForV02Cd(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
const startedAt = new Date(started).toISOString();
|
||
const pipelineRun = v02PipelineRunName(sourceCommit);
|
||
let lastStatus: Record<string, unknown> = {};
|
||
let firstPassedAt: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
lastStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
const summary = summarizeV02CdStatus(lastStatus);
|
||
printEvent("v02.cd.status", { sourceCommit, pipelineRun, summary });
|
||
printV02PrMonitorProgress({ stage: "cd-status", status: "running", sourceCommit, pipelineRun, targetValidationState: summary.targetValidationState ?? null, pipelineStatus: summary.pipelineStatus ?? null, pendingFlush: summary.pendingFlush ?? null });
|
||
if (v02PipelineSucceeded(lastStatus) && summary.pendingFlush === true) {
|
||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||
if (record(flush).ok !== true) {
|
||
return {
|
||
ok: false,
|
||
phase: "git-mirror-flush",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summary,
|
||
rawStatus: lastStatus,
|
||
flush,
|
||
};
|
||
}
|
||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
const finalSummary = summarizeV02CdStatus(finalStatus);
|
||
if (v02CdPassed(finalStatus)) {
|
||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: "succeeded", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||
return {
|
||
ok: true,
|
||
phase: "cd-passed",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: finalSummary,
|
||
rawStatus: finalStatus,
|
||
flush,
|
||
};
|
||
}
|
||
lastStatus = finalStatus;
|
||
printEvent("v02.cd.after-flush", { sourceCommit, pipelineRun, flushOk: record(flush).ok, summary: finalSummary });
|
||
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: record(flush).ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
|
||
}
|
||
if (v02CdPassed(lastStatus)) {
|
||
firstPassedAt ??= new Date().toISOString();
|
||
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
|
||
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
printV02PrMonitorProgress({ stage: "cd-status", status: v02CdPassed(finalStatus) ? "succeeded" : "failed", sourceCommit, pipelineRun, targetValidationState: nested(finalStatus, ["targetValidation", "state"]) ?? null });
|
||
return {
|
||
ok: v02CdPassed(finalStatus),
|
||
phase: v02CdPassed(finalStatus) ? "cd-passed" : "git-mirror-flush",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summarizeV02CdStatus(finalStatus),
|
||
rawStatus: finalStatus,
|
||
flush,
|
||
};
|
||
}
|
||
if (v02CdFailed(lastStatus)) {
|
||
printV02PrMonitorProgress({ stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
|
||
return {
|
||
ok: false,
|
||
phase: "cd-failed",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
status: summary,
|
||
rawStatus: lastStatus,
|
||
};
|
||
}
|
||
await sleep(30_000);
|
||
}
|
||
return {
|
||
ok: false,
|
||
phase: "timeout",
|
||
startedAt,
|
||
finishedAt: new Date().toISOString(),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
timeoutSeconds,
|
||
firstPassedAt,
|
||
status: summarizeV02CdStatus(lastStatus),
|
||
rawStatus: lastStatus,
|
||
};
|
||
}
|
||
|
||
async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
|
||
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
|
||
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
|
||
printV02PrMonitorProgress({ stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "merge",
|
||
state: "merge-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: compactCommandResult(merge),
|
||
dryRun: options.dryRun,
|
||
message: mergeRaceState === "closed"
|
||
? "PR preflight 曾通过，但自动合并前 PR 已被关闭且未确认 merged；本轮不会触发 CD。"
|
||
: "PR preflight 已通过，但自动合并失败；需要处理 GitHub 返回的 merge blocker 后重试。",
|
||
});
|
||
return { ok: mergeRaceState === "closed", phase: "merge", action: "merge-race-closed", pr, merge, comment };
|
||
}
|
||
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
|
||
if (!options.dryRun && expectedCommit === null) {
|
||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "merge-commit",
|
||
state: "cd-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit: null,
|
||
pipelineRun: null,
|
||
dryRun: false,
|
||
message: "PR 已合并，但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit；为避免触发错误 source head，本轮未启动 CD。",
|
||
});
|
||
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, degradedReason: "merge-commit-unresolved" };
|
||
}
|
||
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForV02Head(expectedCommit, 120);
|
||
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
|
||
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
|
||
printV02PrMonitorProgress({ stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
|
||
const startedComment = commentV02PullRequest({
|
||
pr,
|
||
phase: options.dryRun ? "dry-run" : "cd-trigger",
|
||
state: "cd-started",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
dryRun: options.dryRun,
|
||
message: options.dryRun
|
||
? "dry-run：PR 已达到自动合并条件；本轮不会写 GitHub merge 或 CD。"
|
||
: mergeRaceState === "merged"
|
||
? "PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 v0.2 CD。后续成功、失败或超时会继续在本 PR 下回复。"
|
||
: "PR 已自动合并，v0.2 CD 准备开始；其他 commit 的运行中 PipelineRun 不会阻塞本轮 CI，旧 run 若发现 source head 已推进会以 superseded/no-op 收口。后续成功、superseded、失败或超时会继续在本 PR 下回复。",
|
||
});
|
||
if (!options.dryRun && record(startedComment).ok !== true) {
|
||
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
|
||
}
|
||
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
|
||
if (sourceCommit === null || record(headWait).ok !== true) {
|
||
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "v02-source-head",
|
||
state: "cd-timeout",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
dryRun: false,
|
||
message: "PR 已合并，但 G14 v0.2 CI/CD source repo 没有在等待窗口内对齐 merge commit；为避免触发旧 head，本轮未启动 CD。",
|
||
});
|
||
return { ok: false, phase: "v02-head", pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, degradedReason: "v02-head-unresolved-after-merge" };
|
||
}
|
||
const before = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
|
||
const beforeSummary = summarizeV02CdStatus(before);
|
||
const activeRuns = activeV02PipelineRuns(before);
|
||
if (activeRuns.length > 0) {
|
||
printEvent("v02.cd.active-runs-observed", { pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, activeRuns: activeRuns.slice(0, 5), latestOnlyPolicy: "do-not-wait-or-cancel-old-runs" });
|
||
}
|
||
const trigger = v02CdPassed(before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerV02Current(Math.min(options.timeoutSeconds, 600));
|
||
printEvent("v02.cd.trigger", { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||
printV02PrMonitorProgress({ stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
|
||
if (record(trigger).ok !== true) {
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "cd-trigger",
|
||
state: "cd-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
cd: beforeSummary,
|
||
dryRun: false,
|
||
message: "PR 已合并，但 v0.2 CD 触发失败；需要查看 trigger 阶段的 degradedReason。",
|
||
});
|
||
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment };
|
||
}
|
||
const cd = await waitForV02Cd(sourceCommit, options.timeoutSeconds);
|
||
const cdStatus = record(cd.status);
|
||
const flush = record(cd.flush);
|
||
const cdOk = cd.ok === true;
|
||
const cdPhase = String(cd.phase ?? "");
|
||
const cdSuperseded = cdOk && cdStatus.targetValidationState === "superseded";
|
||
const finalComment = commentV02PullRequest({
|
||
pr,
|
||
phase: cdPhase,
|
||
state: cdOk ? (cdSuperseded ? "cd-superseded" : "cd-succeeded") : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
merge: commandData(merge),
|
||
sourceCommit,
|
||
pipelineRun,
|
||
cd: cdStatus,
|
||
flush,
|
||
dryRun: false,
|
||
message: cdOk
|
||
? cdSuperseded
|
||
? "v0.2 自动 CD 已收口：本 merge head 的 PipelineRun 已完成，但 source head 已被后续提交推进，本轮按 latest-only 语义 superseded/no-op，未回写旧 GitOps revision。"
|
||
: "v0.2 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。"
|
||
: cdPhase === "timeout"
|
||
? "v0.2 自动 CD 超时未收口；本评论保留最后一次 targetValidation / PipelineRun / Git mirror 状态用于接续排障。"
|
||
: "v0.2 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。",
|
||
});
|
||
return {
|
||
ok: cdOk && record(finalComment).ok === true,
|
||
action: cdOk ? (cdSuperseded ? "merged-and-superseded-v02" : "merged-and-rolled-v02") : "v02-cd-failed",
|
||
phase: cdOk ? "cd-passed" : cdPhase,
|
||
pr,
|
||
sourceCommit,
|
||
pipelineRun,
|
||
trigger,
|
||
cd,
|
||
comment: finalComment,
|
||
};
|
||
}
|
||
|
||
async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||
const started = Date.now();
|
||
const startedAt = new Date(started).toISOString();
|
||
let pipelineText = "";
|
||
let argoText = "";
|
||
let healthOk = false;
|
||
let workloadsReady: Record<string, unknown> = { ready: false };
|
||
let pipelineSucceededAt: string | null = null;
|
||
while (Date.now() - started < timeoutSeconds * 1000) {
|
||
const pipeline = getPipelineStatus(sourceCommit);
|
||
pipelineText = statusText(pipeline);
|
||
printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) });
|
||
if (!pipelineText.startsWith("True\nSucceeded")) {
|
||
if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText };
|
||
await sleep(30_000);
|
||
continue;
|
||
}
|
||
pipelineSucceededAt ??= new Date().toISOString();
|
||
refreshArgoDev();
|
||
const argo = getArgoStatus();
|
||
argoText = statusText(argo);
|
||
printEvent("g14.argo.status", { text: argoText.slice(0, 500) });
|
||
const argoLines = argoText.split(/\r?\n/u);
|
||
const synced = argoLines[1] === "Synced";
|
||
const healthy = argoLines[2] === "Healthy";
|
||
const gitopsRevision = argoLines[0] ?? null;
|
||
const workloads = getDevWorkloads();
|
||
const workloadsText = statusText(workloads);
|
||
workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads));
|
||
const health = getLiveHealth();
|
||
healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout);
|
||
if (synced && healthy && record(workloadsReady).ready === true && healthOk) {
|
||
const finishedAt = new Date().toISOString();
|
||
const pipelineRun = `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
|
||
const ciMetrics = collectPipelineMetrics(pipelineRun);
|
||
return { ok: true, sourceCommit, gitopsRevision, pipelineRun, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk, ciMetrics };
|
||
}
|
||
await sleep(30_000);
|
||
}
|
||
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
|
||
}
|
||
|
||
async function monitorV02Cycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||
printEvent("v02.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||
printV02PrMonitorProgress({ stage: "cycle", status: "started", cycle, dryRun: options.dryRun });
|
||
const listed = listOpenG14PullRequests();
|
||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||
const prs = extractPullRequests(listed, V02_SOURCE_BRANCH);
|
||
printEvent("v02.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||
if (prs.length === 0) {
|
||
printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, pullRequests: 0, intervalSeconds: options.intervalSeconds });
|
||
return { ok: true, cycle, action: "none", lane: "v02", pullRequests: [] };
|
||
}
|
||
const observations: unknown[] = [];
|
||
for (const pr of prs) {
|
||
const startedAt = new Date().toISOString();
|
||
const preflightResult = preflightPullRequest(pr.number);
|
||
const preflight = preflightSummary(preflightResult);
|
||
printEvent("v02.pr.preflight", {
|
||
cycle,
|
||
number: pr.number,
|
||
ok: isCommandSuccess(preflightResult),
|
||
conclusion: preflight.conclusion,
|
||
readyForCommanderMerge: preflight.readyForCommanderMerge,
|
||
conflict: v02PrConflictState(preflight),
|
||
});
|
||
printV02PrMonitorProgress({
|
||
stage: "preflight",
|
||
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
|
||
pr: pr.number,
|
||
conclusion: preflight.conclusion,
|
||
conflict: v02PrConflictState(preflight),
|
||
});
|
||
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
|
||
const conclusion = String(preflight.conclusion ?? "unknown");
|
||
const blocked = conclusion === "blocked" || v02PrConflictState(preflight) === "conflict" || !isCommandSuccess(preflightResult);
|
||
const comment = commentV02PullRequest({
|
||
pr,
|
||
phase: "preflight",
|
||
state: blocked ? "blocked" : "waiting-ci",
|
||
startedAt,
|
||
observedAt: new Date().toISOString(),
|
||
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
|
||
preflight,
|
||
dryRun: options.dryRun,
|
||
message: blocked
|
||
? "v0.2 自动化已暂停在 PR preflight：存在 blocker 或冲突，需要先修复后再继续。"
|
||
: "v0.2 自动化正在等待 GitHub CI / mergeability 收敛；CI 通过且无冲突后会自动合并并触发 CD。",
|
||
});
|
||
observations.push({ pullRequest: pr, preflight, comment });
|
||
printV02PrMonitorProgress({ stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
|
||
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
|
||
continue;
|
||
}
|
||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||
printEvent("v02.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||
printV02PrMonitorProgress({ stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
|
||
const result = await runV02PrAutoCd(pr, preflight, merge, options, startedAt);
|
||
observations.push(result);
|
||
if (record(result).ok !== true) return { ok: false, cycle, lane: "v02", phase: record(result).phase ?? "v02-auto-cd", pullRequest: pr, result, observations };
|
||
return { ok: true, cycle, lane: "v02", action: record(result).action ?? (options.dryRun ? "dry-run-merge" : "merged-and-rolled-v02"), result, observations };
|
||
}
|
||
return { ok: true, cycle, lane: "v02", action: "none", observations };
|
||
}
|
||
|
||
async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
|
||
if (options.lane === "v02") return monitorV02Cycle(options, cycle);
|
||
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
|
||
const precheck = precheckWorkspace();
|
||
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
|
||
const listed = listOpenG14PullRequests();
|
||
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
|
||
const prs = extractPullRequests(listed);
|
||
printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
|
||
if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] };
|
||
const merged: unknown[] = [];
|
||
for (const pr of prs) {
|
||
const preflight = preflightPullRequest(pr.number);
|
||
printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) });
|
||
if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) {
|
||
return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight };
|
||
}
|
||
const merge = mergePullRequest(pr.number, options.dryRun);
|
||
printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
|
||
if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge };
|
||
const sourceCommit = getG14Head();
|
||
const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds);
|
||
const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true
|
||
? { skipped: true, dryRun: options.dryRun }
|
||
: appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout);
|
||
printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null });
|
||
merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief });
|
||
if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged };
|
||
if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged };
|
||
}
|
||
return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged };
|
||
}
|
||
|
||
async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<string, unknown>> {
|
||
const maxCycles = options.once ? 1 : options.maxCycles;
|
||
let cycle = 0;
|
||
const results: unknown[] = [];
|
||
while (maxCycles === 0 || cycle < maxCycles) {
|
||
cycle += 1;
|
||
const result = await monitorCycle(options, cycle);
|
||
results.push(result);
|
||
printEvent(`${options.lane}.monitor.cycle.done`, { cycle, lane: options.lane, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
|
||
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
|
||
if (options.once || (options.lane === "g14" && record(result).action !== "none")) return { ok: true, cycles: cycle, lastResult: result, results };
|
||
printEvent(`${options.lane}.monitor.sleep`, { cycle, lane: options.lane, intervalSeconds: options.intervalSeconds });
|
||
if (options.lane === "v02") printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, intervalSeconds: options.intervalSeconds });
|
||
await sleep(options.intervalSeconds * 1000);
|
||
}
|
||
return { ok: true, cycles: cycle, results };
|
||
}
|
||
|
||
export function hwlabG14Help(): Record<string, unknown> {
|
||
return {
|
||
command: "hwlab g14",
|
||
output: "json",
|
||
usage: [
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs",
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs --once --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --once --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 record-rollout --pr <number> [--source-commit sha]",
|
||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
|
||
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit <full-sha>",
|
||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
|
||
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-device-pod-api-key --key api-key",
|
||
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-device-pod-api-key --key api-key --dry-run",
|
||
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-device-pod-api-key --key api-key --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror status",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm --wait",
|
||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||
],
|
||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.",
|
||
defaults: {
|
||
repo: HWLAB_REPO,
|
||
base: G14_SOURCE_BRANCH,
|
||
v02Base: V02_SOURCE_BRANCH,
|
||
provider: G14_PROVIDER,
|
||
workspace: G14_WORKSPACE,
|
||
v02Workspace: V02_WORKSPACE,
|
||
ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO,
|
||
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
|
||
devApplication: DEV_APP,
|
||
v02Application: V02_APP,
|
||
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
|
||
},
|
||
stateFiles: {
|
||
monitor: ".state/hwlab-g14/latest-monitor-job.json",
|
||
once: ".state/hwlab-g14/latest-once-job.json",
|
||
dryRun: ".state/hwlab-g14/latest-dry-run-job.json",
|
||
onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json",
|
||
v02Monitor: ".state/hwlab-g14/latest-v02-monitor-job.json",
|
||
v02Once: ".state/hwlab-g14/latest-v02-once-job.json",
|
||
v02DryRun: ".state/hwlab-g14/latest-v02-dry-run-job.json",
|
||
v02OnceDryRun: ".state/hwlab-g14/latest-v02-once-dry-run-job.json",
|
||
v02PrCommentSignatures: ".state/hwlab-g14/v02-pr-comment-signatures.json",
|
||
},
|
||
};
|
||
}
|
||
|
||
export async function runHwlabG14Command(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
||
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return { ok: true, ...hwlabG14Help() };
|
||
const [action] = args;
|
||
if (action === "record-rollout") {
|
||
const options = parseRecordRolloutOptions(args.slice(1));
|
||
return appendRolloutBrief(options);
|
||
}
|
||
if (action === "control-plane") {
|
||
const options = parseControlPlaneOptions(args.slice(1));
|
||
if (options.action === "trigger-current" && options.confirm && !options.dryRun && !options.wait) {
|
||
return startControlPlaneTriggerJob(options);
|
||
}
|
||
return runV02ControlPlane(options);
|
||
}
|
||
if (action === "secret") {
|
||
const options = parseSecretOptions(args.slice(1));
|
||
return runG14Secret(options);
|
||
}
|
||
if (action === "tools-image") {
|
||
const options = parseToolsImageOptions(args.slice(1));
|
||
return runG14ToolsImage(options);
|
||
}
|
||
if (action === "git-mirror") {
|
||
const options = parseGitMirrorOptions(args.slice(1));
|
||
if ((options.action === "sync" || options.action === "flush") && options.confirm && !options.dryRun && !options.wait) {
|
||
return startGitMirrorJob(options);
|
||
}
|
||
return runG14GitMirror(options);
|
||
}
|
||
if (action !== "monitor-prs") {
|
||
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 record-rollout, hwlab g14 control-plane, hwlab g14 secret, hwlab g14 git-mirror, hwlab g14 tools-image" };
|
||
}
|
||
const options = parseOptions(args.slice(1));
|
||
if (options.worker) return runMonitorWorker(options);
|
||
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--lane", options.lane, "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
|
||
const jobName = options.lane === "v02" ? "hwlab_g14_v02_pr_monitor" : "hwlab_g14_pr_monitor";
|
||
const jobNote = options.lane === "v02"
|
||
? `Monitor ${HWLAB_REPO} PRs targeting ${V02_SOURCE_BRANCH}, merge ready PRs, trigger v0.2 CD, and comment PR progress`
|
||
: `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`;
|
||
const job = startJob(jobName, command, jobNote);
|
||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
|
||
const stateDir = rootPath(".state", "hwlab-g14");
|
||
mkdirSync(stateDir, { recursive: true });
|
||
const stateFileName = hwlabG14MonitorStateFileName(options);
|
||
const stateFileRole = hwlabG14MonitorStateRole(options);
|
||
const latestPath = join(stateDir, stateFileName);
|
||
const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null;
|
||
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole, lane: options.lane, baseBranch: monitorBaseBranch(options.lane) }, null, 2)}\n`, "utf8");
|
||
return {
|
||
ok: true,
|
||
command: "hwlab g14 monitor-prs",
|
||
lane: options.lane,
|
||
baseBranch: monitorBaseBranch(options.lane),
|
||
mode: "async-job",
|
||
job,
|
||
statusCommand,
|
||
latestPath,
|
||
stateFileName,
|
||
stateFileRole,
|
||
previousLatest,
|
||
next: {
|
||
status: statusCommand,
|
||
tail: `tail -f ${job.stdoutFile}`,
|
||
},
|
||
};
|
||
}
|