import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { createHash } from "node:crypto"; import { repoRoot, rootPath, type Config } from "./config"; import { runCommand } from "./command"; import { startJob } from "./jobs"; const HWLAB_REPO = "pikasTech/HWLAB"; const G14_SOURCE_BRANCH = "G14"; const G14_PROVIDER = "G14"; const G14_WORKSPACE = "/root/hwlab"; const V02_SOURCE_BRANCH = "v0.2"; const V02_WORKSPACE = "/root/hwlab-v02"; const DEV_NAMESPACE = "hwlab-dev"; const CI_NAMESPACE = "hwlab-ci"; const ARGO_NAMESPACE = "argocd"; const DEV_APP = "hwlab-g14-dev"; const V02_APP = "hwlab-g14-v02"; const V02_PIPELINE = "hwlab-v02-ci-image-publish"; const V02_POLLER = "hwlab-v02-branch-poller"; const V02_RECONCILER = "hwlab-v02-control-plane-reconciler"; const V02_PIPELINERUN_PREFIX = "hwlab-v02-ci-poll"; const V02_CONTROL_PLANE_FIELD_MANAGER = "unidesk-hwlab-v02-control-plane"; const V02_GIT_URL = "git@github.com:pikasTech/HWLAB.git"; const V02_GIT_READ_URL = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; const V02_GIT_WRITE_URL = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; const V02_GITOPS_BRANCH = "v0.2-gitops"; const V02_CATALOG_PATH = "deploy/artifact-catalog.v02.json"; const V02_RUNTIME_PATH = "deploy/gitops/g14/runtime-v02"; const V02_REGISTRY_PREFIX = "127.0.0.1:5000/hwlab"; const V02_BASE_IMAGE = "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; const GIT_MIRROR_NAMESPACE = "devops-infra"; const GIT_MIRROR_MANIFEST_FIELD_MANAGER = "unidesk-hwlab-git-mirror"; const GIT_MIRROR_SYNC_JOB_PREFIX = "git-mirror-hwlab-sync-manual"; const GIT_MIRROR_LEGACY_CRONJOB = "git-mirror-hwlab-sync"; const V02_SERVICE_IDS = [ "hwlab-cloud-api", "hwlab-cloud-web", "hwlab-agent-mgr", "hwlab-agent-worker", "hwlab-device-pod", "hwlab-gateway", "hwlab-edge-proxy", "hwlab-agent-skills", ]; const V02_CLOUD_WEB_URL = "http://74.48.78.17:19666"; const V02_CLOUD_API_URL = "http://74.48.78.17:19667"; export function v02PipelineServiceIds(): string[] { return [...V02_SERVICE_IDS]; } const G14_CI_TOOLS_IMAGE_REPO = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools"; const G14_CI_TOOLS_BASE_TAG = "node22-alpine-v1"; const DEFAULT_INTERVAL_SECONDS = 600; const DEFAULT_MAX_CYCLES = 0; const DEFAULT_TIMEOUT_SECONDS = 1800; const G14_BRIEF_INDEX_ISSUE = 7; const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000; interface G14MonitorOptions { intervalSeconds: number; maxCycles: number; once: boolean; dryRun: boolean; worker: boolean; timeoutSeconds: number; } interface G14RecordRolloutOptions { prNumber: number; sourceCommit?: string; pipelineRun?: string; gitopsRevision?: string; mergedAt?: string; pipelineSucceededAt?: string; finishedAt?: string; dryRun: boolean; } interface G14ControlPlaneOptions { action: "status" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration"; lane: "v02" | "g14" | "all"; dryRun: boolean; confirm: boolean; wait: boolean; allowLiveDbRead: boolean; timeoutSeconds: number; minAgeMinutes: number; limit: number; } interface G14ToolsImageOptions { action: "status" | "build"; name: "ci-node-tools"; tag: string; dockerfile: string; dryRun: boolean; confirm: boolean; timeoutSeconds: number; } interface G14GitMirrorOptions { action: "status" | "apply" | "sync" | "flush"; dryRun: boolean; confirm: boolean; wait: boolean; timeoutSeconds: number; } interface CommandJsonResult { ok: boolean; command: string[]; exitCode: number | null; stdout: string; stderr: string; parsed: unknown | null; } interface ShellSection { stdout: string; exitCode: number | null; } interface OpenPullRequest { number: number; title?: string; url?: string; baseRefName?: string; headRefName?: string; mergeable?: string | null; mergeStateStatus?: string | null; draft?: boolean; } interface CiServiceMetric { taskRun: string; serviceId: string; status: string; durationSeconds: number | null; buildBackend: string | null; reusedFrom: string | null; imageTag: string | null; sourceCommitId: string | null; } interface CiPipelineMetrics { ok: boolean; pipelineRun: string; serviceTotal: number; reusedCount: number; rebuildCount: number; reusedServices: string[]; rebuildServices: string[]; services: CiServiceMetric[]; degradedReason?: string; rawText?: string; } export function hwlabG14MonitorStateFileName(options: Pick): string { if (options.once && options.dryRun) return "latest-once-dry-run-job.json"; if (options.once) return "latest-once-job.json"; if (options.dryRun) return "latest-dry-run-job.json"; return "latest-monitor-job.json"; } function hwlabG14MonitorStateRole(options: Pick): string { if (options.once && options.dryRun) return "once-dry-run"; if (options.once) return "once"; if (options.dryRun) return "dry-run-monitor"; return "monitor"; } function parseOptions(args: string[]): G14MonitorOptions { return { intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400), maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000), once: args.includes("--once"), dryRun: args.includes("--dry-run"), worker: args.includes("--worker"), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", DEFAULT_TIMEOUT_SECONDS, 86400), }; } function optionValue(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index === -1) return undefined; const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); return value; } function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions { const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number"); const prNumber = Number(prRaw); if (!Number.isInteger(prNumber) || prNumber <= 0) throw new Error("record-rollout requires --pr "); return { prNumber, sourceCommit: optionValue(args, "--source-commit"), pipelineRun: optionValue(args, "--pipeline-run"), gitopsRevision: optionValue(args, "--gitops-revision"), mergedAt: optionValue(args, "--merged-at"), pipelineSucceededAt: optionValue(args, "--pipeline-succeeded-at"), finishedAt: optionValue(args, "--finished-at"), dryRun: args.includes("--dry-run"), }; } function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { const [rawAction] = args; const actionRaw = rawAction === "rerun-current" ? "trigger-current" : rawAction; if ( actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "trigger-current" && actionRaw !== "cleanup-runs" && actionRaw !== "cleanup-released-pvs" && actionRaw !== "runtime-migration" ) { throw new Error("control-plane usage: status|apply|trigger-current|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); } const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02"); if (actionRaw === "cleanup-runs") { if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all"); } else if (actionRaw === "cleanup-released-pvs") { if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane"); } else if (lane !== "v02") { throw new Error("control-plane status/apply/trigger-current/runtime-migration currently requires --lane v02"); } const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run"); const allowLiveDbRead = args.includes("--allow-live-db-read"); if (allowLiveDbRead && confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm"); if (allowLiveDbRead && actionRaw !== "runtime-migration") throw new Error("--allow-live-db-read is only valid for control-plane runtime-migration"); return { action: actionRaw, lane, confirm, wait: args.includes("--wait"), allowLiveDbRead, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080), limit: positiveIntegerOption(args, "--limit", 20, 200), }; } function parseToolsImageOptions(args: string[]): G14ToolsImageOptions { const [actionRaw] = args; if (actionRaw !== "status" && actionRaw !== "build") { throw new Error("tools-image usage: status|build --name ci-node-tools --tag [--dockerfile path] [--dry-run|--confirm]"); } const name = optionValue(args, "--name") ?? "ci-node-tools"; if (name !== "ci-node-tools") throw new Error("tools-image currently supports --name ci-node-tools"); const tag = optionValue(args, "--tag") ?? G14_CI_TOOLS_BASE_TAG; if (!/^[A-Za-z0-9_.-]+$/u.test(tag)) throw new Error("--tag may only contain letters, numbers, dot, underscore, and dash"); const dockerfile = optionValue(args, "--dockerfile") ?? "deploy/ci/hwlab-ci-node-tools.Dockerfile"; if (dockerfile.startsWith("/") || dockerfile.includes("..")) throw new Error("--dockerfile must be a repo-relative path without '..'"); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run"); return { action: actionRaw, name, tag, dockerfile, confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800), }; } function parseGitMirrorOptions(args: string[]): G14GitMirrorOptions { const [actionRaw] = args; if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "sync" && actionRaw !== "flush") { throw new Error("git-mirror usage: status|apply|sync|flush [--dry-run|--confirm]"); } const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("git-mirror accepts only one of --confirm or --dry-run"); return { action: actionRaw, confirm, wait: args.includes("--wait"), dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "sync" || actionRaw === "flush" ? 300 : 120, 900), }; } function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const index = args.indexOf(name); if (index === -1) return defaultValue; const raw = args[index + 1]; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return Math.min(value, maxValue); } function shellQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; } function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult { const result = runCommand(command, repoRoot, { timeoutMs }); let parsed: unknown | null = null; if (result.stdout.trim().length > 0) { try { parsed = JSON.parse(result.stdout) as unknown; } catch { parsed = null; } } return { ok: result.exitCode === 0, command, exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr, parsed, }; } function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult { return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs); } function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } function nested(value: unknown, keys: string[]): unknown { let current = value; for (const key of keys) current = record(current)[key]; return current; } function expandedParsedRoot(result: CommandJsonResult): Record { const dumpPath = nested(result.parsed, ["data", "dump", "path"]); if (typeof dumpPath === "string" && existsSync(dumpPath)) { try { return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown); } catch { return record(result.parsed); } } return record(result.parsed); } function commandData(result: CommandJsonResult): Record { return record(sanitizeCliOutput(record(expandedParsedRoot(result).data))); } function textHash(value: string): string { return createHash("sha256").update(value).digest("hex").slice(0, 12); } function redactLargePayloads(value: string): string { return value .replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => { return `manifest_b64=''`; }) .replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => { return `manifest_b64=""`; }) .replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => { return `manifest_b64=`; }) .replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => { return ``; }); } function compactInlineText(value: string, maxChars = 4000): string { const redacted = redactLargePayloads(value); if (redacted.length <= maxChars) return redacted; const headChars = Math.floor(maxChars * 0.35); const tailChars = maxChars - headChars; const omittedChars = redacted.length - headChars - tailChars; return [ redacted.slice(0, headChars), ``, redacted.slice(-tailChars), ].join("\n"); } function sanitizeCliOutput(value: unknown): unknown { if (typeof value === "string") return compactInlineText(value); if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item)); if (typeof value !== "object" || value === null) return value; const output: Record = {}; for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item); return output; } function redactedCommand(command: string[]): string[] { return command.map((arg) => compactInlineText(arg, 1200)); } function compactCommandResult(result: CommandJsonResult): Record { const data = record(expandedParsedRoot(result).data); const stdout = String(data.stdout ?? result.stdout ?? ""); const stderr = String(data.stderr ?? result.stderr ?? ""); return { ok: isCommandSuccess(result), exitCode: result.exitCode, command: redactedCommand(result.command), stdout: compactInlineText(stdout.trim(), 4000), stderr: compactInlineText(stderr.trim(), 4000), stdoutBytes: Buffer.byteLength(stdout), stderrBytes: Buffer.byteLength(stderr), }; } function compactControlPlaneRefresh(value: Record): Record { return { ok: value.ok === true, phase: value.phase ?? null, sourceCommit: value.sourceCommit ?? null, renderDir: value.renderDir ?? null, applyOk: nested(value, ["apply", "ok"]) ?? null, cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null, applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200), degradedReason: value.degradedReason ?? null, }; } function compactTriggerCommandResult(value: Record | null): Record | null { if (value === null) return null; return { ok: value.ok ?? null, exitCode: value.exitCode ?? null, stdout: tailText(value.stdout, 1200), stderr: tailText(value.stderr, 1200), stdoutBytes: value.stdoutBytes ?? null, stderrBytes: value.stderrBytes ?? null, }; } function isCommandSuccess(result: CommandJsonResult): boolean { if (!result.ok) return false; const topOk = record(result.parsed).ok; if (topOk === false) return false; const dataOk = nested(result.parsed, ["data", "ok"]); return dataOk !== false; } function extractPullRequests(result: CommandJsonResult): OpenPullRequest[] { const prs = nested(result.parsed, ["data", "pullRequests"]); if (!Array.isArray(prs)) return []; return prs .map((item) => record(item)) .filter((item) => item.baseRefName === G14_SOURCE_BRANCH || record(item.base).ref === G14_SOURCE_BRANCH) .map((item) => ({ number: Number(item.number), title: typeof item.title === "string" ? item.title : undefined, url: typeof item.url === "string" ? item.url : undefined, baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined, headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined, mergeable: typeof item.mergeable === "string" ? item.mergeable : null, mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null, draft: item.draft === true, })) .filter((item) => Number.isInteger(item.number) && item.number > 0); } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } function printEvent(event: string, data: Record = {}): void { process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`); } function printProgressEvent(event: string, data: Record = {}): void { process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`); } function shortSha(sha: string): string { return sha.slice(0, 12); } function precheckWorkspace(): CommandJsonResult { return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000); } function v02WorkspaceScript(script: string, timeoutMs = 120_000): CommandJsonResult { return cliJson(["ssh", `${G14_PROVIDER}:${V02_WORKSPACE}`, "script", "--", script], timeoutMs); } function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult { return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs); } function getV02Head(): string | null { const result = v02WorkspaceScript("git fetch origin v0.2 --prune >/dev/null 2>&1; git rev-parse origin/v0.2", 120_000); if (!isCommandSuccess(result)) return null; const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim(); const match = /[0-9a-f]{40}/iu.exec(output); return match?.[0] ?? null; } function v02PipelineRunName(sourceCommit: string): string { return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`; } function getPipelineRunCompact(name: string): Record { const result = g14K3s([ "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, name, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}", ], 60_000); return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr); } function pipelineRunCompactFromText( name: string, text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, ): Record { const [status = "", reason = "", message = ""] = text.split(/\r?\n/u); const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`); return { ok: commandOk, exists: commandOk || !notFound, pipelineRun: name, status: status || null, reason: reason || null, message: message || null, command: Array.isArray(command) ? redactedCommand(command) : command, exitCode, stderr: stderr.trim().slice(0, 2000), }; } function timestampMs(value: unknown): number | null { if (typeof value !== "string" || value.trim().length === 0) return null; const parsed = Date.parse(value); return Number.isFinite(parsed) ? parsed : null; } function secondsBetween(start: unknown, end: unknown): number | null { const startMs = timestampMs(start); const endMs = timestampMs(end); if (startMs === null || endMs === null || endMs < startMs) return null; return Math.round((endMs - startMs) / 1000); } function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record | null { const [ name = "", sourceCommit = "", createdAt = "", startTime = "", completionTime = "", status = "", reason = "", ] = line.split("\t"); if (!name.startsWith(V02_PIPELINERUN_PREFIX)) return null; const startMs = timestampMs(startTime); const completion = completionTime.trim().length > 0 ? completionTime : null; const conditionStatus = status.trim().length > 0 ? status : null; return { name, sourceCommit: sourceCommit || null, status: conditionStatus, reason: reason || null, createdAt: createdAt || null, startTime: startTime || null, completionTime: completion, durationSeconds: secondsBetween(startTime, completion), elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)), active: completion === null && conditionStatus !== "True" && conditionStatus !== "False", }; } function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now()): Record { const rows = text .split(/\r?\n/u) .map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs)) .filter((item): item is Record => item !== null) .sort((left, right) => { return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0); }); const activeItems = rows.filter((item) => item.active === true); return { ok: true, count: rows.length, activeCount: activeItems.length, items: rows.slice(0, limit), activeItems: activeItems.slice(0, limit), disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete", }; } function pipelineRunRowsJsonPath(): string { return [ "jsonpath={range .items[*]}", "{.metadata.name}", "{\"\\t\"}", "{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}", "{\"\\t\"}", "{.metadata.creationTimestamp}", "{\"\\t\"}", "{.status.startTime}", "{\"\\t\"}", "{.status.completionTime}", "{\"\\t\"}", "{.status.conditions[0].status}", "{\"\\t\"}", "{.status.conditions[0].reason}", "{\"\\n\"}", "{end}", ].join(""); } function listV02PipelineRunsCompact(limit = 8): Record { const result = g14K3s([ "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, "-l", "hwlab.pikastech.local/gitops-target=v02", "-o", pipelineRunRowsJsonPath(), ], 60_000); if (!isCommandSuccess(result)) { return { ok: false, command: redactedCommand(result.command), exitCode: result.exitCode, stderr: commandErrorSummary(result), items: [], activeItems: [], }; } return parsePipelineRunRows(statusText(result), limit); } function parseShellSections(output: string): Record { const sections: Record = {}; let currentName: string | null = null; let currentLines: string[] = []; let currentExitCode: number | null = null; const flush = (): void => { if (currentName === null) return; sections[currentName] = { stdout: currentLines.join("\n").trim(), exitCode: currentExitCode, }; }; for (const line of output.split(/\r?\n/u)) { const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line); if (begin !== null) { flush(); currentName = begin[1] ?? null; currentLines = []; currentExitCode = null; continue; } const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line); if (end !== null && currentName === end[1]) { currentExitCode = Number(end[2]); flush(); currentName = null; currentLines = []; currentExitCode = null; continue; } if (currentName !== null) currentLines.push(line); } flush(); return sections; } function shellSectionOk(section: ShellSection | undefined): boolean { return section?.exitCode === 0; } function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): CommandJsonResult { const sourceCommitLine = sourceCommit === undefined ? `source_commit=$(git -C ${shellQuote(V02_WORKSPACE)} rev-parse origin/v0.2 2>/dev/null || true)` : `source_commit=${shellQuote(sourceCommit ?? "")}`; const script = [ "set +e", sourceCommitLine, "pipeline_run=", `if [ -n "$source_commit" ]; then pipeline_run="${V02_PIPELINERUN_PREFIX}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`, "section() {", " name=\"$1\"", " shift", " printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"", " \"$@\"", " code=$?", " printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"", "}", "section sourceCommit printf '%s\\n' \"$source_commit\"", "section queryNow date -u +%Y-%m-%dT%H:%M:%SZ", `section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`, `section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`, `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`, `if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`, `if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`, `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`, `section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`, ].join("\n"); return g14K3s(["script", "--", script], 60_000); } function v02WebAssetsProbeScript(): string { return [ "set +e", `base=${shellQuote(V02_CLOUD_WEB_URL)}`, `api=${shellQuote(V02_CLOUD_API_URL)}`, "fetch_url() {", " if command -v curl >/dev/null 2>&1; then", " curl -fsS --connect-timeout 2 --max-time 5 \"$1\"", " elif command -v wget >/dev/null 2>&1; then", " wget -q -T 5 -O - \"$1\"", " else", " return 127", " fi", "}", "printf 'baseUrl\\t%s\\n' \"$base\"", "printf 'apiUrl\\t%s\\n' \"$api\"", "html=$(fetch_url \"$base/\" 2>/dev/null)", "html_code=$?", "printf 'htmlOk\\t%s\\n' \"$html_code\"", "printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"", "css=$(fetch_url \"$base/styles.css\" 2>/dev/null)", "css_code=$?", "printf 'cssOk\\t%s\\n' \"$css_code\"", "printf 'sidebarFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"", "printf 'workspaceFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"", "printf 'eventPanelFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(132px,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"", "health=$(fetch_url \"$api/health/live\" 2>/dev/null)", "health_code=$?", "printf 'apiHealthOk\\t%s\\n' \"$health_code\"", "printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"", ].join("\n"); } function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record { if (!commandOk) { return { ok: false, pipelineRun, exitCode, stderr: stderr.trim().slice(0, 2000), counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 }, items: [], }; } const items = text .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t"); return { name, status: status || null, reason: reason || null, startTime: startTime || null, completionTime: completionTime || null, durationSeconds: secondsBetween(startTime, completionTime), }; }); const counts = { succeeded: items.filter((item) => item.status === "True").length, failed: items.filter((item) => item.status === "False").length, running: items.filter((item) => item.status === "Unknown").length, unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length, }; return { ok: true, pipelineRun, counts, items, summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}`, disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet", }; } function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: string | null, argoSyncRevision: string | null, exitCode: number | null, stderr: string): Record { const fields: Record = {}; for (const line of text.split(/\r?\n/u)) { const [key = "", ...rest] = line.split("\t"); if (key.length > 0) fields[key] = rest.join("\t"); } const htmlOk = fields.htmlOk === "0"; const cssOk = fields.cssOk === "0"; const apiHealthOk = fields.apiHealthOk === "0"; const readonlyNoteAbsent = fields.readonlyNote === "1"; const sidebarFitCss = fields.sidebarFitCss === "0"; const workspaceFitCss = fields.workspaceFitCss === "0"; const eventPanelFitCss = fields.eventPanelFitCss === "0"; const apiRevision = fields.apiRevision || null; const webChecksPass = htmlOk && cssOk && readonlyNoteAbsent && sidebarFitCss && workspaceFitCss && eventPanelFitCss && apiHealthOk; const failedChecks = Object.entries({ htmlOk, cssOk, readonlyNoteAbsent, sidebarFitCss, workspaceFitCss, eventPanelFitCss, apiHealthOk, }).filter(([, ok]) => !ok).map(([name]) => name); return { ok: commandOk && webChecksPass, summary: commandOk && webChecksPass ? "19666/19667 probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`, baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL, apiUrl: fields.apiUrl || V02_CLOUD_API_URL, sourceCommit, argoSyncRevision: argoSyncRevision || null, checks: { htmlOk, cssOk, readonlyNoteAbsent, sidebarFitCss, workspaceFitCss, eventPanelFitCss, apiHealthOk, }, probeExitCodes: { html: numericField(fields.htmlOk), css: numericField(fields.cssOk), readonlyNoteGrep: numericField(fields.readonlyNote), sidebarFitCssGrep: numericField(fields.sidebarFitCss), workspaceFitCssGrep: numericField(fields.workspaceFitCss), eventPanelFitCssGrep: numericField(fields.eventPanelFitCss), apiHealth: numericField(fields.apiHealthOk), }, apiRevision, note: apiRevision && sourceCommit && apiRevision !== sourceCommit ? "cloud-api image revision can differ when a change only republishes Cloud Web static assets; use webAssets.checks for 19666 frontend asset readiness." : null, exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), }; } function numericField(value: string | undefined): number | null { if (value === undefined || value.trim().length === 0) return null; const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record { if (!commandOk) { return { ok: false, command: Array.isArray(command) ? redactedCommand(command) : command, exitCode, stderr: stderr.trim().slice(0, 4000), items: [], activeItems: [], }; } return parsePipelineRunRows(text, limit, nowMs); } function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] { if (lane === "v02") return ["hwlab-v02-ci-poll-"]; if (lane === "g14") return ["hwlab-g14-ci-poll-"]; return ["hwlab-v02-ci-poll-", "hwlab-g14-ci-poll-"]; } function commandErrorSummary(result: CommandJsonResult): string { const remoteStderr = String(nested(result.parsed, ["data", "stderr"]) ?? "").trim(); const remoteStdout = String(nested(result.parsed, ["data", "stdout"]) ?? "").trim(); const text = [result.stderr.trim(), remoteStderr, remoteStdout, result.stdout.trim()].find((item) => item.length > 0) ?? ""; return text.slice(0, 4000); } function tailText(value: unknown, maxBytes = 2000): string { const text = String(value ?? "").trim(); return text.length <= maxBytes ? text : text.slice(-maxBytes); } function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record[] { const result = g14K3s([ "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, "-o", 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}', ], 60_000); if (!isCommandSuccess(result)) { throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`); } const prefixes = pipelinePrefixesForLane(options.lane); const now = Date.now(); return statusText(result) .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const [name = "", createdAt = "", status = "", reason = ""] = line.split("\t"); const createdMs = Date.parse(createdAt); const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null; return { name, createdAt, ageMinutes, status: status || null, reason: reason || null }; }) .filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix))) .filter((item) => item.status === "True" || item.status === "False") .filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes) .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))) .slice(0, options.limit); } function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record[] { if (pipelineRunNames.length === 0) { return []; } const result = g14K3s([ "kubectl", "get", "pvc", "-n", CI_NAMESPACE, "-o", 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}', ], 60_000); if (!isCommandSuccess(result)) { throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`); } const wanted = new Set(pipelineRunNames); return statusText(result) .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t"); return { name, volume: volume || null, phase: phase || null, ownerKind: ownerKind || null, owner: owner || null, }; }) .filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner)); } function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult { if (names.length === 0) { return { ok: true, command: [], exitCode: 0, stdout: "no candidates", stderr: "", parsed: null, }; } return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs); } function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record[] { const result = g14K3s([ "kubectl", "get", "pv", "-o", 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\n"}{end}', ], 60_000); if (!isCommandSuccess(result)) { throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`); } return statusText(result) .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t"); return { name, createdAt, phase, storageClass, reclaimPolicy, claimNamespace, claimName, capacity, hostPath: claimNamespace === CI_NAMESPACE && claimName.length > 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null, }; }) .filter((item) => item.phase === "Released") .filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete") .filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName)) .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))) .slice(0, options.limit); } function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult { if (names.length === 0) { return { ok: true, command: [], exitCode: 0, stdout: "no candidates", stderr: "", parsed: null, }; } return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs); } function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record { const candidates = listReleasedCiWorkspacePvs(options); const candidateNames = candidates.map((item) => String(item.name)); if (options.dryRun) { return { ok: true, command: "hwlab g14 control-plane cleanup-released-pvs", mode: "dry-run", lane: options.lane, limit: options.limit, candidates, candidateCount: candidates.length, mutation: false, next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` }, }; } const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000); return { ok: isCommandSuccess(deletion), command: "hwlab g14 control-plane cleanup-released-pvs", mode: "confirmed-cleanup", lane: options.lane, limit: options.limit, deletedPersistentVolumes: candidateNames, deletedPersistentVolumeCount: candidateNames.length, candidatesBefore: candidates, deletion, followUp: { diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'", storage: "bun scripts/cli.ts ssh G14 script -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'", }, }; } function runV02RuntimeMigration(options: G14ControlPlaneOptions, sourceCommit: string): Record { const reportPath = `/tmp/hwlab-v02-runtime-migration-${shortSha(sourceCommit)}.json`; const migrationArgs = options.dryRun ? [ ...(options.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]), "--report", reportPath, ] : [ "--apply", "--confirm-dev", "--confirmed-non-production", "--report", reportPath, ]; const command = [ "exec", "-n", "hwlab-v02", "deployment/hwlab-cloud-api", "-c", "hwlab-cloud-api", "--", "bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs, ]; const result = g14K3s(["kubectl", ...command], options.timeoutSeconds * 1000); const ok = isCommandSuccess(result); return { ok, command: "hwlab g14 control-plane runtime-migration --lane v02", lane: "v02", mode: options.dryRun ? options.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply", sourceCommit, runtimeNamespace: "hwlab-v02", target: "deployment/hwlab-cloud-api -c hwlab-cloud-api", migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs], reportPath, mutation: !options.dryRun, result, next: options.dryRun ? { liveReadDryRun: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run", apply: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm", } : { health: "curl -fsS --max-time 20 http://74.48.78.17:19667/health/live", triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm", }, }; } function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record { const candidates = listCleanupPipelineRuns(options); const candidateNames = candidates.map((item) => String(item.name)); const ownedPvcs = listOwnedWorkspacePvcs(candidateNames); if (options.dryRun) { return { ok: true, command: "hwlab g14 control-plane cleanup-runs", mode: "dry-run", lane: options.lane, minAgeMinutes: options.minAgeMinutes, limit: options.limit, candidates, candidateCount: candidates.length, ownedPvcs, ownedPvcCount: ownedPvcs.length, mutation: false, next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane ${options.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` }, }; } const deletion = deletePipelineRuns(candidateNames, options.timeoutSeconds * 1000); return { ok: isCommandSuccess(deletion), command: "hwlab g14 control-plane cleanup-runs", mode: "confirmed-cleanup", lane: options.lane, minAgeMinutes: options.minAgeMinutes, limit: options.limit, deletedPipelineRuns: candidateNames, deletedPipelineRunCount: candidateNames.length, ownedPvcsBefore: ownedPvcs, ownedPvcCountBefore: ownedPvcs.length, deletion, followUp: { status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02", diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'", }, }; } function runV02RenderToTemp(sourceCommit: string): CommandJsonResult { const renderDir = v02RenderDir(sourceCommit); return v02WorkspaceScript([ "set -eu", "git fetch origin v0.2 --prune", "git checkout v0.2 >/dev/null 2>&1 || true", "git merge --ff-only origin/v0.2", `test "$(git rev-parse HEAD)" = ${shellQuote(sourceCommit)}`, `rm -rf ${shellQuote(renderDir)}`, `mkdir -p ${shellQuote(renderDir)}`, `node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out ${shellQuote(renderDir)}`, ].join("\n"), 180_000); } function v02RenderDir(sourceCommit: string): string { return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}`; } function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { const renderDir = v02RenderDir(sourceCommit); return g14K3s([ "kubectl", "apply", "--server-side", "--force-conflicts", `--field-manager=${V02_CONTROL_PLANE_FIELD_MANAGER}`, ...(dryRun ? ["--dry-run=server"] : []), "-f", `${renderDir}/tekton-v02/rbac.yaml`, "-f", `${renderDir}/tekton-v02/pipeline.yaml`, "-f", `${renderDir}/argocd/project.yaml`, "-f", `${renderDir}/argocd/application-v02.yaml`, ], timeoutSeconds * 1000); } function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record { const render = runV02RenderToTemp(sourceCommit); if (!isCommandSuccess(render)) { return { ok: false, phase: "source-render", sourceCommit, renderDir: v02RenderDir(sourceCommit), render, degradedReason: "control-plane-render-failed", }; } const apply = applyV02ControlPlaneFiles(sourceCommit, false, timeoutSeconds); const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null; return { ok: isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs)), phase: "control-plane-refresh", sourceCommit, renderDir: v02RenderDir(sourceCommit), render: commandData(render), apply: compactCommandResult(apply), cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs), degradedReason: !isCommandSuccess(apply) ? "control-plane-apply-failed" : cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs) ? "obsolete-cronjob-cleanup-failed" : undefined, }; } function getV02ObsoleteCronJobs(): CommandJsonResult { return g14K3s([ "kubectl", "get", "cronjob", "-n", CI_NAMESPACE, V02_POLLER, V02_RECONCILER, "--ignore-not-found", "-o", "name", ], 60_000); } function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult { return g14K3s([ "kubectl", "delete", "cronjob", "-n", CI_NAMESPACE, V02_POLLER, V02_RECONCILER, "--ignore-not-found=true", ...(dryRun ? ["--dry-run=server", "-o", "name"] : []), ], 60_000); } function v02PipelineRunManifest(sourceCommit: string): Record { const pipelineRun = v02PipelineRunName(sourceCommit); return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, namespace: CI_NAMESPACE, labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": "v02", "hwlab.pikastech.local/source-commit": sourceCommit, "hwlab.pikastech.local/trigger": "manual-cli", }, annotations: { "hwlab.pikastech.local/source-branch": V02_SOURCE_BRANCH, "hwlab.pikastech.local/gitops-branch": V02_GITOPS_BRANCH, "hwlab.pikastech.local/triggered-by": "unidesk-cli", }, }, spec: { pipelineRef: { name: V02_PIPELINE }, taskRunTemplate: { serviceAccountName: "hwlab-v02-tekton-runner", podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, }, }, params: [ { name: "git-url", value: V02_GIT_URL }, { name: "git-read-url", value: V02_GIT_READ_URL }, { name: "source-branch", value: V02_SOURCE_BRANCH }, { name: "gitops-branch", value: V02_GITOPS_BRANCH }, { name: "lane", value: "v02" }, { name: "catalog-path", value: V02_CATALOG_PATH }, { name: "image-tag-mode", value: "full" }, { name: "runtime-path", value: V02_RUNTIME_PATH }, { name: "revision", value: sourceCommit }, { name: "registry-prefix", value: V02_REGISTRY_PREFIX }, { name: "services", value: V02_SERVICE_IDS.join(",") }, { name: "base-image", value: V02_BASE_IMAGE }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, ], }, }; } function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult { const manifest = JSON.stringify(v02PipelineRunManifest(sourceCommit)); const manifestB64 = Buffer.from(manifest, "utf8").toString("base64"); const pipelineRun = v02PipelineRunName(sourceCommit); const script = [ "set -eu", `manifest_b64=${shellQuote(manifestB64)}`, `manifest_path=/tmp/${pipelineRun}.json`, "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", "kubectl create -f \"$manifest_path\"", `kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`, ].join("\n"); return g14K3s(["script", "--", script], timeoutSeconds * 1000); } function deleteV02PipelineRun(pipelineRun: string): CommandJsonResult { return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, pipelineRun, "--ignore-not-found=true"], 60_000); } function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record { const bundle = v02ControlPlaneStatusBundle(sourceCommitInput); const sections = parseShellSections(statusText(bundle)); const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null; const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit); const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now(); const controlPlane = sections.controlPlane; const obsoleteCronJobs = sections.obsoleteCronJobs; const argo = sections.argo; const pipelineRunSection = sections.pipelineRun; const taskRunsSection = sections.taskRuns; const webAssetsSection = sections.webAssets; const recentPipelineRuns = listV02PipelineRunsCompactFromText( sections.recentPipelineRuns?.stdout ?? "", shellSectionOk(sections.recentPipelineRuns), "kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=", sections.recentPipelineRuns?.exitCode ?? null, bundle.stderr, 8, queryNowMs, ); const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : []; const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u); return { ok: sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo), command: "hwlab g14 control-plane status --lane v02", lane: "v02", sourceCommit, sourceCommitSource: "cached origin/v0.2; write operations fetch before acting", expected: { workspace: V02_WORKSPACE, branch: V02_SOURCE_BRANCH, namespace: CI_NAMESPACE, runtimeNamespace: "hwlab-v02", pipeline: V02_PIPELINE, trigger: "manual UniDesk CLI PipelineRun creation", argoApplication: V02_APP, }, controlPlane: { ok: shellSectionOk(controlPlane), names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), exitCode: controlPlane?.exitCode ?? null, }, obsoleteCronJobs: { ok: shellSectionOk(obsoleteCronJobs), names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), exitCode: obsoleteCronJobs?.exitCode ?? null, cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm", }, argo: { ok: shellSectionOk(argo), raw: argo?.stdout ?? "", fields: { targetRevision, path, syncRevision, syncStatus, health }, exitCode: argo?.exitCode ?? null, }, pipelineRun: pipelineRun === null ? null : pipelineRunCompactFromText( pipelineRun, pipelineRunSection?.stdout ?? "", shellSectionOk(pipelineRunSection), `kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`, pipelineRunSection?.exitCode ?? null, bundle.stderr, ), taskRuns: taskRunsCompactFromText( taskRunsSection?.stdout ?? "", shellSectionOk(taskRunsSection), pipelineRun, taskRunsSection?.exitCode ?? null, bundle.stderr, ), webAssets: v02WebAssetsFromText( webAssetsSection?.stdout ?? "", shellSectionOk(webAssetsSection), sourceCommit, syncRevision, webAssetsSection?.exitCode ?? null, bundle.stderr, ), activePipelineRuns, recentPipelineRuns, query: { ok: isCommandSuccess(bundle), exitCode: bundle.exitCode, stderr: bundle.stderr.trim().slice(0, 2000), }, visibilityHint: activePipelineRuns.length > 0 ? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger" : "no active v02 PipelineRun observed", }; } function runV02ControlPlane(options: G14ControlPlaneOptions): Record { if (options.action === "cleanup-runs") return runControlPlaneCleanup(options); if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options); const sourceCommit = options.action === "status" ? undefined : getV02Head(); if (sourceCommit === null) { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE }; } if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit); if (options.action === "status") return v02ControlPlaneStatus(); if (options.action === "apply") { const render = runV02RenderToTemp(sourceCommit); if (!isCommandSuccess(render)) { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, phase: "source-render", sourceCommit, render: compactCommandResult(render), }; } const apply = applyV02ControlPlaneFiles(sourceCommit, options.dryRun, options.timeoutSeconds); return { ok: isCommandSuccess(apply), command: "hwlab g14 control-plane apply --lane v02", lane: "v02", mode: options.dryRun ? "dry-run" : "confirmed-apply", sourceCommit, renderDir: v02RenderDir(sourceCommit), render: commandData(render), apply: compactCommandResult(apply), cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false)), status: v02ControlPlaneStatus(sourceCommit), next: options.dryRun ? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }, }; } const before = getPipelineRunCompact(v02PipelineRunName(sourceCommit)); if (options.dryRun) { const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options); return { ok: true, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "dry-run", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), before, gitMirrorPreSync, controlPlaneRefresh: { mode: "skipped-dry-run", wouldRefreshBeforeCreate: true, resources: ["tekton-v02/rbac.yaml", "tekton-v02/pipeline.yaml", "argocd/project.yaml", "argocd/application-v02.yaml"], }, manifest: v02PipelineRunManifest(sourceCommit), next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }, }; } if (before.status === "True" || before.status === "Unknown") { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), before, degradedReason: "refuse-active-or-successful-pipelinerun", }; } printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) }); const controlPlaneRefresh = refreshV02ControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds); printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: controlPlaneRefresh.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), degradedReason: record(controlPlaneRefresh).degradedReason ?? null }); if (controlPlaneRefresh.ok !== true) { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", phase: "control-plane-refresh", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), before, controlPlaneRefresh, degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed", }; } printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) }); const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options); printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: gitMirrorPreSync.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), mode: record(gitMirrorPreSync).mode ?? null, degradedReason: record(gitMirrorPreSync).degradedReason ?? null }); if (gitMirrorPreSync.ok !== true) { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", phase: "git-mirror-pre-sync", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), before, controlPlaneRefresh, gitMirrorPreSync, degradedReason: "git-mirror-pre-sync-failed", }; } printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) }); const deletePipelineRun = deleteV02PipelineRun(v02PipelineRunName(sourceCommit)); printProgressEvent("hwlab.v02.trigger.progress", { stage: "delete-existing-pipelinerun", status: isCommandSuccess(deletePipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: deletePipelineRun.exitCode }); printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) }); const createPipelineRun = isCommandSuccess(deletePipelineRun) ? createV02PipelineRun(sourceCommit, options.timeoutSeconds) : null; printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: createPipelineRun !== null && isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: createPipelineRun?.exitCode ?? null }); return { ok: isCommandSuccess(deletePipelineRun) && createPipelineRun !== null && isCommandSuccess(createPipelineRun), command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), before, controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh), gitMirrorPreSync, deletePipelineRun: compactTriggerCommandResult(compactCommandResult(deletePipelineRun)), createPipelineRun: createPipelineRun === null ? null : compactTriggerCommandResult(compactCommandResult(createPipelineRun)), after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)), disclosure: { fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.", }, }; } function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult { return g14K3s([ "kubectl", "delete", "cronjob", "-n", GIT_MIRROR_NAMESPACE, GIT_MIRROR_LEGACY_CRONJOB, "--ignore-not-found=true", ...(dryRun ? ["--dry-run=server", "-o", "name"] : []), ], 60_000); } function applyGitMirrorManifestFile(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { const renderDir = v02RenderDir(sourceCommit); return g14K3s([ "kubectl", "apply", "--server-side", "--force-conflicts", `--field-manager=${GIT_MIRROR_MANIFEST_FIELD_MANAGER}`, ...(dryRun ? ["--dry-run=server"] : []), "-f", `${renderDir}/devops-infra/git-mirror.yaml`, ], timeoutSeconds * 1000); } function gitMirrorSyncJobName(): string { return `${GIT_MIRROR_SYNC_JOB_PREFIX}-${Date.now().toString(36)}`.slice(0, 63); } function gitMirrorFlushJobName(): string { return `git-mirror-hwlab-flush-manual-${Date.now().toString(36)}`.slice(0, 63); } export function gitMirrorSyncJobManifest(name: string): Record { return { apiVersion: "batch/v1", kind: "Job", metadata: { name, namespace: GIT_MIRROR_NAMESPACE, labels: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/part-of": "devops-infra", "app.kubernetes.io/component": "sync-controller", "hwlab.pikastech.local/trigger": "manual-cli", }, }, spec: { backoffLimit: 0, activeDeadlineSeconds: 300, ttlSecondsAfterFinished: 3600, template: { metadata: { labels: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/part-of": "devops-infra", "app.kubernetes.io/component": "sync-controller", "hwlab.pikastech.local/trigger": "manual-cli", }, }, spec: { restartPolicy: "Never", hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", volumes: [ { name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } }, { name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } }, { name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } }, ], containers: [{ name: "sync", image: "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1", imagePullPolicy: "IfNotPresent", command: ["/script/sync.sh"], volumeMounts: [ { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, { name: "script", mountPath: "/script", readOnly: true }, ], }], }, }, }, }; } export function gitMirrorFlushJobManifest(name: string): Record { const manifest = gitMirrorSyncJobManifest(name); record(record(manifest.metadata).labels)["app.kubernetes.io/component"] = "flush-controller"; const template = record(record(manifest.spec).template); record(record(template.metadata).labels)["app.kubernetes.io/component"] = "flush-controller"; const podSpec = record(template.spec); const containers = Array.isArray(podSpec.containers) ? podSpec.containers : []; const first = record(containers[0]); first.name = "flush"; first.command = ["/script/flush.sh"]; podSpec.containers = [first]; return manifest; } export function parseGitMirrorStatusRefs(raw: string): { refs: Record; pendingFlush: boolean | null } { const line = raw.split(/\r?\n/u).find((item) => item.startsWith("refs=")); if (line === undefined) return { refs: {}, pendingFlush: null }; try { const parsed = record(JSON.parse(line.slice("refs=".length)) as unknown); const refs = record(parsed.refs); return { refs: { localV02: stringOrNull(refs.localV02), localG14: stringOrNull(refs.localG14), localGitops: stringOrNull(refs.localGitops), githubGitops: stringOrNull(refs.githubGitops), }, pendingFlush: typeof parsed.pendingFlush === "boolean" ? parsed.pendingFlush : null, }; } catch { return { refs: {}, pendingFlush: null }; } } export function gitMirrorV02SyncRequirement(sourceCommit: string, rawStatus: string): Record { const parsed = parseGitMirrorStatusRefs(rawStatus); const localV02 = parsed.refs.localV02 ?? null; return { required: localV02 !== sourceCommit, sourceCommit, localV02, pendingFlush: parsed.pendingFlush, refs: parsed.refs, reason: localV02 === sourceCommit ? "local-v02-current" : localV02 === null ? "local-v02-unresolved" : "local-v02-stale", }; } function gitMirrorStatusCacheRaw(status: Record): string { return String(nested(status, ["cache", "raw"]) ?? ""); } function compactGitMirrorStatus(status: Record, sourceCommit: string): Record { const requirement = gitMirrorV02SyncRequirement(sourceCommit, gitMirrorStatusCacheRaw(status)); return { ok: status.ok === true, readUrl: status.readUrl ?? V02_GIT_READ_URL, writeUrl: status.writeUrl ?? V02_GIT_WRITE_URL, legacyCronJobExists: nested(status, ["legacyCronJob", "exists"]) === true, required: requirement.required, sourceCommit, localV02: requirement.localV02 ?? null, pendingFlush: requirement.pendingFlush ?? null, reason: requirement.reason, cacheOk: nested(status, ["cache", "ok"]) === true, cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000), resourcesOk: nested(status, ["resources", "ok"]) === true, }; } function compactGitMirrorSync(sync: Record): Record { return { ok: sync.ok === true, command: sync.command ?? "hwlab g14 git-mirror sync", mode: sync.mode ?? null, namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE, jobName: sync.jobName ?? null, exitCode: nested(sync, ["result", "exitCode"]) ?? null, stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600), stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200), stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null, stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null, }; } function preSyncV02GitMirror(sourceCommit: string, options: Pick): Record { const before = runGitMirrorStatus(); const beforeSummary = compactGitMirrorStatus(before, sourceCommit); if (options.dryRun) { return { ok: true, mode: "dry-run", sourceCommit, before: beforeSummary, action: beforeSummary.required === true ? "would-sync-before-trigger" : "already-current", next: beforeSummary.required === true ? { syncAndTrigger: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" } : undefined, }; } if (beforeSummary.required !== true) { return { ok: true, mode: "already-current", sourceCommit, before: beforeSummary, }; } const sync = runGitMirrorSync({ action: "sync", confirm: true, dryRun: false, timeoutSeconds: options.timeoutSeconds, }); const after = runGitMirrorStatus(); const afterSummary = compactGitMirrorStatus(after, sourceCommit); const ok = sync.ok === true && afterSummary.required === false; return { ok, mode: "auto-sync-before-trigger", sourceCommit, before: beforeSummary, sync: compactGitMirrorSync(sync), after: afterSummary, degradedReason: ok ? undefined : "git-mirror-local-v02-not-current-after-sync", }; } function runGitMirrorStatus(): Record { const resources = g14K3s([ "kubectl", "get", "deploy,svc,pvc,cm", "-n", GIT_MIRROR_NAMESPACE, "-l", "app.kubernetes.io/name=git-mirror", "-o", "name", ], 60_000); const legacyCronJob = g14K3s([ "kubectl", "get", "cronjob", "-n", GIT_MIRROR_NAMESPACE, GIT_MIRROR_LEGACY_CRONJOB, "--ignore-not-found", "-o", "name", ], 60_000); const jobs = g14K3s([ "kubectl", "get", "job", "-n", GIT_MIRROR_NAMESPACE, "-l", "app.kubernetes.io/name=git-mirror", "--sort-by=.metadata.creationTimestamp", "-o", "custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime", "--no-headers", ], 60_000); const cache = g14K3s([ "kubectl", "exec", "-n", GIT_MIRROR_NAMESPACE, "deploy/git-mirror-http", "--", "sh", "-lc", [ "printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'", "printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'", "printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'", "printf 'refs='", "node - <<'NODE' 2>/dev/null || true", "const { execFileSync } = require('node:child_process');", "const repo = '/cache/pikasTech/HWLAB.git';", "function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim(); } catch { return null; } }", "const localGitops = rev('refs/heads/v0.2-gitops');", "const githubGitops = rev('refs/mirror-stage/heads/v0.2-gitops');", "console.log(JSON.stringify({ refs: { localV02: rev('refs/heads/v0.2'), localG14: rev('refs/heads/G14'), localGitops, githubGitops }, pendingFlush: Boolean(localGitops && githubGitops && localGitops !== githubGitops) }));", "NODE", ].join("\n"), ], 60_000); return { ok: isCommandSuccess(resources), command: "hwlab g14 git-mirror status", namespace: GIT_MIRROR_NAMESPACE, readUrl: V02_GIT_READ_URL, writeUrl: V02_GIT_WRITE_URL, resources: { ok: isCommandSuccess(resources), names: statusText(resources).split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), stderr: resources.stderr.trim().slice(0, 2000), }, legacyCronJob: { ok: isCommandSuccess(legacyCronJob), exists: statusText(legacyCronJob).trim().length > 0, name: statusText(legacyCronJob).trim() || null, cleanupCommand: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm", }, jobs: { ok: isCommandSuccess(jobs), raw: statusText(jobs), stderr: jobs.stderr.trim().slice(0, 2000), }, cache: { ok: isCommandSuccess(cache), raw: statusText(cache), stderr: cache.stderr.trim().slice(0, 2000), }, next: { apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm", sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm", flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm", }, }; } function runGitMirrorApply(options: G14GitMirrorOptions): Record { const sourceCommit = getV02Head(); if (sourceCommit === null) { return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE }; } const render = runV02RenderToTemp(sourceCommit); if (!isCommandSuccess(render)) { return { ok: false, command: "hwlab g14 git-mirror apply", phase: "source-render", sourceCommit, render: compactCommandResult(render), }; } const apply = applyGitMirrorManifestFile(sourceCommit, options.dryRun, options.timeoutSeconds); const cleanupLegacyCronJob = isCommandSuccess(apply) ? deleteLegacyGitMirrorCronJob(options.dryRun) : { ok: false, command: [], exitCode: null, stdout: "", stderr: "skipped because apply failed", parsed: null, }; return { ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob), command: "hwlab g14 git-mirror apply", mode: options.dryRun ? "dry-run" : "confirmed-apply", sourceCommit, manifest: `${v02RenderDir(sourceCommit)}/devops-infra/git-mirror.yaml`, render: commandData(render), apply: compactCommandResult(apply), cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob), status: runGitMirrorStatus(), next: options.dryRun ? { apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm" } : { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" }, }; } function runGitMirrorSync(options: G14GitMirrorOptions): Record { const jobName = gitMirrorSyncJobName(); const manifest = gitMirrorSyncJobManifest(jobName); const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); const script = [ "set -eu", `job=${shellQuote(jobName)}`, `manifest_b64=${shellQuote(manifestB64)}`, "manifest_path=\"/tmp/$job.json\"", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", options.dryRun ? "kubectl create --dry-run=server -f \"$manifest_path\" -o name" : [ `kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`, "kubectl create -f \"$manifest_path\"", `deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`, "while :; do", ` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='{.status.succeeded} {.status.failed}' 2>/dev/null || true)`, " succeeded=$(printf '%s\\n' \"$status\" | awk '{print $1}')", " failed=$(printf '%s\\n' \"$status\" | awk '{print $2}')", " if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi", " if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then", ` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`, " exit 44", " fi", " if [ \"$(date +%s)\" -ge \"$deadline\" ]; then", ` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`, " exit 45", " fi", " sleep 2", "done", `kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`, `kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2 refs/heads/v0.2-gitops refs/heads/G14 2>/dev/null || true'`, ].join("\n"), ].join("\n"); const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000); return { ok: isCommandSuccess(result), command: "hwlab g14 git-mirror sync", mode: options.dryRun ? "dry-run" : "confirmed-sync", namespace: GIT_MIRROR_NAMESPACE, jobName, manifest: options.dryRun ? manifest : undefined, result: compactCommandResult(result), status: options.dryRun ? undefined : runGitMirrorStatus(), next: options.dryRun ? { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }, }; } function runGitMirrorFlush(options: G14GitMirrorOptions): Record { const jobName = gitMirrorFlushJobName(); const manifest = gitMirrorFlushJobManifest(jobName); const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); const script = [ "set -eu", `job=${shellQuote(jobName)}`, `manifest_b64=${shellQuote(manifestB64)}`, "manifest_path=\"/tmp/$job.json\"", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", options.dryRun ? "kubectl create --dry-run=server -f \"$manifest_path\" -o name" : [ `kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`, "kubectl create -f \"$manifest_path\"", `deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`, "while :; do", ` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='{.status.succeeded} {.status.failed}' 2>/dev/null || true)`, " succeeded=$(printf '%s\n' \"$status\" | awk '{print $1}')", " failed=$(printf '%s\n' \"$status\" | awk '{print $2}')", " if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi", " if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then", ` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`, " exit 54", " fi", " if [ \"$(date +%s)\" -ge \"$deadline\" ]; then", ` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`, " exit 55", " fi", " sleep 2", "done", `kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`, `kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2-gitops refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true'`, ].join("\n"), ].join("\n"); const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000); return { ok: isCommandSuccess(result), command: "hwlab g14 git-mirror flush", mode: options.dryRun ? "dry-run" : "confirmed-flush", namespace: GIT_MIRROR_NAMESPACE, jobName, manifest: options.dryRun ? manifest : undefined, result: compactCommandResult(result), status: options.dryRun ? undefined : runGitMirrorStatus(), next: options.dryRun ? { flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" } : { status: "bun scripts/cli.ts hwlab g14 git-mirror status" }, }; } function runG14GitMirror(options: G14GitMirrorOptions): Record { if (options.action === "status") return runGitMirrorStatus(); if (options.action === "apply") return runGitMirrorApply(options); if (options.action === "flush") return runGitMirrorFlush(options); return runGitMirrorSync(options); } function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record { const job = startJob(name, command, note); const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`; return { ok: true, mode: "async-job", job, statusCommand, tailCommand: `tail -f ${job.stdoutFile}`, next: { status: statusCommand, tail: `tail -f ${job.stdoutFile}`, }, }; } function startControlPlaneTriggerJob(options: G14ControlPlaneOptions): Record { const command = [ "bun", "scripts/cli.ts", "hwlab", "g14", "control-plane", "trigger-current", "--lane", options.lane, "--confirm", "--timeout-seconds", String(options.timeoutSeconds), "--wait", ]; return { command: "hwlab g14 control-plane trigger-current --lane v02", lane: options.lane, reason: "confirmed trigger can spend tens of seconds syncing git mirror and creating PipelineRun; default is fire-and-forget to avoid silent blocking", waitCommand: command.join(" "), ...startAsyncHwlabG14Job( "hwlab_g14_v02_trigger_current", command, "Trigger HWLAB v0.2 current commit PipelineRun with git mirror pre-sync through G14 control-plane", ), }; } function startGitMirrorJob(options: G14GitMirrorOptions): Record { const command = [ "bun", "scripts/cli.ts", "hwlab", "g14", "git-mirror", options.action, "--confirm", "--timeout-seconds", String(options.timeoutSeconds), "--wait", ]; return { command: `hwlab g14 git-mirror ${options.action}`, namespace: GIT_MIRROR_NAMESPACE, reason: "manual git mirror sync/flush waits for a Kubernetes Job; default is fire-and-forget to keep CLI output immediately visible", waitCommand: command.join(" "), ...startAsyncHwlabG14Job( `hwlab_g14_git_mirror_${options.action}`, command, `Run HWLAB devops-infra git mirror ${options.action} through a bounded manual Kubernetes Job`, ), }; } function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult { return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs); } function g14CiToolsImage(tag: string): string { return `${G14_CI_TOOLS_IMAGE_REPO}:${tag}`; } function runG14ToolsImageStatus(options: G14ToolsImageOptions): Record { const image = g14CiToolsImage(options.tag); const script = [ "set -u", `image=${shellQuote(image)}`, "exists=false", "image_id=", "created=", "size=", "node_version=", "npm_version=", "bun_version=", "git_version=", "docker_version=", "python_version=", "registry_tags=", "if docker image inspect \"$image\" >/tmp/hwlab-tools-image-inspect.json 2>/tmp/hwlab-tools-image-inspect.err; then", " exists=true", " image_id=$(docker image inspect \"$image\" --format '{{.Id}}' 2>/dev/null || true)", " created=$(docker image inspect \"$image\" --format '{{.Created}}' 2>/dev/null || true)", " size=$(docker image inspect \"$image\" --format '{{.Size}}' 2>/dev/null || true)", " node_version=$(docker run --rm \"$image\" node --version 2>/dev/null || true)", " npm_version=$(docker run --rm \"$image\" npm --version 2>/dev/null || true)", " bun_version=$(docker run --rm \"$image\" bun --version 2>/dev/null || true)", " git_version=$(docker run --rm \"$image\" git --version 2>/dev/null || true)", " docker_version=$(docker run --rm \"$image\" docker --version 2>/dev/null || true)", " python_version=$(docker run --rm \"$image\" python3 --version 2>/dev/null || true)", "fi", `registry_tags=$(curl -fsS --max-time 10 http://127.0.0.1:5000/v2/hwlab/hwlab-ci-node-tools/tags/list 2>/dev/null || true)`, "export image exists image_id created size node_version npm_version bun_version git_version docker_version python_version registry_tags", "node - <<'NODE'", "const fs = require('node:fs');", "const env = process.env;", "const payload = {", " image: env.image,", " exists: env.exists === 'true',", " imageId: env.image_id || null,", " created: env.created || null,", " sizeBytes: env.size ? Number(env.size) : null,", " tools: {", " node: env.node_version || null,", " npm: env.npm_version || null,", " bun: env.bun_version || null,", " git: env.git_version || null,", " docker: env.docker_version || null,", " python: env.python_version || null", " },", " registryTagsRaw: env.registry_tags || null", "};", "console.log(JSON.stringify(payload, null, 2));", "NODE", ].join("\n"); const result = g14HostScript(script, 180_000); let parsedStatus: unknown = null; const text = statusText(result); if (text.length > 0) { try { parsedStatus = JSON.parse(text) as unknown; } catch { parsedStatus = null; } } return { ok: isCommandSuccess(result) && record(parsedStatus).exists === true, command: "hwlab g14 tools-image status --name ci-node-tools", name: options.name, image, status: parsedStatus ?? text, result, }; } function runG14ToolsImageBuild(options: G14ToolsImageOptions): Record { const image = g14CiToolsImage(options.tag); const script = [ "set -eu", `cd ${shellQuote(V02_WORKSPACE)}`, "git fetch origin v0.2 --prune", "git checkout v0.2 >/dev/null 2>&1 || true", "git merge --ff-only origin/v0.2", `test -f ${shellQuote(options.dockerfile)}`, `image=${shellQuote(image)}`, `dockerfile=${shellQuote(options.dockerfile)}`, "echo \"{\\\"phase\\\":\\\"preflight\\\",\\\"image\\\":\\\"$image\\\",\\\"dockerfile\\\":\\\"$dockerfile\\\",\\\"commit\\\":\\\"$(git rev-parse HEAD)\\\"}\"", "export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808", "export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000", "export no_proxy=$NO_PROXY", "docker build --pull --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg http_proxy --build-arg https_proxy --build-arg NO_PROXY --build-arg no_proxy -f \"$dockerfile\" -t \"$image\" .", "docker run --rm \"$image\" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V'", "docker push \"$image\"", "digest=$(docker image inspect \"$image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)", "echo \"{\\\"phase\\\":\\\"published\\\",\\\"image\\\":\\\"$image\\\",\\\"digest\\\":\\\"$digest\\\"}\"", ].join("\n"); if (options.dryRun) { return { ok: true, command: "hwlab g14 tools-image build --name ci-node-tools", mode: "dry-run", image, workspace: V02_WORKSPACE, dockerfile: options.dockerfile, buildScriptPreview: script.split(/\n/u), next: { build: `bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag ${options.tag} --confirm` }, }; } const result = g14HostScript(script, options.timeoutSeconds * 1000); return { ok: isCommandSuccess(result), command: "hwlab g14 tools-image build --name ci-node-tools", mode: "confirmed-build", image, workspace: V02_WORKSPACE, dockerfile: options.dockerfile, result, status: runG14ToolsImageStatus({ ...options, action: "status", dryRun: true, confirm: false }), }; } function runG14ToolsImage(options: G14ToolsImageOptions): Record { if (options.action === "status") return runG14ToolsImageStatus(options); return runG14ToolsImageBuild(options); } function listOpenG14PullRequests(): CommandJsonResult { return cliJson(["gh", "pr", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "30", "--json", "number,title,state,url,head,base,draft,headRefName,baseRefName"], 60_000); } function preflightPullRequest(number: number): CommandJsonResult { return cliJson(["gh", "pr", "preflight", String(number), "--repo", HWLAB_REPO], 80_000); } function mergePullRequest(number: number, dryRun: boolean): CommandJsonResult { return cliJson(["gh", "pr", "merge", String(number), "--repo", HWLAB_REPO, "--merge", "--delete-branch", ...(dryRun ? ["--dry-run"] : [])], 100_000); } function getG14Head(): string | null { const result = cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "git fetch origin G14 --prune >/dev/null 2>&1; git rev-parse origin/G14"], 120_000); if (!isCommandSuccess(result)) return null; const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim(); const match = /[0-9a-f]{40}/iu.exec(output); return match?.[0] ?? null; } function refreshArgoDev(): void { cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "annotate", "application", "-n", ARGO_NAMESPACE, DEV_APP, "argocd.argoproj.io/refresh=hard", "--overwrite"], 60_000); } function getPipelineStatus(sourceCommit: string): CommandJsonResult { return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}"], 60_000); } function getArgoStatus(): CommandJsonResult { return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "application", "-n", ARGO_NAMESPACE, DEV_APP, "-o", "jsonpath={.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.operationState.phase}{\"\\n\"}{.status.operationState.syncResult.revision}{\"\\n\"}"], 60_000); } function getDevWorkloads(): CommandJsonResult { return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "deploy,statefulset,pod", "-n", DEV_NAMESPACE, "-o", "wide"], 60_000); } function getLiveHealth(): CommandJsonResult { return commandJson(["curl", "-fsS", "--max-time", "20", "http://74.48.78.17:17667/health/live"], 30_000); } function statusText(result: CommandJsonResult): string { return String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim(); } function workloadReadiness(workloadsText: string, commandOk: boolean): Record { const blockers: string[] = []; const ignoredPods: string[] = []; const readyWorkloads: string[] = []; const allowedZeroDeployments = new Set(["deployment.apps/hwlab-gateway", "deployment.apps/hwlab-tunnel-client"]); for (const line of workloadsText.split(/\r?\n/u)) { const trimmed = line.trim(); if (trimmed.length === 0 || trimmed.startsWith("NAME ")) continue; const fields = trimmed.split(/\s+/u); const name = fields[0] ?? ""; const ready = fields[1] ?? ""; if (name.startsWith("deployment.apps/") || name.startsWith("statefulset.apps/")) { if (ready === "0/0" && allowedZeroDeployments.has(name)) { readyWorkloads.push(`${name}:${ready}:scaled-zero`); continue; } const match = /^(\d+)\/(\d+)$/u.exec(ready); if (match === null) { blockers.push(`${name}:unparseable-ready:${ready}`); continue; } const readyCount = Number(match[1]); const desiredCount = Number(match[2]); if (desiredCount > 0 && readyCount < desiredCount) blockers.push(`${name}:not-ready:${ready}`); else readyWorkloads.push(`${name}:${ready}`); continue; } if (name.startsWith("pod/")) { const status = fields[2] ?? ""; if (status === "Completed" || status === "Succeeded") ignoredPods.push(`${name}:${status}`); if (/^(CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError|RunContainerError)$/u.test(status)) { blockers.push(`${name}:pod-status:${status}`); } } } return { ready: commandOk && blockers.length === 0, blockers, readyWorkloads: readyWorkloads.slice(0, 20), ignoredPods: ignoredPods.slice(0, 20), }; } function beijingParts(date = new Date()): { date: string; time: string; iso: string } { const shifted = new Date(date.getTime() + BEIJING_OFFSET_MS).toISOString(); return { date: shifted.slice(0, 10), time: shifted.slice(11, 19), iso: shifted }; } function durationSeconds(start?: string | null, end?: string | null): number | null { if (start === undefined || start === null || end === undefined || end === null) return null; const startMs = Date.parse(start); const endMs = Date.parse(end); if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) return null; return Math.round((endMs - startMs) / 1000); } function formatDuration(seconds: number | null): string { if (seconds === null) return "n/a"; const minutes = Math.floor(seconds / 60); const rest = seconds % 60; if (minutes === 0) return `${rest}s`; return `${minutes}m${String(rest).padStart(2, "0")}s`; } function pipelineTaskRunResultsJsonPath(): string { return [ "jsonpath=", "{range .items[*]}", "{.metadata.name}{\"\\t\"}", "{.metadata.labels.tekton\\.dev/pipelineTask}{\"\\t\"}", "{.status.startTime}{\"\\t\"}", "{.status.completionTime}{\"\\t\"}", "{range .status.results[*]}{.name}{\"=\"}{.value}{\";\"}{end}", "{\"\\n\"}", "{end}", ].join(""); } function getPipelineTaskRunResults(pipelineRun: string): CommandJsonResult { return cliJson([ "ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "taskrun", "-n", CI_NAMESPACE, "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", pipelineTaskRunResultsJsonPath(), ], 80_000); } function parseResultMap(raw: string): Record { const values: Record = {}; for (const item of raw.split(";")) { if (item.length === 0) continue; const separator = item.indexOf("="); if (separator <= 0) continue; values[item.slice(0, separator)] = item.slice(separator + 1); } return values; } function serviceMetricSort(left: CiServiceMetric, right: CiServiceMetric): number { return left.serviceId.localeCompare(right.serviceId); } export function parsePipelineTaskRunMetrics(pipelineRun: string, text: string): CiPipelineMetrics { const services: CiServiceMetric[] = []; for (const line of text.split(/\r?\n/u)) { const trimmed = line.trimEnd(); if (trimmed.length === 0) continue; const [taskRunName = "", pipelineTask = "", startTime = "", completionTime = "", resultsRaw = ""] = trimmed.split("\t"); if (!pipelineTask.startsWith("build-")) continue; const results = parseResultMap(resultsRaw); const serviceId = results["service-id"] ?? pipelineTask.replace(/^build-/u, ""); if (serviceId.length === 0) continue; services.push({ taskRun: taskRunName, serviceId, status: results.status ?? "unknown", durationSeconds: durationSeconds(startTime, completionTime), buildBackend: results["build-backend"] ?? null, reusedFrom: results["reused-from"] ?? null, imageTag: results["image-tag"] ?? null, sourceCommitId: results["source-commit-id"] ?? null, }); } services.sort(serviceMetricSort); const reusedServices = services.filter((service) => service.status === "reused").map((service) => service.serviceId); const rebuildServices = services.filter((service) => service.status !== "reused").map((service) => service.serviceId); return { ok: services.length > 0, pipelineRun, serviceTotal: services.length, reusedCount: reusedServices.length, rebuildCount: rebuildServices.length, reusedServices, rebuildServices, services, ...(services.length === 0 ? { degradedReason: "no-build-task-results", rawText: text.slice(0, 2000) } : {}), }; } function ciMetricsFromValue(value: unknown): CiPipelineMetrics | null { const data = record(value); if (data.ok !== true || typeof data.pipelineRun !== "string") return null; const services = Array.isArray(data.services) ? data.services.map((item) => { const service = record(item); return { taskRun: String(service.taskRun ?? ""), serviceId: String(service.serviceId ?? ""), status: String(service.status ?? "unknown"), durationSeconds: typeof service.durationSeconds === "number" ? service.durationSeconds : null, buildBackend: typeof service.buildBackend === "string" ? service.buildBackend : null, reusedFrom: typeof service.reusedFrom === "string" ? service.reusedFrom : null, imageTag: typeof service.imageTag === "string" ? service.imageTag : null, sourceCommitId: typeof service.sourceCommitId === "string" ? service.sourceCommitId : null, }; }).filter((service) => service.serviceId.length > 0).sort(serviceMetricSort) : []; const reusedServices = Array.isArray(data.reusedServices) ? data.reusedServices.map(String) : services.filter((service) => service.status === "reused").map((service) => service.serviceId); const rebuildServices = Array.isArray(data.rebuildServices) ? data.rebuildServices.map(String) : services.filter((service) => service.status !== "reused").map((service) => service.serviceId); return { ok: true, pipelineRun: data.pipelineRun, serviceTotal: typeof data.serviceTotal === "number" ? data.serviceTotal : services.length, reusedCount: typeof data.reusedCount === "number" ? data.reusedCount : reusedServices.length, rebuildCount: typeof data.rebuildCount === "number" ? data.rebuildCount : rebuildServices.length, reusedServices, rebuildServices, services, }; } function collectPipelineMetrics(pipelineRun: string): CiPipelineMetrics { const result = getPipelineTaskRunResults(pipelineRun); if (!isCommandSuccess(result)) { return { ok: false, pipelineRun, serviceTotal: 0, reusedCount: 0, rebuildCount: 0, reusedServices: [], rebuildServices: [], services: [], degradedReason: "taskrun-query-failed", rawText: `${result.stderr}\n${result.stdout}`.trim().slice(0, 2000), }; } return parsePipelineTaskRunMetrics(pipelineRun, statusText(result)); } function ciMetricLines(metrics: CiPipelineMetrics | null): string[] { if (metrics === null) { return [ "- CI/CD 关键指标:", " - lazy build reused: n/a", " - reused services: n/a", " - rebuild services: n/a", " - service durations: n/a", ]; } if (metrics.ok !== true) { return [ "- CI/CD 关键指标:", ` - lazy build reused: unavailable (${metrics.degradedReason ?? "unknown"})`, " - reused services: n/a", " - rebuild services: n/a", " - service durations: n/a", ]; } const lines = [ "- CI/CD 关键指标:", ` - lazy build reused: ${metrics.reusedCount}/${metrics.serviceTotal}`, ` - reused services: ${metrics.reusedServices.length > 0 ? metrics.reusedServices.join(", ") : "none"}`, ` - rebuild services: ${metrics.rebuildServices.length > 0 ? metrics.rebuildServices.join(", ") : "none"}`, " - service durations:", ]; for (const service of metrics.services) { const backend = service.buildBackend === null ? "" : `, backend=${service.buildBackend}`; lines.push(` - ${service.serviceId}: ${service.status}, ${formatDuration(service.durationSeconds)}${backend}`); } return lines; } function stripMarkdownInline(value: string): string { return value .replace(/\[([^\]]+)\]\([^)]+\)/gu, "$1") .replace(/\*\*([^*]+)\*\*/gu, "$1") .replace(/\*([^*]+)\*/gu, "$1") .replace(/__([^_]+)__/gu, "$1") .replace(/_([^_]+)_/gu, "$1") .replace(/\s+/gu, " ") .trim(); } function markdownSection(body: string, headings: string[]): string { const normalizedHeadings = new Set(headings.map((heading) => heading.toLowerCase())); const lines = body.split(/\r?\n/u); let capturing = false; const captured: string[] = []; for (const line of lines) { const heading = /^(#{1,6})\s+(.+?)\s*$/u.exec(line); if (heading !== null) { const title = stripMarkdownInline(heading[2] ?? "").replace(/[::]\s*$/u, "").toLowerCase(); if (capturing) break; capturing = normalizedHeadings.has(title); continue; } if (capturing) captured.push(line); } return captured.join("\n").trim(); } function firstParagraph(section: string): string | null { for (const paragraph of section.split(/\n\s*\n/u)) { const trimmed = stripMarkdownInline(paragraph.replace(/^[-*]\s+/gmu, "")); if (trimmed.length > 0) return trimmed; } return null; } function topLevelBullets(section: string, limit = 8): string[] { const bullets: string[] = []; for (const line of section.split(/\r?\n/u)) { const match = /^[-*]\s+(.+)$/u.exec(line); if (match === null) continue; const text = stripMarkdownInline(match[1] ?? ""); if (text.length > 0) bullets.push(text); if (bullets.length >= limit) break; } return bullets; } function conventionalTitleSummary(title: string): string { const match = /^(feat|fix|docs|test|refactor|perf|ci|build|chore)(?:\([^)]+\))?:\s*(.+)$/iu.exec(title.trim()); if (match === null) return stripMarkdownInline(title); const kind = String(match[1] ?? "").toLowerCase(); const subject = stripMarkdownInline(match[2] ?? title); const labels: Record = { feat: "功能", fix: "修复", docs: "文档", test: "测试", refactor: "重构", perf: "性能", ci: "CI/CD", build: "构建", chore: "维护", }; return `${labels[kind] ?? "变更"}:${subject}`; } function fileHeuristicBullets(fileSummary: Record): string[] { const keyFiles = Array.isArray(fileSummary.keyFiles) ? fileSummary.keyFiles.map(String) : []; const bullets: string[] = []; if (keyFiles.some((file) => file.includes("web/hwlab-cloud-web/"))) { bullets.push("前端:更新 HWLAB Cloud Web / Workbench 交互行为。"); } if (keyFiles.some((file) => /\.(test|spec)\./u.test(file) || file.includes("/test"))) { bullets.push("验证:补充或更新回归测试,覆盖本次变更路径。"); } if (keyFiles.some((file) => file.includes("deploy/") || file.includes("k8s") || file.includes("gitops"))) { bullets.push("部署:更新 G14 Kubernetes/GitOps 发布声明或发布计划。"); } if (keyFiles.some((file) => file.includes("internal/cloud/") || file.includes("code-agent"))) { bullets.push("运行时:调整 Cloud API / Code Agent 运行链路。"); } return bullets; } export function semanticChangelogBullets(title: string, body: string, fileSummary: Record): string[] { const bullets: string[] = []; const background = firstParagraph(markdownSection(body, ["背景", "问题", "problem", "background", "context"])); if (background !== null && /^(fix|perf)(?:\([^)]+\))?:/iu.test(title)) { bullets.push(`修复目标:${background}`); } const changeSection = markdownSection(body, ["上线 changelog", "changelog", "摘要", "概述", "总结", "修改", "变更", "改动", "changes", "what changed", "summary"]); bullets.push(...topLevelBullets(changeSection)); if (bullets.length === 0) bullets.push(conventionalTitleSummary(title)); for (const bullet of fileHeuristicBullets(fileSummary)) { if (bullets.length >= 8) break; if (!bullets.includes(bullet)) bullets.push(bullet); } return bullets.slice(0, 8); } function readIssue(issueNumber: number): CommandJsonResult { return cliJson(["gh", "issue", "read", String(issueNumber), "--repo", HWLAB_REPO, "--json", "title,body,state,updatedAt", "--raw"], 80_000); } function issueFromRead(result: CommandJsonResult): Record { return record(commandData(result).issue); } function parseBriefIssueFromIndex(indexBody: string, date: string): { number: number; url: string } | null { const escapedDate = date.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); const linePattern = new RegExp(`^\\|\\s*${escapedDate}\\s*\\|([^\\n]+)$`, "mu"); const line = linePattern.exec(indexBody)?.[0]; if (line === undefined) return null; const match = /\[#(\d+)\]\((https:\/\/github\.com\/pikasTech\/HWLAB\/issues\/\d+)\)/u.exec(line); if (match === null) return null; return { number: Number(match[1]), url: match[2] }; } function dailyBriefTitle(date: string): string { return `${date} 指挥简报(北京时间)`; } function writeStateFile(name: string, content: string): string { const dir = rootPath(".state", "hwlab-g14"); mkdirSync(dir, { recursive: true }); const path = join(dir, name); writeFileSync(path, content, "utf8"); return path; } function listIssues(): CommandJsonResult { return cliJson(["gh", "issue", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "100", "--json", "number,title,state,url"], 80_000); } function findBriefIssueInList(date: string): { number: number; url: string } | null { const listed = listIssues(); if (!isCommandSuccess(listed)) return null; const issues = commandData(listed).issues; if (!Array.isArray(issues)) return null; const title = dailyBriefTitle(date); const match = issues.map((item) => record(item)).find((issue) => issue.title === title); if (match === undefined) return null; return { number: Number(match.number), url: String(match.url) }; } function newDailyBriefBody(date: string): string { return [ `# ${dailyBriefTitle(date)}`, "", "## 当日更新", "", "## 常驻观察与长期建议", "", "- G14 HWLAB source truth 固定为 G14 `/root/hwlab` 与 `origin/G14`;DEV rollout 只接受 G14 Tekton/GitOps/Argo 和公网 health 证据。", "- GitHub issue/PR 写操作必须走 UniDesk `gh` 子命令;G14 k3s 操作必须走 UniDesk `ssh G14:k3s`。", ].join("\n"); } function insertBriefIndexRow(indexBody: string, date: string, brief: { number: number; url: string }): string { if (parseBriefIssueFromIndex(indexBody, date) !== null) return indexBody; const row = `| ${date} | [#${brief.number}](${brief.url}) | 已创建当日 G14 DEV rollout 自动简报入口;后续每次 DEV 上线记录 CI/CD 耗时和上线 changelog。 | 继续监控 HWLAB base=G14 PR,ready 后自动合并并滚动到 G14 DEV。 |`; const marker = "### 指挥简报索引"; const markerIndex = indexBody.indexOf(marker); if (markerIndex === -1) return `${indexBody.trimEnd()}\n\n${marker}\n\n| 日期(北京时间) | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n`; const afterMarker = indexBody.slice(markerIndex); const separator = "| --- | --- | --- | --- |"; const separatorInBody = afterMarker.indexOf(separator); if (separatorInBody === -1) return `${indexBody.slice(0, markerIndex + marker.length)}\n\n| 日期(北京时间) | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n${indexBody.slice(markerIndex + marker.length)}`; const insertAt = markerIndex + separatorInBody + separator.length; return `${indexBody.slice(0, insertAt)}\n${row}${indexBody.slice(insertAt)}`; } function ensureDailyBriefIssue(date: string, dryRun: boolean): Record { const indexRead = readIssue(G14_BRIEF_INDEX_ISSUE); if (!isCommandSuccess(indexRead)) return { ok: false, phase: "read-index", indexRead }; const indexIssue = issueFromRead(indexRead); const indexBody = String(indexIssue.body ?? ""); let brief = parseBriefIssueFromIndex(indexBody, date) ?? findBriefIssueInList(date); const actions: Record[] = []; if (brief === null) { const bodyPath = writeStateFile(`daily-brief-${date}.md`, `${newDailyBriefBody(date)}\n`); if (dryRun) { brief = { number: 0, url: `https://github.com/pikasTech/HWLAB/issues/new?title=${encodeURIComponent(dailyBriefTitle(date))}` }; actions.push({ action: "would-create-daily-brief", title: dailyBriefTitle(date), bodyPath }); } else { const created = cliJson(["gh", "issue", "create", "--repo", HWLAB_REPO, "--title", dailyBriefTitle(date), "--body-file", bodyPath], 80_000); if (!isCommandSuccess(created)) return { ok: false, phase: "create-daily-brief", created }; const createdIssue = record(commandData(created).issue); brief = { number: Number(createdIssue.number), url: String(createdIssue.url) }; actions.push({ action: "created-daily-brief", issue: brief, bodyPath }); } } if (brief.number > 0 && parseBriefIssueFromIndex(indexBody, date) === null) { const nextBody = insertBriefIndexRow(indexBody, date, brief); const bodyPath = writeStateFile(`issue-${G14_BRIEF_INDEX_ISSUE}-brief-index-${date}.md`, `${nextBody.trimEnd()}\n`); if (dryRun) { actions.push({ action: "would-update-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath }); } else { const bodySha = String(indexIssue.bodySha ?? ""); const update = cliJson(["gh", "issue", "update", String(G14_BRIEF_INDEX_ISSUE), "--repo", HWLAB_REPO, "--mode", "replace", "--body-file", bodyPath, ...(bodySha.length > 0 ? ["--expect-body-sha", bodySha] : [])], 100_000); if (!isCommandSuccess(update)) return { ok: false, phase: "update-brief-index", update, bodyPath }; actions.push({ action: "updated-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath }); } } return { ok: true, date, brief, actions }; } function readPullRequest(number: number): CommandJsonResult { return cliJson(["gh", "pr", "read", String(number), "--repo", HWLAB_REPO, "--json", "title,body,url,stateDetail,merged,mergedAt,mergeCommit,headRefName,baseRefName"], 80_000); } function readPullRequestFiles(number: number): CommandJsonResult { return cliJson(["gh", "pr", "files", String(number), "--repo", HWLAB_REPO, "--limit", "30"], 80_000); } function summarizePrFiles(filesResult: CommandJsonResult): Record { if (!isCommandSuccess(filesResult)) return { ok: false, filesResult }; const data = commandData(filesResult); const files = Array.isArray(data.files) ? data.files.map((item) => record(item)) : []; return { ok: true, summary: data.summary ?? null, keyFiles: files.slice(0, 8).map((file) => `${file.status ?? "changed"} ${file.filename ?? ""}`), }; } function mergeCommitFromPr(prData: Record): string | null { const mergeCommit = record(record(prData.json).mergeCommit); const fromJson = mergeCommit.oid; if (typeof fromJson === "string") return fromJson; const fromSummary = record(record(prData.pullRequest).mergeCommit).oid; return typeof fromSummary === "string" ? fromSummary : null; } function currentGitopsRevision(): string | null { const argo = getArgoStatus(); if (!isCommandSuccess(argo)) return null; return statusText(argo).split(/\r?\n/u)[0] ?? null; } export function rolloutRecordBody(input: { pr: OpenPullRequest; prData: Record; fileSummary: Record; sourceCommit: string; pipelineRun: string; gitopsRevision: string | null; mergedAt: string | null; pipelineSucceededAt: string | null; finishedAt: string | null; rollout: Record; ciMetrics?: CiPipelineMetrics | null; }): string { const now = beijingParts(); const title = String(record(input.prData.json).title ?? input.pr.title ?? `PR #${input.pr.number}`); const prBody = String(record(input.prData.json).body ?? record(input.prData.pullRequest).body ?? ""); const url = String(record(input.prData.json).url ?? input.pr.url ?? `https://github.com/pikasTech/HWLAB/pull/${input.pr.number}`); const summary = record(input.fileSummary.summary); const keyFiles = Array.isArray(input.fileSummary.keyFiles) ? input.fileSummary.keyFiles.map(String) : []; const semanticBullets = semanticChangelogBullets(title, prBody, input.fileSummary); const mergeToPipeline = durationSeconds(input.mergedAt, input.pipelineSucceededAt); const pipelineToDev = durationSeconds(input.pipelineSucceededAt, input.finishedAt); const mergeToDev = durationSeconds(input.mergedAt, input.finishedAt); const metrics = input.ciMetrics ?? ciMetricsFromValue(input.rollout.ciMetrics); return [ "", "", `## 更新 ${now.date} ${now.time.slice(0, 5)} 北京时间`, "", `### G14 DEV rollout:PR #${input.pr.number}`, "", `- PR: [#${input.pr.number} ${title}](${url})`, `- 合并 commit: \`${input.sourceCommit}\``, `- GitOps revision: \`${input.gitopsRevision ?? "unknown"}\``, `- PipelineRun: \`${input.pipelineRun}\``, "- CI/CD 耗时:", ` - merge -> pipeline succeeded: ${formatDuration(mergeToPipeline)}`, ` - pipeline succeeded -> DEV Healthy: ${formatDuration(pipelineToDev)}`, ` - merge -> DEV Healthy: ${formatDuration(mergeToDev)}`, ...ciMetricLines(metrics), "- 上线 changelog(语义化):", ...semanticBullets.map((bullet) => ` - ${bullet}`), "- 自动 diff 摘要:", ` - changed files: ${String(summary.files ?? "n/a")}; +${String(summary.additions ?? "n/a")} / -${String(summary.deletions ?? "n/a")}; commits: ${String(summary.commits ?? "n/a")}`, ...keyFiles.map((file) => ` - ${file}`), "- DEV 验证:", ` - Tekton: ${String(input.rollout.pipelineText ?? "Succeeded").split(/\r?\n/u).join(" / ")}`, ` - Argo: ${String(input.rollout.argoText ?? "Synced / Healthy").split(/\r?\n/u).slice(0, 5).join(" / ")}`, ` - health/live: ${input.rollout.healthOk === false ? "not-ok" : "ok"}`, " - workload readiness: Deployment/StatefulSet ready;历史 Completed smoke/debug pod 不作为 DEV rollout blocker。", ].join("\n"); } function appendRolloutBrief(options: G14RecordRolloutOptions, rollout: Record = {}): Record { const now = beijingParts(); const ensured = ensureDailyBriefIssue(now.date, options.dryRun); if (record(ensured).ok !== true) return ensured; const brief = record(record(ensured).brief); const prRead = readPullRequest(options.prNumber); if (!isCommandSuccess(prRead)) return { ok: false, phase: "read-pr", prRead }; const prData = commandData(prRead); const sourceCommit = options.sourceCommit ?? mergeCommitFromPr(prData); if (sourceCommit === null || sourceCommit === undefined) return { ok: false, phase: "source-commit", message: "source commit unavailable", prData }; const pipelineRun = options.pipelineRun ?? `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`; const files = summarizePrFiles(readPullRequestFiles(options.prNumber)); const rolloutGitopsRevision = typeof rollout.gitopsRevision === "string" && rollout.gitopsRevision.length > 0 ? rollout.gitopsRevision : null; const gitopsRevision = options.gitopsRevision ?? rolloutGitopsRevision ?? currentGitopsRevision(); const prJson = record(prData.json); const prSummary = record(prData.pullRequest); const prMergedAt = typeof prJson.mergedAt === "string" && prJson.mergedAt.length > 0 ? prJson.mergedAt : typeof prSummary.mergedAt === "string" && prSummary.mergedAt.length > 0 ? prSummary.mergedAt : null; const rolloutPipelineSucceededAt = typeof rollout.pipelineSucceededAt === "string" && rollout.pipelineSucceededAt.length > 0 ? rollout.pipelineSucceededAt : null; const rolloutFinishedAt = typeof rollout.finishedAt === "string" && rollout.finishedAt.length > 0 ? rollout.finishedAt : null; const mergedAt = options.mergedAt ?? prMergedAt; const pipelineSucceededAt = options.pipelineSucceededAt ?? rolloutPipelineSucceededAt; const finishedAt = options.finishedAt ?? rolloutFinishedAt; const ciMetrics = ciMetricsFromValue(rollout.ciMetrics) ?? collectPipelineMetrics(pipelineRun); const pr: OpenPullRequest = { number: options.prNumber, title: String(record(prData.json).title ?? ""), url: String(record(prData.json).url ?? ""), baseRefName: String(record(prData.json).baseRefName ?? ""), headRefName: String(record(prData.json).headRefName ?? ""), }; const body = rolloutRecordBody({ pr, prData, fileSummary: files, sourceCommit, pipelineRun, gitopsRevision, mergedAt, pipelineSucceededAt, finishedAt, rollout, ciMetrics }); if (Number(brief.number) <= 0 || options.dryRun) { return { ok: true, dryRun: true, date: now.date, brief, wouldAppend: { bodyPreview: body.slice(0, 1000), bodyChars: body.length }, ensured }; } const currentBrief = readIssue(Number(brief.number)); if (!isCommandSuccess(currentBrief)) return { ok: false, phase: "read-daily-brief", currentBrief, brief }; const currentBody = String(issueFromRead(currentBrief).body ?? ""); if (currentBody.includes(`PR #${options.prNumber}`) && currentBody.includes(sourceCommit)) { return { ok: true, skipped: true, reason: "rollout-brief-already-recorded", brief, sourceCommit, ensured }; } const bodyPath = writeStateFile(`rollout-pr-${options.prNumber}-${shortSha(sourceCommit)}.md`, `${body}\n`); const update = cliJson(["gh", "issue", "update", String(brief.number), "--repo", HWLAB_REPO, "--mode", "append", "--body-file", bodyPath, "--body-profile", "commander-brief"], 100_000); if (!isCommandSuccess(update)) return { ok: false, phase: "append-rollout-brief", update, bodyPath, brief, ensured }; return { ok: true, date: now.date, brief, sourceCommit, pipelineRun, gitopsRevision, ciMetrics, bodyPath, update: commandData(update), ensured }; } async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise> { const started = Date.now(); const startedAt = new Date(started).toISOString(); let pipelineText = ""; let argoText = ""; let healthOk = false; let workloadsReady: Record = { ready: false }; let pipelineSucceededAt: string | null = null; while (Date.now() - started < timeoutSeconds * 1000) { const pipeline = getPipelineStatus(sourceCommit); pipelineText = statusText(pipeline); printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) }); if (!pipelineText.startsWith("True\nSucceeded")) { if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText }; await sleep(30_000); continue; } pipelineSucceededAt ??= new Date().toISOString(); refreshArgoDev(); const argo = getArgoStatus(); argoText = statusText(argo); printEvent("g14.argo.status", { text: argoText.slice(0, 500) }); const argoLines = argoText.split(/\r?\n/u); const synced = argoLines[1] === "Synced"; const healthy = argoLines[2] === "Healthy"; const gitopsRevision = argoLines[0] ?? null; const workloads = getDevWorkloads(); const workloadsText = statusText(workloads); workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads)); const health = getLiveHealth(); healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout); if (synced && healthy && record(workloadsReady).ready === true && healthOk) { const finishedAt = new Date().toISOString(); const pipelineRun = `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`; const ciMetrics = collectPipelineMetrics(pipelineRun); return { ok: true, sourceCommit, gitopsRevision, pipelineRun, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk, ciMetrics }; } await sleep(30_000); } return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds }; } async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise> { printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun }); const precheck = precheckWorkspace(); if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck }; const listed = listOpenG14PullRequests(); if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed }; const prs = extractPullRequests(listed); printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs }); if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] }; const merged: unknown[] = []; for (const pr of prs) { const preflight = preflightPullRequest(pr.number); printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) }); if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) { return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight }; } const merge = mergePullRequest(pr.number, options.dryRun); printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) }); if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge }; const sourceCommit = getG14Head(); const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds); const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true ? { skipped: true, dryRun: options.dryRun } : appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout); printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null }); merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief }); if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged }; if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged }; } return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged }; } async function runMonitorWorker(options: G14MonitorOptions): Promise> { const maxCycles = options.once ? 1 : options.maxCycles; let cycle = 0; const results: unknown[] = []; while (maxCycles === 0 || cycle < maxCycles) { cycle += 1; const result = await monitorCycle(options, cycle); results.push(result); printEvent("g14.monitor.cycle.done", { cycle, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null }); if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results }; if (options.once || record(result).action !== "none") return { ok: true, cycles: cycle, lastResult: result, results }; printEvent("g14.monitor.sleep", { cycle, intervalSeconds: options.intervalSeconds }); await sleep(options.intervalSeconds * 1000); } return { ok: true, cycles: cycle, results }; } export function hwlabG14Help(): Record { return { command: "hwlab g14", output: "json", usage: [ "bun scripts/cli.ts hwlab g14 monitor-prs", "bun scripts/cli.ts hwlab g14 monitor-prs --once --dry-run", "bun scripts/cli.ts hwlab g14 record-rollout --pr [--source-commit sha]", "bun scripts/cli.ts hwlab g14 control-plane status --lane v02", "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm", "bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm", "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run", "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 git-mirror status", "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm", "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm", "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm", "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait", "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm --wait", "bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1", "bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm", "bun scripts/cli.ts job status --tail-bytes 30000", ], description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.", defaults: { repo: HWLAB_REPO, base: G14_SOURCE_BRANCH, provider: G14_PROVIDER, workspace: G14_WORKSPACE, v02Workspace: V02_WORKSPACE, ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO, intervalSeconds: DEFAULT_INTERVAL_SECONDS, devApplication: DEV_APP, v02Application: V02_APP, briefIndexIssue: G14_BRIEF_INDEX_ISSUE, }, stateFiles: { monitor: ".state/hwlab-g14/latest-monitor-job.json", once: ".state/hwlab-g14/latest-once-job.json", dryRun: ".state/hwlab-g14/latest-dry-run-job.json", onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json", }, }; } export async function runHwlabG14Command(_config: Config, args: string[]): Promise> { if (args.length === 0 || args.includes("--help") || args.includes("-h")) return { ok: true, ...hwlabG14Help() }; const [action] = args; if (action === "record-rollout") { const options = parseRecordRolloutOptions(args.slice(1)); return appendRolloutBrief(options); } if (action === "control-plane") { const options = parseControlPlaneOptions(args.slice(1)); if (options.action === "trigger-current" && options.confirm && !options.dryRun && !options.wait) { return startControlPlaneTriggerJob(options); } return runV02ControlPlane(options); } if (action === "tools-image") { const options = parseToolsImageOptions(args.slice(1)); return runG14ToolsImage(options); } if (action === "git-mirror") { const options = parseGitMirrorOptions(args.slice(1)); if ((options.action === "sync" || options.action === "flush") && options.confirm && !options.dryRun && !options.wait) { return startGitMirrorJob(options); } return runG14GitMirror(options); } if (action !== "monitor-prs") { return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 record-rollout, hwlab g14 control-plane, hwlab g14 git-mirror, hwlab g14 tools-image" }; } const options = parseOptions(args.slice(1)); if (options.worker) return runMonitorWorker(options); const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])]; const job = startJob("hwlab_g14_pr_monitor", command, `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`); const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`; const stateDir = rootPath(".state", "hwlab-g14"); mkdirSync(stateDir, { recursive: true }); const stateFileName = hwlabG14MonitorStateFileName(options); const stateFileRole = hwlabG14MonitorStateRole(options); const latestPath = join(stateDir, stateFileName); const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null; writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole }, null, 2)}\n`, "utf8"); return { ok: true, command: "hwlab g14 monitor-prs", mode: "async-job", job, statusCommand, latestPath, stateFileName, stateFileRole, previousLatest, next: { status: statusCommand, tail: `tail -f ${job.stdoutFile}`, }, }; }