// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. v02-status module for scripts/src/hwlab-g14.ts. // Moved mechanically from scripts/src/hwlab-g14.ts:1309-2787 for #903. import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand } from "../command"; import { cancelJob, listJobs, readJob, startJob } from "../jobs"; import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes"; import type { CommandJsonResult, ShellSection, V02ControlPlaneStatusTarget, V02StatusTargetMode } from "./types"; import { commandErrorSummary } from "./cleanup"; import { gitMirrorCacheProbeScript } from "./git-mirror"; import { conditionStatus } from "./observability"; import { durationSeconds, statusText } from "./pr-monitor"; import { fieldBoolean, g14K3s, isCommandSuccess, nested, record, redactedCommand, shellQuote, shortSha, stringOrNull } from "./remote"; import { v02PipelineRunName } from "./source"; import { ARGO_NAMESPACE, CI_NAMESPACE, GIT_MIRROR_NAMESPACE, V02_APP, V02_BUILD_TASKRUN_CRITICAL_SECONDS, V02_BUILD_TASKRUN_WARNING_SECONDS, V02_CICD_REPO, V02_CLOUD_API_URL, V02_CLOUD_WEB_URL, V02_PIPELINERUN_PREFIX, V02_POLLER, V02_RECONCILER, V02_WORKSPACE } from "./types"; export interface V02TriggerSnapshot { sourceCommit: string | null; pipelineRun: string | null; before: Record | null; result: CommandJsonResult; observedAtMs: number; } export function parseV02TriggerSnapshot(result: CommandJsonResult, observedAtMs: number): V02TriggerSnapshot { const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout); const fields = keyValueLinesFromText(output); const sourceCommit = stringOrNull(fields.sourceCommit?.match(/^[0-9a-f]{40}$/iu)?.[0]); const pipelineRun = stringOrNull(fields.pipelineRun); const pipelineRunExitCode = numericField(fields.pipelineRunExitCode); const before = pipelineRun !== null && pipelineRunExitCode !== null ? pipelineRunCompactFromText( pipelineRun, [fields.status ?? "", fields.reason ?? "", fields.message ?? ""].join("\n"), pipelineRunExitCode === 0, "hwlab-v02-trigger-snapshot:pipelinerun", pipelineRunExitCode, fields.pipelineRunStderr ?? "", ) : null; return { sourceCommit, pipelineRun, before, result, observedAtMs, }; } export function getV02TriggerSnapshot(): V02TriggerSnapshot { const script = [ "set +e", "one_line() { tr '\\r\\n\\t' ' ' | sed 's/[[:space:]][[:space:]]*/ /g' | cut -c1-2000; }", "tmp_dir=$(mktemp -d /tmp/hwlab-v02-trigger-snapshot.XXXXXX)", "trap 'rm -rf \"$tmp_dir\"' EXIT", `cicd_repo=${shellQuote(V02_CICD_REPO)}`, "source_commit=", "if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", " source_commit=$(git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)", "elif [ -e \"$cicd_repo\" ]; then", " printf 'snapshotError\\t%s\\n' \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\"", "fi", "pipeline_run=", `if [ -n "$source_commit" ]; then pipeline_run=${shellQuote(V02_PIPELINERUN_PREFIX)}-$(printf '%s' "$source_commit" | cut -c1-12); fi`, "if [ -n \"$pipeline_run\" ]; then", ` (kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}' >\"$tmp_dir/pipelinerun.out\" 2>\"$tmp_dir/pipelinerun.err\"; printf '%s' "$?" >\"$tmp_dir/pipelinerun.status\") &`, " pipeline_pid=$!", "else", " printf '' >\"$tmp_dir/pipelinerun.out\"", " printf '' >\"$tmp_dir/pipelinerun.err\"", " printf '' >\"$tmp_dir/pipelinerun.status\"", " pipeline_pid=", "fi", "if [ -n \"$pipeline_pid\" ]; then wait \"$pipeline_pid\" 2>/dev/null || true; pipeline_pid=; fi", "status_text=$(cat \"$tmp_dir/pipelinerun.out\" 2>/dev/null)", "pipeline_status=$(cat \"$tmp_dir/pipelinerun.status\" 2>/dev/null)", "printf 'sourceCommit\\t%s\\n' \"$source_commit\"", "printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"", "printf 'pipelineRunExitCode\\t%s\\n' \"$pipeline_status\"", "printf 'status\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '1p')\"", "printf 'reason\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '2p')\"", "printf 'message\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '3p')\"", "printf 'pipelineRunStderr\\t%s\\n' \"$(one_line < \"$tmp_dir/pipelinerun.err\")\"", ].join("\n"); const result = g14K3s(["sh", "--", script], 180_000); return parseV02TriggerSnapshot(result, Date.now()); } export function v02ExistingPipelineRunReuseDecision(input: { sourceCommit: string; before: Record; latestSourceCommit: string | null; }): Record { const status = stringOrNull(input.before.status); const exists = input.before.exists === true; const latestSourceCommit = input.latestSourceCommit; const alreadyUsable = exists && (status === "True" || status === "Unknown"); if (latestSourceCommit === null) { return { reusable: false, alreadyUsable, reason: "source-head-recheck-unresolved-before-existing-pipelinerun-reuse", sourceCommit: input.sourceCommit, latestSourceCommit, previousPipelineRun: v02PipelineRunName(input.sourceCommit), }; } if (latestSourceCommit !== null && latestSourceCommit !== input.sourceCommit) { return { reusable: false, alreadyUsable, reason: "source-head-advanced-before-existing-pipelinerun-reuse", sourceCommit: input.sourceCommit, latestSourceCommit, previousPipelineRun: v02PipelineRunName(input.sourceCommit), latestPipelineRun: v02PipelineRunName(latestSourceCommit), }; } return { reusable: exists && status !== null, alreadyUsable, reason: alreadyUsable ? "existing-pipelinerun-reused" : "existing-pipelinerun-terminal-failed", sourceCommit: input.sourceCommit, latestSourceCommit, previousPipelineRun: v02PipelineRunName(input.sourceCommit), }; } export function getPipelineRunCompact(name: string): Record { const result = g14K3s([ "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, name, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}", ], 60_000); return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr); } export function pipelineRunCompactFromText( name: string, text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, ): Record { const [status = "", reason = "", message = ""] = text.split(/\r?\n/u); const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`); return { ok: commandOk, exists: commandOk || !notFound, pipelineRun: name, status: status || null, reason: reason || null, message: message || null, command: Array.isArray(command) ? redactedCommand(command) : command, exitCode, stderr: stderr.trim().slice(0, 2000), }; } export function runtimeLaneTaskRunDiagnosticsScript(): string { return [ "set +e", "node <<'NODE'", "const cp = require('node:child_process');", "const namespace = process.env.CI_NAMESPACE || 'hwlab-ci';", "const pipelineRun = process.env.PIPELINE_RUN || '';", "const tailLines = Math.max(20, Math.min(Number(process.env.TASKRUN_LOG_TAIL_LINES || 60), 200));", "const maxLogChars = Math.max(1000, Math.min(Number(process.env.TASKRUN_LOG_MAX_CHARS || 4000), 20000));", "function kubectl(args, timeoutMs, maxBuffer) {", " const out = cp.spawnSync('kubectl', args, { encoding: 'utf8', timeout: timeoutMs, maxBuffer });", " const err = out.error;", " return { exitCode: out.status, stdout: out.stdout || '', stderr: out.stderr || (err ? String(err.message || err) : ''), timedOut: Boolean(err && err.code === 'ETIMEDOUT') };", "}", "function parseJson(text) { try { return JSON.parse(text); } catch { return null; } }", "function oneLine(value, limit) { return String(value || '').replace(/[\\r\\n\\t]+/g, ' ').replace(/\\s+/g, ' ').trim().slice(0, limit); }", "function timestampMs(value) { const parsed = Date.parse(String(value || '')); return Number.isFinite(parsed) ? parsed : null; }", "function durationSeconds(start, end) { const s = timestampMs(start); const e = timestampMs(end); return s === null || e === null || e < s ? null : Math.round((e - s) / 1000); }", "function condition(item) { const conditions = Array.isArray(item && item.status && item.status.conditions) ? item.status.conditions : []; return conditions.find((entry) => entry && entry.type === 'Succeeded') || conditions[0] || {}; }", "function redact(text) {", " return String(text || '')", " .replace(/Bearer\\s+[A-Za-z0-9._~+\\/-]+=*/giu, 'Bearer ')", " .replace(/((?:api[_-]?key|authorization|token|password|secret|database_url|dsn)\\s*[:=]\\s*)\\S+/giu, '$1');", "}", "function trimTail(text) {", " const redacted = redact(text);", " let tail = redacted.split(/\\r?\\n/).slice(-tailLines).join('\\n').trimEnd();", " if (tail.length > maxLogChars) {", " tail = tail.slice(tail.length - maxLogChars);", " const lineBreak = tail.indexOf('\\n');", " if (lineBreak >= 0) tail = tail.slice(lineBreak + 1);", " }", " return tail;", "}", "function stateSummary(state) {", " if (!state || typeof state !== 'object') return { state: null, reason: null, exitCode: null };", " const key = Object.keys(state)[0] || null;", " const value = key ? state[key] || {} : {};", " return { state: key, reason: value.reason || null, exitCode: typeof value.exitCode === 'number' ? value.exitCode : null };", "}", "function podContainerStates(pod) {", " const status = (pod && pod.status) || {};", " const containers = [...(status.initContainerStatuses || []), ...(status.containerStatuses || [])];", " return containers.map((item) => {", " const summary = stateSummary(item.state);", " return { name: item.name || null, ready: item.ready === true, restartCount: item.restartCount || 0, state: summary.state, reason: summary.reason, exitCode: summary.exitCode };", " }).filter((item) => item.state !== 'running' || item.restartCount > 0).slice(0, 12);", "}", "function findPodName(taskRunName) {", " const out = kubectl(['-n', namespace, 'get', 'pod', '-l', 'tekton.dev/taskRun=' + taskRunName, '-o', 'jsonpath={.items[0].metadata.name}'], 10000, 65536);", " return out.exitCode === 0 && out.stdout.trim() ? out.stdout.trim() : null;", "}", "function collectLogTail(taskRun) {", " const podName = taskRun.podName || findPodName(taskRun.name);", " if (!podName) return { taskRun: taskRun.name, pipelineTask: taskRun.pipelineTask, podName: null, ok: false, reason: 'pod-not-found', logTail: '' };", " const podResult = kubectl(['-n', namespace, 'get', 'pod', podName, '-o', 'json'], 10000, 262144);", " const pod = parseJson(podResult.stdout);", " const logResult = kubectl(['-n', namespace, 'logs', podName, '--all-containers=true', '--tail=' + String(tailLines), '--prefix=true'], 20000, Math.max(262144, maxLogChars * 8));", " const raw = logResult.stdout || logResult.stderr;", " return {", " taskRun: taskRun.name,", " pipelineTask: taskRun.pipelineTask,", " podName,", " ok: logResult.exitCode === 0,", " exitCode: logResult.exitCode,", " timedOut: logResult.timedOut,", " containers: podContainerStates(pod),", " logTail: trimTail(raw),", " };", "}", "if (!pipelineRun) {", " console.log(JSON.stringify({ ok: true, pipelineRun: null, namespace, skipped: true, reason: 'pipeline-run-empty', total: 0, items: [], failedTaskRuns: [], activeTaskRuns: [], logTails: [], valuesPrinted: false }));", " process.exit(0);", "}", "const taskRunResult = kubectl(['-n', namespace, 'get', 'taskrun', '-l', 'tekton.dev/pipelineRun=' + pipelineRun, '-o', 'json'], 25000, 2 * 1024 * 1024);", "const taskData = parseJson(taskRunResult.stdout);", "const rawItems = Array.isArray(taskData && taskData.items) ? taskData.items : [];", "const items = rawItems.map((item) => {", " const metadata = item.metadata || {};", " const labels = metadata.labels || {};", " const status = item.status || {};", " const c = condition(item);", " return {", " name: metadata.name || null,", " pipelineTask: labels['tekton.dev/pipelineTask'] || null,", " status: c.status || null,", " reason: c.reason || null,", " message: oneLine(c.message, 360) || null,", " startTime: status.startTime || null,", " completionTime: status.completionTime || null,", " durationSeconds: durationSeconds(status.startTime, status.completionTime),", " podName: status.podName || null,", " };", "}).filter((item) => item.name).sort((left, right) => (timestampMs(left.startTime) || 0) - (timestampMs(right.startTime) || 0) || String(left.name).localeCompare(String(right.name)));", "const failedTaskRuns = items.filter((item) => item.status === 'False' || /fail|error|timeout/i.test(String(item.reason || '')));", "const activeTaskRuns = items.filter((item) => item.status !== 'True' && item.status !== 'False');", "const succeededCount = items.filter((item) => item.status === 'True').length;", "const logTails = failedTaskRuns.slice(0, 3).map(collectLogTail);", "console.log(JSON.stringify({", " ok: taskRunResult.exitCode === 0,", " pipelineRun,", " namespace,", " total: items.length,", " succeededCount,", " failedCount: failedTaskRuns.length,", " activeCount: activeTaskRuns.length,", " failedTaskRuns: failedTaskRuns.slice(0, 8),", " activeTaskRuns: activeTaskRuns.slice(0, 8),", " recentTaskRuns: items.slice(-8),", " logTails,", " query: { taskRunsExitCode: taskRunResult.exitCode, taskRunsTimedOut: taskRunResult.timedOut, taskRunsStderr: oneLine(taskRunResult.stderr, 500) || null },", " bounded: { maxFailedTaskRuns: 8, maxLogTaskRuns: 3, tailLines, maxLogChars, redacted: true },", " valuesPrinted: false,", "}));", "NODE", ].join("\n"); } export function runtimeLaneTaskRunDiagnosticsFromText( pipelineRun: string, section: ShellSection | undefined, stderr: string, ): Record { const raw = String(section?.stdout ?? "").trim(); if (raw.length === 0) { return { ok: shellSectionOk(section), pipelineRun, total: 0, failedTaskRuns: [], activeTaskRuns: [], logTails: [], sectionExitCode: section?.exitCode ?? null, stderr: stderr.trim().slice(0, 1000), }; } try { const data = record(JSON.parse(raw) as unknown); return { ...data, pipelineRun: stringOrNull(data.pipelineRun) ?? pipelineRun, sectionExitCode: section?.exitCode ?? null, }; } catch { return { ok: false, pipelineRun, sectionExitCode: section?.exitCode ?? null, raw: raw.slice(0, 2000), stderr: stderr.trim().slice(0, 1000), degradedReason: "taskrun-diagnostics-json-parse-failed", }; } } export function timestampMs(value: unknown): number | null { if (typeof value !== "string" || value.trim().length === 0) return null; const parsed = Date.parse(value); return Number.isFinite(parsed) ? parsed : null; } export function secondsBetween(start: unknown, end: unknown): number | null { const startMs = timestampMs(start); const endMs = timestampMs(end); if (startMs === null || endMs === null || endMs < startMs) return null; return Math.round((endMs - startMs) / 1000); } export function pipelineRunStatusRowFromLine(line: string, nowMs: number, pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record | null { const [ name = "", sourceCommit = "", createdAt = "", startTime = "", completionTime = "", status = "", reason = "", ] = line.split("\t"); if (!name.startsWith(pipelineRunPrefix)) return null; const startMs = timestampMs(startTime); const completion = completionTime.trim().length > 0 ? completionTime : null; const conditionStatus = status.trim().length > 0 ? status : null; return { name, sourceCommit: sourceCommit || null, status: conditionStatus, reason: reason || null, createdAt: createdAt || null, startTime: startTime || null, completionTime: completion, durationSeconds: secondsBetween(startTime, completion), elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)), active: completion === null && conditionStatus !== "True" && conditionStatus !== "False", }; } export function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now(), pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record { const rows = text .split(/\r?\n/u) .map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs, pipelineRunPrefix)) .filter((item): item is Record => item !== null) .sort((left, right) => { return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0); }); const activeItems = rows.filter((item) => item.active === true); return { ok: true, count: rows.length, activeCount: activeItems.length, items: rows.slice(0, limit), activeItems: activeItems.slice(0, limit), disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete", }; } export function pipelineRunRowsJsonPath(): string { return [ "jsonpath={range .items[*]}", "{.metadata.name}", "{\"\\t\"}", "{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}", "{\"\\t\"}", "{.metadata.creationTimestamp}", "{\"\\t\"}", "{.status.startTime}", "{\"\\t\"}", "{.status.completionTime}", "{\"\\t\"}", "{.status.conditions[0].status}", "{\"\\t\"}", "{.status.conditions[0].reason}", "{\"\\n\"}", "{end}", ].join(""); } export function listV02PipelineRunsCompact(limit = 8): Record { const result = g14K3s([ "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, "-l", "hwlab.pikastech.local/gitops-target=v02", "-o", pipelineRunRowsJsonPath(), ], 60_000); if (!isCommandSuccess(result)) { return { ok: false, command: redactedCommand(result.command), exitCode: result.exitCode, stderr: commandErrorSummary(result), items: [], activeItems: [], }; } return parsePipelineRunRows(statusText(result), limit); } export function parseShellSections(output: string): Record { const sections: Record = {}; let currentName: string | null = null; let currentLines: string[] = []; let currentExitCode: number | null = null; const flush = (): void => { if (currentName === null) return; sections[currentName] = { stdout: currentLines.join("\n").trim(), exitCode: currentExitCode, }; }; for (const line of output.split(/\r?\n/u)) { const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line); if (begin !== null) { flush(); currentName = begin[1] ?? null; currentLines = []; currentExitCode = null; continue; } const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line); if (end !== null && currentName === end[1]) { currentExitCode = Number(end[2]); flush(); currentName = null; currentLines = []; currentExitCode = null; continue; } if (currentName !== null) currentLines.push(line); } flush(); return sections; } export function shellSectionOk(section: ShellSection | undefined): boolean { return section?.exitCode === 0; } export function v02ControlPlaneStatusBundle(target: V02ControlPlaneStatusTarget = {}): CommandJsonResult { const targetMode: V02StatusTargetMode = target.mode ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null ? [ `pipeline_run=${shellQuote(target.pipelineRun)}`, `source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`, `if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | cut -d- -f5-); fi`, ].join("\n") : [ target.sourceCommit === undefined ? `source_commit=$(git --git-dir=${shellQuote(V02_CICD_REPO)} rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)` : `source_commit=${shellQuote(target.sourceCommit ?? "")}`, "pipeline_run=", ].join("\n"); const script = [ "set +e", targetInit, `target_mode=${shellQuote(targetMode)}`, `if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${V02_PIPELINERUN_PREFIX}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`, "section() {", " name=\"$1\"", " shift", " printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"", " \"$@\"", " code=$?", " printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"", "}", "section statusTarget printf 'mode\\t%s\\nsourceCommit\\t%s\\npipelineRun\\t%s\\n' \"$target_mode\" \"$source_commit\" \"$pipeline_run\"", "section sourceCommit printf '%s\\n' \"$source_commit\"", "section pipelineRunName printf '%s\\n' \"$pipeline_run\"", `section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())} sh "$source_commit"`, "section queryNow date -u +%Y-%m-%dT%H:%M:%SZ", `section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`, `section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`, `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`, `if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`, `if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`, `if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`, `section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`, `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`, `section gitMirrorCache kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(gitMirrorCacheProbeScript())}`, `section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`, ].join("\n"); return g14K3s(["sh", "--", script], 60_000); } export function v02SourceHeadsProbeScript(): string { return [ "set +e", "target_source_commit=${1:-}", `cicd_repo=${shellQuote(V02_CICD_REPO)}`, `workspace=${shellQuote(V02_WORKSPACE)}`, "rev_cicd() { git --git-dir=\"$cicd_repo\" rev-parse \"$1\" 2>/dev/null || true; }", "rev_workspace() { git -C \"$workspace\" rev-parse \"$1\" 2>/dev/null || true; }", "origin_head=$(rev_cicd refs/remotes/origin/v0.2)", "printf 'cicdRepo\\t%s\\n' \"$cicd_repo\"", "printf 'cicdRepoExists\\t%s\\n' \"$([ -d \"$cicd_repo/objects\" ] && printf yes || printf no)\"", "printf 'cicdSourceHead\\t%s\\n' \"$origin_head\"", "printf 'originHead\\t%s\\n' \"$origin_head\"", "printf 'targetSourceCommit\\t%s\\n' \"$target_source_commit\"", "target_object_exists=", "target_ancestor=", "target_ancestor_exit=", "if [ -n \"$target_source_commit\" ] && git --git-dir=\"$cicd_repo\" cat-file -e \"$target_source_commit^{commit}\" 2>/dev/null; then", " target_object_exists=true", " if [ -n \"$origin_head\" ]; then", " git --git-dir=\"$cicd_repo\" merge-base --is-ancestor \"$target_source_commit\" \"$origin_head\" >/dev/null 2>&1", " target_ancestor_exit=$?", " if [ \"$target_ancestor_exit\" = 0 ]; then target_ancestor=true; elif [ \"$target_ancestor_exit\" = 1 ]; then target_ancestor=false; else target_ancestor=unknown; fi", " fi", "elif [ -n \"$target_source_commit\" ]; then", " target_object_exists=false", "fi", "printf 'targetObjectExists\\t%s\\n' \"$target_object_exists\"", "printf 'targetAncestorOfOriginHead\\t%s\\n' \"$target_ancestor\"", "printf 'targetAncestorExitCode\\t%s\\n' \"$target_ancestor_exit\"", "printf 'workspacePath\\t%s\\n' \"$workspace\"", "printf 'workspaceBranch\\t%s\\n' \"$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)\"", "printf 'workspaceHead\\t%s\\n' \"$(rev_workspace HEAD)\"", "printf 'workspaceOriginHead\\t%s\\n' \"$(rev_workspace refs/remotes/origin/v0.2)\"", "printf 'workspaceDirtyCount\\t%s\\n' \"$(git -C \"$workspace\" status --porcelain=v1 2>/dev/null | wc -l | tr -d ' ')\"", ].join("\n"); } export function v02PlanArtifactsLogScript(): string { return [ "pipeline_run=\"$1\"", `namespace=${shellQuote(CI_NAMESPACE)}`, "[ -n \"$pipeline_run\" ] || exit 0", "pods=$(kubectl -n \"$namespace\" get pods -l \"tekton.dev/pipelineRun=$pipeline_run,tekton.dev/pipelineTask=plan-artifacts\" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)", "for pod in $pods; do", " printf '__POD__\\t%s\\n' \"$pod\"", " kubectl -n \"$namespace\" logs \"$pod\" --all-containers --tail=200 2>/dev/null || true", "done", ].join("\n"); } export function v02RuntimeWorkloadsColumns(): string { return [ "custom-columns=KIND:.kind", "NAME:.metadata.name", "SERVICE:.metadata.labels.hwlab\\.pikastech\\.local/service-id", "ARTIFACT:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/artifact-source-commit", "SOURCE:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/source-commit", "IMAGE:.spec.template.spec.containers[0].image", "READY:.status.readyReplicas", "REPLICAS:.status.replicas", ].join(","); } export function v02WebAssetsProbeScript(): string { return [ "set +e", `base=${shellQuote(V02_CLOUD_WEB_URL)}`, `api=${shellQuote(V02_CLOUD_API_URL)}`, "fetch_url() {", " if command -v curl >/dev/null 2>&1; then", " curl -fsS --connect-timeout 2 --max-time 10 \"$1\"", " elif command -v wget >/dev/null 2>&1; then", " wget -q -T 10 -O - \"$1\"", " else", " return 127", " fi", "}", "printf 'baseUrl\\t%s\\n' \"$base\"", "printf 'apiUrl\\t%s\\n' \"$api\"", "probe_start_s=$(date +%s 2>/dev/null || printf '0')", "html=$(fetch_url \"$base/\" 2>/dev/null)", "html_code=$?", "printf 'htmlOk\\t%s\\n' \"$html_code\"", "css_path=$(printf '%s' \"$html\" | sed -n 's/.*href=\"\\([^\"]*\\.css\\)\".*/\\1/p' | head -1)", "printf 'cssPath\\t%s\\n' \"$css_path\"", "case \"$css_path\" in", " http://*|https://*) css_url=\"$css_path\" ;;", " /*) css_url=\"$base$css_path\" ;;", " \"\") css_url=\"\" ;;", " *) css_url=\"$base/$css_path\" ;;", "esac", "printf 'htmlAppScript\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eq 'src=\"/app\\.js\"|src=\"app\\.js\"'; printf '%s' \"$?\")\"", "printf 'htmlCssLink\\t%s\\n' \"$(test -n \"$css_path\"; printf '%s' \"$?\")\"", "printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"", "if [ -n \"$css_url\" ]; then css=$(fetch_url \"$css_url\" 2>/dev/null); else css=\"\"; false; fi", "css_code=$?", "printf 'cssOk\\t%s\\n' \"$css_code\"", "printf 'cssBytes\\t%s\\n' \"$(printf '%s' \"$css\" | wc -c | tr -d ' ')\"", "app_js=$(fetch_url \"$base/app.js\" 2>/dev/null)", "app_js_code=$?", "probe_end_s=$(date +%s 2>/dev/null || printf '0')", "printf 'appJsOk\\t%s\\n' \"$app_js_code\"", "printf 'appJsBytes\\t%s\\n' \"$(printf '%s' \"$app_js\" | wc -c | tr -d ' ')\"", "printf 'probeElapsedMs\\t%s\\n' \"$(((probe_end_s - probe_start_s) * 1000))\"", "fetch_url \"$base/styles.css\" >/dev/null 2>&1", "legacy_styles_code=$?", "if [ \"$legacy_styles_code\" = \"0\" ]; then legacy_styles_absent=1; else legacy_styles_absent=0; fi", "printf 'legacyStylesAbsent\\t%s\\n' \"$legacy_styles_absent\"", "printf 'reactWorkbenchCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'workbench-shell|activity-rail|session-sidebar|conversation-panel'; printf '%s' \"$?\")\"", "health=$(fetch_url \"$api/health/live\" 2>/dev/null)", "health_code=$?", "printf 'apiHealthOk\\t%s\\n' \"$health_code\"", "printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"", ].join("\n"); } export function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record { if (!commandOk) { return { ok: false, pipelineRun, exitCode, stderr: stderr.trim().slice(0, 2000), counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 }, items: [], performance: v02TaskRunPerformanceSummary([]), }; } const items = text .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t"); return { name, status: status || null, reason: reason || null, startTime: startTime || null, completionTime: completionTime || null, durationSeconds: secondsBetween(startTime, completionTime), }; }); const counts = { succeeded: items.filter((item) => item.status === "True").length, failed: items.filter((item) => item.status === "False").length, running: items.filter((item) => item.status === "Unknown").length, unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length, }; const performance = v02TaskRunPerformanceSummary(items); const performanceWarning = performance.ok === false ? `; ${String(performance.summary ?? "")}` : ""; return { ok: true, pipelineRun, counts, items, performance, summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}${performanceWarning}`, disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet", }; } export function v02TaskRunPipelineTaskName(name: string): string | null { const buildIndex = name.lastIndexOf("-build-"); if (buildIndex >= 0) return name.slice(buildIndex + 1); const knownSuffixes = [ "prepare-source", "plan-artifacts", "publish-artifact-catalog", "gitops-render", "gitops-promote", "runtime-ready", "collect-artifacts", ]; return knownSuffixes.find((suffix) => name.endsWith(`-${suffix}`)) ?? null; } export function v02TaskRunPerformanceSummary(taskRuns: unknown[]): Record { const slowTaskRuns: Record[] = []; for (const itemRaw of taskRuns) { const item = record(itemRaw); const name = stringOrNull(item.name) ?? ""; const pipelineTask = stringOrNull(item.pipelineTask) ?? v02TaskRunPipelineTaskName(name); const durationSeconds = typeof item.durationSeconds === "number" ? item.durationSeconds : null; if (!pipelineTask?.startsWith("build-") || durationSeconds === null || durationSeconds <= V02_BUILD_TASKRUN_WARNING_SECONDS) continue; const serviceId = pipelineTask.slice("build-".length) || null; slowTaskRuns.push({ name, pipelineTask, serviceId, status: stringOrNull(item.status), reason: stringOrNull(item.reason), durationSeconds, budgetSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS, severity: durationSeconds >= V02_BUILD_TASKRUN_CRITICAL_SECONDS ? "critical" : "warning", message: `${pipelineTask} took ${durationSeconds}s, above v0.2 build TaskRun warning budget ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`, }); } slowTaskRuns.sort((left, right) => Number(right.durationSeconds ?? 0) - Number(left.durationSeconds ?? 0)); const worst = slowTaskRuns[0]; return { ok: slowTaskRuns.length === 0, warningCount: slowTaskRuns.length, thresholds: { buildTaskRunWarningSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS, buildTaskRunCriticalSeconds: V02_BUILD_TASKRUN_CRITICAL_SECONDS, }, slowTaskRuns, summary: worst ? `slow build taskruns=${slowTaskRuns.length}; worst=${String(worst.pipelineTask)} ${String(worst.durationSeconds)}s budget=${V02_BUILD_TASKRUN_WARNING_SECONDS}s` : `no build taskrun over ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`, }; } export function v02WebAssetsFromText( text: string, commandOk: boolean, sourceCommit: string | null, argoSyncRevision: string | null, exitCode: number | null, stderr: string, activePipelineRuns: unknown[] = [], ): Record { const fields: Record = {}; for (const line of text.split(/\r?\n/u)) { const [key = "", ...rest] = line.split("\t"); if (key.length > 0) fields[key] = rest.join("\t"); } const htmlOk = fields.htmlOk === "0"; const htmlAppScript = fields.htmlAppScript === "0"; const htmlCssLink = fields.htmlCssLink === "0"; const cssOk = fields.cssOk === "0"; const appJsOk = fields.appJsOk === "0"; const apiHealthOk = fields.apiHealthOk === "0"; const readonlyNoteAbsent = fields.readonlyNote === "1"; const legacyStylesAbsent = fields.legacyStylesAbsent === "0"; const reactWorkbenchCss = fields.reactWorkbenchCss === "0"; const apiRevision = fields.apiRevision || null; const webChecksPass = htmlOk && htmlAppScript && htmlCssLink && cssOk && appJsOk && readonlyNoteAbsent && legacyStylesAbsent && reactWorkbenchCss && apiHealthOk; const failedChecks = Object.entries({ htmlOk, htmlAppScript, htmlCssLink, cssOk, appJsOk, readonlyNoteAbsent, legacyStylesAbsent, reactWorkbenchCss, apiHealthOk, }).filter(([, ok]) => !ok).map(([name]) => name); return { ok: commandOk && webChecksPass, summary: commandOk && webChecksPass ? "19666/19667 React/Vite asset probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`, baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL, apiUrl: fields.apiUrl || V02_CLOUD_API_URL, sourceCommit, argoSyncRevision: argoSyncRevision || null, cssPath: fields.cssPath || null, checks: { htmlOk, htmlAppScript, htmlCssLink, cssOk, appJsOk, readonlyNoteAbsent, legacyStylesAbsent, reactWorkbenchCss, apiHealthOk, }, probeExitCodes: { html: numericField(fields.htmlOk), htmlAppScript: numericField(fields.htmlAppScript), htmlCssLink: numericField(fields.htmlCssLink), css: numericField(fields.cssOk), appJs: numericField(fields.appJsOk), readonlyNoteGrep: numericField(fields.readonlyNote), legacyStylesAbsent, reactWorkbenchCssGrep: numericField(fields.reactWorkbenchCss), apiHealth: numericField(fields.apiHealthOk), }, assetBytes: { css: numericField(fields.cssBytes), appJs: numericField(fields.appJsBytes), }, probeElapsedMs: numericField(fields.probeElapsedMs), apiRevision, note: webAssetsRevisionNote(apiRevision, sourceCommit, activePipelineRuns), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), }; } export function v02PlanArtifactsFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record { const events: Record[] = []; const pods: string[] = []; for (const rawLine of text.split(/\r?\n/u)) { const line = rawLine.trim(); if (line.startsWith("__POD__")) { const [, pod = ""] = line.split("\t"); if (pod.length > 0) pods.push(pod); continue; } if (!line.startsWith("{")) continue; try { const event = record(JSON.parse(line) as unknown); if (event.event === "g14-ci-plan") events.push(event); } catch { // Ignore non-JSON log lines; the raw log remains visible through pod logs. } } const latest = events.at(-1) ?? null; if (!commandOk || latest === null) { return { ok: false, pipelineRun, pods, eventFound: latest !== null, degradedReason: commandOk ? "g14-ci-plan-event-not-found" : "plan-artifacts-log-query-failed", exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), }; } const audit = record(latest.artifactProvenanceAudit); const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : []; const provenanceRebuildServices = Array.isArray(audit.provenanceRebuildServices) ? audit.provenanceRebuildServices : []; return { ok: true, pipelineRun, pods, sourceCommitId: stringOrNull(latest.sourceCommitId), affectedServices: stringArray(latest.affectedServices), rolloutServices: stringArray(latest.rolloutServices), buildServices: stringArray(latest.buildServices), reusedServices: stringArray(latest.reusedServices), buildSkippedCount: numericValue(latest.buildSkippedCount), artifactProvenanceAudit: Object.keys(audit).length > 0 ? audit : null, summary: `build=${stringArray(latest.buildServices).length} reuse=${stringArray(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`, disclosure: "parsed from plan-artifacts g14-ci-plan log event", }; } export function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record { if (!commandOk) { return { ok: false, items: [], exitCode, stderr: stderr.trim().slice(0, 2000), }; } const items = text .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const [kind = "", name = "", serviceId = "", artifactSourceCommit = "", sourceCommit = "", image = "", ready = "", replicas = ""] = line.split(/\s+/u); return { kind, name, serviceId: serviceId === "" ? null : serviceId, artifactSourceCommit: artifactSourceCommit === "" ? null : artifactSourceCommit, sourceCommit: sourceCommit === "" ? null : sourceCommit, image: image === "" ? null : image, readyReplicas: numericField(ready), replicas: numericField(replicas), }; }); return { ok: true, items, summary: `runtime deployments=${items.length}`, }; } export function keyValueLinesFromText(text: string): Record { const fields: Record = {}; for (const line of text.split(/\r?\n/u)) { const [key = "", ...rest] = line.split("\t"); if (key.trim().length > 0) fields[key.trim()] = rest.join("\t").trim(); } return fields; } export function v02SourceHeadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record { const fields = keyValueLinesFromText(text); const dirtyCount = numericField(fields.workspaceDirtyCount); const targetSourceCommit = fields.targetSourceCommit || null; return { ok: commandOk, cicdRepo: fields.cicdRepo || V02_CICD_REPO, cicdRepoExists: fields.cicdRepoExists === "yes", cicdSourceHead: fields.cicdSourceHead || null, originHead: fields.originHead || null, target: { sourceCommit: targetSourceCommit, objectExists: fieldBoolean(fields.targetObjectExists), ancestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead), ancestorExitCode: numericField(fields.targetAncestorExitCode), }, targetObjectExists: fieldBoolean(fields.targetObjectExists), targetAncestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead), workspace: { path: fields.workspacePath || V02_WORKSPACE, branch: fields.workspaceBranch || null, head: fields.workspaceHead || null, originHead: fields.workspaceOriginHead || null, dirtyCount, dirty: typeof dirtyCount === "number" ? dirtyCount > 0 : null, isolatedFromCicd: true, }, exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), }; } export function v02FalseGreenGuard(input: { sourceCommit: string | null; pipelineRun: Record | null; taskRuns: Record; planArtifacts: Record; runtimeWorkloads: Record; }): Record { const status = stringOrNull(input.pipelineRun?.status); if (input.sourceCommit === null || input.pipelineRun === null || input.pipelineRun.exists === false) { return { ok: null, state: "not-started", summary: "current source commit has no PipelineRun yet" }; } if (status !== "True") { return { ok: null, state: "pipeline-not-final", summary: `PipelineRun status=${status ?? "unknown"}` }; } if (input.planArtifacts.ok !== true) { return { ok: false, state: "missing-plan-artifacts-evidence", summary: "PipelineRun succeeded but plan-artifacts g14-ci-plan evidence was not visible", planArtifacts: input.planArtifacts, }; } const sourceCommit = input.sourceCommit; const buildServices = stringArray(input.planArtifacts.buildServices); const reusedServices = stringArray(input.planArtifacts.reusedServices); const taskItems = Array.isArray(input.taskRuns.items) ? input.taskRuns.items.map((item) => record(item)) : []; const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : []; const workloadByService = new Map(workloadItems.map((item) => [String(item.serviceId ?? ""), item])); const missingBuildTaskRuns = buildServices.filter((serviceId) => !taskItems.some((item) => String(item.name ?? "").endsWith(`build-${serviceId}`) && item.status === "True")); const runtimeMismatches = buildServices .map((serviceId) => { const workload = workloadByService.get(serviceId); if (!workload) return { serviceId, reason: "runtime-workload-missing" }; if (workload.artifactSourceCommit !== sourceCommit) { return { serviceId, reason: "artifact-source-commit-mismatch", runtimeArtifactSourceCommit: workload.artifactSourceCommit ?? null, expectedSourceCommit: sourceCommit, image: workload.image ?? null, }; } return null; }) .filter((item): item is Record => item !== null); const audit = record(input.planArtifacts.artifactProvenanceAudit); const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : []; const auditPresent = input.planArtifacts.artifactProvenanceAudit !== null; const provenanceWarning = !auditPresent && reusedServices.length > 0 ? { reason: "artifact-provenance-audit-missing-for-reuse", reusedServices } : null; const provenanceOk = auditPresent ? audit.ok !== false && unsafeReuseServices.length === 0 : true; const failures = [ ...(input.runtimeWorkloads.ok === true ? [] : [{ reason: "runtime-workload-query-failed" }]), ...(provenanceOk ? [] : [{ reason: "artifact-provenance-audit-failed", unsafeReuseServices }]), ...missingBuildTaskRuns.map((serviceId) => ({ serviceId, reason: "expected-build-taskrun-not-succeeded" })), ...runtimeMismatches, ]; return { ok: failures.length === 0, state: failures.length === 0 ? "passed" : "failed", summary: failures.length === 0 ? `artifact provenance and built runtime services align with ${shortSha(sourceCommit)}` : `false-green risk: ${failures.length} invariant violation(s)`, sourceCommit, buildServices, reusedServices, provenanceOk, provenanceAuditPresent: auditPresent, provenanceWarnings: provenanceWarning ? [provenanceWarning] : [], missingBuildTaskRuns, runtimeMismatches, failures, }; } export function v02TargetValidation(input: { targetMode: V02StatusTargetMode; sourceCommit: string | null; pipelineRun: Record | null; argo: Record; planArtifacts: Record; runtimeWorkloads: Record; webAssets: Record; gitMirror: Record; recentPipelineRuns: Record; }): Record { if (input.targetMode === "latest-source-head") { return { applicable: false, state: "latest-source-head", summary: "default status uses strict latest source head alignment; use --pipeline-run or --source-commit for historical run validation", }; } const sourceCommit = input.sourceCommit; const argoFields = record(input.argo.fields); const gitMirrorSummary = record(input.gitMirror.summary); const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : []; const serviceSourceCommits = Object.fromEntries(workloadItems .filter((item) => item.serviceId === "hwlab-cloud-api" || item.serviceId === "hwlab-cloud-web") .map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)])); const rolloutServices = stringArray(input.planArtifacts.rolloutServices) .filter((serviceId) => serviceId === "hwlab-cloud-api" || serviceId === "hwlab-cloud-web"); const requiredRuntimeServices = rolloutServices; const recentItems = Array.isArray(input.recentPipelineRuns.items) ? input.recentPipelineRuns.items.map((item) => record(item)) : []; const targetPipelineRunName = stringOrNull(input.pipelineRun?.pipelineRun) ?? stringOrNull(input.pipelineRun?.name); const targetRecentIndex = recentItems.findIndex((item) => ( (targetPipelineRunName !== null && item.name === targetPipelineRunName) || (sourceCommit !== null && item.sourceCommit === sourceCommit) )); const newerCandidates = targetRecentIndex >= 0 ? recentItems.slice(0, targetRecentIndex) : recentItems; const newerSucceededRuns = sourceCommit === null ? [] : newerCandidates.filter((item) => item.sourceCommit !== sourceCommit && item.status === "True"); const failures: Record[] = []; const supersededRuntimeServices: Record[] = []; if (sourceCommit === null) failures.push({ reason: "source-commit-unresolved" }); if (input.pipelineRun === null || input.pipelineRun.exists === false) { failures.push({ reason: "pipeline-run-missing" }); } else if (input.pipelineRun.status !== "True") { failures.push({ reason: "pipeline-run-not-succeeded", status: input.pipelineRun.status ?? null, reasonDetail: input.pipelineRun.reason ?? null }); } if (input.argo.ok !== true || argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") { failures.push({ reason: "argo-not-synced-healthy", syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null }); } if (input.webAssets.ok !== true) failures.push({ reason: "web-assets-probe-failed", summary: input.webAssets.summary ?? null }); if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) { failures.push({ reason: "git-mirror-not-flushed", pendingFlush: gitMirrorSummary.pendingFlush ?? null, githubInSync: gitMirrorSummary.githubInSync ?? null, }); } for (const serviceId of requiredRuntimeServices) { const serviceCommit = serviceSourceCommits[serviceId]; if (sourceCommit !== null && serviceCommit !== sourceCommit) { const mismatch = { reason: "runtime-service-source-mismatch", serviceId, expectedSourceCommit: sourceCommit, actualSourceCommit: serviceCommit ?? null }; if (newerSucceededRuns.length > 0) supersededRuntimeServices.push(mismatch); else failures.push(mismatch); } } const superseded = failures.length === 0 && supersededRuntimeServices.length > 0; return { applicable: true, ok: failures.length === 0, state: failures.length === 0 ? (superseded ? "superseded" : "passed") : "failed", summary: failures.length === 0 ? superseded ? `target ${input.targetMode} completed for ${shortSha(sourceCommit ?? "")}; runtime now reflects newer v0.2 PipelineRun` : requiredRuntimeServices.length === 0 ? `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}; no runtime service rollout required` : `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}` : `target ${input.targetMode} validation failed with ${failures.length} issue(s)`, sourceCommit, pipelineRun: input.pipelineRun === null ? null : { name: input.pipelineRun.pipelineRun ?? null, status: input.pipelineRun.status ?? null, reason: input.pipelineRun.reason ?? null }, argo: { syncRevision: argoFields.syncRevision ?? null, syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null }, runtimeServices: serviceSourceCommits, requiredRuntimeServices, rolloutServices: stringArray(input.planArtifacts.rolloutServices), reusedServices: stringArray(input.planArtifacts.reusedServices), superseded, supersededRuntimeServices, newerSucceededRuns: newerSucceededRuns.slice(0, 3).map((item) => ({ name: item.name ?? null, sourceCommit: item.sourceCommit ?? null, createdAt: item.createdAt ?? null, durationSeconds: item.durationSeconds ?? null, })), webAssets: { ok: input.webAssets.ok ?? null, summary: input.webAssets.summary ?? null, appJsBytes: record(input.webAssets.assetBytes).appJs ?? null }, gitMirror: { pendingFlush: gitMirrorSummary.pendingFlush ?? null, githubInSync: gitMirrorSummary.githubInSync ?? null }, failures, }; } export function v02LatestOnlyTargetValidation(input: { targetMode: string; sourceCommit: string | null; pipelineRun: Record | null; commitAlignment: Record; targetValidation: Record; }): Record { if (input.targetMode === "latest-source-head") return input.targetValidation; if (input.sourceCommit === null) return input.targetValidation; if (input.pipelineRun === null || input.pipelineRun.status !== "True") return input.targetValidation; const validation = record(input.targetValidation); const staleReasons = stringArray(input.commitAlignment.staleReasons); const sourceHeadAdvancedReasons = staleReasons.filter((reason) => ( reason === "origin-head-mismatch" || reason === "cicd-source-repo-stale" || reason === "latest-pipelinerun-not-current" )); if (sourceHeadAdvancedReasons.length === 0) return input.targetValidation; const failures = Array.isArray(validation.failures) ? validation.failures : []; return { ...validation, ok: true, state: "superseded", superseded: true, latestOnlySuperseded: true, latestOnlyReasons: sourceHeadAdvancedReasons, originalState: validation.state ?? null, summary: validation.ok === true || validation.state === "passed" ? `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and is superseded by the newer v0.2 source head` : `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`, supersededFailures: failures.slice(0, 10), failures: [], }; } export function v02CloseoutRecommendedNext(input: { closeable: boolean; blockedReasons: string[]; pendingReasons: string[]; statusCommand: string; sourceCommit: string | null; gitMirrorSummary: Record; }): Record { const rawFlushCommand = stringOrNull(input.gitMirrorSummary.flushCommand) ?? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm"; const flushCommand = rawFlushCommand.includes("--wait") ? rawFlushCommand : `${rawFlushCommand} --wait`; if (input.closeable) { return { action: "comment-and-close-issue", command: input.sourceCommit === null ? input.statusCommand : `bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit ${input.sourceCommit}`, }; } if (input.pendingReasons.includes("git-mirror-not-flushed")) { return { action: "flush-git-mirror", command: flushCommand }; } if (input.pendingReasons.includes("argo-not-healthy")) { return { action: "wait-and-recheck", command: input.statusCommand }; } if (input.pendingReasons.includes("pipeline-run-not-final")) { return { action: "wait-pipelinerun", command: input.statusCommand }; } if (input.blockedReasons.includes("target-pipelinerun-missing")) { return { action: "trigger-current-or-inspect-target", command: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }; } return { action: "inspect-status", command: input.statusCommand }; } export function v02CloseoutVerdict(status: Record): Record { const statusTarget = record(status.statusTarget); const targetMode = stringOrNull(statusTarget.mode) ?? "latest-source-head"; const sourceCommit = stringOrNull(status.sourceCommit) ?? stringOrNull(statusTarget.sourceCommit); const pipelineRun = record(status.pipelineRun); const targetValidation = record(status.targetValidation); const targetValidationState = stringOrNull(targetValidation.state); const argoFields = record(record(status.argo).fields); const gitMirrorSummary = record(record(status.gitMirror).summary); const sourceHeadsTarget = record(record(status.sourceHeads).target); const commitAlignment = record(status.commitAlignment); const latestHead = stringOrNull(commitAlignment.originHead) ?? stringOrNull(record(status.sourceHeads).originHead); const ancestorOfLatest = sourceHeadsTarget.ancestorOfOriginHead ?? status.sourceHeadsTargetAncestorOfOriginHead ?? null; const activePipelineRuns = Array.isArray(status.activePipelineRuns) ? status.activePipelineRuns.map((item) => record(item)) : []; const pipelineStatus = stringOrNull(pipelineRun.status); const pipelineExists = pipelineRun.exists !== false && Object.keys(pipelineRun).length > 0; const pendingReasons: string[] = []; const blockedReasons: string[] = []; if (targetMode === "latest-source-head") { pendingReasons.push("target-not-explicit"); } if (!pipelineExists) { blockedReasons.push("target-pipelinerun-missing"); } else if (pipelineStatus === "Unknown" || pipelineStatus === null) { pendingReasons.push("pipeline-run-not-final"); } else if (pipelineStatus === "False") { blockedReasons.push("pipeline-run-failed"); } if (targetValidationState !== "passed" && targetValidationState !== "superseded") { const failures = Array.isArray(targetValidation.failures) ? targetValidation.failures.map((item) => record(item)) : []; for (const failure of failures) { const reason = stringOrNull(failure.reason); if (reason === "git-mirror-not-flushed") pendingReasons.push(reason); else if (reason === "argo-not-synced-healthy") pendingReasons.push("argo-not-healthy"); else if (reason === "pipeline-run-not-succeeded" && pipelineStatus === "Unknown") pendingReasons.push("pipeline-run-not-final"); else if (reason !== null) blockedReasons.push(reason); } if (failures.length === 0 && targetMode !== "latest-source-head") blockedReasons.push("target-validation-not-passed"); } if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) { pendingReasons.push("git-mirror-not-flushed"); } if (argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") { pendingReasons.push("argo-not-healthy"); } if (targetValidationState === "superseded" && ancestorOfLatest !== true) { if (ancestorOfLatest === false) blockedReasons.push("target-not-ancestor-of-latest-head"); else pendingReasons.push("ancestor-check-unresolved"); } const uniquePendingReasons = Array.from(new Set(pendingReasons)); const uniqueBlockedReasons = Array.from(new Set(blockedReasons)); const terminalState = targetValidationState === "passed" || targetValidationState === "superseded"; const state = uniqueBlockedReasons.length > 0 ? "blocked" : uniquePendingReasons.length > 0 ? "pending" : terminalState ? targetValidationState : "blocked"; const closeable = (state === "passed" || state === "superseded") && uniquePendingReasons.length === 0 && uniqueBlockedReasons.length === 0; const rawStatusCommand = String(status.command ?? ( sourceCommit === null ? "bun scripts/cli.ts hwlab g14 control-plane status --lane v02" : `bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}` )); const statusCommand = rawStatusCommand.startsWith("bun ") ? rawStatusCommand : `bun scripts/cli.ts ${rawStatusCommand}`; const ancestorCheck = { required: state === "superseded", sourceCommit, latestHead, objectExists: sourceHeadsTarget.objectExists ?? null, ancestorOfLatest, exitCode: sourceHeadsTarget.ancestorExitCode ?? null, command: sourceCommit === null ? null : `trans G14:/root/hwlab-v02 sh -- 'git fetch origin v0.2 && git merge-base --is-ancestor ${sourceCommit} origin/v0.2'`, }; const recommendedNext = v02CloseoutRecommendedNext({ closeable, blockedReasons: uniqueBlockedReasons, pendingReasons: uniquePendingReasons, statusCommand, sourceCommit, gitMirrorSummary, }); const summary = closeable ? state === "superseded" ? `closeout ready: target ${shortSha(sourceCommit ?? "")} passed and is superseded by latest v0.2 head ${shortSha(latestHead ?? "")}` : `closeout ready: target ${shortSha(sourceCommit ?? "")} passed on v0.2` : uniquePendingReasons.length > 0 ? `closeout pending: ${uniquePendingReasons.join(",")}` : `closeout blocked: ${uniqueBlockedReasons.join(",") || "unknown"}`; const issueCommentMarkdown = [ `v0.2 closeout verdict: \`${state}\` (${summary})`, "", `- target sourceCommit: \`${sourceCommit ?? "n/a"}\``, `- target PipelineRun: \`${String(pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? "n/a")}\`; status=\`${pipelineStatus ?? "n/a"}\`; reason=\`${String(pipelineRun.reason ?? "n/a")}\``, `- latest head: \`${latestHead ?? "n/a"}\`; ancestorOfLatest=\`${String(ancestorCheck.ancestorOfLatest ?? "n/a")}\``, `- targetValidation: \`${targetValidationState ?? "n/a"}\`; pending=\`${uniquePendingReasons.join(",") || "none"}\`; blocked=\`${uniqueBlockedReasons.join(",") || "none"}\``, `- Git mirror: pendingFlush=\`${String(gitMirrorSummary.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(gitMirrorSummary.githubInSync ?? "n/a")}\`, flushCommand=\`${String(gitMirrorSummary.flushCommand ?? "none")}\``, `- Argo: sync=\`${String(argoFields.syncStatus ?? "n/a")}\`, health=\`${String(argoFields.health ?? "n/a")}\`, revision=\`${String(argoFields.syncRevision ?? "n/a")}\``, `- active v0.2 PipelineRuns: \`${activePipelineRuns.length}\``, `- recommended next: \`${String(recommendedNext.action ?? "inspect-status")}\` -> \`${String(recommendedNext.command ?? statusCommand)}\``, ].join("\n"); return { ok: closeable, closeable, state, summary, targetMode, sourceCommit, latestHead, pipelineRun: { name: pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? null, exists: pipelineExists, status: pipelineStatus, reason: pipelineRun.reason ?? null, }, targetValidation: { state: targetValidationState, ok: targetValidation.ok ?? null, failures: Array.isArray(targetValidation.failures) ? targetValidation.failures.slice(0, 10) : [], }, latestOnly: { superseded: targetValidation.latestOnlySuperseded === true || state === "superseded", reasons: stringArray(targetValidation.latestOnlyReasons), ancestorCheck, }, gitMirror: { pendingFlush: gitMirrorSummary.pendingFlush ?? null, githubInSync: gitMirrorSummary.githubInSync ?? null, flushNeeded: gitMirrorSummary.flushNeeded ?? null, flushCommand: gitMirrorSummary.flushCommand ?? null, lastSync: gitMirrorSummary.lastSync ?? null, lastWrite: gitMirrorSummary.lastWrite ?? null, lastFlush: gitMirrorSummary.lastFlush ?? null, }, argo: { syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null, syncRevision: argoFields.syncRevision ?? null, targetRevision: argoFields.targetRevision ?? null, path: argoFields.path ?? null, }, runtime: { webAssetsOk: nested(status, ["webAssets", "ok"]) ?? null, webAssetsSummary: nested(status, ["webAssets", "summary"]) ?? null, apiRevision: nested(status, ["webAssets", "apiRevision"]) ?? nested(status, ["commitAlignment", "apiRevision"]) ?? null, serviceSourceCommits: nested(status, ["commitAlignment", "serviceSourceCommits"]) ?? null, }, activePipelineRuns, recentPipelineRuns: status.recentPipelineRuns ?? null, pendingReasons: uniquePendingReasons, blockedReasons: uniqueBlockedReasons, recommendedNext, issueCommentMarkdown, }; } export function v02CommitAlignment(input: { expectedSourceHead: string | null; sourceHeads: Record; gitMirrorSummary: Record; pipelineRun: Record | null; recentPipelineRuns: Record; planArtifacts?: Record; runtimeWorkloads: Record; webAssets: Record; }): Record { const expectedSourceHead = input.expectedSourceHead; const cicdSourceHead = stringOrNull(input.sourceHeads.cicdSourceHead); const originHead = stringOrNull(input.sourceHeads.originHead) ?? expectedSourceHead; const workspace = record(input.sourceHeads.workspace); const workspaceHead = stringOrNull(workspace.head); const workspaceOriginHead = stringOrNull(workspace.originHead); const mirrorSourceHead = stringOrNull(input.gitMirrorSummary.localV02); const mirrorGithubSourceHead = stringOrNull(input.gitMirrorSummary.githubV02); const recentItems = Array.isArray(input.recentPipelineRuns.items) ? input.recentPipelineRuns.items.map((item) => record(item)) : []; const latestPipelineRun = recentItems[0] ?? null; const latestPipelineSourceCommit = latestPipelineRun === null ? null : stringOrNull(latestPipelineRun.sourceCommit); const currentPipelineStatus = stringOrNull(input.pipelineRun?.status); const apiRevision = stringOrNull(input.webAssets.apiRevision); const planArtifacts = record(input.planArtifacts); const planSourceCommit = stringOrNull(planArtifacts.sourceCommitId); const rolloutServices = stringArray(planArtifacts.rolloutServices); const apiRevisionRequired = expectedSourceHead !== null && planSourceCommit === expectedSourceHead && rolloutServices.includes("hwlab-cloud-api"); const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : []; const serviceSourceCommits = Object.fromEntries(workloadItems .filter((item) => typeof item.serviceId === "string" && item.serviceId.length > 0) .map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)])); const staleReasons: string[] = []; if (expectedSourceHead === null) staleReasons.push("expected-source-head-unresolved"); if (expectedSourceHead !== null && originHead !== null && originHead !== expectedSourceHead) staleReasons.push("origin-head-mismatch"); if (expectedSourceHead !== null && cicdSourceHead !== null && cicdSourceHead !== expectedSourceHead) staleReasons.push("cicd-source-repo-stale"); if (expectedSourceHead !== null && mirrorSourceHead !== expectedSourceHead) staleReasons.push("mirror-source-stale"); if (expectedSourceHead !== null && latestPipelineSourceCommit !== expectedSourceHead) staleReasons.push("latest-pipelinerun-not-current"); if (apiRevisionRequired && apiRevision !== expectedSourceHead) staleReasons.push("runtime-api-revision-stale"); const workspaceWarnings: string[] = []; if (expectedSourceHead !== null && workspaceHead !== null && workspaceHead !== expectedSourceHead) workspaceWarnings.push("workspace-head-differs-from-latest-source-but-isolated"); if (expectedSourceHead !== null && workspaceOriginHead !== null && workspaceOriginHead !== expectedSourceHead) workspaceWarnings.push("workspace-origin-ref-stale-but-isolated"); if (workspace.dirty === true) workspaceWarnings.push("workspace-dirty-but-isolated-from-cicd"); const runtimeWarnings: string[] = []; if (!apiRevisionRequired && expectedSourceHead !== null && apiRevision !== null && apiRevision !== expectedSourceHead) { runtimeWarnings.push("api-revision-differs-without-current-cloud-api-rollout"); } const latestPipelineSucceeded = latestPipelineRun !== null && latestPipelineRun.status === "True"; const aligned = staleReasons.length === 0; const state = aligned ? "aligned" : latestPipelineSucceeded ? "stale-success" : currentPipelineStatus === "Unknown" ? "in-progress" : "stale"; return { aligned, state, expectedSourceHead, originHead, cicdSourceHead, cicdRepo: input.sourceHeads.cicdRepo ?? V02_CICD_REPO, mirrorSourceHead, mirrorGithubSourceHead, latestPipelineRun: latestPipelineRun === null ? null : { name: latestPipelineRun.name ?? null, sourceCommit: latestPipelineSourceCommit, status: latestPipelineRun.status ?? null, reason: latestPipelineRun.reason ?? null, createdAt: latestPipelineRun.createdAt ?? null, durationSeconds: latestPipelineRun.durationSeconds ?? null, }, latestPipelineSourceCommit, currentPipelineRun: input.pipelineRun === null ? null : { name: input.pipelineRun.pipelineRun ?? null, status: input.pipelineRun.status ?? null, reason: input.pipelineRun.reason ?? null, }, runtimeSourceCommit: apiRevision, apiRevision, apiRevisionRequired, planSourceCommit, rolloutServices, serviceSourceCommits, sourceInSync: input.gitMirrorSummary.sourceInSync ?? null, gitopsInSync: input.gitMirrorSummary.gitopsInSync ?? null, staleReasons, workspace: { path: workspace.path ?? V02_WORKSPACE, branch: workspace.branch ?? null, head: workspaceHead, originHead: workspaceOriginHead, dirty: workspace.dirty ?? null, dirtyCount: workspace.dirtyCount ?? null, isolatedFromCicd: true, }, workspaceWarnings, runtimeWarnings, summary: aligned ? apiRevisionRequired ? `v0.2 CI/CD source, mirror, PipelineRun, and cloud-api runtime align with ${shortSha(expectedSourceHead ?? "")}` : `v0.2 CI/CD source, mirror, and PipelineRun align with ${shortSha(expectedSourceHead ?? "")}; cloud-api runtime rollout not required` : `v0.2 CI/CD alignment state=${state}; staleReasons=${staleReasons.join(",")}`, }; } export function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null { if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null; const activeItems = activePipelineRuns.map((item) => record(item)); const sourceIsActive = activeItems.some((item) => item.sourceCommit === sourceCommit || String(item.name ?? "").includes(shortSha(sourceCommit))); if (sourceIsActive) { return `runtime rollout still in progress; cloud-api apiRevision=${apiRevision} has not reached sourceCommit=${sourceCommit} yet`; } if (activeItems.length > 0) { return `runtime apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; active v02 PipelineRun observed, wait for rollout before judging runtime revision`; } return `cloud-api apiRevision=${apiRevision} is service-scoped and differs from sourceCommit=${sourceCommit}; when web asset probes pass, this is not proof that Cloud Web is stale`; } export function numericField(value: string | undefined): number | null { if (value === undefined || value.trim().length === 0) return null; if (value === "") return null; const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } export function numericValue(value: unknown): number | null { const parsed = typeof value === "number" ? value : Number(value); return Number.isFinite(parsed) ? parsed : null; } export function stringArray(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : []; } export function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record { if (!commandOk) { return { ok: false, command: Array.isArray(command) ? redactedCommand(command) : command, exitCode, stderr: stderr.trim().slice(0, 4000), items: [], activeItems: [], }; } return parsePipelineRunRows(text, limit, nowMs); }