Codex
1500 lines
70 KiB
TypeScript
1500 lines
70 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. v02-status module for scripts/src/hwlab-g14.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-g14.ts:1309-2787 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult, ShellSection, V02ControlPlaneStatusTarget, V02StatusTargetMode } from "./types";
|
|
import { commandErrorSummary } from "./cleanup";
|
|
import { gitMirrorCacheProbeScript } from "./git-mirror";
|
|
import { conditionStatus } from "./observability";
|
|
import { durationSeconds, statusText } from "./pr-monitor";
|
|
import { fieldBoolean, g14K3s, isCommandSuccess, nested, record, redactedCommand, shellQuote, shortSha, stringOrNull } from "./remote";
|
|
import { v02PipelineRunName } from "./source";
|
|
import { ARGO_NAMESPACE, CI_NAMESPACE, GIT_MIRROR_NAMESPACE, V02_APP, V02_BUILD_TASKRUN_CRITICAL_SECONDS, V02_BUILD_TASKRUN_WARNING_SECONDS, V02_CICD_REPO, V02_CLOUD_API_URL, V02_CLOUD_WEB_URL, V02_PIPELINERUN_PREFIX, V02_POLLER, V02_RECONCILER, V02_WORKSPACE } from "./types";
|
|
|
|
export interface V02TriggerSnapshot {
|
|
sourceCommit: string | null;
|
|
pipelineRun: string | null;
|
|
before: Record<string, unknown> | null;
|
|
result: CommandJsonResult;
|
|
observedAtMs: number;
|
|
}
|
|
|
|
export function parseV02TriggerSnapshot(result: CommandJsonResult, observedAtMs: number): V02TriggerSnapshot {
|
|
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout);
|
|
const fields = keyValueLinesFromText(output);
|
|
const sourceCommit = stringOrNull(fields.sourceCommit?.match(/^[0-9a-f]{40}$/iu)?.[0]);
|
|
const pipelineRun = stringOrNull(fields.pipelineRun);
|
|
const pipelineRunExitCode = numericField(fields.pipelineRunExitCode);
|
|
const before = pipelineRun !== null && pipelineRunExitCode !== null
|
|
? pipelineRunCompactFromText(
|
|
pipelineRun,
|
|
[fields.status ?? "", fields.reason ?? "", fields.message ?? ""].join("\n"),
|
|
pipelineRunExitCode === 0,
|
|
"hwlab-v02-trigger-snapshot:pipelinerun",
|
|
pipelineRunExitCode,
|
|
fields.pipelineRunStderr ?? "",
|
|
)
|
|
: null;
|
|
return {
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
result,
|
|
observedAtMs,
|
|
};
|
|
}
|
|
|
|
export function getV02TriggerSnapshot(): V02TriggerSnapshot {
|
|
const script = [
|
|
"set +e",
|
|
"one_line() { tr '\\r\\n\\t' ' ' | sed 's/[[:space:]][[:space:]]*/ /g' | cut -c1-2000; }",
|
|
"tmp_dir=$(mktemp -d /tmp/hwlab-v02-trigger-snapshot.XXXXXX)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
|
"source_commit=",
|
|
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
|
" source_commit=$(git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)",
|
|
"elif [ -e \"$cicd_repo\" ]; then",
|
|
" printf 'snapshotError\\t%s\\n' \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\"",
|
|
"fi",
|
|
"pipeline_run=",
|
|
`if [ -n "$source_commit" ]; then pipeline_run=${shellQuote(V02_PIPELINERUN_PREFIX)}-$(printf '%s' "$source_commit" | cut -c1-12); fi`,
|
|
"if [ -n \"$pipeline_run\" ]; then",
|
|
` (kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}' >\"$tmp_dir/pipelinerun.out\" 2>\"$tmp_dir/pipelinerun.err\"; printf '%s' "$?" >\"$tmp_dir/pipelinerun.status\") &`,
|
|
" pipeline_pid=$!",
|
|
"else",
|
|
" printf '' >\"$tmp_dir/pipelinerun.out\"",
|
|
" printf '' >\"$tmp_dir/pipelinerun.err\"",
|
|
" printf '' >\"$tmp_dir/pipelinerun.status\"",
|
|
" pipeline_pid=",
|
|
"fi",
|
|
"if [ -n \"$pipeline_pid\" ]; then wait \"$pipeline_pid\" 2>/dev/null || true; pipeline_pid=; fi",
|
|
"status_text=$(cat \"$tmp_dir/pipelinerun.out\" 2>/dev/null)",
|
|
"pipeline_status=$(cat \"$tmp_dir/pipelinerun.status\" 2>/dev/null)",
|
|
"printf 'sourceCommit\\t%s\\n' \"$source_commit\"",
|
|
"printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"",
|
|
"printf 'pipelineRunExitCode\\t%s\\n' \"$pipeline_status\"",
|
|
"printf 'status\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '1p')\"",
|
|
"printf 'reason\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '2p')\"",
|
|
"printf 'message\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '3p')\"",
|
|
"printf 'pipelineRunStderr\\t%s\\n' \"$(one_line < \"$tmp_dir/pipelinerun.err\")\"",
|
|
].join("\n");
|
|
const result = g14K3s(["sh", "--", script], 180_000);
|
|
return parseV02TriggerSnapshot(result, Date.now());
|
|
}
|
|
|
|
export function v02ExistingPipelineRunReuseDecision(input: {
|
|
sourceCommit: string;
|
|
before: Record<string, unknown>;
|
|
latestSourceCommit: string | null;
|
|
}): Record<string, unknown> {
|
|
const status = stringOrNull(input.before.status);
|
|
const exists = input.before.exists === true;
|
|
const latestSourceCommit = input.latestSourceCommit;
|
|
const alreadyUsable = exists && (status === "True" || status === "Unknown");
|
|
if (latestSourceCommit === null) {
|
|
return {
|
|
reusable: false,
|
|
alreadyUsable,
|
|
reason: "source-head-recheck-unresolved-before-existing-pipelinerun-reuse",
|
|
sourceCommit: input.sourceCommit,
|
|
latestSourceCommit,
|
|
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
|
|
};
|
|
}
|
|
if (latestSourceCommit !== null && latestSourceCommit !== input.sourceCommit) {
|
|
return {
|
|
reusable: false,
|
|
alreadyUsable,
|
|
reason: "source-head-advanced-before-existing-pipelinerun-reuse",
|
|
sourceCommit: input.sourceCommit,
|
|
latestSourceCommit,
|
|
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
|
|
latestPipelineRun: v02PipelineRunName(latestSourceCommit),
|
|
};
|
|
}
|
|
return {
|
|
reusable: exists && status !== null,
|
|
alreadyUsable,
|
|
reason: alreadyUsable ? "existing-pipelinerun-reused" : "existing-pipelinerun-terminal-failed",
|
|
sourceCommit: input.sourceCommit,
|
|
latestSourceCommit,
|
|
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
|
|
};
|
|
}
|
|
|
|
export function getPipelineRunCompact(name: string): Record<string, unknown> {
|
|
const result = g14K3s([
|
|
"kubectl",
|
|
"get",
|
|
"pipelinerun",
|
|
"-n",
|
|
CI_NAMESPACE,
|
|
name,
|
|
"-o",
|
|
"jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}",
|
|
], 60_000);
|
|
return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr);
|
|
}
|
|
|
|
export function pipelineRunCompactFromText(
|
|
name: string,
|
|
text: string,
|
|
commandOk: boolean,
|
|
command: string[] | string,
|
|
exitCode: number | null,
|
|
stderr: string,
|
|
): Record<string, unknown> {
|
|
const [status = "", reason = "", message = ""] = text.split(/\r?\n/u);
|
|
const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`);
|
|
return {
|
|
ok: commandOk,
|
|
exists: commandOk || !notFound,
|
|
pipelineRun: name,
|
|
status: status || null,
|
|
reason: reason || null,
|
|
message: message || null,
|
|
command: Array.isArray(command) ? redactedCommand(command) : command,
|
|
exitCode,
|
|
stderr: stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
|
|
export function runtimeLaneTaskRunDiagnosticsScript(): string {
|
|
return [
|
|
"set +e",
|
|
"node <<'NODE'",
|
|
"const cp = require('node:child_process');",
|
|
"const namespace = process.env.CI_NAMESPACE || 'hwlab-ci';",
|
|
"const pipelineRun = process.env.PIPELINE_RUN || '';",
|
|
"const tailLines = Math.max(20, Math.min(Number(process.env.TASKRUN_LOG_TAIL_LINES || 60), 200));",
|
|
"const maxLogChars = Math.max(1000, Math.min(Number(process.env.TASKRUN_LOG_MAX_CHARS || 4000), 20000));",
|
|
"function kubectl(args, timeoutMs, maxBuffer) {",
|
|
" const out = cp.spawnSync('kubectl', args, { encoding: 'utf8', timeout: timeoutMs, maxBuffer });",
|
|
" const err = out.error;",
|
|
" return { exitCode: out.status, stdout: out.stdout || '', stderr: out.stderr || (err ? String(err.message || err) : ''), timedOut: Boolean(err && err.code === 'ETIMEDOUT') };",
|
|
"}",
|
|
"function parseJson(text) { try { return JSON.parse(text); } catch { return null; } }",
|
|
"function oneLine(value, limit) { return String(value || '').replace(/[\\r\\n\\t]+/g, ' ').replace(/\\s+/g, ' ').trim().slice(0, limit); }",
|
|
"function timestampMs(value) { const parsed = Date.parse(String(value || '')); return Number.isFinite(parsed) ? parsed : null; }",
|
|
"function durationSeconds(start, end) { const s = timestampMs(start); const e = timestampMs(end); return s === null || e === null || e < s ? null : Math.round((e - s) / 1000); }",
|
|
"function condition(item) { const conditions = Array.isArray(item && item.status && item.status.conditions) ? item.status.conditions : []; return conditions.find((entry) => entry && entry.type === 'Succeeded') || conditions[0] || {}; }",
|
|
"function redact(text) {",
|
|
" return String(text || '')",
|
|
" .replace(/Bearer\\s+[A-Za-z0-9._~+\\/-]+=*/giu, 'Bearer <redacted>')",
|
|
" .replace(/((?:api[_-]?key|authorization|token|password|secret|database_url|dsn)\\s*[:=]\\s*)\\S+/giu, '$1<redacted>');",
|
|
"}",
|
|
"function trimTail(text) {",
|
|
" const redacted = redact(text);",
|
|
" let tail = redacted.split(/\\r?\\n/).slice(-tailLines).join('\\n').trimEnd();",
|
|
" if (tail.length > maxLogChars) {",
|
|
" tail = tail.slice(tail.length - maxLogChars);",
|
|
" const lineBreak = tail.indexOf('\\n');",
|
|
" if (lineBreak >= 0) tail = tail.slice(lineBreak + 1);",
|
|
" }",
|
|
" return tail;",
|
|
"}",
|
|
"function stateSummary(state) {",
|
|
" if (!state || typeof state !== 'object') return { state: null, reason: null, exitCode: null };",
|
|
" const key = Object.keys(state)[0] || null;",
|
|
" const value = key ? state[key] || {} : {};",
|
|
" return { state: key, reason: value.reason || null, exitCode: typeof value.exitCode === 'number' ? value.exitCode : null };",
|
|
"}",
|
|
"function podContainerStates(pod) {",
|
|
" const status = (pod && pod.status) || {};",
|
|
" const containers = [...(status.initContainerStatuses || []), ...(status.containerStatuses || [])];",
|
|
" return containers.map((item) => {",
|
|
" const summary = stateSummary(item.state);",
|
|
" return { name: item.name || null, ready: item.ready === true, restartCount: item.restartCount || 0, state: summary.state, reason: summary.reason, exitCode: summary.exitCode };",
|
|
" }).filter((item) => item.state !== 'running' || item.restartCount > 0).slice(0, 12);",
|
|
"}",
|
|
"function findPodName(taskRunName) {",
|
|
" const out = kubectl(['-n', namespace, 'get', 'pod', '-l', 'tekton.dev/taskRun=' + taskRunName, '-o', 'jsonpath={.items[0].metadata.name}'], 10000, 65536);",
|
|
" return out.exitCode === 0 && out.stdout.trim() ? out.stdout.trim() : null;",
|
|
"}",
|
|
"function collectLogTail(taskRun) {",
|
|
" const podName = taskRun.podName || findPodName(taskRun.name);",
|
|
" if (!podName) return { taskRun: taskRun.name, pipelineTask: taskRun.pipelineTask, podName: null, ok: false, reason: 'pod-not-found', logTail: '' };",
|
|
" const podResult = kubectl(['-n', namespace, 'get', 'pod', podName, '-o', 'json'], 10000, 262144);",
|
|
" const pod = parseJson(podResult.stdout);",
|
|
" const logResult = kubectl(['-n', namespace, 'logs', podName, '--all-containers=true', '--tail=' + String(tailLines), '--prefix=true'], 20000, Math.max(262144, maxLogChars * 8));",
|
|
" const raw = logResult.stdout || logResult.stderr;",
|
|
" return {",
|
|
" taskRun: taskRun.name,",
|
|
" pipelineTask: taskRun.pipelineTask,",
|
|
" podName,",
|
|
" ok: logResult.exitCode === 0,",
|
|
" exitCode: logResult.exitCode,",
|
|
" timedOut: logResult.timedOut,",
|
|
" containers: podContainerStates(pod),",
|
|
" logTail: trimTail(raw),",
|
|
" };",
|
|
"}",
|
|
"if (!pipelineRun) {",
|
|
" console.log(JSON.stringify({ ok: true, pipelineRun: null, namespace, skipped: true, reason: 'pipeline-run-empty', total: 0, items: [], failedTaskRuns: [], activeTaskRuns: [], logTails: [], valuesPrinted: false }));",
|
|
" process.exit(0);",
|
|
"}",
|
|
"const taskRunResult = kubectl(['-n', namespace, 'get', 'taskrun', '-l', 'tekton.dev/pipelineRun=' + pipelineRun, '-o', 'json'], 25000, 2 * 1024 * 1024);",
|
|
"const taskData = parseJson(taskRunResult.stdout);",
|
|
"const rawItems = Array.isArray(taskData && taskData.items) ? taskData.items : [];",
|
|
"const items = rawItems.map((item) => {",
|
|
" const metadata = item.metadata || {};",
|
|
" const labels = metadata.labels || {};",
|
|
" const status = item.status || {};",
|
|
" const c = condition(item);",
|
|
" return {",
|
|
" name: metadata.name || null,",
|
|
" pipelineTask: labels['tekton.dev/pipelineTask'] || null,",
|
|
" status: c.status || null,",
|
|
" reason: c.reason || null,",
|
|
" message: oneLine(c.message, 360) || null,",
|
|
" startTime: status.startTime || null,",
|
|
" completionTime: status.completionTime || null,",
|
|
" durationSeconds: durationSeconds(status.startTime, status.completionTime),",
|
|
" podName: status.podName || null,",
|
|
" };",
|
|
"}).filter((item) => item.name).sort((left, right) => (timestampMs(left.startTime) || 0) - (timestampMs(right.startTime) || 0) || String(left.name).localeCompare(String(right.name)));",
|
|
"const failedTaskRuns = items.filter((item) => item.status === 'False' || /fail|error|timeout/i.test(String(item.reason || '')));",
|
|
"const activeTaskRuns = items.filter((item) => item.status !== 'True' && item.status !== 'False');",
|
|
"const succeededCount = items.filter((item) => item.status === 'True').length;",
|
|
"const logTails = failedTaskRuns.slice(0, 3).map(collectLogTail);",
|
|
"console.log(JSON.stringify({",
|
|
" ok: taskRunResult.exitCode === 0,",
|
|
" pipelineRun,",
|
|
" namespace,",
|
|
" total: items.length,",
|
|
" succeededCount,",
|
|
" failedCount: failedTaskRuns.length,",
|
|
" activeCount: activeTaskRuns.length,",
|
|
" failedTaskRuns: failedTaskRuns.slice(0, 8),",
|
|
" activeTaskRuns: activeTaskRuns.slice(0, 8),",
|
|
" recentTaskRuns: items.slice(-8),",
|
|
" logTails,",
|
|
" query: { taskRunsExitCode: taskRunResult.exitCode, taskRunsTimedOut: taskRunResult.timedOut, taskRunsStderr: oneLine(taskRunResult.stderr, 500) || null },",
|
|
" bounded: { maxFailedTaskRuns: 8, maxLogTaskRuns: 3, tailLines, maxLogChars, redacted: true },",
|
|
" valuesPrinted: false,",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function runtimeLaneTaskRunDiagnosticsFromText(
|
|
pipelineRun: string,
|
|
section: ShellSection | undefined,
|
|
stderr: string,
|
|
): Record<string, unknown> {
|
|
const raw = String(section?.stdout ?? "").trim();
|
|
if (raw.length === 0) {
|
|
return {
|
|
ok: shellSectionOk(section),
|
|
pipelineRun,
|
|
total: 0,
|
|
failedTaskRuns: [],
|
|
activeTaskRuns: [],
|
|
logTails: [],
|
|
sectionExitCode: section?.exitCode ?? null,
|
|
stderr: stderr.trim().slice(0, 1000),
|
|
};
|
|
}
|
|
try {
|
|
const data = record(JSON.parse(raw) as unknown);
|
|
return {
|
|
...data,
|
|
pipelineRun: stringOrNull(data.pipelineRun) ?? pipelineRun,
|
|
sectionExitCode: section?.exitCode ?? null,
|
|
};
|
|
} catch {
|
|
return {
|
|
ok: false,
|
|
pipelineRun,
|
|
sectionExitCode: section?.exitCode ?? null,
|
|
raw: raw.slice(0, 2000),
|
|
stderr: stderr.trim().slice(0, 1000),
|
|
degradedReason: "taskrun-diagnostics-json-parse-failed",
|
|
};
|
|
}
|
|
}
|
|
|
|
export function timestampMs(value: unknown): number | null {
|
|
if (typeof value !== "string" || value.trim().length === 0) return null;
|
|
const parsed = Date.parse(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
export function secondsBetween(start: unknown, end: unknown): number | null {
|
|
const startMs = timestampMs(start);
|
|
const endMs = timestampMs(end);
|
|
if (startMs === null || endMs === null || endMs < startMs) return null;
|
|
return Math.round((endMs - startMs) / 1000);
|
|
}
|
|
|
|
export function pipelineRunStatusRowFromLine(line: string, nowMs: number, pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record<string, unknown> | null {
|
|
const [
|
|
name = "",
|
|
sourceCommit = "",
|
|
createdAt = "",
|
|
startTime = "",
|
|
completionTime = "",
|
|
status = "",
|
|
reason = "",
|
|
] = line.split("\t");
|
|
if (!name.startsWith(pipelineRunPrefix)) return null;
|
|
const startMs = timestampMs(startTime);
|
|
const completion = completionTime.trim().length > 0 ? completionTime : null;
|
|
const conditionStatus = status.trim().length > 0 ? status : null;
|
|
return {
|
|
name,
|
|
sourceCommit: sourceCommit || null,
|
|
status: conditionStatus,
|
|
reason: reason || null,
|
|
createdAt: createdAt || null,
|
|
startTime: startTime || null,
|
|
completionTime: completion,
|
|
durationSeconds: secondsBetween(startTime, completion),
|
|
elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)),
|
|
active: completion === null && conditionStatus !== "True" && conditionStatus !== "False",
|
|
};
|
|
}
|
|
|
|
export function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now(), pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record<string, unknown> {
|
|
const rows = text
|
|
.split(/\r?\n/u)
|
|
.map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs, pipelineRunPrefix))
|
|
.filter((item): item is Record<string, unknown> => item !== null)
|
|
.sort((left, right) => {
|
|
return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0);
|
|
});
|
|
const activeItems = rows.filter((item) => item.active === true);
|
|
return {
|
|
ok: true,
|
|
count: rows.length,
|
|
activeCount: activeItems.length,
|
|
items: rows.slice(0, limit),
|
|
activeItems: activeItems.slice(0, limit),
|
|
disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete",
|
|
};
|
|
}
|
|
|
|
export function pipelineRunRowsJsonPath(): string {
|
|
return [
|
|
"jsonpath={range .items[*]}",
|
|
"{.metadata.name}",
|
|
"{\"\\t\"}",
|
|
"{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}",
|
|
"{\"\\t\"}",
|
|
"{.metadata.creationTimestamp}",
|
|
"{\"\\t\"}",
|
|
"{.status.startTime}",
|
|
"{\"\\t\"}",
|
|
"{.status.completionTime}",
|
|
"{\"\\t\"}",
|
|
"{.status.conditions[0].status}",
|
|
"{\"\\t\"}",
|
|
"{.status.conditions[0].reason}",
|
|
"{\"\\n\"}",
|
|
"{end}",
|
|
].join("");
|
|
}
|
|
|
|
export function listV02PipelineRunsCompact(limit = 8): Record<string, unknown> {
|
|
const result = g14K3s([
|
|
"kubectl",
|
|
"get",
|
|
"pipelinerun",
|
|
"-n",
|
|
CI_NAMESPACE,
|
|
"-l",
|
|
"hwlab.pikastech.local/gitops-target=v02",
|
|
"-o",
|
|
pipelineRunRowsJsonPath(),
|
|
], 60_000);
|
|
if (!isCommandSuccess(result)) {
|
|
return {
|
|
ok: false,
|
|
command: redactedCommand(result.command),
|
|
exitCode: result.exitCode,
|
|
stderr: commandErrorSummary(result),
|
|
items: [],
|
|
activeItems: [],
|
|
};
|
|
}
|
|
return parsePipelineRunRows(statusText(result), limit);
|
|
}
|
|
|
|
export function parseShellSections(output: string): Record<string, ShellSection> {
|
|
const sections: Record<string, ShellSection> = {};
|
|
let currentName: string | null = null;
|
|
let currentLines: string[] = [];
|
|
let currentExitCode: number | null = null;
|
|
const flush = (): void => {
|
|
if (currentName === null) return;
|
|
sections[currentName] = {
|
|
stdout: currentLines.join("\n").trim(),
|
|
exitCode: currentExitCode,
|
|
};
|
|
};
|
|
for (const line of output.split(/\r?\n/u)) {
|
|
const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line);
|
|
if (begin !== null) {
|
|
flush();
|
|
currentName = begin[1] ?? null;
|
|
currentLines = [];
|
|
currentExitCode = null;
|
|
continue;
|
|
}
|
|
const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line);
|
|
if (end !== null && currentName === end[1]) {
|
|
currentExitCode = Number(end[2]);
|
|
flush();
|
|
currentName = null;
|
|
currentLines = [];
|
|
currentExitCode = null;
|
|
continue;
|
|
}
|
|
if (currentName !== null) currentLines.push(line);
|
|
}
|
|
flush();
|
|
return sections;
|
|
}
|
|
|
|
export function shellSectionOk(section: ShellSection | undefined): boolean {
|
|
return section?.exitCode === 0;
|
|
}
|
|
|
|
export function v02ControlPlaneStatusBundle(target: V02ControlPlaneStatusTarget = {}): CommandJsonResult {
|
|
const targetMode: V02StatusTargetMode = target.mode
|
|
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
|
const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null
|
|
? [
|
|
`pipeline_run=${shellQuote(target.pipelineRun)}`,
|
|
`source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`,
|
|
`if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | cut -d- -f5-); fi`,
|
|
].join("\n")
|
|
: [
|
|
target.sourceCommit === undefined
|
|
? `source_commit=$(git --git-dir=${shellQuote(V02_CICD_REPO)} rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)`
|
|
: `source_commit=${shellQuote(target.sourceCommit ?? "")}`,
|
|
"pipeline_run=",
|
|
].join("\n");
|
|
const script = [
|
|
"set +e",
|
|
targetInit,
|
|
`target_mode=${shellQuote(targetMode)}`,
|
|
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${V02_PIPELINERUN_PREFIX}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`,
|
|
"section() {",
|
|
" name=\"$1\"",
|
|
" shift",
|
|
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
|
" \"$@\"",
|
|
" code=$?",
|
|
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
|
"}",
|
|
"section statusTarget printf 'mode\\t%s\\nsourceCommit\\t%s\\npipelineRun\\t%s\\n' \"$target_mode\" \"$source_commit\" \"$pipeline_run\"",
|
|
"section sourceCommit printf '%s\\n' \"$source_commit\"",
|
|
"section pipelineRunName printf '%s\\n' \"$pipeline_run\"",
|
|
`section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())} sh "$source_commit"`,
|
|
"section queryNow date -u +%Y-%m-%dT%H:%M:%SZ",
|
|
`section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`,
|
|
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
|
|
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
|
`if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
|
|
`if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
|
|
`if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`,
|
|
`section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`,
|
|
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
|
|
`section gitMirrorCache kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(gitMirrorCacheProbeScript())}`,
|
|
`section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
|
|
].join("\n");
|
|
return g14K3s(["sh", "--", script], 60_000);
|
|
}
|
|
|
|
export function v02SourceHeadsProbeScript(): string {
|
|
return [
|
|
"set +e",
|
|
"target_source_commit=${1:-}",
|
|
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
|
`workspace=${shellQuote(V02_WORKSPACE)}`,
|
|
"rev_cicd() { git --git-dir=\"$cicd_repo\" rev-parse \"$1\" 2>/dev/null || true; }",
|
|
"rev_workspace() { git -C \"$workspace\" rev-parse \"$1\" 2>/dev/null || true; }",
|
|
"origin_head=$(rev_cicd refs/remotes/origin/v0.2)",
|
|
"printf 'cicdRepo\\t%s\\n' \"$cicd_repo\"",
|
|
"printf 'cicdRepoExists\\t%s\\n' \"$([ -d \"$cicd_repo/objects\" ] && printf yes || printf no)\"",
|
|
"printf 'cicdSourceHead\\t%s\\n' \"$origin_head\"",
|
|
"printf 'originHead\\t%s\\n' \"$origin_head\"",
|
|
"printf 'targetSourceCommit\\t%s\\n' \"$target_source_commit\"",
|
|
"target_object_exists=",
|
|
"target_ancestor=",
|
|
"target_ancestor_exit=",
|
|
"if [ -n \"$target_source_commit\" ] && git --git-dir=\"$cicd_repo\" cat-file -e \"$target_source_commit^{commit}\" 2>/dev/null; then",
|
|
" target_object_exists=true",
|
|
" if [ -n \"$origin_head\" ]; then",
|
|
" git --git-dir=\"$cicd_repo\" merge-base --is-ancestor \"$target_source_commit\" \"$origin_head\" >/dev/null 2>&1",
|
|
" target_ancestor_exit=$?",
|
|
" if [ \"$target_ancestor_exit\" = 0 ]; then target_ancestor=true; elif [ \"$target_ancestor_exit\" = 1 ]; then target_ancestor=false; else target_ancestor=unknown; fi",
|
|
" fi",
|
|
"elif [ -n \"$target_source_commit\" ]; then",
|
|
" target_object_exists=false",
|
|
"fi",
|
|
"printf 'targetObjectExists\\t%s\\n' \"$target_object_exists\"",
|
|
"printf 'targetAncestorOfOriginHead\\t%s\\n' \"$target_ancestor\"",
|
|
"printf 'targetAncestorExitCode\\t%s\\n' \"$target_ancestor_exit\"",
|
|
"printf 'workspacePath\\t%s\\n' \"$workspace\"",
|
|
"printf 'workspaceBranch\\t%s\\n' \"$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)\"",
|
|
"printf 'workspaceHead\\t%s\\n' \"$(rev_workspace HEAD)\"",
|
|
"printf 'workspaceOriginHead\\t%s\\n' \"$(rev_workspace refs/remotes/origin/v0.2)\"",
|
|
"printf 'workspaceDirtyCount\\t%s\\n' \"$(git -C \"$workspace\" status --porcelain=v1 2>/dev/null | wc -l | tr -d ' ')\"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function v02PlanArtifactsLogScript(): string {
|
|
return [
|
|
"pipeline_run=\"$1\"",
|
|
`namespace=${shellQuote(CI_NAMESPACE)}`,
|
|
"[ -n \"$pipeline_run\" ] || exit 0",
|
|
"pods=$(kubectl -n \"$namespace\" get pods -l \"tekton.dev/pipelineRun=$pipeline_run,tekton.dev/pipelineTask=plan-artifacts\" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)",
|
|
"for pod in $pods; do",
|
|
" printf '__POD__\\t%s\\n' \"$pod\"",
|
|
" kubectl -n \"$namespace\" logs \"$pod\" --all-containers --tail=200 2>/dev/null || true",
|
|
"done",
|
|
].join("\n");
|
|
}
|
|
|
|
export function v02RuntimeWorkloadsColumns(): string {
|
|
return [
|
|
"custom-columns=KIND:.kind",
|
|
"NAME:.metadata.name",
|
|
"SERVICE:.metadata.labels.hwlab\\.pikastech\\.local/service-id",
|
|
"ARTIFACT:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/artifact-source-commit",
|
|
"SOURCE:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/source-commit",
|
|
"IMAGE:.spec.template.spec.containers[0].image",
|
|
"READY:.status.readyReplicas",
|
|
"REPLICAS:.status.replicas",
|
|
].join(",");
|
|
}
|
|
|
|
export function v02WebAssetsProbeScript(): string {
|
|
return [
|
|
"set +e",
|
|
`base=${shellQuote(V02_CLOUD_WEB_URL)}`,
|
|
`api=${shellQuote(V02_CLOUD_API_URL)}`,
|
|
"fetch_url() {",
|
|
" if command -v curl >/dev/null 2>&1; then",
|
|
" curl -fsS --connect-timeout 2 --max-time 10 \"$1\"",
|
|
" elif command -v wget >/dev/null 2>&1; then",
|
|
" wget -q -T 10 -O - \"$1\"",
|
|
" else",
|
|
" return 127",
|
|
" fi",
|
|
"}",
|
|
"printf 'baseUrl\\t%s\\n' \"$base\"",
|
|
"printf 'apiUrl\\t%s\\n' \"$api\"",
|
|
"probe_start_s=$(date +%s 2>/dev/null || printf '0')",
|
|
"html=$(fetch_url \"$base/\" 2>/dev/null)",
|
|
"html_code=$?",
|
|
"printf 'htmlOk\\t%s\\n' \"$html_code\"",
|
|
"css_path=$(printf '%s' \"$html\" | sed -n 's/.*href=\"\\([^\"]*\\.css\\)\".*/\\1/p' | head -1)",
|
|
"printf 'cssPath\\t%s\\n' \"$css_path\"",
|
|
"case \"$css_path\" in",
|
|
" http://*|https://*) css_url=\"$css_path\" ;;",
|
|
" /*) css_url=\"$base$css_path\" ;;",
|
|
" \"\") css_url=\"\" ;;",
|
|
" *) css_url=\"$base/$css_path\" ;;",
|
|
"esac",
|
|
"printf 'htmlAppScript\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eq 'src=\"/app\\.js\"|src=\"app\\.js\"'; printf '%s' \"$?\")\"",
|
|
"printf 'htmlCssLink\\t%s\\n' \"$(test -n \"$css_path\"; printf '%s' \"$?\")\"",
|
|
"printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"",
|
|
"if [ -n \"$css_url\" ]; then css=$(fetch_url \"$css_url\" 2>/dev/null); else css=\"\"; false; fi",
|
|
"css_code=$?",
|
|
"printf 'cssOk\\t%s\\n' \"$css_code\"",
|
|
"printf 'cssBytes\\t%s\\n' \"$(printf '%s' \"$css\" | wc -c | tr -d ' ')\"",
|
|
"app_js=$(fetch_url \"$base/app.js\" 2>/dev/null)",
|
|
"app_js_code=$?",
|
|
"probe_end_s=$(date +%s 2>/dev/null || printf '0')",
|
|
"printf 'appJsOk\\t%s\\n' \"$app_js_code\"",
|
|
"printf 'appJsBytes\\t%s\\n' \"$(printf '%s' \"$app_js\" | wc -c | tr -d ' ')\"",
|
|
"printf 'probeElapsedMs\\t%s\\n' \"$(((probe_end_s - probe_start_s) * 1000))\"",
|
|
"fetch_url \"$base/styles.css\" >/dev/null 2>&1",
|
|
"legacy_styles_code=$?",
|
|
"if [ \"$legacy_styles_code\" = \"0\" ]; then legacy_styles_absent=1; else legacy_styles_absent=0; fi",
|
|
"printf 'legacyStylesAbsent\\t%s\\n' \"$legacy_styles_absent\"",
|
|
"printf 'reactWorkbenchCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'workbench-shell|activity-rail|session-sidebar|conversation-panel'; printf '%s' \"$?\")\"",
|
|
"health=$(fetch_url \"$api/health/live\" 2>/dev/null)",
|
|
"health_code=$?",
|
|
"printf 'apiHealthOk\\t%s\\n' \"$health_code\"",
|
|
"printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
|
if (!commandOk) {
|
|
return {
|
|
ok: false,
|
|
pipelineRun,
|
|
exitCode,
|
|
stderr: stderr.trim().slice(0, 2000),
|
|
counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
|
|
items: [],
|
|
performance: v02TaskRunPerformanceSummary([]),
|
|
};
|
|
}
|
|
const items = text
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.map((line) => {
|
|
const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t");
|
|
return {
|
|
name,
|
|
status: status || null,
|
|
reason: reason || null,
|
|
startTime: startTime || null,
|
|
completionTime: completionTime || null,
|
|
durationSeconds: secondsBetween(startTime, completionTime),
|
|
};
|
|
});
|
|
const counts = {
|
|
succeeded: items.filter((item) => item.status === "True").length,
|
|
failed: items.filter((item) => item.status === "False").length,
|
|
running: items.filter((item) => item.status === "Unknown").length,
|
|
unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
|
|
};
|
|
const performance = v02TaskRunPerformanceSummary(items);
|
|
const performanceWarning = performance.ok === false ? `; ${String(performance.summary ?? "")}` : "";
|
|
return {
|
|
ok: true,
|
|
pipelineRun,
|
|
counts,
|
|
items,
|
|
performance,
|
|
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}${performanceWarning}`,
|
|
disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
|
|
};
|
|
}
|
|
|
|
export function v02TaskRunPipelineTaskName(name: string): string | null {
|
|
const buildIndex = name.lastIndexOf("-build-");
|
|
if (buildIndex >= 0) return name.slice(buildIndex + 1);
|
|
const knownSuffixes = [
|
|
"prepare-source",
|
|
"plan-artifacts",
|
|
"publish-artifact-catalog",
|
|
"gitops-render",
|
|
"gitops-promote",
|
|
"runtime-ready",
|
|
"collect-artifacts",
|
|
];
|
|
return knownSuffixes.find((suffix) => name.endsWith(`-${suffix}`)) ?? null;
|
|
}
|
|
|
|
export function v02TaskRunPerformanceSummary(taskRuns: unknown[]): Record<string, unknown> {
|
|
const slowTaskRuns: Record<string, unknown>[] = [];
|
|
for (const itemRaw of taskRuns) {
|
|
const item = record(itemRaw);
|
|
const name = stringOrNull(item.name) ?? "";
|
|
const pipelineTask = stringOrNull(item.pipelineTask) ?? v02TaskRunPipelineTaskName(name);
|
|
const durationSeconds = typeof item.durationSeconds === "number" ? item.durationSeconds : null;
|
|
if (!pipelineTask?.startsWith("build-") || durationSeconds === null || durationSeconds <= V02_BUILD_TASKRUN_WARNING_SECONDS) continue;
|
|
const serviceId = pipelineTask.slice("build-".length) || null;
|
|
slowTaskRuns.push({
|
|
name,
|
|
pipelineTask,
|
|
serviceId,
|
|
status: stringOrNull(item.status),
|
|
reason: stringOrNull(item.reason),
|
|
durationSeconds,
|
|
budgetSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
|
severity: durationSeconds >= V02_BUILD_TASKRUN_CRITICAL_SECONDS ? "critical" : "warning",
|
|
message: `${pipelineTask} took ${durationSeconds}s, above v0.2 build TaskRun warning budget ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
|
});
|
|
}
|
|
slowTaskRuns.sort((left, right) => Number(right.durationSeconds ?? 0) - Number(left.durationSeconds ?? 0));
|
|
const worst = slowTaskRuns[0];
|
|
return {
|
|
ok: slowTaskRuns.length === 0,
|
|
warningCount: slowTaskRuns.length,
|
|
thresholds: {
|
|
buildTaskRunWarningSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
|
buildTaskRunCriticalSeconds: V02_BUILD_TASKRUN_CRITICAL_SECONDS,
|
|
},
|
|
slowTaskRuns,
|
|
summary: worst
|
|
? `slow build taskruns=${slowTaskRuns.length}; worst=${String(worst.pipelineTask)} ${String(worst.durationSeconds)}s budget=${V02_BUILD_TASKRUN_WARNING_SECONDS}s`
|
|
: `no build taskrun over ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
|
};
|
|
}
|
|
|
|
export function v02WebAssetsFromText(
|
|
text: string,
|
|
commandOk: boolean,
|
|
sourceCommit: string | null,
|
|
argoSyncRevision: string | null,
|
|
exitCode: number | null,
|
|
stderr: string,
|
|
activePipelineRuns: unknown[] = [],
|
|
): Record<string, unknown> {
|
|
const fields: Record<string, string> = {};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const [key = "", ...rest] = line.split("\t");
|
|
if (key.length > 0) fields[key] = rest.join("\t");
|
|
}
|
|
const htmlOk = fields.htmlOk === "0";
|
|
const htmlAppScript = fields.htmlAppScript === "0";
|
|
const htmlCssLink = fields.htmlCssLink === "0";
|
|
const cssOk = fields.cssOk === "0";
|
|
const appJsOk = fields.appJsOk === "0";
|
|
const apiHealthOk = fields.apiHealthOk === "0";
|
|
const readonlyNoteAbsent = fields.readonlyNote === "1";
|
|
const legacyStylesAbsent = fields.legacyStylesAbsent === "0";
|
|
const reactWorkbenchCss = fields.reactWorkbenchCss === "0";
|
|
const apiRevision = fields.apiRevision || null;
|
|
const webChecksPass = htmlOk && htmlAppScript && htmlCssLink && cssOk && appJsOk && readonlyNoteAbsent && legacyStylesAbsent && reactWorkbenchCss && apiHealthOk;
|
|
const failedChecks = Object.entries({
|
|
htmlOk,
|
|
htmlAppScript,
|
|
htmlCssLink,
|
|
cssOk,
|
|
appJsOk,
|
|
readonlyNoteAbsent,
|
|
legacyStylesAbsent,
|
|
reactWorkbenchCss,
|
|
apiHealthOk,
|
|
}).filter(([, ok]) => !ok).map(([name]) => name);
|
|
return {
|
|
ok: commandOk && webChecksPass,
|
|
summary: commandOk && webChecksPass ? "19666/19667 React/Vite asset probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`,
|
|
baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL,
|
|
apiUrl: fields.apiUrl || V02_CLOUD_API_URL,
|
|
sourceCommit,
|
|
argoSyncRevision: argoSyncRevision || null,
|
|
cssPath: fields.cssPath || null,
|
|
checks: {
|
|
htmlOk,
|
|
htmlAppScript,
|
|
htmlCssLink,
|
|
cssOk,
|
|
appJsOk,
|
|
readonlyNoteAbsent,
|
|
legacyStylesAbsent,
|
|
reactWorkbenchCss,
|
|
apiHealthOk,
|
|
},
|
|
probeExitCodes: {
|
|
html: numericField(fields.htmlOk),
|
|
htmlAppScript: numericField(fields.htmlAppScript),
|
|
htmlCssLink: numericField(fields.htmlCssLink),
|
|
css: numericField(fields.cssOk),
|
|
appJs: numericField(fields.appJsOk),
|
|
readonlyNoteGrep: numericField(fields.readonlyNote),
|
|
legacyStylesAbsent,
|
|
reactWorkbenchCssGrep: numericField(fields.reactWorkbenchCss),
|
|
apiHealth: numericField(fields.apiHealthOk),
|
|
},
|
|
assetBytes: {
|
|
css: numericField(fields.cssBytes),
|
|
appJs: numericField(fields.appJsBytes),
|
|
},
|
|
probeElapsedMs: numericField(fields.probeElapsedMs),
|
|
apiRevision,
|
|
note: webAssetsRevisionNote(apiRevision, sourceCommit, activePipelineRuns),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
|
|
export function v02PlanArtifactsFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
|
const events: Record<string, unknown>[] = [];
|
|
const pods: string[] = [];
|
|
for (const rawLine of text.split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (line.startsWith("__POD__")) {
|
|
const [, pod = ""] = line.split("\t");
|
|
if (pod.length > 0) pods.push(pod);
|
|
continue;
|
|
}
|
|
if (!line.startsWith("{")) continue;
|
|
try {
|
|
const event = record(JSON.parse(line) as unknown);
|
|
if (event.event === "g14-ci-plan") events.push(event);
|
|
} catch {
|
|
// Ignore non-JSON log lines; the raw log remains visible through pod logs.
|
|
}
|
|
}
|
|
const latest = events.at(-1) ?? null;
|
|
if (!commandOk || latest === null) {
|
|
return {
|
|
ok: false,
|
|
pipelineRun,
|
|
pods,
|
|
eventFound: latest !== null,
|
|
degradedReason: commandOk ? "g14-ci-plan-event-not-found" : "plan-artifacts-log-query-failed",
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
const audit = record(latest.artifactProvenanceAudit);
|
|
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
|
|
const provenanceRebuildServices = Array.isArray(audit.provenanceRebuildServices) ? audit.provenanceRebuildServices : [];
|
|
return {
|
|
ok: true,
|
|
pipelineRun,
|
|
pods,
|
|
sourceCommitId: stringOrNull(latest.sourceCommitId),
|
|
affectedServices: stringArray(latest.affectedServices),
|
|
rolloutServices: stringArray(latest.rolloutServices),
|
|
buildServices: stringArray(latest.buildServices),
|
|
reusedServices: stringArray(latest.reusedServices),
|
|
buildSkippedCount: numericValue(latest.buildSkippedCount),
|
|
artifactProvenanceAudit: Object.keys(audit).length > 0 ? audit : null,
|
|
summary: `build=${stringArray(latest.buildServices).length} reuse=${stringArray(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`,
|
|
disclosure: "parsed from plan-artifacts g14-ci-plan log event",
|
|
};
|
|
}
|
|
|
|
export function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
|
if (!commandOk) {
|
|
return {
|
|
ok: false,
|
|
items: [],
|
|
exitCode,
|
|
stderr: stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
const items = text
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.map((line) => {
|
|
const [kind = "", name = "", serviceId = "", artifactSourceCommit = "", sourceCommit = "", image = "", ready = "", replicas = ""] = line.split(/\s+/u);
|
|
return {
|
|
kind,
|
|
name,
|
|
serviceId: serviceId === "<none>" ? null : serviceId,
|
|
artifactSourceCommit: artifactSourceCommit === "<none>" ? null : artifactSourceCommit,
|
|
sourceCommit: sourceCommit === "<none>" ? null : sourceCommit,
|
|
image: image === "<none>" ? null : image,
|
|
readyReplicas: numericField(ready),
|
|
replicas: numericField(replicas),
|
|
};
|
|
});
|
|
return {
|
|
ok: true,
|
|
items,
|
|
summary: `runtime deployments=${items.length}`,
|
|
};
|
|
}
|
|
|
|
export function keyValueLinesFromText(text: string): Record<string, string> {
|
|
const fields: Record<string, string> = {};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const [key = "", ...rest] = line.split("\t");
|
|
if (key.trim().length > 0) fields[key.trim()] = rest.join("\t").trim();
|
|
}
|
|
return fields;
|
|
}
|
|
|
|
export function v02SourceHeadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
|
|
const fields = keyValueLinesFromText(text);
|
|
const dirtyCount = numericField(fields.workspaceDirtyCount);
|
|
const targetSourceCommit = fields.targetSourceCommit || null;
|
|
return {
|
|
ok: commandOk,
|
|
cicdRepo: fields.cicdRepo || V02_CICD_REPO,
|
|
cicdRepoExists: fields.cicdRepoExists === "yes",
|
|
cicdSourceHead: fields.cicdSourceHead || null,
|
|
originHead: fields.originHead || null,
|
|
target: {
|
|
sourceCommit: targetSourceCommit,
|
|
objectExists: fieldBoolean(fields.targetObjectExists),
|
|
ancestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead),
|
|
ancestorExitCode: numericField(fields.targetAncestorExitCode),
|
|
},
|
|
targetObjectExists: fieldBoolean(fields.targetObjectExists),
|
|
targetAncestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead),
|
|
workspace: {
|
|
path: fields.workspacePath || V02_WORKSPACE,
|
|
branch: fields.workspaceBranch || null,
|
|
head: fields.workspaceHead || null,
|
|
originHead: fields.workspaceOriginHead || null,
|
|
dirtyCount,
|
|
dirty: typeof dirtyCount === "number" ? dirtyCount > 0 : null,
|
|
isolatedFromCicd: true,
|
|
},
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
|
|
export function v02FalseGreenGuard(input: {
|
|
sourceCommit: string | null;
|
|
pipelineRun: Record<string, unknown> | null;
|
|
taskRuns: Record<string, unknown>;
|
|
planArtifacts: Record<string, unknown>;
|
|
runtimeWorkloads: Record<string, unknown>;
|
|
}): Record<string, unknown> {
|
|
const status = stringOrNull(input.pipelineRun?.status);
|
|
if (input.sourceCommit === null || input.pipelineRun === null || input.pipelineRun.exists === false) {
|
|
return { ok: null, state: "not-started", summary: "current source commit has no PipelineRun yet" };
|
|
}
|
|
if (status !== "True") {
|
|
return { ok: null, state: "pipeline-not-final", summary: `PipelineRun status=${status ?? "unknown"}` };
|
|
}
|
|
if (input.planArtifacts.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
state: "missing-plan-artifacts-evidence",
|
|
summary: "PipelineRun succeeded but plan-artifacts g14-ci-plan evidence was not visible",
|
|
planArtifacts: input.planArtifacts,
|
|
};
|
|
}
|
|
const sourceCommit = input.sourceCommit;
|
|
const buildServices = stringArray(input.planArtifacts.buildServices);
|
|
const reusedServices = stringArray(input.planArtifacts.reusedServices);
|
|
const taskItems = Array.isArray(input.taskRuns.items) ? input.taskRuns.items.map((item) => record(item)) : [];
|
|
const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : [];
|
|
const workloadByService = new Map(workloadItems.map((item) => [String(item.serviceId ?? ""), item]));
|
|
const missingBuildTaskRuns = buildServices.filter((serviceId) => !taskItems.some((item) => String(item.name ?? "").endsWith(`build-${serviceId}`) && item.status === "True"));
|
|
const runtimeMismatches = buildServices
|
|
.map((serviceId) => {
|
|
const workload = workloadByService.get(serviceId);
|
|
if (!workload) return { serviceId, reason: "runtime-workload-missing" };
|
|
if (workload.artifactSourceCommit !== sourceCommit) {
|
|
return {
|
|
serviceId,
|
|
reason: "artifact-source-commit-mismatch",
|
|
runtimeArtifactSourceCommit: workload.artifactSourceCommit ?? null,
|
|
expectedSourceCommit: sourceCommit,
|
|
image: workload.image ?? null,
|
|
};
|
|
}
|
|
return null;
|
|
})
|
|
.filter((item): item is Record<string, unknown> => item !== null);
|
|
const audit = record(input.planArtifacts.artifactProvenanceAudit);
|
|
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
|
|
const auditPresent = input.planArtifacts.artifactProvenanceAudit !== null;
|
|
const provenanceWarning = !auditPresent && reusedServices.length > 0
|
|
? { reason: "artifact-provenance-audit-missing-for-reuse", reusedServices }
|
|
: null;
|
|
const provenanceOk = auditPresent ? audit.ok !== false && unsafeReuseServices.length === 0 : true;
|
|
const failures = [
|
|
...(input.runtimeWorkloads.ok === true ? [] : [{ reason: "runtime-workload-query-failed" }]),
|
|
...(provenanceOk ? [] : [{ reason: "artifact-provenance-audit-failed", unsafeReuseServices }]),
|
|
...missingBuildTaskRuns.map((serviceId) => ({ serviceId, reason: "expected-build-taskrun-not-succeeded" })),
|
|
...runtimeMismatches,
|
|
];
|
|
return {
|
|
ok: failures.length === 0,
|
|
state: failures.length === 0 ? "passed" : "failed",
|
|
summary: failures.length === 0
|
|
? `artifact provenance and built runtime services align with ${shortSha(sourceCommit)}`
|
|
: `false-green risk: ${failures.length} invariant violation(s)`,
|
|
sourceCommit,
|
|
buildServices,
|
|
reusedServices,
|
|
provenanceOk,
|
|
provenanceAuditPresent: auditPresent,
|
|
provenanceWarnings: provenanceWarning ? [provenanceWarning] : [],
|
|
missingBuildTaskRuns,
|
|
runtimeMismatches,
|
|
failures,
|
|
};
|
|
}
|
|
|
|
export function v02TargetValidation(input: {
|
|
targetMode: V02StatusTargetMode;
|
|
sourceCommit: string | null;
|
|
pipelineRun: Record<string, unknown> | null;
|
|
argo: Record<string, unknown>;
|
|
planArtifacts: Record<string, unknown>;
|
|
runtimeWorkloads: Record<string, unknown>;
|
|
webAssets: Record<string, unknown>;
|
|
gitMirror: Record<string, unknown>;
|
|
recentPipelineRuns: Record<string, unknown>;
|
|
}): Record<string, unknown> {
|
|
if (input.targetMode === "latest-source-head") {
|
|
return {
|
|
applicable: false,
|
|
state: "latest-source-head",
|
|
summary: "default status uses strict latest source head alignment; use --pipeline-run or --source-commit for historical run validation",
|
|
};
|
|
}
|
|
const sourceCommit = input.sourceCommit;
|
|
const argoFields = record(input.argo.fields);
|
|
const gitMirrorSummary = record(input.gitMirror.summary);
|
|
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
|
|
? input.runtimeWorkloads.items.map((item) => record(item))
|
|
: [];
|
|
const serviceSourceCommits = Object.fromEntries(workloadItems
|
|
.filter((item) => item.serviceId === "hwlab-cloud-api" || item.serviceId === "hwlab-cloud-web")
|
|
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
|
|
const rolloutServices = stringArray(input.planArtifacts.rolloutServices)
|
|
.filter((serviceId) => serviceId === "hwlab-cloud-api" || serviceId === "hwlab-cloud-web");
|
|
const requiredRuntimeServices = rolloutServices;
|
|
const recentItems = Array.isArray(input.recentPipelineRuns.items)
|
|
? input.recentPipelineRuns.items.map((item) => record(item))
|
|
: [];
|
|
const targetPipelineRunName = stringOrNull(input.pipelineRun?.pipelineRun) ?? stringOrNull(input.pipelineRun?.name);
|
|
const targetRecentIndex = recentItems.findIndex((item) => (
|
|
(targetPipelineRunName !== null && item.name === targetPipelineRunName)
|
|
|| (sourceCommit !== null && item.sourceCommit === sourceCommit)
|
|
));
|
|
const newerCandidates = targetRecentIndex >= 0 ? recentItems.slice(0, targetRecentIndex) : recentItems;
|
|
const newerSucceededRuns = sourceCommit === null
|
|
? []
|
|
: newerCandidates.filter((item) => item.sourceCommit !== sourceCommit && item.status === "True");
|
|
const failures: Record<string, unknown>[] = [];
|
|
const supersededRuntimeServices: Record<string, unknown>[] = [];
|
|
if (sourceCommit === null) failures.push({ reason: "source-commit-unresolved" });
|
|
if (input.pipelineRun === null || input.pipelineRun.exists === false) {
|
|
failures.push({ reason: "pipeline-run-missing" });
|
|
} else if (input.pipelineRun.status !== "True") {
|
|
failures.push({ reason: "pipeline-run-not-succeeded", status: input.pipelineRun.status ?? null, reasonDetail: input.pipelineRun.reason ?? null });
|
|
}
|
|
if (input.argo.ok !== true || argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") {
|
|
failures.push({ reason: "argo-not-synced-healthy", syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null });
|
|
}
|
|
if (input.webAssets.ok !== true) failures.push({ reason: "web-assets-probe-failed", summary: input.webAssets.summary ?? null });
|
|
if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) {
|
|
failures.push({
|
|
reason: "git-mirror-not-flushed",
|
|
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
|
|
githubInSync: gitMirrorSummary.githubInSync ?? null,
|
|
});
|
|
}
|
|
for (const serviceId of requiredRuntimeServices) {
|
|
const serviceCommit = serviceSourceCommits[serviceId];
|
|
if (sourceCommit !== null && serviceCommit !== sourceCommit) {
|
|
const mismatch = { reason: "runtime-service-source-mismatch", serviceId, expectedSourceCommit: sourceCommit, actualSourceCommit: serviceCommit ?? null };
|
|
if (newerSucceededRuns.length > 0) supersededRuntimeServices.push(mismatch);
|
|
else failures.push(mismatch);
|
|
}
|
|
}
|
|
const superseded = failures.length === 0 && supersededRuntimeServices.length > 0;
|
|
return {
|
|
applicable: true,
|
|
ok: failures.length === 0,
|
|
state: failures.length === 0 ? (superseded ? "superseded" : "passed") : "failed",
|
|
summary: failures.length === 0
|
|
? superseded
|
|
? `target ${input.targetMode} completed for ${shortSha(sourceCommit ?? "")}; runtime now reflects newer v0.2 PipelineRun`
|
|
: requiredRuntimeServices.length === 0
|
|
? `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}; no runtime service rollout required`
|
|
: `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}`
|
|
: `target ${input.targetMode} validation failed with ${failures.length} issue(s)`,
|
|
sourceCommit,
|
|
pipelineRun: input.pipelineRun === null
|
|
? null
|
|
: { name: input.pipelineRun.pipelineRun ?? null, status: input.pipelineRun.status ?? null, reason: input.pipelineRun.reason ?? null },
|
|
argo: { syncRevision: argoFields.syncRevision ?? null, syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null },
|
|
runtimeServices: serviceSourceCommits,
|
|
requiredRuntimeServices,
|
|
rolloutServices: stringArray(input.planArtifacts.rolloutServices),
|
|
reusedServices: stringArray(input.planArtifacts.reusedServices),
|
|
superseded,
|
|
supersededRuntimeServices,
|
|
newerSucceededRuns: newerSucceededRuns.slice(0, 3).map((item) => ({
|
|
name: item.name ?? null,
|
|
sourceCommit: item.sourceCommit ?? null,
|
|
createdAt: item.createdAt ?? null,
|
|
durationSeconds: item.durationSeconds ?? null,
|
|
})),
|
|
webAssets: { ok: input.webAssets.ok ?? null, summary: input.webAssets.summary ?? null, appJsBytes: record(input.webAssets.assetBytes).appJs ?? null },
|
|
gitMirror: { pendingFlush: gitMirrorSummary.pendingFlush ?? null, githubInSync: gitMirrorSummary.githubInSync ?? null },
|
|
failures,
|
|
};
|
|
}
|
|
|
|
export function v02LatestOnlyTargetValidation(input: {
|
|
targetMode: string;
|
|
sourceCommit: string | null;
|
|
pipelineRun: Record<string, unknown> | null;
|
|
commitAlignment: Record<string, unknown>;
|
|
targetValidation: Record<string, unknown>;
|
|
}): Record<string, unknown> {
|
|
if (input.targetMode === "latest-source-head") return input.targetValidation;
|
|
if (input.sourceCommit === null) return input.targetValidation;
|
|
if (input.pipelineRun === null || input.pipelineRun.status !== "True") return input.targetValidation;
|
|
const validation = record(input.targetValidation);
|
|
const staleReasons = stringArray(input.commitAlignment.staleReasons);
|
|
const sourceHeadAdvancedReasons = staleReasons.filter((reason) => (
|
|
reason === "origin-head-mismatch"
|
|
|| reason === "cicd-source-repo-stale"
|
|
|| reason === "latest-pipelinerun-not-current"
|
|
));
|
|
if (sourceHeadAdvancedReasons.length === 0) return input.targetValidation;
|
|
const failures = Array.isArray(validation.failures) ? validation.failures : [];
|
|
return {
|
|
...validation,
|
|
ok: true,
|
|
state: "superseded",
|
|
superseded: true,
|
|
latestOnlySuperseded: true,
|
|
latestOnlyReasons: sourceHeadAdvancedReasons,
|
|
originalState: validation.state ?? null,
|
|
summary: validation.ok === true || validation.state === "passed"
|
|
? `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and is superseded by the newer v0.2 source head`
|
|
: `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`,
|
|
supersededFailures: failures.slice(0, 10),
|
|
failures: [],
|
|
};
|
|
}
|
|
|
|
export function v02CloseoutRecommendedNext(input: {
|
|
closeable: boolean;
|
|
blockedReasons: string[];
|
|
pendingReasons: string[];
|
|
statusCommand: string;
|
|
sourceCommit: string | null;
|
|
gitMirrorSummary: Record<string, unknown>;
|
|
}): Record<string, unknown> {
|
|
const rawFlushCommand = stringOrNull(input.gitMirrorSummary.flushCommand) ?? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm";
|
|
const flushCommand = rawFlushCommand.includes("--wait") ? rawFlushCommand : `${rawFlushCommand} --wait`;
|
|
if (input.closeable) {
|
|
return {
|
|
action: "comment-and-close-issue",
|
|
command: input.sourceCommit === null
|
|
? input.statusCommand
|
|
: `bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit ${input.sourceCommit}`,
|
|
};
|
|
}
|
|
if (input.pendingReasons.includes("git-mirror-not-flushed")) {
|
|
return { action: "flush-git-mirror", command: flushCommand };
|
|
}
|
|
if (input.pendingReasons.includes("argo-not-healthy")) {
|
|
return { action: "wait-and-recheck", command: input.statusCommand };
|
|
}
|
|
if (input.pendingReasons.includes("pipeline-run-not-final")) {
|
|
return { action: "wait-pipelinerun", command: input.statusCommand };
|
|
}
|
|
if (input.blockedReasons.includes("target-pipelinerun-missing")) {
|
|
return { action: "trigger-current-or-inspect-target", command: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" };
|
|
}
|
|
return { action: "inspect-status", command: input.statusCommand };
|
|
}
|
|
|
|
export function v02CloseoutVerdict(status: Record<string, unknown>): Record<string, unknown> {
|
|
const statusTarget = record(status.statusTarget);
|
|
const targetMode = stringOrNull(statusTarget.mode) ?? "latest-source-head";
|
|
const sourceCommit = stringOrNull(status.sourceCommit) ?? stringOrNull(statusTarget.sourceCommit);
|
|
const pipelineRun = record(status.pipelineRun);
|
|
const targetValidation = record(status.targetValidation);
|
|
const targetValidationState = stringOrNull(targetValidation.state);
|
|
const argoFields = record(record(status.argo).fields);
|
|
const gitMirrorSummary = record(record(status.gitMirror).summary);
|
|
const sourceHeadsTarget = record(record(status.sourceHeads).target);
|
|
const commitAlignment = record(status.commitAlignment);
|
|
const latestHead = stringOrNull(commitAlignment.originHead) ?? stringOrNull(record(status.sourceHeads).originHead);
|
|
const ancestorOfLatest = sourceHeadsTarget.ancestorOfOriginHead ?? status.sourceHeadsTargetAncestorOfOriginHead ?? null;
|
|
const activePipelineRuns = Array.isArray(status.activePipelineRuns)
|
|
? status.activePipelineRuns.map((item) => record(item))
|
|
: [];
|
|
const pipelineStatus = stringOrNull(pipelineRun.status);
|
|
const pipelineExists = pipelineRun.exists !== false && Object.keys(pipelineRun).length > 0;
|
|
const pendingReasons: string[] = [];
|
|
const blockedReasons: string[] = [];
|
|
if (targetMode === "latest-source-head") {
|
|
pendingReasons.push("target-not-explicit");
|
|
}
|
|
if (!pipelineExists) {
|
|
blockedReasons.push("target-pipelinerun-missing");
|
|
} else if (pipelineStatus === "Unknown" || pipelineStatus === null) {
|
|
pendingReasons.push("pipeline-run-not-final");
|
|
} else if (pipelineStatus === "False") {
|
|
blockedReasons.push("pipeline-run-failed");
|
|
}
|
|
if (targetValidationState !== "passed" && targetValidationState !== "superseded") {
|
|
const failures = Array.isArray(targetValidation.failures) ? targetValidation.failures.map((item) => record(item)) : [];
|
|
for (const failure of failures) {
|
|
const reason = stringOrNull(failure.reason);
|
|
if (reason === "git-mirror-not-flushed") pendingReasons.push(reason);
|
|
else if (reason === "argo-not-synced-healthy") pendingReasons.push("argo-not-healthy");
|
|
else if (reason === "pipeline-run-not-succeeded" && pipelineStatus === "Unknown") pendingReasons.push("pipeline-run-not-final");
|
|
else if (reason !== null) blockedReasons.push(reason);
|
|
}
|
|
if (failures.length === 0 && targetMode !== "latest-source-head") blockedReasons.push("target-validation-not-passed");
|
|
}
|
|
if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) {
|
|
pendingReasons.push("git-mirror-not-flushed");
|
|
}
|
|
if (argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") {
|
|
pendingReasons.push("argo-not-healthy");
|
|
}
|
|
if (targetValidationState === "superseded" && ancestorOfLatest !== true) {
|
|
if (ancestorOfLatest === false) blockedReasons.push("target-not-ancestor-of-latest-head");
|
|
else pendingReasons.push("ancestor-check-unresolved");
|
|
}
|
|
const uniquePendingReasons = Array.from(new Set(pendingReasons));
|
|
const uniqueBlockedReasons = Array.from(new Set(blockedReasons));
|
|
const terminalState = targetValidationState === "passed" || targetValidationState === "superseded";
|
|
const state = uniqueBlockedReasons.length > 0
|
|
? "blocked"
|
|
: uniquePendingReasons.length > 0
|
|
? "pending"
|
|
: terminalState
|
|
? targetValidationState
|
|
: "blocked";
|
|
const closeable = (state === "passed" || state === "superseded") && uniquePendingReasons.length === 0 && uniqueBlockedReasons.length === 0;
|
|
const rawStatusCommand = String(status.command ?? (
|
|
sourceCommit === null
|
|
? "bun scripts/cli.ts hwlab g14 control-plane status --lane v02"
|
|
: `bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}`
|
|
));
|
|
const statusCommand = rawStatusCommand.startsWith("bun ") ? rawStatusCommand : `bun scripts/cli.ts ${rawStatusCommand}`;
|
|
const ancestorCheck = {
|
|
required: state === "superseded",
|
|
sourceCommit,
|
|
latestHead,
|
|
objectExists: sourceHeadsTarget.objectExists ?? null,
|
|
ancestorOfLatest,
|
|
exitCode: sourceHeadsTarget.ancestorExitCode ?? null,
|
|
command: sourceCommit === null
|
|
? null
|
|
: `trans G14:/root/hwlab-v02 sh -- 'git fetch origin v0.2 && git merge-base --is-ancestor ${sourceCommit} origin/v0.2'`,
|
|
};
|
|
const recommendedNext = v02CloseoutRecommendedNext({
|
|
closeable,
|
|
blockedReasons: uniqueBlockedReasons,
|
|
pendingReasons: uniquePendingReasons,
|
|
statusCommand,
|
|
sourceCommit,
|
|
gitMirrorSummary,
|
|
});
|
|
const summary = closeable
|
|
? state === "superseded"
|
|
? `closeout ready: target ${shortSha(sourceCommit ?? "")} passed and is superseded by latest v0.2 head ${shortSha(latestHead ?? "")}`
|
|
: `closeout ready: target ${shortSha(sourceCommit ?? "")} passed on v0.2`
|
|
: uniquePendingReasons.length > 0
|
|
? `closeout pending: ${uniquePendingReasons.join(",")}`
|
|
: `closeout blocked: ${uniqueBlockedReasons.join(",") || "unknown"}`;
|
|
const issueCommentMarkdown = [
|
|
`v0.2 closeout verdict: \`${state}\` (${summary})`,
|
|
"",
|
|
`- target sourceCommit: \`${sourceCommit ?? "n/a"}\``,
|
|
`- target PipelineRun: \`${String(pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? "n/a")}\`; status=\`${pipelineStatus ?? "n/a"}\`; reason=\`${String(pipelineRun.reason ?? "n/a")}\``,
|
|
`- latest head: \`${latestHead ?? "n/a"}\`; ancestorOfLatest=\`${String(ancestorCheck.ancestorOfLatest ?? "n/a")}\``,
|
|
`- targetValidation: \`${targetValidationState ?? "n/a"}\`; pending=\`${uniquePendingReasons.join(",") || "none"}\`; blocked=\`${uniqueBlockedReasons.join(",") || "none"}\``,
|
|
`- Git mirror: pendingFlush=\`${String(gitMirrorSummary.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(gitMirrorSummary.githubInSync ?? "n/a")}\`, flushCommand=\`${String(gitMirrorSummary.flushCommand ?? "none")}\``,
|
|
`- Argo: sync=\`${String(argoFields.syncStatus ?? "n/a")}\`, health=\`${String(argoFields.health ?? "n/a")}\`, revision=\`${String(argoFields.syncRevision ?? "n/a")}\``,
|
|
`- active v0.2 PipelineRuns: \`${activePipelineRuns.length}\``,
|
|
`- recommended next: \`${String(recommendedNext.action ?? "inspect-status")}\` -> \`${String(recommendedNext.command ?? statusCommand)}\``,
|
|
].join("\n");
|
|
return {
|
|
ok: closeable,
|
|
closeable,
|
|
state,
|
|
summary,
|
|
targetMode,
|
|
sourceCommit,
|
|
latestHead,
|
|
pipelineRun: {
|
|
name: pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? null,
|
|
exists: pipelineExists,
|
|
status: pipelineStatus,
|
|
reason: pipelineRun.reason ?? null,
|
|
},
|
|
targetValidation: {
|
|
state: targetValidationState,
|
|
ok: targetValidation.ok ?? null,
|
|
failures: Array.isArray(targetValidation.failures) ? targetValidation.failures.slice(0, 10) : [],
|
|
},
|
|
latestOnly: {
|
|
superseded: targetValidation.latestOnlySuperseded === true || state === "superseded",
|
|
reasons: stringArray(targetValidation.latestOnlyReasons),
|
|
ancestorCheck,
|
|
},
|
|
gitMirror: {
|
|
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
|
|
githubInSync: gitMirrorSummary.githubInSync ?? null,
|
|
flushNeeded: gitMirrorSummary.flushNeeded ?? null,
|
|
flushCommand: gitMirrorSummary.flushCommand ?? null,
|
|
lastSync: gitMirrorSummary.lastSync ?? null,
|
|
lastWrite: gitMirrorSummary.lastWrite ?? null,
|
|
lastFlush: gitMirrorSummary.lastFlush ?? null,
|
|
},
|
|
argo: {
|
|
syncStatus: argoFields.syncStatus ?? null,
|
|
health: argoFields.health ?? null,
|
|
syncRevision: argoFields.syncRevision ?? null,
|
|
targetRevision: argoFields.targetRevision ?? null,
|
|
path: argoFields.path ?? null,
|
|
},
|
|
runtime: {
|
|
webAssetsOk: nested(status, ["webAssets", "ok"]) ?? null,
|
|
webAssetsSummary: nested(status, ["webAssets", "summary"]) ?? null,
|
|
apiRevision: nested(status, ["webAssets", "apiRevision"]) ?? nested(status, ["commitAlignment", "apiRevision"]) ?? null,
|
|
serviceSourceCommits: nested(status, ["commitAlignment", "serviceSourceCommits"]) ?? null,
|
|
},
|
|
activePipelineRuns,
|
|
recentPipelineRuns: status.recentPipelineRuns ?? null,
|
|
pendingReasons: uniquePendingReasons,
|
|
blockedReasons: uniqueBlockedReasons,
|
|
recommendedNext,
|
|
issueCommentMarkdown,
|
|
};
|
|
}
|
|
|
|
export function v02CommitAlignment(input: {
|
|
expectedSourceHead: string | null;
|
|
sourceHeads: Record<string, unknown>;
|
|
gitMirrorSummary: Record<string, unknown>;
|
|
pipelineRun: Record<string, unknown> | null;
|
|
recentPipelineRuns: Record<string, unknown>;
|
|
planArtifacts?: Record<string, unknown>;
|
|
runtimeWorkloads: Record<string, unknown>;
|
|
webAssets: Record<string, unknown>;
|
|
}): Record<string, unknown> {
|
|
const expectedSourceHead = input.expectedSourceHead;
|
|
const cicdSourceHead = stringOrNull(input.sourceHeads.cicdSourceHead);
|
|
const originHead = stringOrNull(input.sourceHeads.originHead) ?? expectedSourceHead;
|
|
const workspace = record(input.sourceHeads.workspace);
|
|
const workspaceHead = stringOrNull(workspace.head);
|
|
const workspaceOriginHead = stringOrNull(workspace.originHead);
|
|
const mirrorSourceHead = stringOrNull(input.gitMirrorSummary.localV02);
|
|
const mirrorGithubSourceHead = stringOrNull(input.gitMirrorSummary.githubV02);
|
|
const recentItems = Array.isArray(input.recentPipelineRuns.items)
|
|
? input.recentPipelineRuns.items.map((item) => record(item))
|
|
: [];
|
|
const latestPipelineRun = recentItems[0] ?? null;
|
|
const latestPipelineSourceCommit = latestPipelineRun === null ? null : stringOrNull(latestPipelineRun.sourceCommit);
|
|
const currentPipelineStatus = stringOrNull(input.pipelineRun?.status);
|
|
const apiRevision = stringOrNull(input.webAssets.apiRevision);
|
|
const planArtifacts = record(input.planArtifacts);
|
|
const planSourceCommit = stringOrNull(planArtifacts.sourceCommitId);
|
|
const rolloutServices = stringArray(planArtifacts.rolloutServices);
|
|
const apiRevisionRequired = expectedSourceHead !== null && planSourceCommit === expectedSourceHead && rolloutServices.includes("hwlab-cloud-api");
|
|
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
|
|
? input.runtimeWorkloads.items.map((item) => record(item))
|
|
: [];
|
|
const serviceSourceCommits = Object.fromEntries(workloadItems
|
|
.filter((item) => typeof item.serviceId === "string" && item.serviceId.length > 0)
|
|
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
|
|
const staleReasons: string[] = [];
|
|
if (expectedSourceHead === null) staleReasons.push("expected-source-head-unresolved");
|
|
if (expectedSourceHead !== null && originHead !== null && originHead !== expectedSourceHead) staleReasons.push("origin-head-mismatch");
|
|
if (expectedSourceHead !== null && cicdSourceHead !== null && cicdSourceHead !== expectedSourceHead) staleReasons.push("cicd-source-repo-stale");
|
|
if (expectedSourceHead !== null && mirrorSourceHead !== expectedSourceHead) staleReasons.push("mirror-source-stale");
|
|
if (expectedSourceHead !== null && latestPipelineSourceCommit !== expectedSourceHead) staleReasons.push("latest-pipelinerun-not-current");
|
|
if (apiRevisionRequired && apiRevision !== expectedSourceHead) staleReasons.push("runtime-api-revision-stale");
|
|
const workspaceWarnings: string[] = [];
|
|
if (expectedSourceHead !== null && workspaceHead !== null && workspaceHead !== expectedSourceHead) workspaceWarnings.push("workspace-head-differs-from-latest-source-but-isolated");
|
|
if (expectedSourceHead !== null && workspaceOriginHead !== null && workspaceOriginHead !== expectedSourceHead) workspaceWarnings.push("workspace-origin-ref-stale-but-isolated");
|
|
if (workspace.dirty === true) workspaceWarnings.push("workspace-dirty-but-isolated-from-cicd");
|
|
const runtimeWarnings: string[] = [];
|
|
if (!apiRevisionRequired && expectedSourceHead !== null && apiRevision !== null && apiRevision !== expectedSourceHead) {
|
|
runtimeWarnings.push("api-revision-differs-without-current-cloud-api-rollout");
|
|
}
|
|
const latestPipelineSucceeded = latestPipelineRun !== null && latestPipelineRun.status === "True";
|
|
const aligned = staleReasons.length === 0;
|
|
const state = aligned
|
|
? "aligned"
|
|
: latestPipelineSucceeded
|
|
? "stale-success"
|
|
: currentPipelineStatus === "Unknown"
|
|
? "in-progress"
|
|
: "stale";
|
|
return {
|
|
aligned,
|
|
state,
|
|
expectedSourceHead,
|
|
originHead,
|
|
cicdSourceHead,
|
|
cicdRepo: input.sourceHeads.cicdRepo ?? V02_CICD_REPO,
|
|
mirrorSourceHead,
|
|
mirrorGithubSourceHead,
|
|
latestPipelineRun: latestPipelineRun === null
|
|
? null
|
|
: {
|
|
name: latestPipelineRun.name ?? null,
|
|
sourceCommit: latestPipelineSourceCommit,
|
|
status: latestPipelineRun.status ?? null,
|
|
reason: latestPipelineRun.reason ?? null,
|
|
createdAt: latestPipelineRun.createdAt ?? null,
|
|
durationSeconds: latestPipelineRun.durationSeconds ?? null,
|
|
},
|
|
latestPipelineSourceCommit,
|
|
currentPipelineRun: input.pipelineRun === null
|
|
? null
|
|
: {
|
|
name: input.pipelineRun.pipelineRun ?? null,
|
|
status: input.pipelineRun.status ?? null,
|
|
reason: input.pipelineRun.reason ?? null,
|
|
},
|
|
runtimeSourceCommit: apiRevision,
|
|
apiRevision,
|
|
apiRevisionRequired,
|
|
planSourceCommit,
|
|
rolloutServices,
|
|
serviceSourceCommits,
|
|
sourceInSync: input.gitMirrorSummary.sourceInSync ?? null,
|
|
gitopsInSync: input.gitMirrorSummary.gitopsInSync ?? null,
|
|
staleReasons,
|
|
workspace: {
|
|
path: workspace.path ?? V02_WORKSPACE,
|
|
branch: workspace.branch ?? null,
|
|
head: workspaceHead,
|
|
originHead: workspaceOriginHead,
|
|
dirty: workspace.dirty ?? null,
|
|
dirtyCount: workspace.dirtyCount ?? null,
|
|
isolatedFromCicd: true,
|
|
},
|
|
workspaceWarnings,
|
|
runtimeWarnings,
|
|
summary: aligned
|
|
? apiRevisionRequired
|
|
? `v0.2 CI/CD source, mirror, PipelineRun, and cloud-api runtime align with ${shortSha(expectedSourceHead ?? "")}`
|
|
: `v0.2 CI/CD source, mirror, and PipelineRun align with ${shortSha(expectedSourceHead ?? "")}; cloud-api runtime rollout not required`
|
|
: `v0.2 CI/CD alignment state=${state}; staleReasons=${staleReasons.join(",")}`,
|
|
};
|
|
}
|
|
|
|
export function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null {
|
|
if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null;
|
|
const activeItems = activePipelineRuns.map((item) => record(item));
|
|
const sourceIsActive = activeItems.some((item) => item.sourceCommit === sourceCommit || String(item.name ?? "").includes(shortSha(sourceCommit)));
|
|
if (sourceIsActive) {
|
|
return `runtime rollout still in progress; cloud-api apiRevision=${apiRevision} has not reached sourceCommit=${sourceCommit} yet`;
|
|
}
|
|
if (activeItems.length > 0) {
|
|
return `runtime apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; active v02 PipelineRun observed, wait for rollout before judging runtime revision`;
|
|
}
|
|
return `cloud-api apiRevision=${apiRevision} is service-scoped and differs from sourceCommit=${sourceCommit}; when web asset probes pass, this is not proof that Cloud Web is stale`;
|
|
}
|
|
|
|
export function numericField(value: string | undefined): number | null {
|
|
if (value === undefined || value.trim().length === 0) return null;
|
|
if (value === "<none>") return null;
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
export function numericValue(value: unknown): number | null {
|
|
const parsed = typeof value === "number" ? value : Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
export function stringArray(value: unknown): string[] {
|
|
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : [];
|
|
}
|
|
|
|
export function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
|
|
if (!commandOk) {
|
|
return {
|
|
ok: false,
|
|
command: Array.isArray(command) ? redactedCommand(command) : command,
|
|
exitCode,
|
|
stderr: stderr.trim().slice(0, 4000),
|
|
items: [],
|
|
activeItems: [],
|
|
};
|
|
}
|
|
return parsePipelineRunRows(text, limit, nowMs);
|
|
}
|