mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
cf1dc4802abbfa784113b734d16a51ef25187827
pikasTech-unidesk/scripts/src/hwlab-g14.ts
T
Codex cf1dc4802a fix: expose hwpod platform db secret status
2026-06-09 07:07:30 +00:00

10465 lines
477 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { createHash, randomBytes } from "node:crypto";
import { repoRoot, rootPath, type Config } from "./config";
import { runCommand } from "./command";
import { cancelJob, listJobs, readJob, startJob } from "./jobs";
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
const HWLAB_REPO = "pikasTech/HWLAB";
const G14_SOURCE_BRANCH = "G14";
const G14_PROVIDER = "G14";
const G14_WORKSPACE = "/root/hwlab";
const V02_LANE_SPEC = hwlabRuntimeLaneSpec("v02");
const V02_SOURCE_BRANCH = V02_LANE_SPEC.sourceBranch;
const V02_WORKSPACE = V02_LANE_SPEC.workspace;
const V02_CICD_REPO = V02_LANE_SPEC.cicdRepo;
const DEV_NAMESPACE = "hwlab-dev";
const PROD_NAMESPACE = "hwlab-prod";
const CI_NAMESPACE = "hwlab-ci";
const ARGO_NAMESPACE = "argocd";
const DEV_APP = "hwlab-g14-dev";
const PROD_APP = "hwlab-g14-prod";
const V02_APP = V02_LANE_SPEC.app;
const V02_PIPELINE = V02_LANE_SPEC.pipeline;
const V02_POLLER = "hwlab-v02-branch-poller";
const V02_RECONCILER = "hwlab-v02-control-plane-reconciler";
const V02_PIPELINERUN_PREFIX = V02_LANE_SPEC.pipelineRunPrefix;
const V02_CONTROL_PLANE_FIELD_MANAGER = V02_LANE_SPEC.controlPlaneFieldManager;
const V02_GIT_URL = V02_LANE_SPEC.gitUrl;
const V02_GIT_READ_URL = V02_LANE_SPEC.gitReadUrl;
const V02_GIT_WRITE_URL = V02_LANE_SPEC.gitWriteUrl;
const V02_GITOPS_BRANCH = V02_LANE_SPEC.gitopsBranch;
const V02_CATALOG_PATH = V02_LANE_SPEC.catalogPath;
const V02_RUNTIME_PATH = V02_LANE_SPEC.runtimePath;
const V02_RUNTIME_NAMESPACE = V02_LANE_SPEC.runtimeNamespace;
const V02_OPENFGA_SECRET = "hwlab-v02-openfga";
const V02_OPENFGA_AUTHN_SECRET_KEY = "authn-preshared-key";
const V02_OPENFGA_DATASTORE_URI_SECRET_KEY = "datastore-uri";
const V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY = "postgres-password";
const V02_OPENFGA_DB_NAME = "hwlab_openfga";
const V02_OPENFGA_DB_USER = "hwlab_openfga";
const V02_MASTER_ADMIN_API_KEY_SECRET = "hwlab-v02-master-server-admin-api-key";
const V02_MASTER_ADMIN_API_KEY_SECRET_KEY = "api-key";
const V02_MASTER_ADMIN_API_KEY_LOCAL_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env";
const V02_REGISTRY_PREFIX = V02_LANE_SPEC.registryPrefix;
const V02_BASE_IMAGE = V02_LANE_SPEC.baseImage;
const GIT_MIRROR_NAMESPACE = "devops-infra";
const GIT_MIRROR_MANIFEST_FIELD_MANAGER = "unidesk-hwlab-git-mirror";
const GIT_MIRROR_SYNC_JOB_PREFIX = "git-mirror-hwlab-sync-manual";
const GIT_MIRROR_LEGACY_CRONJOB = "git-mirror-hwlab-sync";
const G14_OBSERVABILITY_NAMESPACE = "devops-infra";
const G14_OBSERVABILITY_FIELD_MANAGER = "unidesk-g14-observability";
const G14_PROMETHEUS_OPERATOR_VERSION = "v0.91.0";
const G14_PROMETHEUS_VERSION = "v3.12.0";
const G14_PROMETHEUS_NAME = "g14-shared";
const G14_PROMETHEUS_SERVICE = "prometheus-g14-shared";
const G14_PROMETHEUS_SERVICE_ACCOUNT = "g14-observability-prometheus";
const G14_PROMETHEUS_OPERATOR_RELEASE_ASSET = `https://github.com/prometheus-operator/prometheus-operator/releases/download/${G14_PROMETHEUS_OPERATOR_VERSION}/bundle.yaml`;
const V02_SERVICE_IDS = [...V02_LANE_SPEC.serviceIds];
const V02_CLOUD_WEB_URL = V02_LANE_SPEC.publicWebUrl;
const V02_CLOUD_API_URL = V02_LANE_SPEC.publicApiUrl;
const V02_OBSERVABILITY_SERVICE_IDS = [
"hwlab-agent-skills",
"hwlab-cloud-api",
"hwlab-cloud-web",
"hwlab-deepseek-proxy",
"hwlab-edge-proxy",
];
const V02_OBSERVABILITY_EXPECTED_TARGET_COUNT = V02_OBSERVABILITY_SERVICE_IDS.length;
const V02_OBSERVABILITY_QUERIES = {
scrapeReachable: 'up{namespace="hwlab-v02"}',
sidecarServing: 'hwlab_service_up{namespace="hwlab-v02"}',
businessHealthProbe: 'hwlab_service_health_probe_success{namespace="hwlab-v02"}',
healthProbeConfigured: 'hwlab_service_health_probe_configured{namespace="hwlab-v02"}',
healthProbeStatusCode: 'hwlab_service_health_probe_status_code{namespace="hwlab-v02"}',
healthProbeDurationSeconds: 'hwlab_service_health_probe_duration_seconds{namespace="hwlab-v02"}',
processUptimeSeconds: 'hwlab_service_process_uptime_seconds{namespace="hwlab-v02"}',
scrapeDurationSeconds: 'scrape_duration_seconds{namespace="hwlab-v02"}',
scrapeSamplesScraped: 'scrape_samples_scraped{namespace="hwlab-v02"}',
};
const V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES = ["scrapeReachable", "sidecarServing", "businessHealthProbe"] as const;
const V02_OBSERVABILITY_BOOLEAN_QUERY_NAMES = [...V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES, "healthProbeConfigured"];
export function v02PipelineServiceIds(): string[] {
return [...V02_SERVICE_IDS];
}
const G14_CI_TOOLS_IMAGE_REPO = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools";
const G14_CI_TOOLS_BASE_TAG = "node22-alpine-v1";
const DEFAULT_INTERVAL_SECONDS = 600;
const DEFAULT_MAX_CYCLES = 0;
const DEFAULT_TIMEOUT_SECONDS = 1800;
const G14_BRIEF_INDEX_ISSUE = 7;
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000;
const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000;
const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000;
const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000;
const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000;
export type G14MonitorLane = "g14" | HwlabRuntimeLane;
interface G14MonitorOptions {
lane: G14MonitorLane;
intervalSeconds: number;
maxCycles: number;
once: boolean;
dryRun: boolean;
worker: boolean;
timeoutSeconds: number;
}
interface G14RecordRolloutOptions {
prNumber: number;
sourceCommit?: string;
pipelineRun?: string;
gitopsRevision?: string;
mergedAt?: string;
pipelineSucceededAt?: string;
finishedAt?: string;
dryRun: boolean;
}
interface G14ControlPlaneOptions {
action: "status" | "closeout" | "apply" | "trigger-current" | "refresh" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration";
lane: HwlabRuntimeLane | "g14" | "all";
dryRun: boolean;
confirm: boolean;
wait: boolean;
allowLiveDbRead: boolean;
timeoutSeconds: number;
minAgeMinutes: number;
limit: number;
sourceCommit?: string;
pipelineRun?: string;
history: boolean;
}
interface G14LegacyRetirementOptions {
action: "status" | "plan" | "execute";
dryRun: boolean;
confirm: boolean;
wait: boolean;
timeoutSeconds: number;
reason: string;
}
type V02StatusTargetMode = "latest-source-head" | "source-commit" | "pipeline-run";
interface V02ControlPlaneStatusTarget {
sourceCommit?: string | null;
pipelineRun?: string | null;
mode?: V02StatusTargetMode;
includeHistory?: boolean;
}
interface G14ToolsImageOptions {
action: "status" | "build";
name: "ci-node-tools";
tag: string;
dockerfile: string;
dryRun: boolean;
confirm: boolean;
timeoutSeconds: number;
}
interface G14UpstreamImageOptions {
action: "status" | "ensure";
name: "openfga";
tag: string;
dryRun: boolean;
confirm: boolean;
timeoutSeconds: number;
}
interface G14GitMirrorOptions {
action: "status" | "apply" | "sync" | "flush";
lane: HwlabRuntimeLane;
dryRun: boolean;
confirm: boolean;
wait: boolean;
timeoutSeconds: number;
}
interface G14ObservabilityOptions {
action: "status" | "apply" | "query" | "targets" | "boundary" | "closeout";
lane: "v02";
dryRun: boolean;
confirm: boolean;
wait: boolean;
timeoutSeconds: number;
query: string;
expectCount?: number;
expectValue?: string;
}
interface G14SecretOptions {
action: "status" | "ensure" | "delete";
lane: HwlabRuntimeLane;
dryRun: boolean;
confirm: boolean;
name: string;
key?: string;
preset: "openfga" | "master-server-admin-api-key" | "generic-delete";
timeoutSeconds: number;
}
interface CommandJsonResult {
ok: boolean;
command: string[];
exitCode: number | null;
stdout: string;
stderr: string;
parsed: unknown | null;
}
interface RemoteAsyncCommandSpec {
script: string;
timeoutSeconds: number;
label: string;
token: string;
command: string[];
}
interface ShellSection {
stdout: string;
exitCode: number | null;
}
export interface OpenPullRequest {
number: number;
title?: string;
url?: string;
baseRefName?: string;
headRefName?: string;
mergeable?: string | null;
mergeStateStatus?: string | null;
draft?: boolean;
}
interface CiServiceMetric {
taskRun: string;
serviceId: string;
status: string;
durationSeconds: number | null;
buildBackend: string | null;
reusedFrom: string | null;
imageTag: string | null;
sourceCommitId: string | null;
}
interface CiPipelineMetrics {
ok: boolean;
pipelineRun: string;
serviceTotal: number;
reusedCount: number;
rebuildCount: number;
reusedServices: string[];
rebuildServices: string[];
services: CiServiceMetric[];
degradedReason?: string;
rawText?: string;
}
export interface V02PrCommentInput {
pr: OpenPullRequest;
phase: string;
state: "waiting-ci" | "blocked" | "merge-failed" | "cd-started" | "cd-succeeded" | "cd-superseded" | "cd-failed" | "cd-timeout" | "cd-blocked";
startedAt: string;
observedAt: string;
elapsedSeconds: number | null;
preflight?: Record<string, unknown>;
merge?: Record<string, unknown>;
sourceCommit?: string | null;
pipelineRun?: string | null;
cd?: Record<string, unknown>;
flush?: Record<string, unknown>;
dryRun?: boolean;
message?: string;
}
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
const prefix = options.lane !== undefined && options.lane !== "g14" ? `latest-${options.lane}-` : "latest-";
if (options.once && options.dryRun) return `${prefix}once-dry-run-job.json`;
if (options.once) return `${prefix}once-job.json`;
if (options.dryRun) return `${prefix}dry-run-job.json`;
return `${prefix}monitor-job.json`;
}
function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
const lanePrefix = options.lane !== undefined && options.lane !== "g14" ? `${options.lane}-` : "";
if (options.once && options.dryRun) return `${lanePrefix}once-dry-run`;
if (options.once) return `${lanePrefix}once`;
if (options.dryRun) return `${lanePrefix}dry-run-monitor`;
return `${lanePrefix}monitor`;
}
function parseMonitorLane(args: string[]): G14MonitorLane {
const lane = optionValue(args, "--lane") ?? "g14";
if (lane !== "g14" && !isHwlabRuntimeLane(lane)) throw new Error(`monitor-prs --lane must be g14 or ${hwlabRuntimeLaneIds().join("|")}`);
return lane;
}
function parseOptions(args: string[]): G14MonitorOptions {
return {
lane: parseMonitorLane(args),
intervalSeconds: positiveIntegerOption(args, "--interval-seconds", DEFAULT_INTERVAL_SECONDS, 86400),
maxCycles: positiveIntegerOption(args, "--max-cycles", DEFAULT_MAX_CYCLES, 100000),
once: args.includes("--once"),
dryRun: args.includes("--dry-run"),
worker: args.includes("--worker"),
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", DEFAULT_TIMEOUT_SECONDS, 86400),
};
}
function optionValue(args: string[], name: string): string | undefined {
const index = args.indexOf(name);
if (index === -1) return undefined;
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
return value;
}
function validateFullShaOption(value: string, name: string): string {
if (!/^[0-9a-f]{40}$/iu.test(value)) throw new Error(`${name} must be a full 40-character git SHA`);
return value.toLowerCase();
}
function validateV02PipelineRunOption(value: string): string {
if (!/^hwlab-v02-ci-poll-[0-9a-f]{7,40}$/iu.test(value)) throw new Error("--pipeline-run must be a hwlab-v02-ci-poll-<sha> PipelineRun name");
return value.toLowerCase();
}
function validateRuntimeLanePipelineRunOption(lane: HwlabRuntimeLane, value: string): string {
const spec = hwlabRuntimeLaneSpec(lane);
const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) {
throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-<sha>[-rerun] PipelineRun name for --lane ${lane}`);
}
return value.toLowerCase();
}
function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number");
const prNumber = Number(prRaw);
if (!Number.isInteger(prNumber) || prNumber <= 0) throw new Error("record-rollout requires --pr <number>");
return {
prNumber,
sourceCommit: optionValue(args, "--source-commit"),
pipelineRun: optionValue(args, "--pipeline-run"),
gitopsRevision: optionValue(args, "--gitops-revision"),
mergedAt: optionValue(args, "--merged-at"),
pipelineSucceededAt: optionValue(args, "--pipeline-succeeded-at"),
finishedAt: optionValue(args, "--finished-at"),
dryRun: args.includes("--dry-run"),
};
}
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
const [rawAction] = args;
const actionRaw = rawAction === "rerun-current" ? "trigger-current" : rawAction;
if (
actionRaw !== "status" &&
actionRaw !== "closeout" &&
actionRaw !== "apply" &&
actionRaw !== "trigger-current" &&
actionRaw !== "refresh" &&
actionRaw !== "cleanup-runs" &&
actionRaw !== "cleanup-released-pvs" &&
actionRaw !== "runtime-migration"
) {
throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|v03|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
}
const laneRaw = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
let lane: G14ControlPlaneOptions["lane"];
if (actionRaw === "cleanup-runs") {
if (laneRaw !== "g14" && laneRaw !== "all" && !isHwlabRuntimeLane(laneRaw)) throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all");
lane = laneRaw;
} else if (actionRaw === "cleanup-released-pvs") {
if (laneRaw !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
lane = laneRaw;
} else if (actionRaw === "closeout" || actionRaw === "runtime-migration") {
if (laneRaw !== "v02") throw new Error("control-plane closeout/runtime-migration currently requires --lane v02");
lane = laneRaw;
} else if (!isHwlabRuntimeLane(laneRaw)) {
throw new Error(`control-plane ${actionRaw} requires --lane ${hwlabRuntimeLaneIds().join("|")}`);
} else {
lane = laneRaw;
}
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run");
const allowLiveDbRead = args.includes("--allow-live-db-read");
if (allowLiveDbRead && confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
if (allowLiveDbRead && actionRaw !== "runtime-migration") throw new Error("--allow-live-db-read is only valid for control-plane runtime-migration");
const sourceCommitRaw = optionValue(args, "--source-commit");
const pipelineRunRaw = optionValue(args, "--pipeline-run");
if ((sourceCommitRaw !== undefined || pipelineRunRaw !== undefined) && actionRaw !== "status" && actionRaw !== "closeout" && actionRaw !== "cleanup-runs") {
throw new Error("--source-commit and --pipeline-run are only valid for control-plane status/closeout/cleanup-runs");
}
if (actionRaw === "closeout" && sourceCommitRaw === undefined && pipelineRunRaw === undefined) {
throw new Error("control-plane closeout requires --source-commit <full-sha> or --pipeline-run hwlab-v02-ci-poll-<sha>");
}
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) throw new Error("control-plane status/closeout/cleanup-runs accepts only one of --source-commit or --pipeline-run");
return {
action: actionRaw,
lane,
confirm,
wait: args.includes("--wait"),
allowLiveDbRead,
dryRun: actionRaw === "status" || actionRaw === "closeout" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
limit: positiveIntegerOption(args, "--limit", 20, 200),
sourceCommit: sourceCommitRaw === undefined ? undefined : validateFullShaOption(sourceCommitRaw, "--source-commit"),
pipelineRun: pipelineRunRaw === undefined
? undefined
: isHwlabRuntimeLane(lane)
? validateRuntimeLanePipelineRunOption(lane, pipelineRunRaw)
: validateV02PipelineRunOption(pipelineRunRaw),
history: args.includes("--history"),
};
}
function parseLegacyRetirementOptions(args: string[]): G14LegacyRetirementOptions {
const [actionRaw = "status"] = args;
if (actionRaw !== "status" && actionRaw !== "plan" && actionRaw !== "execute") {
throw new Error("retirement usage: status|plan|execute [--dry-run|--confirm] [--wait] [--reason text] [--timeout-seconds N]");
}
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("retirement accepts only one of --confirm or --dry-run");
if (confirm && actionRaw !== "execute") throw new Error("--confirm is only valid for retirement execute");
if (args.includes("--wait") && actionRaw !== "execute") throw new Error("--wait is only valid for retirement execute");
const reason = optionValue(args, "--reason") ?? "user requested immediate retirement of legacy G14 DEV/PROD";
if (reason.length > 240) throw new Error("--reason is limited to 240 characters");
return {
action: actionRaw,
confirm,
wait: args.includes("--wait"),
dryRun: actionRaw !== "execute" || explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "execute" ? 60 : 30, 300),
reason,
};
}
function parseToolsImageOptions(args: string[]): G14ToolsImageOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "build") {
throw new Error("tools-image usage: status|build --name ci-node-tools --tag <tag> [--dockerfile path] [--dry-run|--confirm]");
}
const name = optionValue(args, "--name") ?? "ci-node-tools";
if (name !== "ci-node-tools") throw new Error("tools-image currently supports --name ci-node-tools");
const tag = optionValue(args, "--tag") ?? G14_CI_TOOLS_BASE_TAG;
if (!/^[A-Za-z0-9_.-]+$/u.test(tag)) throw new Error("--tag may only contain letters, numbers, dot, underscore, and dash");
const dockerfile = optionValue(args, "--dockerfile") ?? "deploy/ci/hwlab-ci-node-tools.Dockerfile";
if (dockerfile.startsWith("/") || dockerfile.includes("..")) throw new Error("--dockerfile must be a repo-relative path without '..'");
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run");
return {
action: actionRaw,
name,
tag,
dockerfile,
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
};
}
function parseUpstreamImageOptions(args: string[]): G14UpstreamImageOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "ensure") {
throw new Error("upstream-image usage: status|ensure --name openfga --tag v1.17.0 [--dry-run|--confirm]");
}
const name = optionValue(args, "--name") ?? "openfga";
if (name !== "openfga") throw new Error("upstream-image currently supports --name openfga");
const tag = optionValue(args, "--tag") ?? "v1.17.0";
if (tag !== "v1.17.0") throw new Error("upstream-image currently supports OpenFGA tag v1.17.0");
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("upstream-image accepts only one of --confirm or --dry-run");
return {
action: actionRaw,
name,
tag,
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 600, 1800),
};
}
function parseGitMirrorOptions(args: string[]): G14GitMirrorOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "sync" && actionRaw !== "flush") {
throw new Error("git-mirror usage: status|apply|sync|flush [--lane v02|v03] [--dry-run|--confirm]");
}
const laneRaw = optionValue(args, "--lane") ?? "v02";
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`git-mirror --lane must be ${hwlabRuntimeLaneIds().join("|")}`);
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("git-mirror accepts only one of --confirm or --dry-run");
return {
action: actionRaw,
lane: laneRaw,
confirm,
wait: args.includes("--wait"),
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "sync" || actionRaw === "flush" ? 300 : 120, 900),
};
}
function parseObservabilityOptions(args: string[]): G14ObservabilityOptions {
const [actionRaw] = args;
if (
actionRaw !== "status" &&
actionRaw !== "apply" &&
actionRaw !== "query" &&
actionRaw !== "targets" &&
actionRaw !== "boundary" &&
actionRaw !== "closeout"
) {
throw new Error("observability usage: status|apply|query|targets|boundary|closeout [--lane v02] [--promql <expr>] [--expect-count N] [--expect-value V] [--dry-run|--confirm]");
}
const valueOptions = new Set(["--lane", "--promql", "--query", "--expect-count", "--expect-value", "--timeout-seconds"]);
const booleanOptions = new Set(["--confirm", "--dry-run"]);
for (let i = 1; i < args.length; i += 1) {
const arg = args[i];
if (valueOptions.has(arg)) {
i += 1;
if (i >= args.length || args[i].startsWith("--")) throw new Error(`${arg} requires a value`);
continue;
}
if (booleanOptions.has(arg)) continue;
if (arg === "--wait") throw new Error("observability does not support --wait; commands are bounded short CLI calls");
if (arg.startsWith("--")) throw new Error(`unsupported observability option: ${arg}`);
throw new Error(`unexpected observability argument: ${arg}`);
}
const lane = optionValue(args, "--lane") ?? "v02";
if (lane !== "v02") throw new Error("observability currently supports --lane v02");
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("observability accepts only one of --confirm or --dry-run");
if ((confirm || explicitDryRun) && actionRaw !== "apply") throw new Error("--confirm and --dry-run are only supported by observability apply");
const query = optionValue(args, "--promql") ?? optionValue(args, "--query") ?? 'up{namespace="hwlab-v02"}';
if (query.length > 500) throw new Error("--promql is limited to 500 characters");
if (query.includes("\n") || query.includes("\r")) throw new Error("--promql must be a single-line expression");
const expectCountRaw = optionValue(args, "--expect-count");
const expectCount = expectCountRaw === undefined ? undefined : Number(expectCountRaw);
if (expectCountRaw !== undefined && (!Number.isInteger(expectCount) || expectCount < 0)) throw new Error("--expect-count must be a non-negative integer");
const expectValue = optionValue(args, "--expect-value");
if ((expectCount !== undefined || expectValue !== undefined) && actionRaw !== "query") throw new Error("--expect-count and --expect-value are only supported by observability query");
if (expectValue !== undefined && expectValue.trim().length === 0) throw new Error("--expect-value must not be blank");
return {
action: actionRaw,
lane,
confirm,
wait: args.includes("--wait"),
dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", actionRaw === "apply" ? 240 : 120, 900),
query,
expectCount,
expectValue,
};
}
function parseSecretOptions(args: string[]): G14SecretOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "delete") {
throw new Error("secret usage: status|ensure --lane v02|v03 --name hwlab-v0x-openfga|hwlab-v0x-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name <obsolete-secret> [--dry-run|--confirm]");
}
const laneRaw = optionValue(args, "--lane") ?? "v02";
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`secret --lane must be one of ${hwlabRuntimeLaneIds().join(", ")}`);
const lane = laneRaw;
const spec = hwlabRuntimeLaneSpec(lane);
const openFgaSecret = runtimeLaneOpenFgaSecretName(spec);
const masterAdminSecret = runtimeLaneMasterAdminApiKeySecretName(spec);
const name = optionValue(args, "--name") ?? openFgaSecret;
const key = optionValue(args, "--key");
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
if (actionRaw === "delete") {
if (lane !== "v02") throw new Error("secret delete is currently limited to --lane v02 obsolete cleanup");
if (key !== undefined) throw new Error("secret delete does not accept --key; it deletes a whole obsolete Secret object");
if (!/^hwlab-v02-[a-z0-9-]+$/u.test(name)) throw new Error("secret delete requires a hwlab-v02-* Secret name");
if (name === V02_OPENFGA_SECRET || name === V02_MASTER_ADMIN_API_KEY_SECRET || name === "hwlab-v02-postgres") throw new Error(`secret delete refuses required v0.2 Secret ${name}`);
return {
action: actionRaw,
lane,
confirm,
dryRun: explicitDryRun || !confirm,
name,
preset: "generic-delete",
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
};
}
if (name === masterAdminSecret) {
if (key !== undefined && key !== V02_MASTER_ADMIN_API_KEY_SECRET_KEY) throw new Error(`secret ${masterAdminSecret} supports only key ${V02_MASTER_ADMIN_API_KEY_SECRET_KEY}`);
return {
action: actionRaw,
lane,
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
name,
key: key ?? V02_MASTER_ADMIN_API_KEY_SECRET_KEY,
preset: "master-server-admin-api-key",
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
};
}
if (name !== openFgaSecret) {
throw new Error(`secret status/ensure currently supports only --name ${openFgaSecret} or ${masterAdminSecret} for --lane ${lane}; use secret delete for obsolete v02 Secret objects`);
}
if (key !== undefined && key !== V02_OPENFGA_AUTHN_SECRET_KEY && key !== V02_OPENFGA_DATASTORE_URI_SECRET_KEY && key !== V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) {
throw new Error(`secret ${openFgaSecret} supports keys ${V02_OPENFGA_AUTHN_SECRET_KEY}, ${V02_OPENFGA_DATASTORE_URI_SECRET_KEY}, and ${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY}`);
}
if (lane !== "v02" && key === V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) throw new Error(`${openFgaSecret}/${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY} is derived from ${postgresSecret}/POSTGRES_PASSWORD`);
return {
action: actionRaw,
lane,
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
name,
key,
preset: "openfga",
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
};
}
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
const index = args.indexOf(name);
if (index === -1) return defaultValue;
const raw = args[index + 1];
const value = Number(raw);
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
return Math.min(value, maxValue);
}
function shellQuote(value: string): string {
return `'${value.replace(/'/gu, `'"'"'`)}'`;
}
function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
const result = runCommand(command, repoRoot, { timeoutMs });
let parsed: unknown | null = null;
if (result.stdout.trim().length > 0) {
try {
parsed = JSON.parse(result.stdout) as unknown;
} catch {
parsed = null;
}
}
return {
ok: result.exitCode === 0,
command,
exitCode: result.exitCode,
stdout: result.stdout,
stderr: result.stderr,
parsed,
};
}
function commandJsonWithInput(command: string[], input: string, timeoutMs = 60_000): CommandJsonResult {
const result = runCommand(command, repoRoot, { timeoutMs, input });
let parsed: unknown | null = null;
if (result.stdout.trim().length > 0) {
try {
parsed = JSON.parse(result.stdout) as unknown;
} catch {
parsed = null;
}
}
return {
ok: result.exitCode === 0,
command,
exitCode: result.exitCode,
stdout: result.stdout,
stderr: result.stderr,
parsed,
};
}
function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult {
return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs);
}
function record(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
function stringOrNull(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function fieldBoolean(value: unknown): boolean | null {
if (value === true || value === false) return value;
if (typeof value !== "string") return null;
const normalized = value.trim().toLowerCase();
if (normalized === "true" || normalized === "yes" || normalized === "1") return true;
if (normalized === "false" || normalized === "no" || normalized === "0") return false;
return null;
}
function nested(value: unknown, keys: string[]): unknown {
let current = value;
for (const key of keys) current = record(current)[key];
return current;
}
function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown> {
const dumpPath = nested(result.parsed, ["data", "dump", "path"]);
if (typeof dumpPath === "string" && existsSync(dumpPath)) {
try {
return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown);
} catch {
return record(result.parsed);
}
}
return record(result.parsed);
}
function commandData(result: CommandJsonResult): Record<string, unknown> {
return record(sanitizeCliOutput(record(expandedParsedRoot(result).data)));
}
function compactCommandMetadata(result: CommandJsonResult, includeTail = false, includeCommand = false): Record<string, unknown> {
return {
ok: result.ok,
exitCode: result.exitCode,
command: includeCommand ? redactedCommand(result.command) : undefined,
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
stdoutTail: includeTail ? compactInlineText(result.stdout, 4000) : undefined,
stderrTail: includeTail ? compactInlineText(result.stderr, 4000) : undefined,
};
}
function textHash(value: string): string {
return createHash("sha256").update(value).digest("hex").slice(0, 12);
}
function generateHwlabApiKey(): string {
return `hwl_live_${randomBytes(32).toString("base64url")}`;
}
function hwlabApiKeyPrefix(value: string): string {
const text = value.trim();
if (!text.startsWith("hwl_live_")) return "";
const remainder = text.slice("hwl_live_".length);
const dot = remainder.indexOf(".");
const segment = dot === -1 ? remainder : remainder.slice(0, dot);
return `hwl_live_${segment}`.slice(0, 24);
}
function runtimeLaneSecretFieldManager(spec: HwlabRuntimeLaneSpec): string {
return `unidesk-hwlab-${spec.lane}-secret`;
}
function runtimeLanePostgresSecretName(spec: HwlabRuntimeLaneSpec): string {
return `${spec.runtimeNamespace}-postgres`;
}
function runtimeLanePrimaryDbName(spec: HwlabRuntimeLaneSpec): string {
return `hwlab_${spec.lane}`;
}
function runtimeLaneOpenFgaSecretName(spec: HwlabRuntimeLaneSpec): string {
return `hwlab-${spec.lane}-openfga`;
}
function runtimeLaneMasterAdminApiKeySecretName(spec: HwlabRuntimeLaneSpec): string {
return `hwlab-${spec.lane}-master-server-admin-api-key`;
}
function runtimeLaneMasterAdminApiKeyLocalEnv(spec: HwlabRuntimeLaneSpec): string {
if (spec.lane === "v02") return V02_MASTER_ADMIN_API_KEY_LOCAL_ENV;
return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`;
}
function localMasterAdminApiKeyStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
if (!existsSync(envPath)) {
return { exists: false, path: envPath, mode: null, valueBytes: 0, keyPrefix: null };
}
const stat = statSync(envPath);
const content = readFileSync(envPath, "utf8");
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
const value = match?.[1]?.trim() ?? "";
return {
exists: true,
path: envPath,
mode: `0${(stat.mode & 0o777).toString(8)}`,
valueBytes: Buffer.byteLength(value, "utf8"),
keyPrefix: value ? hwlabApiKeyPrefix(value) : null,
validPrefix: value.startsWith("hwl_live_"),
};
}
function readOrCreateLocalMasterAdminApiKey(spec: HwlabRuntimeLaneSpec, dryRun: boolean): { key: string | null; created: boolean; status: Record<string, unknown> } {
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
const existing = localMasterAdminApiKeyStatus(spec);
if (existing.exists === true) {
const content = readFileSync(envPath, "utf8");
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
const key = match?.[1]?.trim() ?? "";
if (!key.startsWith("hwl_live_")) throw new Error(`${envPath} exists but does not contain a hwl_live_ HWLAB_API_KEY`);
if (!dryRun && existing.mode !== "0600") chmodSync(envPath, 0o600);
return { key, created: false, status: localMasterAdminApiKeyStatus(spec) };
}
if (dryRun) return { key: null, created: false, status: existing };
const key = generateHwlabApiKey();
mkdirSync(dirname(envPath), { recursive: true, mode: 0o700 });
writeFileSync(envPath, `# HWLAB ${spec.version} master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 });
return { key, created: true, status: localMasterAdminApiKeyStatus(spec) };
}
function redactLargePayloads(value: string): string {
return value
.replace(/hwl_live_[A-Za-z0-9._:-]{12,}/gu, (payload: string) => {
return `hwl_live_<redacted chars=${payload.length} sha256=${textHash(payload)}>`;
})
.replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => {
return `manifest_b64='<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>'`;
})
.replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => {
return `manifest_b64="<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>"`;
})
.replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => {
return `manifest_b64=<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
})
.replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => {
return `<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
});
}
function compactInlineText(value: string, maxChars = 4000): string {
const redacted = redactLargePayloads(value);
if (redacted.length <= maxChars) return redacted;
const headChars = Math.floor(maxChars * 0.35);
const tailChars = maxChars - headChars;
const omittedChars = redacted.length - headChars - tailChars;
return [
redacted.slice(0, headChars),
`<truncated chars=${omittedChars} sha256=${textHash(redacted)}>`,
redacted.slice(-tailChars),
].join("\n");
}
function sanitizeCliOutput(value: unknown): unknown {
if (typeof value === "string") return compactInlineText(value);
if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item));
if (typeof value !== "object" || value === null) return value;
const output: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item);
return output;
}
function redactedCommand(command: string[]): string[] {
return command.map((arg) => compactInlineText(arg, 1200));
}
function compactCommandResult(result: CommandJsonResult): Record<string, unknown> {
const data = record(expandedParsedRoot(result).data);
const stdout = String(data.stdout ?? result.stdout ?? "");
const stderr = String(data.stderr ?? result.stderr ?? "");
return {
ok: isCommandSuccess(result),
exitCode: result.exitCode,
command: redactedCommand(result.command),
stdout: compactInlineText(stdout.trim(), 4000),
stderr: compactInlineText(stderr.trim(), 4000),
stdoutBytes: Buffer.byteLength(stdout),
stderrBytes: Buffer.byteLength(stderr),
};
}
function compactControlPlaneRefresh(value: Record<string, unknown>): Record<string, unknown> {
return {
ok: value.ok === true,
phase: value.phase ?? null,
sourceCommit: value.sourceCommit ?? null,
renderDir: value.renderDir ?? null,
applyOk: nested(value, ["apply", "ok"]) ?? null,
cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null,
applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200),
degradedReason: value.degradedReason ?? null,
};
}
function compactTriggerCommandResult(value: Record<string, unknown> | null): Record<string, unknown> | null {
if (value === null) return null;
return {
ok: value.ok ?? null,
exitCode: value.exitCode ?? null,
stdout: tailText(value.stdout, 1200),
stderr: tailText(value.stderr, 1200),
stdoutBytes: value.stdoutBytes ?? null,
stderrBytes: value.stderrBytes ?? null,
};
}
function isCommandSuccess(result: CommandJsonResult): boolean {
if (!result.ok) return false;
const topOk = record(result.parsed).ok;
if (topOk === false) return false;
const dataOk = nested(result.parsed, ["data", "ok"]);
return dataOk !== false;
}
function monitorBaseBranch(lane: G14MonitorLane): string {
return lane === "g14" ? G14_SOURCE_BRANCH : hwlabRuntimeLaneSpec(lane).sourceBranch;
}
function extractPullRequests(result: CommandJsonResult, baseBranch = G14_SOURCE_BRANCH): OpenPullRequest[] {
const prs = nested(result.parsed, ["data", "pullRequests"]);
if (!Array.isArray(prs)) return [];
return prs
.map((item) => record(item))
.filter((item) => item.baseRefName === baseBranch || record(item.base).ref === baseBranch)
.map((item) => ({
number: Number(item.number),
title: typeof item.title === "string" ? item.title : undefined,
url: typeof item.url === "string" ? item.url : undefined,
baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined,
headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined,
mergeable: typeof item.mergeable === "string" ? item.mergeable : null,
mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null,
draft: item.draft === true,
}))
.filter((item) => Number.isInteger(item.number) && item.number > 0);
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function printEvent(event: string, data: Record<string, unknown> = {}): void {
process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
}
function printProgressEvent(event: string, data: Record<string, unknown> = {}): void {
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
}
function printV02PrMonitorProgress(data: Record<string, unknown> = {}): void {
printProgressEvent("hwlab.v02.pr-monitor.progress", data);
}
function printRuntimeLanePrMonitorProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
printProgressEvent("hwlab.runtime-lane.pr-monitor.progress", {
lane: spec.lane,
node: spec.nodeId,
...data,
});
}
function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
printProgressEvent("hwlab.runtime-lane.trigger.progress", {
lane: spec.lane,
node: spec.nodeId,
...data,
});
}
function shortSha(sha: string): string {
return sha.slice(0, 12);
}
function precheckWorkspace(): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
}
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
}
function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs);
}
function g14K3sInlineScriptWithInput(script: string, input: string, timeoutMs = 60_000): CommandJsonResult {
return commandJsonWithInput(["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "script", "--", script], input, timeoutMs);
}
function remoteAsyncStatusDir(label: string): string {
return `/tmp/hwlab-${label}-async`;
}
function remoteAsyncStatusPath(label: string, token: string): string {
return `${remoteAsyncStatusDir(label)}/${token}.json`;
}
function remoteAsyncStartScript(spec: RemoteAsyncCommandSpec): string {
const statusPath = remoteAsyncStatusPath(spec.label, spec.token);
const donePath = `${statusPath}.done`;
const stdoutPath = `${statusPath}.stdout`;
const stderrPath = `${statusPath}.stderr`;
const commandB64 = Buffer.from(spec.script, "utf8").toString("base64");
return [
"set -eu",
`status_dir=${shellQuote(remoteAsyncStatusDir(spec.label))}`,
`status_path=${shellQuote(statusPath)}`,
`done_path=${shellQuote(donePath)}`,
`stdout_path=${shellQuote(stdoutPath)}`,
`stderr_path=${shellQuote(stderrPath)}`,
`command_b64=${shellQuote(commandB64)}`,
"mkdir -p \"$status_dir\"",
"rm -f \"$status_path\" \"$done_path\" \"$stdout_path\" \"$stderr_path\"",
"command_path=\"$status_path.command.sh\"",
"printf '%s' \"$command_b64\" | base64 -d > \"$command_path\"",
"chmod +x \"$command_path\"",
"node -e 'const fs=require(\"fs\"); const p=process.argv[1]; const payload={ok:null,state:\"running\",pid:Number(process.argv[2]),startedAt:new Date().toISOString(),finishedAt:null,exitCode:null,stdout:\"\",stderr:\"\"}; fs.writeFileSync(p, JSON.stringify(payload));' \"$status_path\" $$",
"(",
" set +e",
" sh \"$command_path\" >\"$stdout_path\" 2>\"$stderr_path\"",
" code=$?",
" node - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'NODE'",
"const fs = require('fs');",
"const [statusPath, stdoutPath, stderrPath, codeRaw] = process.argv.slice(2);",
"const code = Number(codeRaw);",
"const read = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };",
"fs.writeFileSync(statusPath, JSON.stringify({",
" ok: code === 0,",
" state: 'finished',",
" pid: null,",
" startedAt: null,",
" finishedAt: new Date().toISOString(),",
" exitCode: code,",
" stdout: read(stdoutPath),",
" stderr: read(stderrPath),",
"}));",
"NODE",
" touch \"$done_path\"",
") >/dev/null 2>&1 &",
"pid=$!",
"node - \"$status_path\" \"$pid\" <<'NODE'",
"const fs = require('fs');",
"const [statusPath, pidRaw] = process.argv.slice(2);",
"const current = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
"current.pid = Number(pidRaw);",
"fs.writeFileSync(statusPath, JSON.stringify(current));",
"console.log(JSON.stringify({ ok: true, state: current.state, pid: current.pid, statusPath }));",
"NODE",
].join("\n");
}
function remoteAsyncStatusScript(label: string, token: string): string {
const statusPath = remoteAsyncStatusPath(label, token);
const donePath = `${statusPath}.done`;
return [
"set -eu",
`status_path=${shellQuote(statusPath)}`,
`done_path=${shellQuote(donePath)}`,
"if [ ! -f \"$status_path\" ]; then",
" printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing\"}'",
" exit 0",
"fi",
"if [ -f \"$done_path\" ]; then",
" cat \"$status_path\"",
" exit 0",
"fi",
"node - \"$status_path\" <<'NODE'",
"const fs = require('fs');",
"const statusPath = process.argv[2];",
"const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
"payload.state = payload.state || 'running';",
"payload.ok = payload.ok ?? null;",
"payload.stdout = payload.stdout || '';",
"payload.stderr = payload.stderr || '';",
"console.log(JSON.stringify(payload));",
"NODE",
].join("\n");
}
function remoteAsyncKillScript(label: string, token: string): string {
const statusPath = remoteAsyncStatusPath(label, token);
const donePath = `${statusPath}.done`;
return [
"set -eu",
`status_path=${shellQuote(statusPath)}`,
`done_path=${shellQuote(donePath)}`,
"if [ ! -f \"$status_path\" ]; then",
" printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing during kill\"}'",
" exit 0",
"fi",
"node - \"$status_path\" <<'NODE'",
"const fs = require('fs');",
"const statusPath = process.argv[2];",
"const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
"if (Number.isInteger(payload.pid) && payload.pid > 0) {",
" try { process.kill(payload.pid, 'TERM'); } catch {}",
"}",
"payload.ok = false;",
"payload.state = 'timeout-killed';",
"payload.exitCode = 124;",
"payload.finishedAt = new Date().toISOString();",
"payload.stderr = `${payload.stderr || ''}\\nremote async command exceeded local poll timeout and was terminated`;",
"fs.writeFileSync(statusPath, JSON.stringify(payload));",
"console.log(JSON.stringify(payload));",
"NODE",
"touch \"$done_path\"",
].join("\n");
}
function parseRemoteAsyncPayload(result: CommandJsonResult): Record<string, unknown> {
const text = result.stdout.trim();
if (text.length === 0) return {};
try {
return record(JSON.parse(text) as unknown);
} catch {
return {};
}
}
function runG14K3sRemoteAsync(spec: RemoteAsyncCommandSpec): CommandJsonResult {
const startedAtMs = Date.now();
const start = g14K3s(["script", "--", remoteAsyncStartScript(spec)], 55_000);
const startPayload = parseRemoteAsyncPayload(start);
if (!isCommandSuccess(start) || startPayload.ok !== true) return start;
const deadlineMs = startedAtMs + Math.max(1, spec.timeoutSeconds) * 1000;
let attempts = 0;
let lastStatus: CommandJsonResult | null = null;
while (Date.now() <= deadlineMs) {
attempts += 1;
const wait = runCommand(["sleep", attempts <= 1 ? "1" : "2"], repoRoot, { timeoutMs: 3_000 });
if (wait.exitCode !== 0 && wait.timedOut) break;
const status = g14K3s(["script", "--", remoteAsyncStatusScript(spec.label, spec.token)], 55_000);
lastStatus = status;
const payload = parseRemoteAsyncPayload(status);
const state = typeof payload.state === "string" ? payload.state : null;
if (state === "finished" || payload.ok === true || payload.ok === false) {
return {
ok: payload.ok === true,
command: spec.command,
exitCode: typeof payload.exitCode === "number" ? payload.exitCode : status.exitCode,
stdout: typeof payload.stdout === "string" ? payload.stdout : status.stdout,
stderr: typeof payload.stderr === "string" ? payload.stderr : status.stderr,
parsed: {
ok: payload.ok === true,
data: {
mode: "remote-async",
label: spec.label,
token: spec.token,
statusPath: remoteAsyncStatusPath(spec.label, spec.token),
attempts,
elapsedMs: Date.now() - startedAtMs,
state,
pid: payload.pid ?? null,
},
},
};
}
}
const killed = g14K3s(["script", "--", remoteAsyncKillScript(spec.label, spec.token)], 55_000);
const payload = parseRemoteAsyncPayload(killed);
return {
ok: false,
command: spec.command,
exitCode: 124,
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
stderr: [
`remote async command timed out after ${spec.timeoutSeconds}s`,
typeof payload.stderr === "string" ? payload.stderr : "",
lastStatus === null ? "" : `last status stderr: ${lastStatus.stderr.trim()}`,
killed.stderr.trim(),
].filter(Boolean).join("\n"),
parsed: {
ok: false,
data: {
mode: "remote-async",
label: spec.label,
token: spec.token,
statusPath: remoteAsyncStatusPath(spec.label, spec.token),
attempts,
elapsedMs: Date.now() - startedAtMs,
state: typeof payload.state === "string" ? payload.state : "timeout",
pid: payload.pid ?? null,
killed: compactCommandMetadata(killed, true, false),
},
},
};
}
function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
return [
`cicd_repo=${shellQuote(spec.cicdRepo)}`,
`cicd_url=${shellQuote(spec.gitUrl)}`,
`cicd_branch=${shellQuote(spec.sourceBranch)}`,
`cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`,
"cicd_repo_ensure_fetch() {",
" mkdir -p \"$(dirname \"$cicd_repo\")\" || return $?",
" if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
" :",
" elif [ -e \"$cicd_repo\" ]; then",
` echo "${spec.version} CI/CD repo path exists but is not a bare git repo: $cicd_repo" >&2`,
" return 41",
" else",
" git clone --bare \"$cicd_url\" \"$cicd_repo\" || return $?",
" fi",
" git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\" || return $?",
" git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' || return $?",
" git --git-dir=\"$cicd_repo\" fetch origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune || return $?",
"}",
"cicd_repo_lock_mode=none",
"if command -v flock >/dev/null 2>&1; then",
" exec 9>\"$cicd_repo_lock\"",
` flock -w 120 9 || { echo "timed out waiting for ${spec.version} CI/CD repo lock: $cicd_repo_lock" >&2; exit 42; }`,
" cicd_repo_lock_mode=flock",
"else",
" cicd_repo_lock_dir=\"$cicd_repo_lock.d\"",
" cicd_repo_lock_attempt=0",
" while ! mkdir \"$cicd_repo_lock_dir\" 2>/dev/null; do",
" cicd_repo_lock_attempt=$((cicd_repo_lock_attempt + 1))",
` if [ "$cicd_repo_lock_attempt" -ge 120 ]; then echo "timed out waiting for ${spec.version} CI/CD repo lock: $cicd_repo_lock_dir" >&2; exit 42; fi`,
" sleep 1",
" done",
" cicd_repo_lock_mode=dir",
"fi",
"cicd_repo_status=0",
"cicd_repo_ensure_fetch || cicd_repo_status=$?",
"if [ \"$cicd_repo_lock_mode\" = flock ]; then flock -u 9 >/dev/null 2>&1 || true; fi",
"if [ \"$cicd_repo_lock_mode\" = dir ]; then rmdir \"$cicd_repo_lock_dir\" >/dev/null 2>&1 || true; fi",
`if [ "$cicd_repo_status" -ne 0 ]; then echo "${spec.version} CI/CD repo refresh failed status=$cicd_repo_status repo=$cicd_repo" >&2; exit "$cicd_repo_status"; fi`,
].join("\n");
}
function v02CicdRepoEnsureScript(): string {
return runtimeLaneCicdRepoEnsureScript(V02_LANE_SPEC);
}
function resolveV02Head(): { sourceCommit: string | null; result: CommandJsonResult } {
const result = g14HostScript([
"set -eu",
v02CicdRepoEnsureScript(),
"git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2",
].join("\n"), 180_000);
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
const match = /[0-9a-f]{40}/iu.exec(output);
return { sourceCommit: match?.[0] ?? null, result };
}
function getV02Head(): string | null {
return resolveV02Head().sourceCommit;
}
function resolveV02LatestRemoteHead(): { sourceCommit: string | null; result: CommandJsonResult } {
const result = commandJson(["git", "ls-remote", V02_GIT_URL, "refs/heads/v0.2"], 30_000);
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
const output = statusText(result);
const match = /^[0-9a-f]{40}/imu.exec(output);
return { sourceCommit: match?.[0] ?? null, result };
}
function resolveRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandJsonResult } {
const result = g14HostScript([
"set -eu",
runtimeLaneCicdRepoEnsureScript(spec),
`git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch}`,
].join("\n"), 180_000);
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
const match = /[0-9a-f]{40}/iu.exec(output);
return { sourceCommit: match?.[0] ?? null, result };
}
function resolveRuntimeLaneLatestRemoteHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandJsonResult } {
const result = commandJson(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], 30_000);
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
const output = statusText(result);
const match = /^[0-9a-f]{40}/imu.exec(output);
return { sourceCommit: match?.[0] ?? null, result };
}
function runtimeLanePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
}
function runtimeLaneRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string, now = new Date()): string {
const stamp = now.toISOString().replace(/\D/gu, "").slice(0, 14);
return `${runtimeLanePipelineRunName(spec, sourceCommit)}-r${stamp}`;
}
function v02PipelineRunName(sourceCommit: string): string {
return runtimeLanePipelineRunName(V02_LANE_SPEC, sourceCommit);
}
interface V02TriggerSnapshot {
sourceCommit: string | null;
pipelineRun: string | null;
before: Record<string, unknown> | null;
result: CommandJsonResult;
observedAtMs: number;
}
export function parseV02TriggerSnapshot(result: CommandJsonResult, observedAtMs: number): V02TriggerSnapshot {
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout);
const fields = keyValueLinesFromText(output);
const sourceCommit = stringOrNull(fields.sourceCommit?.match(/^[0-9a-f]{40}$/iu)?.[0]);
const pipelineRun = stringOrNull(fields.pipelineRun);
const pipelineRunExitCode = numericField(fields.pipelineRunExitCode);
const before = pipelineRun !== null && pipelineRunExitCode !== null
? pipelineRunCompactFromText(
pipelineRun,
[fields.status ?? "", fields.reason ?? "", fields.message ?? ""].join("\n"),
pipelineRunExitCode === 0,
"hwlab-v02-trigger-snapshot:pipelinerun",
pipelineRunExitCode,
fields.pipelineRunStderr ?? "",
)
: null;
return {
sourceCommit,
pipelineRun,
before,
result,
observedAtMs,
};
}
function getV02TriggerSnapshot(): V02TriggerSnapshot {
const script = [
"set +e",
"one_line() { tr '\\r\\n\\t' ' ' | sed 's/[[:space:]][[:space:]]*/ /g' | cut -c1-2000; }",
"tmp_dir=$(mktemp -d /tmp/hwlab-v02-trigger-snapshot.XXXXXX)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
"source_commit=",
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
" source_commit=$(git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)",
"elif [ -e \"$cicd_repo\" ]; then",
" printf 'snapshotError\\t%s\\n' \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\"",
"fi",
"pipeline_run=",
`if [ -n "$source_commit" ]; then pipeline_run=${shellQuote(V02_PIPELINERUN_PREFIX)}-$(printf '%s' "$source_commit" | cut -c1-12); fi`,
"if [ -n \"$pipeline_run\" ]; then",
` (kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}' >\"$tmp_dir/pipelinerun.out\" 2>\"$tmp_dir/pipelinerun.err\"; printf '%s' "$?" >\"$tmp_dir/pipelinerun.status\") &`,
" pipeline_pid=$!",
"else",
" printf '' >\"$tmp_dir/pipelinerun.out\"",
" printf '' >\"$tmp_dir/pipelinerun.err\"",
" printf '' >\"$tmp_dir/pipelinerun.status\"",
" pipeline_pid=",
"fi",
"if [ -n \"$pipeline_pid\" ]; then wait \"$pipeline_pid\" 2>/dev/null || true; pipeline_pid=; fi",
"status_text=$(cat \"$tmp_dir/pipelinerun.out\" 2>/dev/null)",
"pipeline_status=$(cat \"$tmp_dir/pipelinerun.status\" 2>/dev/null)",
"printf 'sourceCommit\\t%s\\n' \"$source_commit\"",
"printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"",
"printf 'pipelineRunExitCode\\t%s\\n' \"$pipeline_status\"",
"printf 'status\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '1p')\"",
"printf 'reason\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '2p')\"",
"printf 'message\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '3p')\"",
"printf 'pipelineRunStderr\\t%s\\n' \"$(one_line < \"$tmp_dir/pipelinerun.err\")\"",
].join("\n");
const result = g14K3s(["script", "--", script], 180_000);
return parseV02TriggerSnapshot(result, Date.now());
}
export function v02ExistingPipelineRunReuseDecision(input: {
sourceCommit: string;
before: Record<string, unknown>;
latestSourceCommit: string | null;
}): Record<string, unknown> {
const status = stringOrNull(input.before.status);
const exists = input.before.exists === true;
const latestSourceCommit = input.latestSourceCommit;
const alreadyUsable = exists && (status === "True" || status === "Unknown");
if (latestSourceCommit === null) {
return {
reusable: false,
alreadyUsable,
reason: "source-head-recheck-unresolved-before-existing-pipelinerun-reuse",
sourceCommit: input.sourceCommit,
latestSourceCommit,
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
};
}
if (latestSourceCommit !== null && latestSourceCommit !== input.sourceCommit) {
return {
reusable: false,
alreadyUsable,
reason: "source-head-advanced-before-existing-pipelinerun-reuse",
sourceCommit: input.sourceCommit,
latestSourceCommit,
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
latestPipelineRun: v02PipelineRunName(latestSourceCommit),
};
}
return {
reusable: exists && status !== null,
alreadyUsable,
reason: alreadyUsable ? "existing-pipelinerun-reused" : "existing-pipelinerun-terminal-failed",
sourceCommit: input.sourceCommit,
latestSourceCommit,
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
};
}
function getPipelineRunCompact(name: string): Record<string, unknown> {
const result = g14K3s([
"kubectl",
"get",
"pipelinerun",
"-n",
CI_NAMESPACE,
name,
"-o",
"jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}",
], 60_000);
return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr);
}
function pipelineRunCompactFromText(
name: string,
text: string,
commandOk: boolean,
command: string[] | string,
exitCode: number | null,
stderr: string,
): Record<string, unknown> {
const [status = "", reason = "", message = ""] = text.split(/\r?\n/u);
const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`);
return {
ok: commandOk,
exists: commandOk || !notFound,
pipelineRun: name,
status: status || null,
reason: reason || null,
message: message || null,
command: Array.isArray(command) ? redactedCommand(command) : command,
exitCode,
stderr: stderr.trim().slice(0, 2000),
};
}
function timestampMs(value: unknown): number | null {
if (typeof value !== "string" || value.trim().length === 0) return null;
const parsed = Date.parse(value);
return Number.isFinite(parsed) ? parsed : null;
}
function secondsBetween(start: unknown, end: unknown): number | null {
const startMs = timestampMs(start);
const endMs = timestampMs(end);
if (startMs === null || endMs === null || endMs < startMs) return null;
return Math.round((endMs - startMs) / 1000);
}
function pipelineRunStatusRowFromLine(line: string, nowMs: number, pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record<string, unknown> | null {
const [
name = "",
sourceCommit = "",
createdAt = "",
startTime = "",
completionTime = "",
status = "",
reason = "",
] = line.split("\t");
if (!name.startsWith(pipelineRunPrefix)) return null;
const startMs = timestampMs(startTime);
const completion = completionTime.trim().length > 0 ? completionTime : null;
const conditionStatus = status.trim().length > 0 ? status : null;
return {
name,
sourceCommit: sourceCommit || null,
status: conditionStatus,
reason: reason || null,
createdAt: createdAt || null,
startTime: startTime || null,
completionTime: completion,
durationSeconds: secondsBetween(startTime, completion),
elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)),
active: completion === null && conditionStatus !== "True" && conditionStatus !== "False",
};
}
function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now(), pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record<string, unknown> {
const rows = text
.split(/\r?\n/u)
.map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs, pipelineRunPrefix))
.filter((item): item is Record<string, unknown> => item !== null)
.sort((left, right) => {
return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0);
});
const activeItems = rows.filter((item) => item.active === true);
return {
ok: true,
count: rows.length,
activeCount: activeItems.length,
items: rows.slice(0, limit),
activeItems: activeItems.slice(0, limit),
disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete",
};
}
function pipelineRunRowsJsonPath(): string {
return [
"jsonpath={range .items[*]}",
"{.metadata.name}",
"{\"\\t\"}",
"{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}",
"{\"\\t\"}",
"{.metadata.creationTimestamp}",
"{\"\\t\"}",
"{.status.startTime}",
"{\"\\t\"}",
"{.status.completionTime}",
"{\"\\t\"}",
"{.status.conditions[0].status}",
"{\"\\t\"}",
"{.status.conditions[0].reason}",
"{\"\\n\"}",
"{end}",
].join("");
}
function listV02PipelineRunsCompact(limit = 8): Record<string, unknown> {
const result = g14K3s([
"kubectl",
"get",
"pipelinerun",
"-n",
CI_NAMESPACE,
"-l",
"hwlab.pikastech.local/gitops-target=v02",
"-o",
pipelineRunRowsJsonPath(),
], 60_000);
if (!isCommandSuccess(result)) {
return {
ok: false,
command: redactedCommand(result.command),
exitCode: result.exitCode,
stderr: commandErrorSummary(result),
items: [],
activeItems: [],
};
}
return parsePipelineRunRows(statusText(result), limit);
}
function parseShellSections(output: string): Record<string, ShellSection> {
const sections: Record<string, ShellSection> = {};
let currentName: string | null = null;
let currentLines: string[] = [];
let currentExitCode: number | null = null;
const flush = (): void => {
if (currentName === null) return;
sections[currentName] = {
stdout: currentLines.join("\n").trim(),
exitCode: currentExitCode,
};
};
for (const line of output.split(/\r?\n/u)) {
const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line);
if (begin !== null) {
flush();
currentName = begin[1] ?? null;
currentLines = [];
currentExitCode = null;
continue;
}
const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line);
if (end !== null && currentName === end[1]) {
currentExitCode = Number(end[2]);
flush();
currentName = null;
currentLines = [];
currentExitCode = null;
continue;
}
if (currentName !== null) currentLines.push(line);
}
flush();
return sections;
}
function shellSectionOk(section: ShellSection | undefined): boolean {
return section?.exitCode === 0;
}
function v02ControlPlaneStatusBundle(target: V02ControlPlaneStatusTarget = {}): CommandJsonResult {
const targetMode: V02StatusTargetMode = target.mode
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null
? [
`pipeline_run=${shellQuote(target.pipelineRun)}`,
`source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`,
`if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | cut -d- -f5-); fi`,
].join("\n")
: [
target.sourceCommit === undefined
? `source_commit=$(git --git-dir=${shellQuote(V02_CICD_REPO)} rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)`
: `source_commit=${shellQuote(target.sourceCommit ?? "")}`,
"pipeline_run=",
].join("\n");
const script = [
"set +e",
targetInit,
`target_mode=${shellQuote(targetMode)}`,
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${V02_PIPELINERUN_PREFIX}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`,
"section() {",
" name=\"$1\"",
" shift",
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
" \"$@\"",
" code=$?",
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
"section statusTarget printf 'mode\\t%s\\nsourceCommit\\t%s\\npipelineRun\\t%s\\n' \"$target_mode\" \"$source_commit\" \"$pipeline_run\"",
"section sourceCommit printf '%s\\n' \"$source_commit\"",
"section pipelineRunName printf '%s\\n' \"$pipeline_run\"",
`section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())} sh "$source_commit"`,
"section queryNow date -u +%Y-%m-%dT%H:%M:%SZ",
`section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`,
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
`if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
`if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
`if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`,
`section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`,
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
`section gitMirrorCache kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(gitMirrorCacheProbeScript())}`,
`section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
].join("\n");
return g14K3s(["script", "--", script], 60_000);
}
function v02SourceHeadsProbeScript(): string {
return [
"set +e",
"target_source_commit=${1:-}",
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
`workspace=${shellQuote(V02_WORKSPACE)}`,
"rev_cicd() { git --git-dir=\"$cicd_repo\" rev-parse \"$1\" 2>/dev/null || true; }",
"rev_workspace() { git -C \"$workspace\" rev-parse \"$1\" 2>/dev/null || true; }",
"origin_head=$(rev_cicd refs/remotes/origin/v0.2)",
"printf 'cicdRepo\\t%s\\n' \"$cicd_repo\"",
"printf 'cicdRepoExists\\t%s\\n' \"$([ -d \"$cicd_repo/objects\" ] && printf yes || printf no)\"",
"printf 'cicdSourceHead\\t%s\\n' \"$origin_head\"",
"printf 'originHead\\t%s\\n' \"$origin_head\"",
"printf 'targetSourceCommit\\t%s\\n' \"$target_source_commit\"",
"target_object_exists=",
"target_ancestor=",
"target_ancestor_exit=",
"if [ -n \"$target_source_commit\" ] && git --git-dir=\"$cicd_repo\" cat-file -e \"$target_source_commit^{commit}\" 2>/dev/null; then",
" target_object_exists=true",
" if [ -n \"$origin_head\" ]; then",
" git --git-dir=\"$cicd_repo\" merge-base --is-ancestor \"$target_source_commit\" \"$origin_head\" >/dev/null 2>&1",
" target_ancestor_exit=$?",
" if [ \"$target_ancestor_exit\" = 0 ]; then target_ancestor=true; elif [ \"$target_ancestor_exit\" = 1 ]; then target_ancestor=false; else target_ancestor=unknown; fi",
" fi",
"elif [ -n \"$target_source_commit\" ]; then",
" target_object_exists=false",
"fi",
"printf 'targetObjectExists\\t%s\\n' \"$target_object_exists\"",
"printf 'targetAncestorOfOriginHead\\t%s\\n' \"$target_ancestor\"",
"printf 'targetAncestorExitCode\\t%s\\n' \"$target_ancestor_exit\"",
"printf 'workspacePath\\t%s\\n' \"$workspace\"",
"printf 'workspaceBranch\\t%s\\n' \"$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)\"",
"printf 'workspaceHead\\t%s\\n' \"$(rev_workspace HEAD)\"",
"printf 'workspaceOriginHead\\t%s\\n' \"$(rev_workspace refs/remotes/origin/v0.2)\"",
"printf 'workspaceDirtyCount\\t%s\\n' \"$(git -C \"$workspace\" status --porcelain=v1 2>/dev/null | wc -l | tr -d ' ')\"",
].join("\n");
}
function v02PlanArtifactsLogScript(): string {
return [
"pipeline_run=\"$1\"",
`namespace=${shellQuote(CI_NAMESPACE)}`,
"[ -n \"$pipeline_run\" ] || exit 0",
"pods=$(kubectl -n \"$namespace\" get pods -l \"tekton.dev/pipelineRun=$pipeline_run,tekton.dev/pipelineTask=plan-artifacts\" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)",
"for pod in $pods; do",
" printf '__POD__\\t%s\\n' \"$pod\"",
" kubectl -n \"$namespace\" logs \"$pod\" --all-containers --tail=200 2>/dev/null || true",
"done",
].join("\n");
}
function v02RuntimeWorkloadsColumns(): string {
return [
"custom-columns=KIND:.kind",
"NAME:.metadata.name",
"SERVICE:.metadata.labels.hwlab\\.pikastech\\.local/service-id",
"ARTIFACT:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/artifact-source-commit",
"SOURCE:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/source-commit",
"IMAGE:.spec.template.spec.containers[0].image",
"READY:.status.readyReplicas",
"REPLICAS:.status.replicas",
].join(",");
}
function v02WebAssetsProbeScript(): string {
return [
"set +e",
`base=${shellQuote(V02_CLOUD_WEB_URL)}`,
`api=${shellQuote(V02_CLOUD_API_URL)}`,
"fetch_url() {",
" if command -v curl >/dev/null 2>&1; then",
" curl -fsS --connect-timeout 2 --max-time 10 \"$1\"",
" elif command -v wget >/dev/null 2>&1; then",
" wget -q -T 10 -O - \"$1\"",
" else",
" return 127",
" fi",
"}",
"printf 'baseUrl\\t%s\\n' \"$base\"",
"printf 'apiUrl\\t%s\\n' \"$api\"",
"probe_start_s=$(date +%s 2>/dev/null || printf '0')",
"html=$(fetch_url \"$base/\" 2>/dev/null)",
"html_code=$?",
"printf 'htmlOk\\t%s\\n' \"$html_code\"",
"css_path=$(printf '%s' \"$html\" | sed -n 's/.*href=\"\\([^\"]*\\.css\\)\".*/\\1/p' | head -1)",
"printf 'cssPath\\t%s\\n' \"$css_path\"",
"case \"$css_path\" in",
" http://*|https://*) css_url=\"$css_path\" ;;",
" /*) css_url=\"$base$css_path\" ;;",
" \"\") css_url=\"\" ;;",
" *) css_url=\"$base/$css_path\" ;;",
"esac",
"printf 'htmlAppScript\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eq 'src=\"/app\\.js\"|src=\"app\\.js\"'; printf '%s' \"$?\")\"",
"printf 'htmlCssLink\\t%s\\n' \"$(test -n \"$css_path\"; printf '%s' \"$?\")\"",
"printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"",
"if [ -n \"$css_url\" ]; then css=$(fetch_url \"$css_url\" 2>/dev/null); else css=\"\"; false; fi",
"css_code=$?",
"printf 'cssOk\\t%s\\n' \"$css_code\"",
"printf 'cssBytes\\t%s\\n' \"$(printf '%s' \"$css\" | wc -c | tr -d ' ')\"",
"app_js=$(fetch_url \"$base/app.js\" 2>/dev/null)",
"app_js_code=$?",
"probe_end_s=$(date +%s 2>/dev/null || printf '0')",
"printf 'appJsOk\\t%s\\n' \"$app_js_code\"",
"printf 'appJsBytes\\t%s\\n' \"$(printf '%s' \"$app_js\" | wc -c | tr -d ' ')\"",
"printf 'probeElapsedMs\\t%s\\n' \"$(((probe_end_s - probe_start_s) * 1000))\"",
"fetch_url \"$base/styles.css\" >/dev/null 2>&1",
"legacy_styles_code=$?",
"if [ \"$legacy_styles_code\" = \"0\" ]; then legacy_styles_absent=1; else legacy_styles_absent=0; fi",
"printf 'legacyStylesAbsent\\t%s\\n' \"$legacy_styles_absent\"",
"printf 'reactWorkbenchCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'workbench-shell|activity-rail|session-sidebar|conversation-panel'; printf '%s' \"$?\")\"",
"health=$(fetch_url \"$api/health/live\" 2>/dev/null)",
"health_code=$?",
"printf 'apiHealthOk\\t%s\\n' \"$health_code\"",
"printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"",
].join("\n");
}
function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
if (!commandOk) {
return {
ok: false,
pipelineRun,
exitCode,
stderr: stderr.trim().slice(0, 2000),
counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
items: [],
performance: v02TaskRunPerformanceSummary([]),
};
}
const items = text
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t");
return {
name,
status: status || null,
reason: reason || null,
startTime: startTime || null,
completionTime: completionTime || null,
durationSeconds: secondsBetween(startTime, completionTime),
};
});
const counts = {
succeeded: items.filter((item) => item.status === "True").length,
failed: items.filter((item) => item.status === "False").length,
running: items.filter((item) => item.status === "Unknown").length,
unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
};
const performance = v02TaskRunPerformanceSummary(items);
const performanceWarning = performance.ok === false ? `; ${String(performance.summary ?? "")}` : "";
return {
ok: true,
pipelineRun,
counts,
items,
performance,
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}${performanceWarning}`,
disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
};
}
function v02TaskRunPipelineTaskName(name: string): string | null {
const buildIndex = name.lastIndexOf("-build-");
if (buildIndex >= 0) return name.slice(buildIndex + 1);
const knownSuffixes = [
"prepare-source",
"plan-artifacts",
"publish-artifact-catalog",
"gitops-render",
"gitops-promote",
"runtime-ready",
"collect-artifacts",
];
return knownSuffixes.find((suffix) => name.endsWith(`-${suffix}`)) ?? null;
}
export function v02TaskRunPerformanceSummary(taskRuns: unknown[]): Record<string, unknown> {
const slowTaskRuns: Record<string, unknown>[] = [];
for (const itemRaw of taskRuns) {
const item = record(itemRaw);
const name = stringOrNull(item.name) ?? "";
const pipelineTask = stringOrNull(item.pipelineTask) ?? v02TaskRunPipelineTaskName(name);
const durationSeconds = typeof item.durationSeconds === "number" ? item.durationSeconds : null;
if (!pipelineTask?.startsWith("build-") || durationSeconds === null || durationSeconds <= V02_BUILD_TASKRUN_WARNING_SECONDS) continue;
const serviceId = pipelineTask.slice("build-".length) || null;
slowTaskRuns.push({
name,
pipelineTask,
serviceId,
status: stringOrNull(item.status),
reason: stringOrNull(item.reason),
durationSeconds,
budgetSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
severity: durationSeconds >= V02_BUILD_TASKRUN_CRITICAL_SECONDS ? "critical" : "warning",
message: `${pipelineTask} took ${durationSeconds}s, above v0.2 build TaskRun warning budget ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
});
}
slowTaskRuns.sort((left, right) => Number(right.durationSeconds ?? 0) - Number(left.durationSeconds ?? 0));
const worst = slowTaskRuns[0];
return {
ok: slowTaskRuns.length === 0,
warningCount: slowTaskRuns.length,
thresholds: {
buildTaskRunWarningSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
buildTaskRunCriticalSeconds: V02_BUILD_TASKRUN_CRITICAL_SECONDS,
},
slowTaskRuns,
summary: worst
? `slow build taskruns=${slowTaskRuns.length}; worst=${String(worst.pipelineTask)} ${String(worst.durationSeconds)}s budget=${V02_BUILD_TASKRUN_WARNING_SECONDS}s`
: `no build taskrun over ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
};
}
function v02WebAssetsFromText(
text: string,
commandOk: boolean,
sourceCommit: string | null,
argoSyncRevision: string | null,
exitCode: number | null,
stderr: string,
activePipelineRuns: unknown[] = [],
): Record<string, unknown> {
const fields: Record<string, string> = {};
for (const line of text.split(/\r?\n/u)) {
const [key = "", ...rest] = line.split("\t");
if (key.length > 0) fields[key] = rest.join("\t");
}
const htmlOk = fields.htmlOk === "0";
const htmlAppScript = fields.htmlAppScript === "0";
const htmlCssLink = fields.htmlCssLink === "0";
const cssOk = fields.cssOk === "0";
const appJsOk = fields.appJsOk === "0";
const apiHealthOk = fields.apiHealthOk === "0";
const readonlyNoteAbsent = fields.readonlyNote === "1";
const legacyStylesAbsent = fields.legacyStylesAbsent === "0";
const reactWorkbenchCss = fields.reactWorkbenchCss === "0";
const apiRevision = fields.apiRevision || null;
const webChecksPass = htmlOk && htmlAppScript && htmlCssLink && cssOk && appJsOk && readonlyNoteAbsent && legacyStylesAbsent && reactWorkbenchCss && apiHealthOk;
const failedChecks = Object.entries({
htmlOk,
htmlAppScript,
htmlCssLink,
cssOk,
appJsOk,
readonlyNoteAbsent,
legacyStylesAbsent,
reactWorkbenchCss,
apiHealthOk,
}).filter(([, ok]) => !ok).map(([name]) => name);
return {
ok: commandOk && webChecksPass,
summary: commandOk && webChecksPass ? "19666/19667 React/Vite asset probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`,
baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL,
apiUrl: fields.apiUrl || V02_CLOUD_API_URL,
sourceCommit,
argoSyncRevision: argoSyncRevision || null,
cssPath: fields.cssPath || null,
checks: {
htmlOk,
htmlAppScript,
htmlCssLink,
cssOk,
appJsOk,
readonlyNoteAbsent,
legacyStylesAbsent,
reactWorkbenchCss,
apiHealthOk,
},
probeExitCodes: {
html: numericField(fields.htmlOk),
htmlAppScript: numericField(fields.htmlAppScript),
htmlCssLink: numericField(fields.htmlCssLink),
css: numericField(fields.cssOk),
appJs: numericField(fields.appJsOk),
readonlyNoteGrep: numericField(fields.readonlyNote),
legacyStylesAbsent,
reactWorkbenchCssGrep: numericField(fields.reactWorkbenchCss),
apiHealth: numericField(fields.apiHealthOk),
},
assetBytes: {
css: numericField(fields.cssBytes),
appJs: numericField(fields.appJsBytes),
},
probeElapsedMs: numericField(fields.probeElapsedMs),
apiRevision,
note: webAssetsRevisionNote(apiRevision, sourceCommit, activePipelineRuns),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
};
}
function v02PlanArtifactsFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
const events: Record<string, unknown>[] = [];
const pods: string[] = [];
for (const rawLine of text.split(/\r?\n/u)) {
const line = rawLine.trim();
if (line.startsWith("__POD__")) {
const [, pod = ""] = line.split("\t");
if (pod.length > 0) pods.push(pod);
continue;
}
if (!line.startsWith("{")) continue;
try {
const event = record(JSON.parse(line) as unknown);
if (event.event === "g14-ci-plan") events.push(event);
} catch {
// Ignore non-JSON log lines; the raw log remains visible through pod logs.
}
}
const latest = events.at(-1) ?? null;
if (!commandOk || latest === null) {
return {
ok: false,
pipelineRun,
pods,
eventFound: latest !== null,
degradedReason: commandOk ? "g14-ci-plan-event-not-found" : "plan-artifacts-log-query-failed",
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
};
}
const audit = record(latest.artifactProvenanceAudit);
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
const provenanceRebuildServices = Array.isArray(audit.provenanceRebuildServices) ? audit.provenanceRebuildServices : [];
return {
ok: true,
pipelineRun,
pods,
sourceCommitId: stringOrNull(latest.sourceCommitId),
affectedServices: stringArray(latest.affectedServices),
rolloutServices: stringArray(latest.rolloutServices),
buildServices: stringArray(latest.buildServices),
reusedServices: stringArray(latest.reusedServices),
buildSkippedCount: numericValue(latest.buildSkippedCount),
artifactProvenanceAudit: Object.keys(audit).length > 0 ? audit : null,
summary: `build=${stringArray(latest.buildServices).length} reuse=${stringArray(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`,
disclosure: "parsed from plan-artifacts g14-ci-plan log event",
};
}
function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
if (!commandOk) {
return {
ok: false,
items: [],
exitCode,
stderr: stderr.trim().slice(0, 2000),
};
}
const items = text
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const [kind = "", name = "", serviceId = "", artifactSourceCommit = "", sourceCommit = "", image = "", ready = "", replicas = ""] = line.split(/\s+/u);
return {
kind,
name,
serviceId: serviceId === "<none>" ? null : serviceId,
artifactSourceCommit: artifactSourceCommit === "<none>" ? null : artifactSourceCommit,
sourceCommit: sourceCommit === "<none>" ? null : sourceCommit,
image: image === "<none>" ? null : image,
readyReplicas: numericField(ready),
replicas: numericField(replicas),
};
});
return {
ok: true,
items,
summary: `runtime deployments=${items.length}`,
};
}
function keyValueLinesFromText(text: string): Record<string, string> {
const fields: Record<string, string> = {};
for (const line of text.split(/\r?\n/u)) {
const [key = "", ...rest] = line.split("\t");
if (key.trim().length > 0) fields[key.trim()] = rest.join("\t").trim();
}
return fields;
}
function v02SourceHeadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
const fields = keyValueLinesFromText(text);
const dirtyCount = numericField(fields.workspaceDirtyCount);
const targetSourceCommit = fields.targetSourceCommit || null;
return {
ok: commandOk,
cicdRepo: fields.cicdRepo || V02_CICD_REPO,
cicdRepoExists: fields.cicdRepoExists === "yes",
cicdSourceHead: fields.cicdSourceHead || null,
originHead: fields.originHead || null,
target: {
sourceCommit: targetSourceCommit,
objectExists: fieldBoolean(fields.targetObjectExists),
ancestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead),
ancestorExitCode: numericField(fields.targetAncestorExitCode),
},
targetObjectExists: fieldBoolean(fields.targetObjectExists),
targetAncestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead),
workspace: {
path: fields.workspacePath || V02_WORKSPACE,
branch: fields.workspaceBranch || null,
head: fields.workspaceHead || null,
originHead: fields.workspaceOriginHead || null,
dirtyCount,
dirty: typeof dirtyCount === "number" ? dirtyCount > 0 : null,
isolatedFromCicd: true,
},
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
};
}
export function v02FalseGreenGuard(input: {
sourceCommit: string | null;
pipelineRun: Record<string, unknown> | null;
taskRuns: Record<string, unknown>;
planArtifacts: Record<string, unknown>;
runtimeWorkloads: Record<string, unknown>;
}): Record<string, unknown> {
const status = stringOrNull(input.pipelineRun?.status);
if (input.sourceCommit === null || input.pipelineRun === null || input.pipelineRun.exists === false) {
return { ok: null, state: "not-started", summary: "current source commit has no PipelineRun yet" };
}
if (status !== "True") {
return { ok: null, state: "pipeline-not-final", summary: `PipelineRun status=${status ?? "unknown"}` };
}
if (input.planArtifacts.ok !== true) {
return {
ok: false,
state: "missing-plan-artifacts-evidence",
summary: "PipelineRun succeeded but plan-artifacts g14-ci-plan evidence was not visible",
planArtifacts: input.planArtifacts,
};
}
const sourceCommit = input.sourceCommit;
const buildServices = stringArray(input.planArtifacts.buildServices);
const reusedServices = stringArray(input.planArtifacts.reusedServices);
const taskItems = Array.isArray(input.taskRuns.items) ? input.taskRuns.items.map((item) => record(item)) : [];
const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : [];
const workloadByService = new Map(workloadItems.map((item) => [String(item.serviceId ?? ""), item]));
const missingBuildTaskRuns = buildServices.filter((serviceId) => !taskItems.some((item) => String(item.name ?? "").endsWith(`build-${serviceId}`) && item.status === "True"));
const runtimeMismatches = buildServices
.map((serviceId) => {
const workload = workloadByService.get(serviceId);
if (!workload) return { serviceId, reason: "runtime-workload-missing" };
if (workload.artifactSourceCommit !== sourceCommit) {
return {
serviceId,
reason: "artifact-source-commit-mismatch",
runtimeArtifactSourceCommit: workload.artifactSourceCommit ?? null,
expectedSourceCommit: sourceCommit,
image: workload.image ?? null,
};
}
return null;
})
.filter((item): item is Record<string, unknown> => item !== null);
const audit = record(input.planArtifacts.artifactProvenanceAudit);
const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
const auditPresent = input.planArtifacts.artifactProvenanceAudit !== null;
const provenanceWarning = !auditPresent && reusedServices.length > 0
? { reason: "artifact-provenance-audit-missing-for-reuse", reusedServices }
: null;
const provenanceOk = auditPresent ? audit.ok !== false && unsafeReuseServices.length === 0 : true;
const failures = [
...(input.runtimeWorkloads.ok === true ? [] : [{ reason: "runtime-workload-query-failed" }]),
...(provenanceOk ? [] : [{ reason: "artifact-provenance-audit-failed", unsafeReuseServices }]),
...missingBuildTaskRuns.map((serviceId) => ({ serviceId, reason: "expected-build-taskrun-not-succeeded" })),
...runtimeMismatches,
];
return {
ok: failures.length === 0,
state: failures.length === 0 ? "passed" : "failed",
summary: failures.length === 0
? `artifact provenance and built runtime services align with ${shortSha(sourceCommit)}`
: `false-green risk: ${failures.length} invariant violation(s)`,
sourceCommit,
buildServices,
reusedServices,
provenanceOk,
provenanceAuditPresent: auditPresent,
provenanceWarnings: provenanceWarning ? [provenanceWarning] : [],
missingBuildTaskRuns,
runtimeMismatches,
failures,
};
}
function v02TargetValidation(input: {
targetMode: V02StatusTargetMode;
sourceCommit: string | null;
pipelineRun: Record<string, unknown> | null;
argo: Record<string, unknown>;
planArtifacts: Record<string, unknown>;
runtimeWorkloads: Record<string, unknown>;
webAssets: Record<string, unknown>;
gitMirror: Record<string, unknown>;
recentPipelineRuns: Record<string, unknown>;
}): Record<string, unknown> {
if (input.targetMode === "latest-source-head") {
return {
applicable: false,
state: "latest-source-head",
summary: "default status uses strict latest source head alignment; use --pipeline-run or --source-commit for historical run validation",
};
}
const sourceCommit = input.sourceCommit;
const argoFields = record(input.argo.fields);
const gitMirrorSummary = record(input.gitMirror.summary);
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
? input.runtimeWorkloads.items.map((item) => record(item))
: [];
const serviceSourceCommits = Object.fromEntries(workloadItems
.filter((item) => item.serviceId === "hwlab-cloud-api" || item.serviceId === "hwlab-cloud-web")
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
const rolloutServices = stringArray(input.planArtifacts.rolloutServices)
.filter((serviceId) => serviceId === "hwlab-cloud-api" || serviceId === "hwlab-cloud-web");
const requiredRuntimeServices = rolloutServices;
const recentItems = Array.isArray(input.recentPipelineRuns.items)
? input.recentPipelineRuns.items.map((item) => record(item))
: [];
const targetPipelineRunName = stringOrNull(input.pipelineRun?.pipelineRun) ?? stringOrNull(input.pipelineRun?.name);
const targetRecentIndex = recentItems.findIndex((item) => (
(targetPipelineRunName !== null && item.name === targetPipelineRunName)
|| (sourceCommit !== null && item.sourceCommit === sourceCommit)
));
const newerCandidates = targetRecentIndex >= 0 ? recentItems.slice(0, targetRecentIndex) : recentItems;
const newerSucceededRuns = sourceCommit === null
? []
: newerCandidates.filter((item) => item.sourceCommit !== sourceCommit && item.status === "True");
const failures: Record<string, unknown>[] = [];
const supersededRuntimeServices: Record<string, unknown>[] = [];
if (sourceCommit === null) failures.push({ reason: "source-commit-unresolved" });
if (input.pipelineRun === null || input.pipelineRun.exists === false) {
failures.push({ reason: "pipeline-run-missing" });
} else if (input.pipelineRun.status !== "True") {
failures.push({ reason: "pipeline-run-not-succeeded", status: input.pipelineRun.status ?? null, reasonDetail: input.pipelineRun.reason ?? null });
}
if (input.argo.ok !== true || argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") {
failures.push({ reason: "argo-not-synced-healthy", syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null });
}
if (input.webAssets.ok !== true) failures.push({ reason: "web-assets-probe-failed", summary: input.webAssets.summary ?? null });
if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) {
failures.push({
reason: "git-mirror-not-flushed",
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
githubInSync: gitMirrorSummary.githubInSync ?? null,
});
}
for (const serviceId of requiredRuntimeServices) {
const serviceCommit = serviceSourceCommits[serviceId];
if (sourceCommit !== null && serviceCommit !== sourceCommit) {
const mismatch = { reason: "runtime-service-source-mismatch", serviceId, expectedSourceCommit: sourceCommit, actualSourceCommit: serviceCommit ?? null };
if (newerSucceededRuns.length > 0) supersededRuntimeServices.push(mismatch);
else failures.push(mismatch);
}
}
const superseded = failures.length === 0 && supersededRuntimeServices.length > 0;
return {
applicable: true,
ok: failures.length === 0,
state: failures.length === 0 ? (superseded ? "superseded" : "passed") : "failed",
summary: failures.length === 0
? superseded
? `target ${input.targetMode} completed for ${shortSha(sourceCommit ?? "")}; runtime now reflects newer v0.2 PipelineRun`
: requiredRuntimeServices.length === 0
? `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}; no runtime service rollout required`
: `target ${input.targetMode} validation passed for ${shortSha(sourceCommit ?? "")}`
: `target ${input.targetMode} validation failed with ${failures.length} issue(s)`,
sourceCommit,
pipelineRun: input.pipelineRun === null
? null
: { name: input.pipelineRun.pipelineRun ?? null, status: input.pipelineRun.status ?? null, reason: input.pipelineRun.reason ?? null },
argo: { syncRevision: argoFields.syncRevision ?? null, syncStatus: argoFields.syncStatus ?? null, health: argoFields.health ?? null },
runtimeServices: serviceSourceCommits,
requiredRuntimeServices,
rolloutServices: stringArray(input.planArtifacts.rolloutServices),
reusedServices: stringArray(input.planArtifacts.reusedServices),
superseded,
supersededRuntimeServices,
newerSucceededRuns: newerSucceededRuns.slice(0, 3).map((item) => ({
name: item.name ?? null,
sourceCommit: item.sourceCommit ?? null,
createdAt: item.createdAt ?? null,
durationSeconds: item.durationSeconds ?? null,
})),
webAssets: { ok: input.webAssets.ok ?? null, summary: input.webAssets.summary ?? null, appJsBytes: record(input.webAssets.assetBytes).appJs ?? null },
gitMirror: { pendingFlush: gitMirrorSummary.pendingFlush ?? null, githubInSync: gitMirrorSummary.githubInSync ?? null },
failures,
};
}
export function v02LatestOnlyTargetValidation(input: {
targetMode: string;
sourceCommit: string | null;
pipelineRun: Record<string, unknown> | null;
commitAlignment: Record<string, unknown>;
targetValidation: Record<string, unknown>;
}): Record<string, unknown> {
if (input.targetMode === "latest-source-head") return input.targetValidation;
if (input.sourceCommit === null) return input.targetValidation;
if (input.pipelineRun === null || input.pipelineRun.status !== "True") return input.targetValidation;
const validation = record(input.targetValidation);
const staleReasons = stringArray(input.commitAlignment.staleReasons);
const sourceHeadAdvancedReasons = staleReasons.filter((reason) => (
reason === "origin-head-mismatch"
|| reason === "cicd-source-repo-stale"
|| reason === "latest-pipelinerun-not-current"
));
if (sourceHeadAdvancedReasons.length === 0) return input.targetValidation;
const failures = Array.isArray(validation.failures) ? validation.failures : [];
return {
...validation,
ok: true,
state: "superseded",
superseded: true,
latestOnlySuperseded: true,
latestOnlyReasons: sourceHeadAdvancedReasons,
originalState: validation.state ?? null,
summary: validation.ok === true || validation.state === "passed"
? `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and is superseded by the newer v0.2 source head`
: `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`,
supersededFailures: failures.slice(0, 10),
failures: [],
};
}
function v02CloseoutRecommendedNext(input: {
closeable: boolean;
blockedReasons: string[];
pendingReasons: string[];
statusCommand: string;
sourceCommit: string | null;
gitMirrorSummary: Record<string, unknown>;
}): Record<string, unknown> {
const rawFlushCommand = stringOrNull(input.gitMirrorSummary.flushCommand) ?? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm";
const flushCommand = rawFlushCommand.includes("--wait") ? rawFlushCommand : `${rawFlushCommand} --wait`;
if (input.closeable) {
return {
action: "comment-and-close-issue",
command: input.sourceCommit === null
? input.statusCommand
: `bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit ${input.sourceCommit}`,
};
}
if (input.pendingReasons.includes("git-mirror-not-flushed")) {
return { action: "flush-git-mirror", command: flushCommand };
}
if (input.pendingReasons.includes("argo-not-healthy")) {
return { action: "wait-and-recheck", command: input.statusCommand };
}
if (input.pendingReasons.includes("pipeline-run-not-final")) {
return { action: "wait-pipelinerun", command: input.statusCommand };
}
if (input.blockedReasons.includes("target-pipelinerun-missing")) {
return { action: "trigger-current-or-inspect-target", command: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" };
}
return { action: "inspect-status", command: input.statusCommand };
}
export function v02CloseoutVerdict(status: Record<string, unknown>): Record<string, unknown> {
const statusTarget = record(status.statusTarget);
const targetMode = stringOrNull(statusTarget.mode) ?? "latest-source-head";
const sourceCommit = stringOrNull(status.sourceCommit) ?? stringOrNull(statusTarget.sourceCommit);
const pipelineRun = record(status.pipelineRun);
const targetValidation = record(status.targetValidation);
const targetValidationState = stringOrNull(targetValidation.state);
const argoFields = record(record(status.argo).fields);
const gitMirrorSummary = record(record(status.gitMirror).summary);
const sourceHeadsTarget = record(record(status.sourceHeads).target);
const commitAlignment = record(status.commitAlignment);
const latestHead = stringOrNull(commitAlignment.originHead) ?? stringOrNull(record(status.sourceHeads).originHead);
const ancestorOfLatest = sourceHeadsTarget.ancestorOfOriginHead ?? status.sourceHeadsTargetAncestorOfOriginHead ?? null;
const activePipelineRuns = Array.isArray(status.activePipelineRuns)
? status.activePipelineRuns.map((item) => record(item))
: [];
const pipelineStatus = stringOrNull(pipelineRun.status);
const pipelineExists = pipelineRun.exists !== false && Object.keys(pipelineRun).length > 0;
const pendingReasons: string[] = [];
const blockedReasons: string[] = [];
if (targetMode === "latest-source-head") {
pendingReasons.push("target-not-explicit");
}
if (!pipelineExists) {
blockedReasons.push("target-pipelinerun-missing");
} else if (pipelineStatus === "Unknown" || pipelineStatus === null) {
pendingReasons.push("pipeline-run-not-final");
} else if (pipelineStatus === "False") {
blockedReasons.push("pipeline-run-failed");
}
if (targetValidationState !== "passed" && targetValidationState !== "superseded") {
const failures = Array.isArray(targetValidation.failures) ? targetValidation.failures.map((item) => record(item)) : [];
for (const failure of failures) {
const reason = stringOrNull(failure.reason);
if (reason === "git-mirror-not-flushed") pendingReasons.push(reason);
else if (reason === "argo-not-synced-healthy") pendingReasons.push("argo-not-healthy");
else if (reason === "pipeline-run-not-succeeded" && pipelineStatus === "Unknown") pendingReasons.push("pipeline-run-not-final");
else if (reason !== null) blockedReasons.push(reason);
}
if (failures.length === 0 && targetMode !== "latest-source-head") blockedReasons.push("target-validation-not-passed");
}
if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) {
pendingReasons.push("git-mirror-not-flushed");
}
if (argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") {
pendingReasons.push("argo-not-healthy");
}
if (targetValidationState === "superseded" && ancestorOfLatest !== true) {
if (ancestorOfLatest === false) blockedReasons.push("target-not-ancestor-of-latest-head");
else pendingReasons.push("ancestor-check-unresolved");
}
const uniquePendingReasons = Array.from(new Set(pendingReasons));
const uniqueBlockedReasons = Array.from(new Set(blockedReasons));
const terminalState = targetValidationState === "passed" || targetValidationState === "superseded";
const state = uniqueBlockedReasons.length > 0
? "blocked"
: uniquePendingReasons.length > 0
? "pending"
: terminalState
? targetValidationState
: "blocked";
const closeable = (state === "passed" || state === "superseded") && uniquePendingReasons.length === 0 && uniqueBlockedReasons.length === 0;
const rawStatusCommand = String(status.command ?? (
sourceCommit === null
? "bun scripts/cli.ts hwlab g14 control-plane status --lane v02"
: `bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}`
));
const statusCommand = rawStatusCommand.startsWith("bun ") ? rawStatusCommand : `bun scripts/cli.ts ${rawStatusCommand}`;
const ancestorCheck = {
required: state === "superseded",
sourceCommit,
latestHead,
objectExists: sourceHeadsTarget.objectExists ?? null,
ancestorOfLatest,
exitCode: sourceHeadsTarget.ancestorExitCode ?? null,
command: sourceCommit === null
? null
: `trans G14:/root/hwlab-v02 script -- 'git fetch origin v0.2 && git merge-base --is-ancestor ${sourceCommit} origin/v0.2'`,
};
const recommendedNext = v02CloseoutRecommendedNext({
closeable,
blockedReasons: uniqueBlockedReasons,
pendingReasons: uniquePendingReasons,
statusCommand,
sourceCommit,
gitMirrorSummary,
});
const summary = closeable
? state === "superseded"
? `closeout ready: target ${shortSha(sourceCommit ?? "")} passed and is superseded by latest v0.2 head ${shortSha(latestHead ?? "")}`
: `closeout ready: target ${shortSha(sourceCommit ?? "")} passed on v0.2`
: uniquePendingReasons.length > 0
? `closeout pending: ${uniquePendingReasons.join(",")}`
: `closeout blocked: ${uniqueBlockedReasons.join(",") || "unknown"}`;
const issueCommentMarkdown = [
`v0.2 closeout verdict: \`${state}\` (${summary})`,
"",
`- target sourceCommit: \`${sourceCommit ?? "n/a"}\``,
`- target PipelineRun: \`${String(pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? "n/a")}\`; status=\`${pipelineStatus ?? "n/a"}\`; reason=\`${String(pipelineRun.reason ?? "n/a")}\``,
`- latest head: \`${latestHead ?? "n/a"}\`; ancestorOfLatest=\`${String(ancestorCheck.ancestorOfLatest ?? "n/a")}\``,
`- targetValidation: \`${targetValidationState ?? "n/a"}\`; pending=\`${uniquePendingReasons.join(",") || "none"}\`; blocked=\`${uniqueBlockedReasons.join(",") || "none"}\``,
`- Git mirror: pendingFlush=\`${String(gitMirrorSummary.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(gitMirrorSummary.githubInSync ?? "n/a")}\`, flushCommand=\`${String(gitMirrorSummary.flushCommand ?? "none")}\``,
`- Argo: sync=\`${String(argoFields.syncStatus ?? "n/a")}\`, health=\`${String(argoFields.health ?? "n/a")}\`, revision=\`${String(argoFields.syncRevision ?? "n/a")}\``,
`- active v0.2 PipelineRuns: \`${activePipelineRuns.length}\``,
`- recommended next: \`${String(recommendedNext.action ?? "inspect-status")}\` -> \`${String(recommendedNext.command ?? statusCommand)}\``,
].join("\n");
return {
ok: closeable,
closeable,
state,
summary,
targetMode,
sourceCommit,
latestHead,
pipelineRun: {
name: pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? null,
exists: pipelineExists,
status: pipelineStatus,
reason: pipelineRun.reason ?? null,
},
targetValidation: {
state: targetValidationState,
ok: targetValidation.ok ?? null,
failures: Array.isArray(targetValidation.failures) ? targetValidation.failures.slice(0, 10) : [],
},
latestOnly: {
superseded: targetValidation.latestOnlySuperseded === true || state === "superseded",
reasons: stringArray(targetValidation.latestOnlyReasons),
ancestorCheck,
},
gitMirror: {
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
githubInSync: gitMirrorSummary.githubInSync ?? null,
flushNeeded: gitMirrorSummary.flushNeeded ?? null,
flushCommand: gitMirrorSummary.flushCommand ?? null,
lastSync: gitMirrorSummary.lastSync ?? null,
lastWrite: gitMirrorSummary.lastWrite ?? null,
lastFlush: gitMirrorSummary.lastFlush ?? null,
},
argo: {
syncStatus: argoFields.syncStatus ?? null,
health: argoFields.health ?? null,
syncRevision: argoFields.syncRevision ?? null,
targetRevision: argoFields.targetRevision ?? null,
path: argoFields.path ?? null,
},
runtime: {
webAssetsOk: nested(status, ["webAssets", "ok"]) ?? null,
webAssetsSummary: nested(status, ["webAssets", "summary"]) ?? null,
apiRevision: nested(status, ["webAssets", "apiRevision"]) ?? nested(status, ["commitAlignment", "apiRevision"]) ?? null,
serviceSourceCommits: nested(status, ["commitAlignment", "serviceSourceCommits"]) ?? null,
},
activePipelineRuns,
recentPipelineRuns: status.recentPipelineRuns ?? null,
pendingReasons: uniquePendingReasons,
blockedReasons: uniqueBlockedReasons,
recommendedNext,
issueCommentMarkdown,
};
}
export function v02CommitAlignment(input: {
expectedSourceHead: string | null;
sourceHeads: Record<string, unknown>;
gitMirrorSummary: Record<string, unknown>;
pipelineRun: Record<string, unknown> | null;
recentPipelineRuns: Record<string, unknown>;
planArtifacts?: Record<string, unknown>;
runtimeWorkloads: Record<string, unknown>;
webAssets: Record<string, unknown>;
}): Record<string, unknown> {
const expectedSourceHead = input.expectedSourceHead;
const cicdSourceHead = stringOrNull(input.sourceHeads.cicdSourceHead);
const originHead = stringOrNull(input.sourceHeads.originHead) ?? expectedSourceHead;
const workspace = record(input.sourceHeads.workspace);
const workspaceHead = stringOrNull(workspace.head);
const workspaceOriginHead = stringOrNull(workspace.originHead);
const mirrorSourceHead = stringOrNull(input.gitMirrorSummary.localV02);
const mirrorGithubSourceHead = stringOrNull(input.gitMirrorSummary.githubV02);
const recentItems = Array.isArray(input.recentPipelineRuns.items)
? input.recentPipelineRuns.items.map((item) => record(item))
: [];
const latestPipelineRun = recentItems[0] ?? null;
const latestPipelineSourceCommit = latestPipelineRun === null ? null : stringOrNull(latestPipelineRun.sourceCommit);
const currentPipelineStatus = stringOrNull(input.pipelineRun?.status);
const apiRevision = stringOrNull(input.webAssets.apiRevision);
const planArtifacts = record(input.planArtifacts);
const planSourceCommit = stringOrNull(planArtifacts.sourceCommitId);
const rolloutServices = stringArray(planArtifacts.rolloutServices);
const apiRevisionRequired = expectedSourceHead !== null && planSourceCommit === expectedSourceHead && rolloutServices.includes("hwlab-cloud-api");
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
? input.runtimeWorkloads.items.map((item) => record(item))
: [];
const serviceSourceCommits = Object.fromEntries(workloadItems
.filter((item) => typeof item.serviceId === "string" && item.serviceId.length > 0)
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
const staleReasons: string[] = [];
if (expectedSourceHead === null) staleReasons.push("expected-source-head-unresolved");
if (expectedSourceHead !== null && originHead !== null && originHead !== expectedSourceHead) staleReasons.push("origin-head-mismatch");
if (expectedSourceHead !== null && cicdSourceHead !== null && cicdSourceHead !== expectedSourceHead) staleReasons.push("cicd-source-repo-stale");
if (expectedSourceHead !== null && mirrorSourceHead !== expectedSourceHead) staleReasons.push("mirror-source-stale");
if (expectedSourceHead !== null && latestPipelineSourceCommit !== expectedSourceHead) staleReasons.push("latest-pipelinerun-not-current");
if (apiRevisionRequired && apiRevision !== expectedSourceHead) staleReasons.push("runtime-api-revision-stale");
const workspaceWarnings: string[] = [];
if (expectedSourceHead !== null && workspaceHead !== null && workspaceHead !== expectedSourceHead) workspaceWarnings.push("workspace-head-differs-from-latest-source-but-isolated");
if (expectedSourceHead !== null && workspaceOriginHead !== null && workspaceOriginHead !== expectedSourceHead) workspaceWarnings.push("workspace-origin-ref-stale-but-isolated");
if (workspace.dirty === true) workspaceWarnings.push("workspace-dirty-but-isolated-from-cicd");
const runtimeWarnings: string[] = [];
if (!apiRevisionRequired && expectedSourceHead !== null && apiRevision !== null && apiRevision !== expectedSourceHead) {
runtimeWarnings.push("api-revision-differs-without-current-cloud-api-rollout");
}
const latestPipelineSucceeded = latestPipelineRun !== null && latestPipelineRun.status === "True";
const aligned = staleReasons.length === 0;
const state = aligned
? "aligned"
: latestPipelineSucceeded
? "stale-success"
: currentPipelineStatus === "Unknown"
? "in-progress"
: "stale";
return {
aligned,
state,
expectedSourceHead,
originHead,
cicdSourceHead,
cicdRepo: input.sourceHeads.cicdRepo ?? V02_CICD_REPO,
mirrorSourceHead,
mirrorGithubSourceHead,
latestPipelineRun: latestPipelineRun === null
? null
: {
name: latestPipelineRun.name ?? null,
sourceCommit: latestPipelineSourceCommit,
status: latestPipelineRun.status ?? null,
reason: latestPipelineRun.reason ?? null,
createdAt: latestPipelineRun.createdAt ?? null,
durationSeconds: latestPipelineRun.durationSeconds ?? null,
},
latestPipelineSourceCommit,
currentPipelineRun: input.pipelineRun === null
? null
: {
name: input.pipelineRun.pipelineRun ?? null,
status: input.pipelineRun.status ?? null,
reason: input.pipelineRun.reason ?? null,
},
runtimeSourceCommit: apiRevision,
apiRevision,
apiRevisionRequired,
planSourceCommit,
rolloutServices,
serviceSourceCommits,
sourceInSync: input.gitMirrorSummary.sourceInSync ?? null,
gitopsInSync: input.gitMirrorSummary.gitopsInSync ?? null,
staleReasons,
workspace: {
path: workspace.path ?? V02_WORKSPACE,
branch: workspace.branch ?? null,
head: workspaceHead,
originHead: workspaceOriginHead,
dirty: workspace.dirty ?? null,
dirtyCount: workspace.dirtyCount ?? null,
isolatedFromCicd: true,
},
workspaceWarnings,
runtimeWarnings,
summary: aligned
? apiRevisionRequired
? `v0.2 CI/CD source, mirror, PipelineRun, and cloud-api runtime align with ${shortSha(expectedSourceHead ?? "")}`
: `v0.2 CI/CD source, mirror, and PipelineRun align with ${shortSha(expectedSourceHead ?? "")}; cloud-api runtime rollout not required`
: `v0.2 CI/CD alignment state=${state}; staleReasons=${staleReasons.join(",")}`,
};
}
function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null {
if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null;
const activeItems = activePipelineRuns.map((item) => record(item));
const sourceIsActive = activeItems.some((item) => item.sourceCommit === sourceCommit || String(item.name ?? "").includes(shortSha(sourceCommit)));
if (sourceIsActive) {
return `runtime rollout still in progress; cloud-api apiRevision=${apiRevision} has not reached sourceCommit=${sourceCommit} yet`;
}
if (activeItems.length > 0) {
return `runtime apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; active v02 PipelineRun observed, wait for rollout before judging runtime revision`;
}
return `cloud-api apiRevision=${apiRevision} is service-scoped and differs from sourceCommit=${sourceCommit}; when web asset probes pass, this is not proof that Cloud Web is stale`;
}
function numericField(value: string | undefined): number | null {
if (value === undefined || value.trim().length === 0) return null;
if (value === "<none>") return null;
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function numericValue(value: unknown): number | null {
const parsed = typeof value === "number" ? value : Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function stringArray(value: unknown): string[] {
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : [];
}
function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
if (!commandOk) {
return {
ok: false,
command: Array.isArray(command) ? redactedCommand(command) : command,
exitCode,
stderr: stderr.trim().slice(0, 4000),
items: [],
activeItems: [],
};
}
return parsePipelineRunRows(text, limit, nowMs);
}
interface CleanupPipelineRunRow {
name: string;
createdAt: string;
ageMinutes: number | null;
status: string | null;
reason: string | null;
}
function pipelinePrefixesForLane(lane: G14ControlPlaneOptions["lane"]): string[] {
if (isHwlabRuntimeLane(lane)) return [`${hwlabRuntimeLaneSpec(lane).pipelineRunPrefix}-`];
if (lane === "g14") return ["hwlab-g14-ci-poll-"];
return [...hwlabRuntimeLaneIds().map((runtimeLane) => `${hwlabRuntimeLaneSpec(runtimeLane).pipelineRunPrefix}-`), "hwlab-g14-ci-poll-"];
}
function commandErrorSummary(result: CommandJsonResult): string {
const remoteStderr = String(nested(result.parsed, ["data", "stderr"]) ?? "").trim();
const remoteStdout = String(nested(result.parsed, ["data", "stdout"]) ?? "").trim();
const text = [result.stderr.trim(), remoteStderr, remoteStdout, result.stdout.trim()].find((item) => item.length > 0) ?? "";
return text.slice(0, 4000);
}
function tailText(value: unknown, maxBytes = 2000): string {
const text = String(value ?? "").trim();
return text.length <= maxBytes ? text : text.slice(-maxBytes);
}
function cleanupPipelineRunFieldsJsonPath(range: boolean): string {
const body = [
"{.metadata.name}",
"{\"\\t\"}",
"{.metadata.creationTimestamp}",
"{\"\\t\"}",
"{.status.conditions[0].status}",
"{\"\\t\"}",
"{.status.conditions[0].reason}",
"{\"\\n\"}",
].join("");
return range ? `jsonpath={range .items[*]}${body}{end}` : `jsonpath=${body}`;
}
function parseCleanupPipelineRunLine(line: string, nowMs: number): CleanupPipelineRunRow | null {
const [name = "", createdAt = "", status = "", reason = ""] = line.trim().split("\t");
if (name.length === 0) return null;
const createdMs = Date.parse(createdAt);
const ageMinutes = Number.isFinite(createdMs) ? Math.floor((nowMs - createdMs) / 60000) : null;
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
}
function formatRetentionBytes(bytes: number): string | null {
if (!Number.isFinite(bytes)) return null;
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
let value = bytes;
let unit = 0;
while (value >= 1024 && unit < units.length - 1) {
value /= 1024;
unit += 1;
}
return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`;
}
function storageHostPathFromClaim(volume: string | null, namespace: string, claimName: string | null): string | null {
if (volume === null || claimName === null) return null;
if (!/^pvc-[a-z0-9-]+$/iu.test(volume) || !/^pvc-[a-z0-9]+$/iu.test(claimName)) return null;
return `/var/lib/rancher/k3s/storage/${volume}_${namespace}_${claimName}`;
}
function safeStorageHostPath(value: unknown): string | null {
const path = stringOrNull(value);
return path !== null && path.startsWith("/var/lib/rancher/k3s/storage/") ? path : null;
}
function remoteStoragePathEstimates(paths: Array<string | null | undefined>): Map<string, number | null> {
const uniquePaths = [...new Set(paths.map((item) => safeStorageHostPath(item)).filter((item): item is string => item !== null))];
const estimates = new Map<string, number | null>(uniquePaths.map((path) => [path, null]));
if (uniquePaths.length === 0) return estimates;
const script = [
"set -eu",
`for path in ${uniquePaths.map((path) => shellQuote(path)).join(" ")}; do`,
" bytes=\"\"",
" if [ -e \"$path\" ]; then bytes=$(du -sB1 \"$path\" 2>/dev/null | awk '{print $1}' || true); fi",
" printf '%s\\t%s\\n' \"$path\" \"$bytes\"",
"done",
].join("\n");
const result = g14HostScript(script, 120_000);
if (!isCommandSuccess(result)) return estimates;
for (const line of statusText(result).split(/\r?\n/u)) {
const [path = "", rawBytes = ""] = line.trim().split("\t");
if (!estimates.has(path)) continue;
const bytes = /^\d+$/u.test(rawBytes) ? Number(rawBytes) : null;
estimates.set(path, Number.isFinite(bytes) ? bytes : null);
}
return estimates;
}
function sumEstimatedBytes(items: Record<string, unknown>[]): number {
return items.reduce((sum, item) => {
const bytes = item.estimatedBytes;
return sum + (typeof bytes === "number" && Number.isFinite(bytes) ? bytes : 0);
}, 0);
}
function cleanupPipelineRunTargetCandidate(input: {
targetPipelineRun: string;
text: string;
commandOk: boolean;
exitCode: number | null;
stderr: string;
minAgeMinutes: number;
nowMs: number;
}): Record<string, unknown> {
if (!input.commandOk) {
const queryError = tailText(input.stderr);
const notFound = /notfound|not found/i.test(queryError);
return {
name: input.targetPipelineRun,
createdAt: null,
ageMinutes: null,
status: null,
reason: notFound ? "target-pipelinerun-not-found" : "target-pipelinerun-query-failed",
selected: false,
queryExitCode: input.exitCode,
...(queryError.length > 0 ? { queryError } : {}),
};
}
const target = parseCleanupPipelineRunLine(input.text, input.nowMs);
if (target === null || target.name !== input.targetPipelineRun) {
return {
name: input.targetPipelineRun,
createdAt: null,
ageMinutes: null,
status: null,
reason: "target-pipelinerun-query-empty",
selected: false,
queryOutputPreview: tailText(input.text),
};
}
if (target.status !== "True" && target.status !== "False") {
return {
...target,
selected: false,
selectedReason: "target-pipelinerun-not-terminal",
};
}
const ageMinutes = typeof target.ageMinutes === "number" ? target.ageMinutes : null;
const belowMinAge = ageMinutes === null || ageMinutes < input.minAgeMinutes;
return {
...target,
selected: !belowMinAge,
...(belowMinAge
? { selectedReason: ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }
: {}),
};
}
export function cleanupPipelineRunTargetCandidateFromTextForTest(input: {
targetPipelineRun: string;
text: string;
commandOk?: boolean;
exitCode?: number | null;
stderr?: string;
minAgeMinutes?: number;
nowMs?: number;
}): Record<string, unknown> {
return cleanupPipelineRunTargetCandidate({
targetPipelineRun: input.targetPipelineRun,
text: input.text,
commandOk: input.commandOk ?? true,
exitCode: input.exitCode ?? 0,
stderr: input.stderr ?? "",
minAgeMinutes: input.minAgeMinutes ?? 60,
nowMs: input.nowMs ?? Date.now(),
});
}
function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
if (options.lane !== "g14" && options.lane !== "all" && !isHwlabRuntimeLane(options.lane)) {
throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all");
}
const targetPipelineRun = options.pipelineRun ?? (options.sourceCommit === undefined
? undefined
: isHwlabRuntimeLane(options.lane)
? runtimeLanePipelineRunName(hwlabRuntimeLaneSpec(options.lane), options.sourceCommit)
: v02PipelineRunName(options.sourceCommit));
const now = Date.now();
if (targetPipelineRun !== undefined) {
const targetResult = g14K3s([
"kubectl",
"get",
"pipelinerun",
"-n",
CI_NAMESPACE,
targetPipelineRun,
"-o",
cleanupPipelineRunFieldsJsonPath(false),
], 60_000);
return [cleanupPipelineRunTargetCandidate({
targetPipelineRun,
text: statusText(targetResult),
commandOk: isCommandSuccess(targetResult),
exitCode: targetResult.exitCode,
stderr: commandErrorSummary(targetResult),
minAgeMinutes: options.minAgeMinutes,
nowMs: now,
})];
}
const result = g14K3s([
"kubectl",
"get",
"pipelinerun",
"-n",
CI_NAMESPACE,
"-o",
cleanupPipelineRunFieldsJsonPath(true),
], 60_000);
if (!isCommandSuccess(result)) {
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
}
const prefixes = pipelinePrefixesForLane(options.lane);
const terminalRuns = statusText(result)
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => parseCleanupPipelineRunLine(line, now))
.filter((item): item is CleanupPipelineRunRow => item !== null)
.filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
.filter((item) => item.status === "True" || item.status === "False")
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
const protectedLatestByPrefix = new Map<string, string>();
for (const prefix of prefixes) {
const latest = terminalRuns
.filter((item) => item.name.startsWith(prefix))
.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0];
if (latest !== undefined) protectedLatestByPrefix.set(prefix, latest.name);
}
const candidates = terminalRuns
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
.map((item) => {
const protectedLatest = [...protectedLatestByPrefix.values()].includes(item.name);
return protectedLatest
? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" }
: item;
})
.slice(0, options.limit);
return candidates;
}
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unknown>[] {
if (pipelineRunNames.length === 0) {
return [];
}
const result = g14K3s([
"kubectl",
"get",
"pvc",
"-n",
CI_NAMESPACE,
"-o",
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}',
], 60_000);
if (!isCommandSuccess(result)) {
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
}
const pvResult = g14K3s([
"kubectl",
"get",
"pv",
"-o",
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}',
], 60_000);
if (!isCommandSuccess(pvResult)) {
throw new Error(`failed to list hwlab-ci PipelineRun PV backing paths: ${commandErrorSummary(pvResult)}`);
}
const podResult = g14K3s([
"kubectl",
"get",
"pod",
"-n",
CI_NAMESPACE,
"-o",
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{","}{end}{"\\n"}{end}',
], 60_000);
if (!isCommandSuccess(podResult)) {
throw new Error(`failed to list hwlab-ci active pod PVC mounts: ${commandErrorSummary(podResult)}`);
}
const pvByName = new Map<string, Record<string, unknown>>();
for (const line of statusText(pvResult).split(/\r?\n/u)) {
const [name = "", storageClass = "", reclaimPolicy = "", localPath = "", hostPath = ""] = line.trim().split("\t");
if (name.length === 0) continue;
pvByName.set(name, {
storageClass: storageClass || null,
reclaimPolicy: reclaimPolicy || null,
hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath),
});
}
const activeClaimPods = new Map<string, string[]>();
for (const line of statusText(podResult).split(/\r?\n/u)) {
const [podName = "", phase = "", claims = ""] = line.trim().split("\t");
if (podName.length === 0 || phase === "Succeeded" || phase === "Failed") continue;
for (const claimName of claims.split(",").map((item) => item.trim()).filter(Boolean)) {
const entry = activeClaimPods.get(claimName) ?? [];
entry.push(podName);
activeClaimPods.set(claimName, entry);
}
}
const wanted = new Set(pipelineRunNames);
const ownedPvcs = statusText(result)
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
const pv = volume.length > 0 ? pvByName.get(volume) : undefined;
const hostPath = safeStorageHostPath(pv?.hostPath) ?? storageHostPathFromClaim(volume || null, CI_NAMESPACE, name || null);
return {
name,
volume: volume || null,
phase: phase || null,
ownerKind: ownerKind || null,
owner: owner || null,
storageClass: pv?.storageClass ?? null,
reclaimPolicy: pv?.reclaimPolicy ?? null,
hostPath,
activeMountPods: activeClaimPods.get(name) ?? [],
};
})
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
const estimates = remoteStoragePathEstimates(ownedPvcs.map((item) => stringOrNull(item.hostPath)));
return ownedPvcs.map((item) => ({
...item,
estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null,
}));
}
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
if (names.length === 0) {
return {
ok: true,
command: [],
exitCode: 0,
stdout: "no candidates",
stderr: "",
parsed: null,
};
}
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
}
function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<string, unknown>[] {
const result = g14K3s([
"kubectl",
"get",
"pv",
"-o",
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}',
], 60_000);
if (!isCommandSuccess(result)) {
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
}
const candidates = statusText(result)
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = "", localPath = "", hostPath = ""] = line.split("\t");
return {
name,
createdAt,
phase,
storageClass,
reclaimPolicy,
claimNamespace,
claimName,
capacity,
hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath) ?? (claimNamespace === CI_NAMESPACE ? storageHostPathFromClaim(name || null, claimNamespace, claimName || null) : null),
};
})
.filter((item) => item.phase === "Released")
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
.slice(0, options.limit);
const estimates = remoteStoragePathEstimates(candidates.map((item) => stringOrNull(item.hostPath)));
return candidates.map((item) => ({
...item,
estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null,
}));
}
function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
if (names.length === 0) {
return {
ok: true,
command: [],
exitCode: 0,
stdout: "no candidates",
stderr: "",
parsed: null,
};
}
return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs);
}
function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
const candidates = listReleasedCiWorkspacePvs(options);
const candidateNames = candidates.map((item) => String(item.name));
const estimatedReclaimBytes = sumEstimatedBytes(candidates);
if (options.dryRun) {
return {
ok: true,
command: "hwlab g14 control-plane cleanup-released-pvs",
mode: "dry-run",
lane: options.lane,
limit: options.limit,
candidates,
candidateCount: candidates.length,
selectedPersistentVolumes: candidateNames,
selectedPersistentVolumeCount: candidateNames.length,
estimatedReclaimBytes,
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
mutation: false,
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
};
}
const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000);
return {
ok: isCommandSuccess(deletion),
command: "hwlab g14 control-plane cleanup-released-pvs",
mode: "confirmed-cleanup",
lane: options.lane,
limit: options.limit,
deletedPersistentVolumes: candidateNames,
deletedPersistentVolumeCount: candidateNames.length,
candidatesBefore: candidates,
estimatedReclaimBytes,
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
deletion,
followUp: {
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
storage: "trans G14 script -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'",
},
};
}
function runV02RuntimeMigration(options: G14ControlPlaneOptions, sourceCommit: string): Record<string, unknown> {
const reportPath = `/tmp/hwlab-v02-runtime-migration-${shortSha(sourceCommit)}.json`;
const migrationArgs = options.dryRun
? [
...(options.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
"--report",
reportPath,
]
: [
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--report",
reportPath,
];
const command = [
"exec",
"-n",
"hwlab-v02",
"deployment/hwlab-cloud-api",
"-c",
"hwlab-cloud-api",
"--",
"bun",
"cmd/hwlab-cloud-api/migrate.ts",
...migrationArgs,
];
const result = g14K3s(["kubectl", ...command], options.timeoutSeconds * 1000);
const ok = isCommandSuccess(result);
return {
ok,
command: "hwlab g14 control-plane runtime-migration --lane v02",
lane: "v02",
mode: options.dryRun ? options.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
sourceCommit,
runtimeNamespace: "hwlab-v02",
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
reportPath,
mutation: !options.dryRun,
result,
next: options.dryRun
? {
liveReadDryRun: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
apply: "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
}
: {
health: "curl -fsS --max-time 20 http://74.48.78.17:19667/health/live",
triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
},
};
}
function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
const candidates = listCleanupPipelineRuns(options);
const candidateNames = candidates
.filter((item) => item.selected !== false)
.map((item) => String(item.name));
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
const estimatedReclaimBytes = sumEstimatedBytes(ownedPvcs);
const followUpStatusLane = isHwlabRuntimeLane(options.lane) ? options.lane : "v02";
if (options.dryRun) {
return {
ok: true,
command: "hwlab g14 control-plane cleanup-runs",
mode: "dry-run",
lane: options.lane,
minAgeMinutes: options.minAgeMinutes,
limit: options.limit,
sourceCommit: options.sourceCommit,
pipelineRun: options.pipelineRun,
candidates,
candidateCount: candidates.length,
selectedPipelineRuns: candidateNames,
selectedPipelineRunCount: candidateNames.length,
ownedPvcs,
ownedPvcCount: ownedPvcs.length,
estimatedReclaimBytes,
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
mutation: false,
next: { confirm: [
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs",
`--lane ${options.lane}`,
`--min-age-minutes ${options.minAgeMinutes}`,
`--limit ${options.limit}`,
options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`,
options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`,
"--confirm",
].filter(Boolean).join(" ") },
};
}
const deletion = deletePipelineRuns(candidateNames, options.timeoutSeconds * 1000);
return {
ok: isCommandSuccess(deletion),
command: "hwlab g14 control-plane cleanup-runs",
mode: "confirmed-cleanup",
lane: options.lane,
minAgeMinutes: options.minAgeMinutes,
limit: options.limit,
sourceCommit: options.sourceCommit,
pipelineRun: options.pipelineRun,
deletedPipelineRuns: candidateNames,
deletedPipelineRunCount: candidateNames.length,
ownedPvcsBefore: ownedPvcs,
ownedPvcCountBefore: ownedPvcs.length,
estimatedReclaimBytes,
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
deletion,
followUp: {
status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${followUpStatusLane}`,
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
},
};
}
function runtimeLaneControlPlaneRenderScript(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token = runtimeLaneRenderToken()): string {
const renderDir = runtimeLaneRenderDir(spec, sourceCommit, token);
const worktreeDir = runtimeLaneRenderWorktreeDir(spec, sourceCommit, token);
return [
"set -eu",
runtimeLaneCicdRepoEnsureScript(spec),
`source_commit=${shellQuote(sourceCommit)}`,
`render_dir=${shellQuote(renderDir)}`,
`worktree_dir=${shellQuote(worktreeDir)}`,
`render_lane=${shellQuote(spec.lane)}`,
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
"trap cleanup_render_worktree EXIT",
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = ${shellQuote(sourceCommit)}`,
"cleanup_render_worktree",
"rm -rf \"$render_dir\"",
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
"cd \"$worktree_dir\"",
"if [ ! -d node_modules/yaml ]; then",
" if [ -f package-lock.json ]; then",
" npm ci --ignore-scripts --no-audit --prefer-offline",
" elif [ -f bun.lock ] || [ -f bun.lockb ]; then",
" bun install --frozen-lockfile --ignore-scripts",
" else",
` echo "${spec.version} control-plane render cannot install dependencies: no lockfile found" >&2`,
" exit 42",
" fi",
"fi",
"if [ -f scripts/gitops-render.mjs ]; then",
" render_script=scripts/gitops-render.mjs",
"elif [ \"$render_lane\" = v02 ] && [ -f scripts/g14-gitops-render.mjs ]; then",
" render_script=scripts/g14-gitops-render.mjs",
"else",
` echo "${spec.version} control-plane render script missing: scripts/gitops-render.mjs" >&2`,
" exit 43",
"fi",
"if [ -f scripts/run-bun.mjs ]; then",
` node scripts/run-bun.mjs "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
"else",
` node "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
"fi",
].join("\n");
}
export function v02ControlPlaneRenderScript(sourceCommit: string, token = v02RenderToken()): string {
return runtimeLaneControlPlaneRenderScript(V02_LANE_SPEC, sourceCommit, token);
}
interface V02RenderTempResult {
result: CommandJsonResult;
renderDir: string;
worktreeDir: string;
token: string;
}
function runRuntimeLaneRenderToTemp(spec: HwlabRuntimeLaneSpec, sourceCommit: string): V02RenderTempResult {
const token = runtimeLaneRenderToken();
return {
result: g14HostScript(runtimeLaneControlPlaneRenderScript(spec, sourceCommit, token), 180_000),
renderDir: runtimeLaneRenderDir(spec, sourceCommit, token),
worktreeDir: runtimeLaneRenderWorktreeDir(spec, sourceCommit, token),
token,
};
}
function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult {
return runRuntimeLaneRenderToTemp(V02_LANE_SPEC, sourceCommit);
}
function runtimeLaneRenderToken(): string {
const random = Math.random().toString(16).slice(2, 10);
return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-");
}
function v02RenderToken(): string {
return runtimeLaneRenderToken();
}
function runtimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
return `/tmp/hwlab-${spec.lane}-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
}
function v02RenderDir(sourceCommit: string, token?: string): string {
return runtimeLaneRenderDir(V02_LANE_SPEC, sourceCommit, token);
}
function runtimeLaneRenderWorktreeDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
return `/tmp/hwlab-${spec.lane}-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
}
function v02RenderWorktreeDir(sourceCommit: string, token?: string): string {
return runtimeLaneRenderWorktreeDir(V02_LANE_SPEC, sourceCommit, token);
}
function cleanupRuntimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandJsonResult {
return g14HostScript([
"set -eu",
`render_dir=${shellQuote(renderDir)}`,
"case \"$render_dir\" in",
` /tmp/hwlab-${spec.lane}-control-plane-*) rm -rf "$render_dir" ;;`,
` *) echo "refusing to cleanup unexpected ${spec.version} render dir: $render_dir" >&2; exit 44 ;;`,
"esac",
].join("\n"), 60_000);
}
function cleanupV02RenderDir(renderDir: string): CommandJsonResult {
return cleanupRuntimeLaneRenderDir(V02_LANE_SPEC, renderDir);
}
function legacyRuntimeLaneArgoApplications(spec: HwlabRuntimeLaneSpec): string[] {
if (spec.lane === "v02") return [];
return [`hwlab-g14-${spec.lane}`].filter((name) => name !== spec.app);
}
function applyRuntimeLaneControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
const namespaceFile = `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`;
const rbacFile = `${renderDir}/${spec.tektonDir}/rbac.yaml`;
const pipelineFile = `${renderDir}/${spec.tektonDir}/pipeline.yaml`;
const projectFile = `${renderDir}/argocd/project.yaml`;
const applicationFile = `${renderDir}/argocd/${spec.argoApplicationFile}`;
const resourceFiles = [namespaceFile, rbacFile, pipelineFile, projectFile, applicationFile];
const legacyApplications = legacyRuntimeLaneArgoApplications(spec);
const serverSideArgs = [
"kubectl",
"apply",
"--server-side",
"--force-conflicts",
`--field-manager=${spec.controlPlaneFieldManager}`,
...(dryRun ? ["--dry-run=server"] : []),
...resourceFiles.flatMap((file) => ["-f", file]),
];
const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...serverSideArgs];
const shellCommand = dryRun && spec.lane !== "v02"
? [
"set -eu",
[
...serverSideArgs.slice(0, 6),
"-f",
namespaceFile,
"-f",
pipelineFile,
"-f",
projectFile,
"-f",
applicationFile,
].map(shellQuote).join(" "),
[
"kubectl",
"apply",
"--dry-run=client",
...resourceFiles.flatMap((file) => ["-f", file]),
].map(shellQuote).join(" "),
].join("\n")
: `exec ${serverSideArgs.map(shellQuote).join(" ")}`;
const cleanupCommands = legacyApplications.map((name) => [
"kubectl",
"delete",
"application",
"-n",
"argocd",
name,
"--ignore-not-found=true",
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
].map(shellQuote).join(" "));
const script = cleanupCommands.length > 0
? ["set -eu", shellCommand.replace(/^exec /, ""), ...cleanupCommands].join("\n")
: shellCommand;
const visibleCommand = cleanupCommands.length > 0
? ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "script", "--", script]
: command;
return runG14K3sRemoteAsync({
script,
timeoutSeconds,
label: `${spec.lane}-control-plane-apply`,
token: runtimeLaneRenderToken(),
command: visibleCommand,
});
}
function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
return applyRuntimeLaneControlPlaneFiles(V02_LANE_SPEC, renderDir, dryRun, timeoutSeconds);
}
interface V02RefreshMarker {
ok: boolean;
sourceCommit: string;
refreshedAt: string;
renderScriptHash: string;
renderDir?: string | null;
}
function v02ControlPlaneRefreshStateDir(): string {
return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh");
}
export function v02ControlPlaneRefreshScriptHash(sourceCommit: string): string {
return textHash(v02ControlPlaneRenderScript(sourceCommit, "contract-token"));
}
function v02ControlPlaneRefreshMarkerPath(sourceCommit: string): string {
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`);
}
function v02ControlPlaneRefreshLockDir(sourceCommit: string): string {
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`);
}
function readV02RefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
const path = v02ControlPlaneRefreshMarkerPath(sourceCommit);
if (!existsSync(path)) return null;
try {
const marker = JSON.parse(readFileSync(path, "utf8")) as V02RefreshMarker;
return v02ReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs);
} catch {
return null;
}
}
export function v02ReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
const candidate = record(marker) as V02RefreshMarker;
const refreshedAtMs = timestampMs(candidate.refreshedAt);
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null;
if (refreshedAtMs === null || nowMs - refreshedAtMs > V02_CONTROL_PLANE_REFRESH_TTL_MS) return null;
return candidate;
}
function writeV02RefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): V02RefreshMarker {
const marker = {
ok: true,
sourceCommit,
refreshedAt: new Date().toISOString(),
renderScriptHash: scriptHash,
renderDir,
};
const dir = v02ControlPlaneRefreshStateDir();
mkdirSync(dir, { recursive: true });
writeFileSync(v02ControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
return marker;
}
function acquireV02RefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } {
const stateDir = v02ControlPlaneRefreshStateDir();
mkdirSync(stateDir, { recursive: true });
const lockDir = v02ControlPlaneRefreshLockDir(sourceCommit);
const startedAtMs = Date.now();
for (;;) {
try {
mkdirSync(lockDir);
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
} catch {
const ageMs = (() => {
try {
return Date.now() - statSync(lockDir).mtimeMs;
} catch {
return 0;
}
})();
if (ageMs > V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) {
try {
rmSync(lockDir, { recursive: true, force: true });
continue;
} catch {
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
}
}
if (Date.now() - startedAtMs >= V02_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
}
}
}
function releaseV02RefreshLock(lockDir: string): void {
try {
rmSync(lockDir, { recursive: true, force: true });
} catch {
// Best-effort cleanup; stale lock expiry handles interrupted workers.
}
}
function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
const scriptHash = v02ControlPlaneRefreshScriptHash(sourceCommit);
const existingMarker = readV02RefreshMarker(sourceCommit, scriptHash);
if (existingMarker !== null) {
return {
ok: true,
phase: "control-plane-refresh",
mode: "reused-recent-refresh-marker",
sourceCommit,
renderDir: existingMarker.renderDir ?? null,
marker: existingMarker,
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
};
}
const lock = acquireV02RefreshLock(sourceCommit);
if (!lock.acquired) {
return {
ok: false,
phase: "control-plane-refresh-lock",
mode: "local-refresh-lock",
sourceCommit,
lockDir: lock.lockDir,
waitedMs: lock.waitedMs,
degradedReason: "control-plane-refresh-lock-timeout",
};
}
try {
const markerAfterWait = readV02RefreshMarker(sourceCommit, scriptHash);
if (markerAfterWait !== null) {
return {
ok: true,
phase: "control-plane-refresh",
mode: "waited-for-recent-refresh-marker",
sourceCommit,
renderDir: markerAfterWait.renderDir ?? null,
marker: markerAfterWait,
waitedMs: lock.waitedMs,
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
};
}
const render = runV02RenderToTemp(sourceCommit);
if (!isCommandSuccess(render.result)) {
return {
ok: false,
phase: "source-render",
sourceCommit,
renderDir: render.renderDir,
render: compactCommandResult(render.result),
degradedReason: "control-plane-render-failed",
};
}
const apply = applyV02ControlPlaneFiles(render.renderDir, false, timeoutSeconds);
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs));
const marker = ok ? writeV02RefreshMarker(sourceCommit, scriptHash, render.renderDir) : null;
return {
ok,
phase: "control-plane-refresh",
mode: "refreshed",
sourceCommit,
renderDir: render.renderDir,
render: compactCommandResult(render.result),
apply: compactCommandResult(apply),
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
marker,
waitedMs: lock.waitedMs,
degradedReason: !isCommandSuccess(apply)
? "control-plane-apply-failed"
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
? "obsolete-cronjob-cleanup-failed"
: undefined,
};
} finally {
releaseV02RefreshLock(lock.lockDir);
}
}
function getV02ObsoleteCronJobs(): CommandJsonResult {
return g14K3s([
"kubectl",
"get",
"cronjob",
"-n",
CI_NAMESPACE,
V02_POLLER,
V02_RECONCILER,
"--ignore-not-found",
"-o",
"name",
], 60_000);
}
function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult {
return g14K3s([
"kubectl",
"delete",
"cronjob",
"-n",
CI_NAMESPACE,
V02_POLLER,
V02_RECONCILER,
"--ignore-not-found=true",
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
], 60_000);
}
export function runtimeLanePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit)): Record<string, unknown> {
return {
apiVersion: "tekton.dev/v1",
kind: "PipelineRun",
metadata: {
name: pipelineRun,
namespace: CI_NAMESPACE,
labels: {
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/gitops-target": spec.lane,
"hwlab.pikastech.local/source-commit": sourceCommit,
"hwlab.pikastech.local/trigger": "manual-cli",
},
annotations: {
"hwlab.pikastech.local/source-branch": spec.sourceBranch,
"hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
"hwlab.pikastech.local/node": spec.nodeId,
"hwlab.pikastech.local/network-profile": spec.networkProfileId,
"hwlab.pikastech.local/download-profile": spec.downloadProfileId,
"hwlab.pikastech.local/no-proxy-required": hwlabRequiredNoProxyEntries().join(","),
"hwlab.pikastech.local/triggered-by": "unidesk-cli",
},
},
spec: {
pipelineRef: { name: spec.pipeline },
taskRunTemplate: {
serviceAccountName: spec.serviceAccountName,
podTemplate: {
hostNetwork: true,
dnsPolicy: "ClusterFirstWithHostNet",
securityContext: { fsGroup: 1000 },
},
},
params: [
{ name: "git-url", value: spec.gitUrl },
{ name: "git-read-url", value: spec.gitReadUrl },
{ name: "source-branch", value: spec.sourceBranch },
{ name: "gitops-branch", value: spec.gitopsBranch },
{ name: "lane", value: spec.lane },
{ name: "catalog-path", value: spec.catalogPath },
{ name: "image-tag-mode", value: "full" },
{ name: "runtime-path", value: spec.runtimePath },
{ name: "revision", value: sourceCommit },
{ name: "registry-prefix", value: spec.registryPrefix },
{ name: "services", value: spec.serviceIds.join(",") },
{ name: "base-image", value: spec.baseImage },
],
workspaces: [
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
],
},
};
}
function v02PipelineRunManifest(sourceCommit: string): Record<string, unknown> {
return runtimeLanePipelineRunManifest(V02_LANE_SPEC, sourceCommit);
}
function createRuntimeLanePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number, pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit)): CommandJsonResult {
const manifest = JSON.stringify(runtimeLanePipelineRunManifest(spec, sourceCommit, pipelineRun));
const manifestB64 = Buffer.from(manifest, "utf8").toString("base64");
const script = [
"set -eu",
`manifest_b64=${shellQuote(manifestB64)}`,
`manifest_path=/tmp/${pipelineRun}.json`,
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
"if kubectl create -f \"$manifest_path\"; then",
" :",
"else",
" code=$?",
` if kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} >/dev/null 2>&1; then`,
` printf 'PipelineRun %s already exists; reusing existing object\\n' ${shellQuote(pipelineRun)} >&2`,
" else",
" exit \"$code\"",
" fi",
"fi",
`kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`,
].join("\n");
return g14K3s(["script", "--", script], timeoutSeconds * 1000);
}
function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult {
return createRuntimeLanePipelineRun(V02_LANE_SPEC, sourceCommit, timeoutSeconds);
}
function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record<string, unknown> {
const targetMode: V02StatusTargetMode = target.mode
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
const includeHistory = target.includeHistory === true;
const strictHeadAlignment = targetMode === "latest-source-head";
const bundle = v02ControlPlaneStatusBundle({ ...target, mode: targetMode });
const sections = parseShellSections(statusText(bundle));
const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null;
const pipelineRun = stringOrNull(sections.pipelineRunName?.stdout) ?? (sourceCommit === null ? null : v02PipelineRunName(sourceCommit));
const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? "");
const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now();
const sourceHeadsSection = sections.sourceHeads;
const controlPlane = sections.controlPlane;
const obsoleteCronJobs = sections.obsoleteCronJobs;
const argo = sections.argo;
const pipelineRunSection = sections.pipelineRun;
const taskRunsSection = sections.taskRuns;
const planArtifactsSection = sections.planArtifacts;
const runtimeWorkloadsSection = sections.runtimeWorkloads;
const gitMirrorCacheSection = sections.gitMirrorCache;
const webAssetsSection = sections.webAssets;
const recentPipelineRuns = listV02PipelineRunsCompactFromText(
sections.recentPipelineRuns?.stdout ?? "",
shellSectionOk(sections.recentPipelineRuns),
"kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=<summary-fields>",
sections.recentPipelineRuns?.exitCode ?? null,
bundle.stderr,
8,
queryNowMs,
);
const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
const sourceHeads = v02SourceHeadsFromText(
sourceHeadsSection?.stdout ?? "",
shellSectionOk(sourceHeadsSection),
sourceHeadsSection?.exitCode ?? null,
bundle.stderr,
);
const pipelineRunInfo = pipelineRun === null
? null
: pipelineRunCompactFromText(
pipelineRun,
pipelineRunSection?.stdout ?? "",
shellSectionOk(pipelineRunSection),
`kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
pipelineRunSection?.exitCode ?? null,
bundle.stderr,
);
const taskRuns = taskRunsCompactFromText(
taskRunsSection?.stdout ?? "",
shellSectionOk(taskRunsSection),
pipelineRun,
taskRunsSection?.exitCode ?? null,
bundle.stderr,
);
const planArtifacts = v02PlanArtifactsFromText(
planArtifactsSection?.stdout ?? "",
shellSectionOk(planArtifactsSection),
pipelineRun,
planArtifactsSection?.exitCode ?? null,
bundle.stderr,
);
const runtimeWorkloads = v02RuntimeWorkloadsFromText(
runtimeWorkloadsSection?.stdout ?? "",
shellSectionOk(runtimeWorkloadsSection),
runtimeWorkloadsSection?.exitCode ?? null,
bundle.stderr,
);
const webAssets = v02WebAssetsFromText(
webAssetsSection?.stdout ?? "",
shellSectionOk(webAssetsSection),
sourceCommit,
syncRevision,
webAssetsSection?.exitCode ?? null,
bundle.stderr,
activePipelineRuns,
);
const gitMirror = {
ok: shellSectionOk(gitMirrorCacheSection),
summary: gitMirrorStatusSummary(String(gitMirrorCacheSection?.stdout ?? "").trim()),
raw: String(gitMirrorCacheSection?.stdout ?? "").trim(),
exitCode: gitMirrorCacheSection?.exitCode ?? null,
stderr: shellSectionOk(gitMirrorCacheSection) ? "" : bundle.stderr.trim().slice(0, 2000),
};
const commitAlignment = v02CommitAlignment({
expectedSourceHead: sourceCommit,
sourceHeads,
gitMirrorSummary: record(gitMirror.summary),
pipelineRun: pipelineRunInfo,
recentPipelineRuns,
planArtifacts,
runtimeWorkloads,
webAssets,
});
const targetValidationBase = v02TargetValidation({
targetMode,
sourceCommit,
pipelineRun: pipelineRunInfo,
argo: {
ok: shellSectionOk(argo),
fields: { targetRevision, path, syncRevision, syncStatus, health },
},
planArtifacts,
runtimeWorkloads,
webAssets,
gitMirror,
recentPipelineRuns,
});
const targetValidation = v02LatestOnlyTargetValidation({
targetMode,
sourceCommit,
pipelineRun: pipelineRunInfo,
commitAlignment,
targetValidation: targetValidationBase,
});
const sourceHeadsTarget = record(sourceHeads.target);
const falseGreenGuard = targetValidation.state === "superseded"
? {
ok: null,
state: "superseded",
summary: "target PipelineRun succeeded but runtime now reflects a newer v0.2 PipelineRun; current-runtime false-green guard is not applicable to this historical target",
sourceCommit,
targetValidationState: targetValidation.state,
newerSucceededRuns: targetValidation.newerSucceededRuns ?? [],
supersededRuntimeServices: targetValidation.supersededRuntimeServices ?? [],
}
: v02FalseGreenGuard({ sourceCommit, pipelineRun: pipelineRunInfo, taskRuns, planArtifacts, runtimeWorkloads });
const baseOk = sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo);
const targetPipelineRunOk = strictHeadAlignment
? true
: pipelineRunInfo !== null && pipelineRunInfo.ok === true && pipelineRunInfo.exists !== false;
const targetDegradedReason = !strictHeadAlignment && !targetPipelineRunOk ? "target-pipelinerun-not-found-or-unreadable" : undefined;
const statusCommand = targetMode === "pipeline-run" && pipelineRun !== null
? `hwlab g14 control-plane status --lane v02 --pipeline-run ${pipelineRun}`
: targetMode === "source-commit" && sourceCommit !== null
? `hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}`
: "hwlab g14 control-plane status --lane v02";
const state = strictHeadAlignment ? commitAlignment.state : `target-${targetMode}`;
const sourceCommitSource = targetMode === "pipeline-run"
? "PipelineRun metadata label hwlab.pikastech.local/source-commit"
: targetMode === "source-commit"
? "explicit --source-commit"
: "G14 CI/CD dedicated bare repo refs/remotes/origin/v0.2; workspace checkout is observable but isolated";
const status: Record<string, unknown> = {
ok: baseOk && targetPipelineRunOk && (!strictHeadAlignment || commitAlignment.aligned !== false),
command: statusCommand,
lane: "v02",
state,
degradedReason: strictHeadAlignment && commitAlignment.aligned === false ? commitAlignment.state : targetDegradedReason,
statusTarget: {
mode: statusTargetFields.mode || targetMode,
sourceCommit,
pipelineRun,
strictHeadAlignment,
ancestorOfLatest: sourceHeadsTarget.ancestorOfOriginHead ?? null,
note: strictHeadAlignment
? "default status validates the latest v0.2 source head alignment"
: "targeted status inspects the requested PipelineRun/source commit without failing merely because origin/v0.2 advanced later",
},
sourceCommit,
sourceCommitSource,
expected: {
sourceRepo: V02_CICD_REPO,
workspace: V02_WORKSPACE,
workspaceIsolation: "workspace dirty/stale state must not select CI/CD source commits",
branch: V02_SOURCE_BRANCH,
namespace: CI_NAMESPACE,
runtimeNamespace: "hwlab-v02",
pipeline: V02_PIPELINE,
trigger: "manual UniDesk CLI PipelineRun creation",
argoApplication: V02_APP,
},
commitAlignment,
targetValidation,
sourceHeads,
sourceHeadsTargetAncestorOfOriginHead: sourceHeadsTarget.ancestorOfOriginHead ?? null,
gitMirror,
controlPlane: {
ok: shellSectionOk(controlPlane),
names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
exitCode: controlPlane?.exitCode ?? null,
},
obsoleteCronJobs: {
ok: shellSectionOk(obsoleteCronJobs),
names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
exitCode: obsoleteCronJobs?.exitCode ?? null,
cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
},
argo: {
ok: shellSectionOk(argo),
raw: argo?.stdout ?? "",
fields: { targetRevision, path, syncRevision, syncStatus, health },
exitCode: argo?.exitCode ?? null,
},
pipelineRun: pipelineRunInfo,
taskRuns,
planArtifacts,
runtimeWorkloads,
falseGreenGuard,
webAssets,
activePipelineRuns,
history: {
included: includeHistory,
activePipelineRunCount: activePipelineRuns.length,
recentPipelineRunCount: Array.isArray(recentPipelineRuns.items) ? recentPipelineRuns.items.length : 0,
note: includeHistory
? "recentPipelineRuns is included because --history was requested; historical runs are diagnostic evidence, not the default current-target verdict"
: "recent PipelineRun history is hidden by default so old manifest/service-id failures do not pollute the current-target verdict; rerun with --history to inspect it",
command: statusCommand.includes("--history") ? statusCommand : `${statusCommand} --history`,
},
...(includeHistory ? { recentPipelineRuns } : {}),
query: {
ok: isCommandSuccess(bundle),
exitCode: bundle.exitCode,
stderr: bundle.stderr.trim().slice(0, 2000),
},
visibilityHint: activePipelineRuns.length > 0
? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger"
: "no active v02 PipelineRun observed",
};
return {
...status,
closeout: v02CloseoutVerdict(status),
};
}
function runtimeLaneControlPlaneStatusBundle(spec: HwlabRuntimeLaneSpec, target: V02ControlPlaneStatusTarget = {}): CommandJsonResult {
const targetMode: V02StatusTargetMode = target.mode
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null
? [
`pipeline_run=${shellQuote(target.pipelineRun)}`,
`source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`,
`if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | sed 's/^${spec.pipelineRunPrefix}-//'); fi`,
].join("\n")
: [
target.sourceCommit === undefined
? `source_commit=$(git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch} 2>/dev/null || true)`
: `source_commit=${shellQuote(target.sourceCommit ?? "")}`,
"pipeline_run=",
].join("\n");
const controlPlaneProbe = [
`kubectl get serviceaccount -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.serviceAccountName)} -o name`,
`kubectl get pipeline -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.pipeline)} -o name`,
].join("; ");
const script = [
"set +e",
targetInit,
`target_mode=${shellQuote(targetMode)}`,
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l "hwlab.pikastech.local/gitops-target=${spec.lane},hwlab.pikastech.local/source-commit=$source_commit" --sort-by=.metadata.creationTimestamp -o name 2>/dev/null | sed 's#.*/##' | tail -n 1 || true); fi`,
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${spec.pipelineRunPrefix}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`,
"section() {",
" name=\"$1\"",
" shift",
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
" \"$@\"",
" code=$?",
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
"printf '__UNIDESK_SECTION_BEGIN__ statusTarget\\n'",
"printf 'mode\\t%s\\n' \"$target_mode\"",
"printf 'sourceCommit\\t%s\\n' \"$source_commit\"",
"printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"",
"printf '__UNIDESK_SECTION_END__ statusTarget exit=0\\n'",
`section sourceHead git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch}`,
`section controlPlane sh -lc ${shellQuote(controlPlaneProbe)}`,
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(spec.app)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
`section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'`,
`section runtimeWorkloads kubectl get deploy,statefulset,svc,ingress,configmap -n ${shellQuote(spec.runtimeNamespace)} -l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)} -o name`,
`section publicProbes sh -lc ${shellQuote(runtimeLanePublicProbeScript(spec))}`,
[
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)}`,
`-l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)}`,
"--sort-by=.metadata.creationTimestamp",
"-o",
shellQuote(pipelineRunRowsJsonPath()),
].join(" "),
].join("\n");
return g14K3s(["script", "--", script], 120_000);
}
function runtimeLanePublicProbeScript(spec: HwlabRuntimeLaneSpec): string {
const apiUrl = `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`;
const webUrl = spec.publicWebUrl.replace(/\/+$/u, "/");
return [
"set +e",
`api_url=${shellQuote(apiUrl)}`,
`web_url=${shellQuote(webUrl)}`,
"api_output=$(curl -fsS --connect-timeout 3 --max-time 15 \"$api_url\" 2>&1)",
"api_exit=$?",
"api_status_ok=no",
"if [ \"$api_exit\" = 0 ] && printf '%s' \"$api_output\" | grep -Eq '\"status\"[[:space:]]*:[[:space:]]*\"ok\"'; then api_status_ok=yes; fi",
"web_tmp=$(mktemp /tmp/hwlab-lane-web-probe.XXXXXX)",
"web_output=$(curl -fsS --connect-timeout 3 --max-time 15 -o \"$web_tmp\" \"$web_url\" 2>&1)",
"web_exit=$?",
"web_bytes=0",
"web_has_html=no",
"if [ -f \"$web_tmp\" ]; then",
" web_bytes=$(wc -c < \"$web_tmp\" | tr -d ' ')",
" if grep -Eiq '<html|<script|<div' \"$web_tmp\"; then web_has_html=yes; fi",
"fi",
"printf 'apiUrl\\t%s\\n' \"$api_url\"",
"printf 'apiExitCode\\t%s\\n' \"$api_exit\"",
"printf 'apiStatusOk\\t%s\\n' \"$api_status_ok\"",
"printf 'apiPreview\\t%s\\n' \"$(printf '%s' \"$api_output\" | tr '\\n\\t' ' ' | cut -c1-240)\"",
"printf 'webUrl\\t%s\\n' \"$web_url\"",
"printf 'webExitCode\\t%s\\n' \"$web_exit\"",
"printf 'webBytes\\t%s\\n' \"$web_bytes\"",
"printf 'webHasHtml\\t%s\\n' \"$web_has_html\"",
"printf 'webErrorPreview\\t%s\\n' \"$(printf '%s' \"$web_output\" | tr '\\n\\t' ' ' | cut -c1-240)\"",
"rm -f \"$web_tmp\"",
"exit 0",
].join("\n");
}
function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02ControlPlaneStatusTarget = {}): Record<string, unknown> {
const targetMode: V02StatusTargetMode = target.mode
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
const bundle = runtimeLaneControlPlaneStatusBundle(spec, target);
const sections = parseShellSections(statusText(bundle));
const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? "");
const sourceCommit = stringOrNull(statusTargetFields.sourceCommit) ?? stringOrNull(sections.sourceHead?.stdout) ?? null;
const pipelineRun = stringOrNull(statusTargetFields.pipelineRun) ?? (sourceCommit === null ? null : runtimeLanePipelineRunName(spec, sourceCommit));
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(sections.argo?.stdout ?? "").split(/\r?\n/u);
const pipelineRunInfo = pipelineRun === null
? null
: pipelineRunCompactFromText(
pipelineRun,
sections.pipelineRun?.stdout ?? "",
shellSectionOk(sections.pipelineRun),
`kubectl get pipelinerun -n ${CI_NAMESPACE} ${pipelineRun}`,
sections.pipelineRun?.exitCode ?? null,
bundle.stderr,
);
const runtimeWorkloadNames = String(sections.runtimeWorkloads?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
const publicProbeFields = keyValueLinesFromText(sections.publicProbes?.stdout ?? "");
const publicProbesOk = shellSectionOk(sections.publicProbes)
&& publicProbeFields.apiExitCode === "0"
&& publicProbeFields.apiStatusOk === "yes"
&& publicProbeFields.webExitCode === "0"
&& Number(publicProbeFields.webBytes ?? 0) > 0;
const recentPipelineRuns = parsePipelineRunRows(sections.recentPipelineRuns?.stdout ?? "", 8, Date.now(), spec.pipelineRunPrefix);
const pipelineRunOk = pipelineRunInfo !== null && pipelineRunInfo.status === "True";
const argoOk = shellSectionOk(sections.argo) && syncStatus === "Synced" && health === "Healthy";
const runtimeWorkloadsOk = shellSectionOk(sections.runtimeWorkloads);
const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && pipelineRunOk && argoOk && runtimeWorkloadsOk && publicProbesOk;
return {
ok: baseOk,
command: `hwlab g14 control-plane status --lane ${spec.lane}`,
lane: spec.lane,
mode: "runtime-lane-status",
statusTarget: {
mode: statusTargetFields.mode || targetMode,
sourceCommit,
pipelineRun,
note: `${spec.lane} status is a bootstrap visibility check; v0.2 closeout/false-green verdicts are not reused for new lanes`,
},
sourceCommit,
expected: {
sourceRepo: spec.cicdRepo,
configPath: hwlabRuntimeLaneConfigPath(),
node: spec.nodeId,
nodeRoute: spec.nodeRoute,
workspace: spec.workspace,
branch: spec.sourceBranch,
namespace: CI_NAMESPACE,
runtimeNamespace: spec.runtimeNamespace,
pipeline: spec.pipeline,
serviceAccount: spec.serviceAccountName,
argoApplication: spec.app,
gitopsBranch: spec.gitopsBranch,
runtimePath: spec.runtimePath,
networkProfile: {
id: spec.networkProfileId,
httpProxy: spec.networkProfile.proxy.http,
noProxy: spec.networkProfile.proxy.noProxy.join(","),
dockerBuildHttpProxy: spec.networkProfile.dockerBuildProxy.http,
dockerBuildNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
},
downloadProfile: spec.downloadProfile,
},
sourceHead: {
ok: shellSectionOk(sections.sourceHead),
value: stringOrNull(sections.sourceHead?.stdout),
exitCode: sections.sourceHead?.exitCode ?? null,
},
controlPlane: {
ok: shellSectionOk(sections.controlPlane),
raw: sections.controlPlane?.stdout ?? "",
exitCode: sections.controlPlane?.exitCode ?? null,
},
argo: {
ok: shellSectionOk(sections.argo),
raw: sections.argo?.stdout ?? "",
fields: { targetRevision, path, syncRevision, syncStatus, health },
exitCode: sections.argo?.exitCode ?? null,
},
pipelineRun: pipelineRunInfo,
runtimeWorkloads: {
ok: shellSectionOk(sections.runtimeWorkloads),
namespace: spec.runtimeNamespace,
names: runtimeWorkloadNames,
count: runtimeWorkloadNames.length,
exitCode: sections.runtimeWorkloads?.exitCode ?? null,
},
publicProbes: {
ok: publicProbesOk,
apiUrl: publicProbeFields.apiUrl || `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`,
apiExitCode: numericField(publicProbeFields.apiExitCode),
apiStatusOk: publicProbeFields.apiStatusOk === "yes",
apiPreview: publicProbeFields.apiPreview || null,
webUrl: publicProbeFields.webUrl || spec.publicWebUrl,
webExitCode: numericField(publicProbeFields.webExitCode),
webBytes: numericField(publicProbeFields.webBytes),
webHasHtml: publicProbeFields.webHasHtml === "yes",
webErrorPreview: publicProbeFields.webErrorPreview || null,
exitCode: sections.publicProbes?.exitCode ?? null,
},
recentPipelineRuns,
query: {
ok: isCommandSuccess(bundle),
exitCode: bundle.exitCode,
stderr: bundle.stderr.trim().slice(0, 2000),
},
next: {
apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm`,
refresh: `bun scripts/cli.ts hwlab g14 control-plane refresh --lane ${spec.lane} --confirm`,
triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm`,
},
};
}
function runtimeLaneArgoRefreshScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
const operationPatch = JSON.stringify({ operation: null });
return [
"set +e",
`app=${shellQuote(spec.app)}`,
`argo_namespace=${shellQuote(ARGO_NAMESPACE)}`,
`runtime_namespace=${shellQuote(spec.runtimeNamespace)}`,
`lane=${shellQuote(spec.lane)}`,
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
`operation_patch=${shellQuote(operationPatch)}`,
"app_jsonpath() { kubectl -n \"$argo_namespace\" get application \"$app\" -o \"jsonpath=$1\" 2>/dev/null || true; }",
"job_jsonpath() { kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o \"jsonpath=$1\" 2>/dev/null || true; }",
"before_operation_revision=$(app_jsonpath '{.operation.sync.revision}')",
"before_operation_phase=$(app_jsonpath '{.status.operationState.phase}')",
"before_operation_message=$(app_jsonpath '{.status.operationState.message}')",
"before_sync_revision=$(app_jsonpath '{.status.sync.revision}')",
"before_sync_status=$(app_jsonpath '{.status.sync.status}')",
"before_health=$(app_jsonpath '{.status.health.status}')",
"before_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')",
"before_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)",
"before_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')",
"before_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')",
"before_hook_active=$(job_jsonpath '{.status.active}')",
"before_operation_present=$([ -n \"$before_operation_revision$before_operation_phase$before_operation_message\" ] && printf yes || printf no)",
"patch_exit=",
"patch_output=skipped-no-live-operation",
"annotate_exit=",
"annotate_output=",
"if [ \"$before_operation_present\" = yes ]; then",
" if [ \"$dry_run\" = true ]; then",
" patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" --dry-run=server -o name 2>&1)",
" else",
" patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" -o name 2>&1)",
" fi",
" patch_exit=$?",
"fi",
"if [ \"$dry_run\" = true ]; then",
" annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)",
"else",
" annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)",
"fi",
"annotate_exit=$?",
"if [ \"$dry_run\" != true ]; then sleep 2; fi",
"after_operation_revision=$(app_jsonpath '{.operation.sync.revision}')",
"after_operation_phase=$(app_jsonpath '{.status.operationState.phase}')",
"after_operation_message=$(app_jsonpath '{.status.operationState.message}')",
"after_sync_revision=$(app_jsonpath '{.status.sync.revision}')",
"after_sync_status=$(app_jsonpath '{.status.sync.status}')",
"after_health=$(app_jsonpath '{.status.health.status}')",
"after_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')",
"after_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)",
"after_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')",
"after_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')",
"after_hook_active=$(job_jsonpath '{.status.active}')",
"after_operation_present=$([ -n \"$after_operation_revision$after_operation_phase$after_operation_message\" ] && printf yes || printf no)",
"printf 'app\\t%s\\n' \"$app\"",
"printf 'lane\\t%s\\n' \"$lane\"",
"printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'beforeOperationPresent\\t%s\\n' \"$before_operation_present\"",
"printf 'beforeOperationRevision\\t%s\\n' \"$before_operation_revision\"",
"printf 'beforeOperationPhase\\t%s\\n' \"$before_operation_phase\"",
"printf 'beforeOperationMessage\\t%s\\n' \"$before_operation_message\"",
"printf 'beforeSyncRevision\\t%s\\n' \"$before_sync_revision\"",
"printf 'beforeSyncStatus\\t%s\\n' \"$before_sync_status\"",
"printf 'beforeHealth\\t%s\\n' \"$before_health\"",
"printf 'beforeResourceVersion\\t%s\\n' \"$before_resource_version\"",
"printf 'beforeHookJob\\t%s\\n' \"$before_hook_job\"",
"printf 'beforeHookSourceCommit\\t%s\\n' \"$before_hook_source\"",
"printf 'beforeHookPhase\\t%s\\n' \"$before_hook_phase\"",
"printf 'beforeHookActive\\t%s\\n' \"$before_hook_active\"",
"printf 'patchExitCode\\t%s\\n' \"$patch_exit\"",
"printf 'patchOutput\\t%s\\n' \"$patch_output\"",
"printf 'annotateExitCode\\t%s\\n' \"$annotate_exit\"",
"printf 'annotateOutput\\t%s\\n' \"$annotate_output\"",
"printf 'afterOperationPresent\\t%s\\n' \"$after_operation_present\"",
"printf 'afterOperationRevision\\t%s\\n' \"$after_operation_revision\"",
"printf 'afterOperationPhase\\t%s\\n' \"$after_operation_phase\"",
"printf 'afterOperationMessage\\t%s\\n' \"$after_operation_message\"",
"printf 'afterSyncRevision\\t%s\\n' \"$after_sync_revision\"",
"printf 'afterSyncStatus\\t%s\\n' \"$after_sync_status\"",
"printf 'afterHealth\\t%s\\n' \"$after_health\"",
"printf 'afterResourceVersion\\t%s\\n' \"$after_resource_version\"",
"printf 'afterHookJob\\t%s\\n' \"$after_hook_job\"",
"printf 'afterHookSourceCommit\\t%s\\n' \"$after_hook_source\"",
"printf 'afterHookPhase\\t%s\\n' \"$after_hook_phase\"",
"printf 'afterHookActive\\t%s\\n' \"$after_hook_active\"",
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
"if [ -n \"$annotate_exit\" ] && [ \"$annotate_exit\" != 0 ]; then exit \"$annotate_exit\"; fi",
].join("\n");
}
function refreshRuntimeLaneArgoApplication(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record<string, unknown> {
const result = g14K3s(["script", "--", runtimeLaneArgoRefreshScript(spec, options.dryRun)], options.timeoutSeconds * 1000);
const fields = keyValueLinesFromText(statusText(result));
const ok = isCommandSuccess(result);
return {
ok,
command: `hwlab g14 control-plane refresh --lane ${spec.lane}`,
lane: spec.lane,
mode: options.dryRun ? "dry-run" : "confirmed-refresh",
argoApplication: spec.app,
runtimeNamespace: spec.runtimeNamespace,
dryRun: options.dryRun,
action: {
terminatedOperation: fields.beforeOperationPresent === "yes",
hardRefreshRequested: numericField(fields.annotateExitCode) === 0,
mutation: !options.dryRun && ok,
},
before: {
operationPresent: fields.beforeOperationPresent === "yes",
operationRevision: fields.beforeOperationRevision || null,
operationPhase: fields.beforeOperationPhase || null,
operationMessage: fields.beforeOperationMessage || null,
syncRevision: fields.beforeSyncRevision || null,
syncStatus: fields.beforeSyncStatus || null,
health: fields.beforeHealth || null,
resourceVersion: fields.beforeResourceVersion || null,
hookJob: fields.beforeHookJob || null,
hookSourceCommit: fields.beforeHookSourceCommit || null,
hookPhase: fields.beforeHookPhase || null,
hookActive: fields.beforeHookActive || null,
},
after: {
operationPresent: fields.afterOperationPresent === "yes",
operationRevision: fields.afterOperationRevision || null,
operationPhase: fields.afterOperationPhase || null,
operationMessage: fields.afterOperationMessage || null,
syncRevision: fields.afterSyncRevision || null,
syncStatus: fields.afterSyncStatus || null,
health: fields.afterHealth || null,
resourceVersion: fields.afterResourceVersion || null,
hookJob: fields.afterHookJob || null,
hookSourceCommit: fields.afterHookSourceCommit || null,
hookPhase: fields.afterHookPhase || null,
hookActive: fields.afterHookActive || null,
},
patchExitCode: numericField(fields.patchExitCode),
annotateExitCode: numericField(fields.annotateExitCode),
result: compactCommandResult(result),
next: {
status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane}`,
},
};
}
function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record<string, unknown> {
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-released-pvs") {
return {
ok: false,
command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`,
lane: spec.lane,
degradedReason: "unsupported-runtime-lane-action",
message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh/cleanup-runs`,
};
}
if (options.action === "status" && options.pipelineRun !== undefined) {
return runtimeLaneControlPlaneStatus(spec, { pipelineRun: options.pipelineRun, mode: "pipeline-run", includeHistory: options.history });
}
if (options.action === "status" && options.sourceCommit !== undefined) {
return runtimeLaneControlPlaneStatus(spec, { sourceCommit: options.sourceCommit, mode: "source-commit", includeHistory: options.history });
}
const latestHead = options.action === "trigger-current" ? resolveRuntimeLaneLatestRemoteHead(spec) : null;
const head = resolveRuntimeLaneHead(spec);
let sourceCommit = head.sourceCommit;
if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) {
const refreshedHead = resolveRuntimeLaneHead(spec);
sourceCommit = refreshedHead.sourceCommit;
if (sourceCommit !== latestHead.sourceCommit) {
return {
ok: false,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
phase: "source-head-refresh",
sourceCommit,
latestSourceHeadProbe: {
sourceCommit: latestHead.sourceCommit,
result: compactCommandResult(latestHead.result),
},
refreshedHeadProbe: compactCommandResult(refreshedHead.result),
degradedReason: `${spec.lane}-head-refresh-did-not-reach-latest-source`,
};
}
}
if (sourceCommit === null) {
return {
ok: false,
command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`,
lane: spec.lane,
degradedReason: `${spec.lane}-head-unresolved`,
sourceRepo: spec.cicdRepo,
workspace: spec.workspace,
headProbe: compactCommandResult(head.result),
};
}
if (options.action === "status") {
return runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history });
}
if (options.action === "refresh") {
return refreshRuntimeLaneArgoApplication(spec, options);
}
if (options.action === "apply") {
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
if (!isCommandSuccess(render.result)) {
return {
ok: false,
command: `hwlab g14 control-plane apply --lane ${spec.lane}`,
lane: spec.lane,
phase: "source-render",
sourceCommit,
renderDir: render.renderDir,
render: compactCommandResult(render.result),
};
}
const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, options.dryRun, options.timeoutSeconds);
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
return {
ok: isCommandSuccess(apply),
command: `hwlab g14 control-plane apply --lane ${spec.lane}`,
lane: spec.lane,
mode: options.dryRun ? "dry-run" : "confirmed-apply",
sourceCommit,
renderDir: render.renderDir,
render: compactCommandResult(render.result),
apply: compactCommandResult(apply),
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }),
next: options.dryRun
? { apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm` }
: { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` },
};
}
let pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit);
let before = getPipelineRunCompact(pipelineRun);
if (options.dryRun) {
return {
ok: true,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "dry-run",
sourceCommit,
pipelineRun,
before,
gitMirrorSync: {
mode: "skipped-dry-run",
wouldSyncBeforeTrigger: true,
command: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
},
controlPlaneRefresh: {
mode: "skipped-dry-run",
wouldRefreshBeforeCreate: true,
resources: [`${spec.runtimeRenderDir}/namespace.yaml`, `${spec.tektonDir}/rbac.yaml`, `${spec.tektonDir}/pipeline.yaml`, "argocd/project.yaml", `argocd/${spec.argoApplicationFile}`],
},
manifest: runtimeLanePipelineRunManifest(spec, sourceCommit),
next: { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` },
};
}
const beforeStatus = stringOrNull(before.status);
if (before.exists === true && beforeStatus === "True") {
const currentCd = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
if (runtimeLaneCdPassedWithMirror(spec, currentCd)) {
return {
ok: true,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
sourceCommit,
pipelineRun,
before,
skipped: true,
reason: "existing-pipelinerun-reused",
cd: summarizeRuntimeLaneCdStatus(spec, currentCd),
latestOnlyPolicy: "same source commit is idempotent only after PipelineRun, Argo/runtime, public probes, and Git mirror are already aligned",
};
}
pipelineRun = runtimeLaneRerunPipelineRunName(spec, sourceCommit);
before = getPipelineRunCompact(pipelineRun);
printRuntimeLaneTriggerProgress(spec, {
stage: "existing-pipelinerun",
status: "rerun-required",
sourceCommit,
pipelineRun,
previousPipelineRun: runtimeLanePipelineRunName(spec, sourceCommit),
previousCd: summarizeRuntimeLaneCdStatus(spec, currentCd),
});
}
const effectiveBeforeStatus = stringOrNull(before.status);
if (before.exists === true && (effectiveBeforeStatus === "True" || effectiveBeforeStatus === "Unknown")) {
return {
ok: true,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
sourceCommit,
pipelineRun,
before,
skipped: true,
reason: "existing-rerun-pipelinerun-reused",
latestOnlyPolicy: "same source commit reuses an active or already successful rerun PipelineRun after the deterministic run failed CD alignment",
};
}
if (before.exists === true && effectiveBeforeStatus !== null) {
return {
ok: false,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
sourceCommit,
pipelineRun,
before,
skipped: true,
reason: "existing-pipelinerun-terminal-failed",
degradedReason: "existing-pipelinerun-terminal-failed",
next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
};
}
printRuntimeLaneTriggerProgress(spec, {
stage: "git-mirror-sync",
status: "started",
sourceCommit,
pipelineRun,
});
const gitMirrorPreSyncStatus = runGitMirrorStatus();
const gitMirrorSourceAlreadyCurrent = runtimeLaneGitMirrorSourceInSync(spec, sourceCommit, gitMirrorPreSyncStatus);
const gitMirrorSync = gitMirrorSourceAlreadyCurrent
? {
ok: true,
command: "hwlab nodes git-mirror sync",
mode: "skipped-source-already-current",
skipped: true,
reason: "local-mirror-source-current",
status: gitMirrorPreSyncStatus,
}
: runGitMirrorSync({
action: "sync",
lane: spec.lane,
confirm: true,
dryRun: false,
wait: true,
timeoutSeconds: options.timeoutSeconds,
});
printRuntimeLaneTriggerProgress(spec, {
stage: "git-mirror-sync",
status: gitMirrorSync.ok === true ? "succeeded" : "failed",
sourceCommit,
pipelineRun,
jobName: gitMirrorSync.jobName ?? null,
elapsedMs: gitMirrorSync.elapsedMs ?? null,
});
if (gitMirrorSync.ok !== true) {
return {
ok: false,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
sourceCommit,
pipelineRun,
before,
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
degradedReason: "git-mirror-sync-before-trigger-failed",
};
}
printRuntimeLaneTriggerProgress(spec, {
stage: "source-render",
status: "started",
sourceCommit,
pipelineRun,
});
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
printRuntimeLaneTriggerProgress(spec, {
stage: "source-render",
status: isCommandSuccess(render.result) ? "succeeded" : "failed",
sourceCommit,
pipelineRun,
renderDir: render.renderDir,
exitCode: render.result.exitCode,
});
if (!isCommandSuccess(render.result)) {
return {
ok: false,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
phase: "source-render",
sourceCommit,
pipelineRun,
before,
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
renderDir: render.renderDir,
render: compactCommandResult(render.result),
degradedReason: "control-plane-render-failed",
};
}
printRuntimeLaneTriggerProgress(spec, {
stage: "control-plane-apply",
status: "started",
sourceCommit,
pipelineRun,
renderDir: render.renderDir,
});
const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, false, options.timeoutSeconds);
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
printRuntimeLaneTriggerProgress(spec, {
stage: "control-plane-apply",
status: isCommandSuccess(apply) ? "succeeded" : "failed",
sourceCommit,
pipelineRun,
exitCode: apply.exitCode,
});
if (!isCommandSuccess(apply)) {
return {
ok: false,
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
phase: "control-plane-apply",
sourceCommit,
pipelineRun,
before,
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
renderDir: render.renderDir,
render: compactCommandResult(render.result),
apply: compactCommandResult(apply),
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
degradedReason: "control-plane-apply-before-trigger-failed",
};
}
printRuntimeLaneTriggerProgress(spec, {
stage: "create-pipelinerun",
status: "started",
sourceCommit,
pipelineRun,
});
const create = createRuntimeLanePipelineRun(spec, sourceCommit, options.timeoutSeconds, pipelineRun);
printRuntimeLaneTriggerProgress(spec, {
stage: "create-pipelinerun",
status: isCommandSuccess(create) ? "succeeded" : "failed",
sourceCommit,
pipelineRun,
exitCode: create.exitCode,
});
return {
ok: isCommandSuccess(create),
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
lane: spec.lane,
mode: "confirmed-trigger",
sourceCommit,
pipelineRun,
before,
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
renderDir: render.renderDir,
render: compactCommandResult(render.result),
apply: compactCommandResult(apply),
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
create: compactCommandResult(create),
status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }),
degradedReason: isCommandSuccess(create) ? undefined : "pipelinerun-create-failed",
next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --source-commit ${sourceCommit}` },
};
}
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
if ((options.action === "status" || options.action === "closeout") && options.pipelineRun !== undefined) {
const status = v02ControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run", includeHistory: options.history });
if (options.action === "closeout") return v02ControlPlaneCloseout(status);
return status;
}
if ((options.action === "status" || options.action === "closeout") && options.sourceCommit !== undefined) {
const status = v02ControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit", includeHistory: options.history });
if (options.action === "closeout") return v02ControlPlaneCloseout(status);
return status;
}
if (options.action === "closeout") {
const status = v02ControlPlaneStatus({ includeHistory: options.history });
return v02ControlPlaneCloseout(status);
}
const snapshot = options.action === "trigger-current" ? getV02TriggerSnapshot() : null;
const latestHead = options.action === "trigger-current" ? resolveV02LatestRemoteHead() : null;
const head = snapshot === null
? resolveV02Head()
: snapshot.sourceCommit === null
? resolveV02Head()
: { sourceCommit: snapshot.sourceCommit, result: snapshot.result };
let sourceCommit = head.sourceCommit;
if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) {
const refreshedHead = resolveV02Head();
sourceCommit = refreshedHead.sourceCommit;
if (sourceCommit !== latestHead.sourceCommit) {
return {
ok: false,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
phase: "source-head-refresh",
sourceCommit,
latestSourceHeadProbe: {
sourceCommit: latestHead.sourceCommit,
result: compactCommandResult(latestHead.result),
},
refreshedHeadProbe: compactCommandResult(refreshedHead.result),
degradedReason: "v02-head-refresh-did-not-reach-latest-source",
};
}
}
if (sourceCommit === null) {
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) };
}
if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit);
if (options.action === "status") return v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head", includeHistory: options.history });
if (options.action === "refresh") return refreshRuntimeLaneArgoApplication(V02_LANE_SPEC, options);
if (options.action === "apply") {
const render = runV02RenderToTemp(sourceCommit);
if (!isCommandSuccess(render.result)) {
return {
ok: false,
command: `hwlab g14 control-plane ${options.action} --lane v02`,
phase: "source-render",
sourceCommit,
renderDir: render.renderDir,
render: compactCommandResult(render.result),
};
}
const apply = applyV02ControlPlaneFiles(render.renderDir, options.dryRun, options.timeoutSeconds);
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
return {
ok: isCommandSuccess(apply),
command: "hwlab g14 control-plane apply --lane v02",
lane: "v02",
mode: options.dryRun ? "dry-run" : "confirmed-apply",
sourceCommit,
renderDir: render.renderDir,
render: compactCommandResult(render.result),
apply: compactCommandResult(apply),
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false)),
status: v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head" }),
next: options.dryRun
? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" }
: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
};
}
let before = snapshot !== null && snapshot.sourceCommit === sourceCommit && snapshot.before !== null
? snapshot.before
: getPipelineRunCompact(v02PipelineRunName(sourceCommit));
if (options.dryRun) {
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
return {
ok: true,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "dry-run",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
gitMirrorPreSync,
controlPlaneRefresh: {
mode: "skipped-dry-run",
wouldRefreshBeforeCreate: true,
resources: ["tekton-v02/rbac.yaml", "tekton-v02/pipeline.yaml", "argocd/project.yaml", "argocd/application-v02.yaml"],
},
manifest: v02PipelineRunManifest(sourceCommit),
next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
};
}
if (before.exists === true && before.status !== null) {
const reuseHead = latestHead ?? resolveV02LatestRemoteHead();
const reuseDecision = v02ExistingPipelineRunReuseDecision({ sourceCommit, before, latestSourceCommit: reuseHead.sourceCommit });
if (reuseDecision.reusable === true) {
const alreadyUsable = reuseDecision.alreadyUsable === true;
return {
ok: alreadyUsable,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
skipped: true,
reuseDecision,
reason: reuseDecision.reason,
degradedReason: alreadyUsable ? undefined : "existing-pipelinerun-terminal-failed",
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse",
};
}
if (reuseHead.sourceCommit !== null && reuseHead.sourceCommit !== sourceCommit) {
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "source-head-recheck",
status: "advanced",
sourceCommit,
latestSourceCommit: reuseHead.sourceCommit,
previousPipelineRun: v02PipelineRunName(sourceCommit),
latestPipelineRun: v02PipelineRunName(reuseHead.sourceCommit),
});
sourceCommit = reuseHead.sourceCommit;
before = getPipelineRunCompact(v02PipelineRunName(sourceCommit));
if (before.exists === true && before.status !== null) {
const advancedAlreadyUsable = before.status === "True" || before.status === "Unknown";
return {
ok: advancedAlreadyUsable,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
skipped: true,
reuseDecision: {
...reuseDecision,
advancedSourceCommit: sourceCommit,
advancedPipelineRun: v02PipelineRunName(sourceCommit),
advancedPipelineRunStatus: before.status ?? null,
},
reason: advancedAlreadyUsable ? "advanced-existing-pipelinerun-reused" : "advanced-existing-pipelinerun-terminal-failed",
degradedReason: advancedAlreadyUsable ? undefined : "advanced-existing-pipelinerun-terminal-failed",
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse",
};
}
} else {
return {
ok: false,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
reuseDecision,
degradedReason: "existing-pipelinerun-reuse-rejected",
latestSourceHeadProbe: {
sourceCommit: reuseHead.sourceCommit,
exitCode: reuseHead.result.exitCode,
stderr: reuseHead.result.stderr.trim().slice(0, 2000),
},
};
}
}
if (latestHead !== null && latestHead.sourceCommit !== sourceCommit) {
return {
ok: false,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
phase: "source-head-recheck",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
latestSourceHeadProbe: {
sourceCommit: latestHead.sourceCommit,
exitCode: latestHead.result.exitCode,
stderr: latestHead.result.stderr.trim().slice(0, 2000),
},
degradedReason: latestHead.sourceCommit === null
? "source-head-recheck-unresolved-before-pipelinerun-create"
: "source-head-advanced-before-pipelinerun-create",
};
}
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
const controlPlaneRefresh = refreshV02ControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds);
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "control-plane-refresh",
status: controlPlaneRefresh.ok === true ? "succeeded" : "failed",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
mode: record(controlPlaneRefresh).mode ?? null,
waitedMs: record(controlPlaneRefresh).waitedMs ?? null,
renderDir: record(controlPlaneRefresh).renderDir ?? null,
degradedReason: record(controlPlaneRefresh).degradedReason ?? null,
});
if (controlPlaneRefresh.ok !== true) {
return {
ok: false,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
phase: "control-plane-refresh",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
controlPlaneRefresh,
degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed",
};
}
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync",
status: "started",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
reason: "ensure local git mirror has source commit before creating PipelineRun",
});
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync",
status: gitMirrorPreSync.ok === true ? "succeeded" : "failed",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
mode: record(gitMirrorPreSync).mode ?? null,
required: nested(gitMirrorPreSync, ["before", "required"]) ?? null,
localV02: nested(gitMirrorPreSync, ["before", "localV02"]) ?? null,
finalLocalV02: nested(gitMirrorPreSync, ["final", "localV02"]) ?? nested(gitMirrorPreSync, ["marker", "localV02"]) ?? nested(gitMirrorPreSync, ["after", "localV02"]) ?? null,
pendingFlush: nested(gitMirrorPreSync, ["before", "pendingFlush"]) ?? null,
reason: nested(gitMirrorPreSync, ["before", "reason"]) ?? null,
syncElapsedMs: nested(gitMirrorPreSync, ["sync", "elapsedMs"]) ?? null,
waitElapsedMs: nested(gitMirrorPreSync, ["wait", "elapsedMs"]) ?? null,
waitAttempts: nested(gitMirrorPreSync, ["wait", "attempts"]) ?? null,
lockWaitedMs: record(gitMirrorPreSync).waitedMs ?? null,
degradedReason: record(gitMirrorPreSync).degradedReason ?? null,
});
if (gitMirrorPreSync.ok !== true) {
return {
ok: false,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
phase: "git-mirror-pre-sync",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
controlPlaneRefresh,
gitMirrorPreSync,
degradedReason: "git-mirror-pre-sync-failed",
};
}
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
const createPipelineRun = createV02PipelineRun(sourceCommit, options.timeoutSeconds);
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), exitCode: createPipelineRun.exitCode });
return {
ok: isCommandSuccess(createPipelineRun),
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh),
gitMirrorPreSync,
createPipelineRun: compactTriggerCommandResult(compactCommandResult(createPipelineRun)),
after: getPipelineRunCompact(v02PipelineRunName(sourceCommit)),
latestOnlyPolicy: "no old PipelineRun is canceled; stale commits self-supersede before GitOps writeback",
disclosure: {
fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.",
},
};
}
function v02SecretScript(options: G14SecretOptions): string {
const spec = hwlabRuntimeLaneSpec(options.lane);
if (options.preset === "generic-delete") return v02DeleteSecretScript(options);
if (options.preset === "master-server-admin-api-key") return v02MasterAdminApiKeySecretScript(options, spec);
return [
v02OpenFgaSecretScript(options, spec),
].join("\n");
}
function v02DeleteSecretScript(options: G14SecretOptions): string {
return [
"set +e",
`namespace=${shellQuote(V02_RUNTIME_NAMESPACE)}`,
`name=${shellQuote(options.name)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"preset=generic-delete",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
"before_exists=$(secret_exists_flag)",
"delete_exit=",
"action=observed",
"mutation=false",
"if [ \"$dry_run\" = true ]; then",
" if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
"else",
" kubectl -n \"$namespace\" delete secret \"$name\" --ignore-not-found=true >/tmp/hwlab-secret-delete.out 2>/tmp/hwlab-secret-delete.err",
" delete_exit=$?",
" if [ \"$delete_exit\" -eq 0 ]; then",
" if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
" else",
" action=delete-failed",
" fi",
"fi",
"after_exists=$(secret_exists_flag)",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'deleteExitCode\\t%s\\n' \"$delete_exit\"",
"if [ -n \"$delete_exit\" ] && [ \"$delete_exit\" != 0 ]; then exit \"$delete_exit\"; fi",
].join("\n");
}
function v02OpenFgaSecretScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string {
const selectedKey = options.key ?? "";
const namespace = spec.runtimeNamespace;
const openFgaSecret = runtimeLaneOpenFgaSecretName(spec);
const postgresSecret = runtimeLanePostgresSecretName(spec);
const primaryDbName = runtimeLanePrimaryDbName(spec);
const dedicatedDb = spec.lane === "v02";
const dbMode = dedicatedDb ? "dedicated" : "primary";
const dbName = dedicatedDb ? V02_OPENFGA_DB_NAME : primaryDbName;
const dbUser = dedicatedDb ? V02_OPENFGA_DB_USER : primaryDbName;
const dbHost = `${postgresSecret}.${namespace}.svc.cluster.local`;
const fieldManager = runtimeLaneSecretFieldManager(spec);
return [
"set +e",
`namespace=${shellQuote(namespace)}`,
`name=${shellQuote(openFgaSecret)}`,
`selected_key=${shellQuote(selectedKey)}`,
`authn_key=${shellQuote(V02_OPENFGA_AUTHN_SECRET_KEY)}`,
`datastore_uri_key=${shellQuote(V02_OPENFGA_DATASTORE_URI_SECRET_KEY)}`,
`postgres_password_key=${shellQuote(V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY)}`,
`db_mode=${shellQuote(dbMode)}`,
`db_name=${shellQuote(dbName)}`,
`db_user=${shellQuote(dbUser)}`,
`postgres_secret=${shellQuote(postgresSecret)}`,
`postgres_statefulset=${shellQuote(postgresSecret)}`,
`postgres_admin_user=${shellQuote(primaryDbName)}`,
`db_host=${shellQuote(dbHost)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(fieldManager)}`,
"preset=openfga",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
"probe_db() {",
" if [ \"$db_mode\" = primary ]; then",
" role_result=primary",
" database_result=primary",
" probe_exit=not-required",
" return",
" fi",
" role_result=unknown",
" database_result=unknown",
" probe_exit=missing-postgres-admin-secret",
" if [ -n \"$postgres_admin_password\" ]; then",
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
" role_exit=$?",
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
" database_exit=$?",
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
" fi",
"}",
"before_exists=$(secret_exists_flag)",
"before_authn_b64=$(secret_b64_key \"$authn_key\")",
"before_uri_b64=$(secret_b64_key \"$datastore_uri_key\")",
"before_pg_password_b64=$(secret_b64_key \"$postgres_password_key\")",
"before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)",
"before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)",
"before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)",
"before_authn_bytes=$(decoded_length \"$before_authn_b64\")",
"before_uri_bytes=$(decoded_length \"$before_uri_b64\")",
"before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")",
"authn_value=$(decoded_value \"$before_authn_b64\")",
"datastore_uri=$(decoded_value \"$before_uri_b64\")",
"pg_password=$(decoded_value \"$before_pg_password_b64\")",
"postgres_admin_b64=$(kubectl -n \"$namespace\" get secret \"$postgres_secret\" -o 'go-template={{ index .data \"POSTGRES_PASSWORD\" }}' 2>/dev/null || true)",
"postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)",
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
"probe_db",
"db_role_exists_before=$role_result",
"db_database_exists_before=$database_result",
"db_probe_exit_before=$probe_exit",
"action=observed",
"mutation=false",
"apply_exit=",
"db_ensure_exit=",
"if [ \"$action_request\" = ensure ]; then",
" missing_secret=false",
" [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true",
" [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true",
" [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true",
" missing_db=false",
" if [ \"$db_mode\" = dedicated ]; then",
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
" fi",
" if [ \"$dry_run\" = true ]; then",
" if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
" elif ! command -v openssl >/dev/null 2>&1; then",
" action=openssl-missing",
" apply_exit=127",
" elif [ -z \"$postgres_admin_password\" ]; then",
" action=postgres-admin-secret-missing",
" apply_exit=44",
" else",
" [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)",
" if [ \"$db_mode\" = primary ]; then",
" [ -n \"$pg_password\" ] || pg_password=\"$postgres_admin_password\"",
" else",
" [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
" fi",
" [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then",
" if [ \"$db_mode\" = primary ]; then",
" action=ensured",
" mutation=true",
" else",
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'",
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
"\\gexec",
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
"\\gexec",
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
"SQL",
" db_ensure_exit=$?",
" if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi",
" fi",
" else",
" action=apply-failed",
" fi",
" fi",
"fi",
"after_exists=$(secret_exists_flag)",
"after_authn_b64=$(secret_b64_key \"$authn_key\")",
"after_uri_b64=$(secret_b64_key \"$datastore_uri_key\")",
"after_pg_password_b64=$(secret_b64_key \"$postgres_password_key\")",
"after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)",
"after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)",
"after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)",
"after_authn_bytes=$(decoded_length \"$after_authn_b64\")",
"after_uri_bytes=$(decoded_length \"$after_uri_b64\")",
"after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")",
"probe_db",
"db_role_exists_after=$role_result",
"db_database_exists_after=$database_result",
"db_probe_exit_after=$probe_exit",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'key\\t%s\\n' \"$selected_key\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'dbMode\\t%s\\n' \"$db_mode\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'beforeAuthnPresent\\t%s\\n' \"$before_authn_present\"",
"printf 'beforeAuthnBytes\\t%s\\n' \"$before_authn_bytes\"",
"printf 'beforeDatastoreUriPresent\\t%s\\n' \"$before_uri_present\"",
"printf 'beforeDatastoreUriBytes\\t%s\\n' \"$before_uri_bytes\"",
"printf 'beforePostgresPasswordPresent\\t%s\\n' \"$before_pg_password_present\"",
"printf 'beforePostgresPasswordBytes\\t%s\\n' \"$before_pg_password_bytes\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"",
"printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"",
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"",
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"",
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"",
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"",
"printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"",
"printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"",
"printf 'dbName\\t%s\\n' \"$db_name\"",
"printf 'dbUser\\t%s\\n' \"$db_user\"",
"printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"",
"printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"",
"printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"",
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
"authn_value=",
"datastore_uri=",
"pg_password=",
"postgres_admin_password=",
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
].join("\n");
}
function v02MasterAdminApiKeySecretScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string {
const namespace = spec.runtimeNamespace;
const name = runtimeLaneMasterAdminApiKeySecretName(spec);
const fieldManager = runtimeLaneSecretFieldManager(spec);
return [
"set +e",
`namespace=${shellQuote(namespace)}`,
`name=${shellQuote(name)}`,
`api_key_name=${shellQuote(V02_MASTER_ADMIN_API_KEY_SECRET_KEY)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(fieldManager)}`,
"preset=master-server-admin-api-key",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-24; value=; fi; }",
"before_exists=$(secret_exists_flag)",
"before_api_key_b64=$(secret_b64_key \"$api_key_name\")",
"before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)",
"before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")",
"before_api_key_prefix=$(decoded_prefix \"$before_api_key_b64\")",
"action=observed",
"mutation=false",
"apply_exit=",
"if [ \"$action_request\" = ensure ]; then",
" missing_secret=false",
" [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true",
" if [ \"$dry_run\" = true ]; then",
" if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi",
" else",
" api_key=$(cat)",
" case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac",
" if [ -z \"$apply_exit\" ]; then",
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
" fi",
" api_key=",
" fi",
"fi",
"after_exists=$(secret_exists_flag)",
"after_api_key_b64=$(secret_b64_key \"$api_key_name\")",
"after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)",
"after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")",
"after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'key\\t%s\\n' \"$api_key_name\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'beforeApiKeyPresent\\t%s\\n' \"$before_api_key_present\"",
"printf 'beforeApiKeyBytes\\t%s\\n' \"$before_api_key_bytes\"",
"printf 'beforeApiKeyPrefix\\t%s\\n' \"$before_api_key_prefix\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"",
"printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"",
"printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
].join("\n");
}
function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
const fields = keyValueLinesFromText(text);
if (fields.preset === "generic-delete") {
const absent = fields.afterExists !== "yes";
return {
ok: commandOk && absent,
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
secret: fields.secret || null,
preset: "generic-delete",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
before: {
exists: fields.beforeExists === "yes",
},
after: {
exists: fields.afterExists === "yes",
},
deleteExitCode: numericField(fields.deleteExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: absent
? `${fields.secret || "secret"} absent`
: `${fields.secret || "secret"} still exists`,
};
}
if (fields.preset === "master-server-admin-api-key") {
const afterBytes = numericField(fields.afterApiKeyBytes);
const healthy =
fields.afterExists === "yes" &&
fields.afterApiKeyPresent === "yes" &&
typeof afterBytes === "number" && afterBytes > 0 &&
String(fields.afterApiKeyPrefix ?? "").startsWith("hwl_live_");
return {
ok: commandOk && healthy,
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
secret: fields.secret || V02_MASTER_ADMIN_API_KEY_SECRET,
key: fields.key || V02_MASTER_ADMIN_API_KEY_SECRET_KEY,
preset: "master-server-admin-api-key",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
before: {
exists: fields.beforeExists === "yes",
apiKey: { keyPresent: fields.beforeApiKeyPresent === "yes", valueBytes: numericField(fields.beforeApiKeyBytes), keyPrefix: fields.beforeApiKeyPrefix || null },
},
after: {
exists: fields.afterExists === "yes",
apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null },
},
applyExitCode: numericField(fields.applyExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy
? `${fields.secret || V02_MASTER_ADMIN_API_KEY_SECRET}/${fields.key || V02_MASTER_ADMIN_API_KEY_SECRET_KEY} exists`
: `${fields.secret || V02_MASTER_ADMIN_API_KEY_SECRET}/${fields.key || V02_MASTER_ADMIN_API_KEY_SECRET_KEY} missing`,
};
}
if (fields.preset === "openfga") {
const afterAuthnBytes = numericField(fields.afterAuthnBytes);
const afterUriBytes = numericField(fields.afterDatastoreUriBytes);
const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes);
const keysHealthy =
fields.afterExists === "yes" &&
fields.afterAuthnPresent === "yes" &&
fields.afterDatastoreUriPresent === "yes" &&
fields.afterPostgresPasswordPresent === "yes" &&
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
typeof afterUriBytes === "number" && afterUriBytes > 0 &&
typeof afterPasswordBytes === "number" && afterPasswordBytes > 0;
const databaseHealthy = fields.dbMode === "primary" || (fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t");
const healthy = keysHealthy && databaseHealthy;
return {
ok: commandOk && healthy,
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
secret: fields.secret || V02_OPENFGA_SECRET,
key: fields.key || null,
preset: "openfga",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
dbMode: fields.dbMode || "dedicated",
before: {
exists: fields.beforeExists === "yes",
authnPresharedKey: { keyPresent: fields.beforeAuthnPresent === "yes", valueBytes: numericField(fields.beforeAuthnBytes) },
datastoreUri: { keyPresent: fields.beforeDatastoreUriPresent === "yes", valueBytes: numericField(fields.beforeDatastoreUriBytes) },
postgresPassword: { keyPresent: fields.beforePostgresPasswordPresent === "yes", valueBytes: numericField(fields.beforePostgresPasswordBytes) },
database: {
roleExists: fields.dbRoleExistsBefore || "unknown",
databaseExists: fields.dbDatabaseExistsBefore || "unknown",
probeExitCode: fields.dbProbeExitCodeBefore || null,
},
},
after: {
exists: fields.afterExists === "yes",
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
database: {
roleExists: fields.dbRoleExistsAfter || "unknown",
databaseExists: fields.dbDatabaseExistsAfter || "unknown",
probeExitCode: fields.dbProbeExitCodeAfter || null,
},
},
postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes",
postgresSecret: fields.postgresSecret || null,
dbName: fields.dbName || V02_OPENFGA_DB_NAME,
dbUser: fields.dbUser || V02_OPENFGA_DB_USER,
applyExitCode: numericField(fields.applyExitCode),
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy
? `${fields.secret || V02_OPENFGA_SECRET} keys and Postgres database exist`
: `${fields.secret || V02_OPENFGA_SECRET} keys or Postgres database missing`,
};
}
return {
ok: false,
namespace: fields.namespace || V02_RUNTIME_NAMESPACE,
secret: fields.secret || null,
preset: fields.preset || null,
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
exitCode,
stderr: stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: "unrecognized secret status output",
};
}
function runG14Secret(options: G14SecretOptions): Record<string, unknown> {
const spec = hwlabRuntimeLaneSpec(options.lane);
const script = v02SecretScript(options);
const localAdminApiKey = options.preset === "master-server-admin-api-key"
? readOrCreateLocalMasterAdminApiKey(spec, options.action !== "ensure" || options.dryRun)
: null;
const result = options.preset === "master-server-admin-api-key"
? g14K3sInlineScriptWithInput(script, localAdminApiKey?.key ?? "", options.timeoutSeconds * 1000 + 2000)
: g14K3s(["script", "--", script], options.timeoutSeconds * 1000);
const status = v02SecretStatusFromText(statusText(result), isCommandSuccess(result), result.exitCode, result.stderr);
const dryRunOk = options.action === "ensure" && options.dryRun && isCommandSuccess(result);
const deleteDryRunOk = options.action === "delete" && options.dryRun && isCommandSuccess(result);
const ok = dryRunOk || deleteDryRunOk ? true : status.ok === true;
return {
ok,
command: `hwlab g14 secret ${options.action} --lane ${options.lane}`,
lane: options.lane,
namespace: spec.runtimeNamespace,
secret: options.name,
key: options.key ?? null,
preset: options.preset,
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "delete" ? "confirmed-delete" : "confirmed-ensure",
status,
localMasterServer: localAdminApiKey
? { created: localAdminApiKey.created, envFile: localAdminApiKey.status, valuesRedacted: true }
: undefined,
mutation: status.mutation === true,
result: compactCommandResult(result),
valuesRedacted: true,
next: ok && options.action === "status"
? undefined
: options.action === "delete"
? undefined
: { ensure: `bun scripts/cli.ts hwlab g14 secret ensure --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` },
};
}
function legacyG14RetirementStatePath(): string {
return rootPath(".state", "hwlab-g14", "legacy-g14-retirement.json");
}
function legacyG14RetirementApplications(): string[] {
return [DEV_APP, PROD_APP];
}
function legacyG14RetirementNamespaces(): string[] {
return [DEV_NAMESPACE, PROD_NAMESPACE];
}
function protectedG14RuntimeApplications(): string[] {
return [V02_APP, hwlabRuntimeLaneSpec("v03").app];
}
function protectedG14RuntimeNamespaces(): string[] {
return [V02_RUNTIME_NAMESPACE, hwlabRuntimeLaneSpec("v03").runtimeNamespace];
}
function readLegacyG14RetirementMarker(): Record<string, unknown> | null {
const path = legacyG14RetirementStatePath();
if (!existsSync(path)) return null;
try {
return record(JSON.parse(readFileSync(path, "utf8")) as unknown);
} catch (error) {
return { status: "unreadable", path, error: error instanceof Error ? error.message : String(error) };
}
}
function writeLegacyG14RetirementMarker(status: string, reason: string, details: Record<string, unknown> = {}): Record<string, unknown> {
const path = legacyG14RetirementStatePath();
mkdirSync(dirname(path), { recursive: true });
const previous = readLegacyG14RetirementMarker();
const marker = {
status,
reason,
requestedAt: typeof previous?.requestedAt === "string" ? previous.requestedAt : new Date().toISOString(),
updatedAt: new Date().toISOString(),
legacyApplications: legacyG14RetirementApplications(),
legacyNamespaces: legacyG14RetirementNamespaces(),
protectedApplications: protectedG14RuntimeApplications(),
protectedNamespaces: protectedG14RuntimeNamespaces(),
...details,
};
writeFileSync(path, `${JSON.stringify(marker, null, 2)}\n`, "utf8");
return { ...marker, path };
}
function legacyG14RetirementBlocksMonitor(): Record<string, unknown> | null {
const marker = readLegacyG14RetirementMarker();
if (marker === null) {
return {
status: "retired-by-contract",
path: legacyG14RetirementStatePath(),
reason: "legacy G14 DEV/PROD monitor is retired; inspect retirement status for live cluster evidence",
next: { status: "bun scripts/cli.ts hwlab g14 retirement status", v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02", v03Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v03" },
};
}
const status = String(marker.status ?? "");
return status === "retiring" || status === "retired" || status === "failed" || status === "unreadable" ? marker : null;
}
function k8sMetadataNamespace(item: Record<string, unknown>): string | null {
const namespace = record(item.metadata).namespace;
return typeof namespace === "string" && namespace.length > 0 ? namespace : null;
}
function applicationSummary(item: Record<string, unknown>): Record<string, unknown> {
return {
name: k8sMetadataName(item),
namespace: k8sMetadataNamespace(item) ?? ARGO_NAMESPACE,
deletionTimestamp: record(item.metadata).deletionTimestamp ?? null,
sync: nested(item, ["status", "sync", "status"]) ?? null,
health: nested(item, ["status", "health", "status"]) ?? null,
revision: nested(item, ["status", "sync", "revision"]) ?? null,
};
}
function namespaceSummary(item: Record<string, unknown>): Record<string, unknown> {
return {
name: k8sMetadataName(item),
phase: nested(item, ["status", "phase"]) ?? null,
deletionTimestamp: record(item.metadata).deletionTimestamp ?? null,
};
}
function parseNamespacedResourcePreview(text: string): Record<string, unknown>[] {
const groups: Record<string, string[]> = {};
let current = "";
for (const line of text.split(/\r?\n/u)) {
const trimmed = line.trim();
if (trimmed.length === 0) continue;
const namespaceMatch = /^__namespace\t(.+)$/u.exec(trimmed);
if (namespaceMatch !== null) {
current = namespaceMatch[1] ?? "";
if (current.length > 0) groups[current] = [];
continue;
}
if (current.length > 0) groups[current]?.push(trimmed);
}
return Object.entries(groups).map(([namespace, items]) => ({
namespace,
count: items.length,
preview: items.slice(0, 40),
truncated: items.length > 40,
}));
}
function legacyG14RetirementStatusScript(): string {
const legacyApps = legacyG14RetirementApplications().map(shellQuote).join(" ");
const legacyNamespaces = legacyG14RetirementNamespaces().map(shellQuote).join(" ");
const protectedApps = protectedG14RuntimeApplications().map(shellQuote).join(" ");
const protectedNamespaces = protectedG14RuntimeNamespaces().map(shellQuote).join(" ");
return [
"set +e",
"section() {",
" name=\"$1\"",
" shift",
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
" \"$@\"",
" code=$?",
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
`section legacyApplications kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${legacyApps} --ignore-not-found=true -o json`,
`section legacyNamespaces kubectl get namespace ${legacyNamespaces} --ignore-not-found=true -o json`,
`section protectedApplications kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${protectedApps} --ignore-not-found=true -o json`,
`section protectedNamespaces kubectl get namespace ${protectedNamespaces} --ignore-not-found=true -o json`,
"section legacyResources sh -lc " + shellQuote([
`for ns in ${legacyNamespaces}; do`,
" printf '__namespace\\t%s\\n' \"$ns\"",
" kubectl get deploy,statefulset,svc,pod,pvc,ingress,configmap,secret -n \"$ns\" -o name --ignore-not-found=true 2>/dev/null | sed -n '1,160p'",
"done",
].join("\n")),
].join("\n");
}
function legacyG14RetirementStatus(): Record<string, unknown> {
const result = g14K3s(["script", "--", legacyG14RetirementStatusScript()], 60_000);
const sections = parseShellSections(statusText(result));
const legacyApplications = parseSectionJsonArray(sections.legacyApplications).map(applicationSummary);
const legacyNamespaces = parseSectionJsonArray(sections.legacyNamespaces).map(namespaceSummary);
const protectedApplications = parseSectionJsonArray(sections.protectedApplications).map(applicationSummary);
const protectedNamespaces = parseSectionJsonArray(sections.protectedNamespaces).map(namespaceSummary);
const terminating = legacyApplications.some((item) => item.deletionTimestamp !== null)
|| legacyNamespaces.some((item) => item.deletionTimestamp !== null || item.phase === "Terminating");
const retired = legacyApplications.length === 0 && legacyNamespaces.length === 0;
const retirementState = retired ? "retired" : terminating ? "terminating" : "active";
const protectedAppNames = new Set(protectedApplications.map((item) => item.name).filter((name): name is string => typeof name === "string"));
const protectedNamespaceNames = new Set(protectedNamespaces.map((item) => item.name).filter((name): name is string => typeof name === "string"));
const missingProtectedApplications = protectedG14RuntimeApplications().filter((name) => !protectedAppNames.has(name));
const missingProtectedNamespaces = protectedG14RuntimeNamespaces().filter((name) => !protectedNamespaceNames.has(name));
const sectionsOk =
shellSectionOk(sections.legacyApplications) &&
shellSectionOk(sections.legacyNamespaces) &&
shellSectionOk(sections.protectedApplications) &&
shellSectionOk(sections.protectedNamespaces) &&
shellSectionOk(sections.legacyResources);
return {
ok: isCommandSuccess(result) && sectionsOk,
command: "hwlab g14 retirement status",
provider: G14_PROVIDER,
argoNamespace: ARGO_NAMESPACE,
retirementState,
retired,
legacy: {
applications: legacyApplications,
namespaces: legacyNamespaces,
resources: parseNamespacedResourcePreview(String(sections.legacyResources?.stdout ?? "")),
},
protected: {
applications: protectedApplications,
namespaces: protectedNamespaces,
missingApplications: missingProtectedApplications,
missingNamespaces: missingProtectedNamespaces,
ok: missingProtectedApplications.length === 0 && missingProtectedNamespaces.length === 0,
},
marker: readLegacyG14RetirementMarker(),
result: compactCommandResult(result),
next: retired
? { verifyV02: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02", verifyV03: "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03" }
: { plan: "bun scripts/cli.ts hwlab g14 retirement plan", execute: "bun scripts/cli.ts hwlab g14 retirement execute --confirm" },
};
}
function legacyG14RetirementPlan(status = legacyG14RetirementStatus()): Record<string, unknown> {
return {
ok: status.ok === true,
command: "hwlab g14 retirement plan",
mode: "dry-run",
mutation: false,
reason: "retire legacy G14 DEV/PROD runtime and support surface without touching v0.2/v0.3",
destructiveTargets: {
applications: legacyG14RetirementApplications().map((name) => ({ apiVersion: "argoproj.io/v1alpha1", kind: "Application", namespace: ARGO_NAMESPACE, name })),
namespaces: legacyG14RetirementNamespaces().map((name) => ({ apiVersion: "v1", kind: "Namespace", name })),
},
protectedTargets: {
applications: protectedG14RuntimeApplications().map((name) => ({ apiVersion: "argoproj.io/v1alpha1", kind: "Application", namespace: ARGO_NAMESPACE, name })),
namespaces: protectedG14RuntimeNamespaces().map((name) => ({ apiVersion: "v1", kind: "Namespace", name })),
},
localSupportActions: [
{ action: "write-retirement-marker", path: legacyG14RetirementStatePath() },
{ action: "cancel-active-local-job", jobName: "hwlab_g14_pr_monitor" },
{ action: "block-new-monitor-starts", command: "bun scripts/cli.ts hwlab g14 monitor-prs --lane g14" },
],
status,
next: { execute: "bun scripts/cli.ts hwlab g14 retirement execute --confirm" },
};
}
function legacyG14RecordRolloutRetired(): Record<string, unknown> {
return {
ok: false,
command: "hwlab g14 record-rollout",
phase: "retired",
degradedReason: "legacy-g14-dev-prod-retired",
message: "Legacy G14 DEV rollout recording is retired. Use active runtime lane closeout/status evidence instead.",
retirement: readLegacyG14RetirementMarker(),
next: {
retirementStatus: "bun scripts/cli.ts hwlab g14 retirement status",
v02Status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
v03Status: "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
},
};
}
function cancelLegacyG14MonitorJobs(dryRun: boolean): Record<string, unknown> {
const candidates = listJobs()
.filter((job) => job.name === "hwlab_g14_pr_monitor" && (job.status === "queued" || job.status === "running"))
.map((job) => ({ id: job.id, name: job.name, status: job.status, createdAt: job.createdAt, runnerPid: job.runnerPid ?? null }));
if (dryRun) return { ok: true, dryRun: true, candidates, actions: candidates.map((job) => ({ action: "would-cancel", ...job })) };
const actions = candidates.map((job) => cancelJob(job.id));
return { ok: actions.every((action) => record(action).ok !== false), dryRun: false, candidates, actions };
}
function legacyG14RetirementDeleteScript(): string {
const legacyApps = legacyG14RetirementApplications();
const legacyNamespaces = legacyG14RetirementNamespaces();
return [
"set +e",
"overall=0",
"printf '__UNIDESK_SECTION_BEGIN__ deleteApplications\\n'",
...legacyApps.map((app) => [
`kubectl delete application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(app)} --ignore-not-found=true --wait=false`,
"code=$?",
"[ \"$code\" -eq 0 ] || overall=1",
`printf 'application\\t%s\\texit=%s\\n' ${shellQuote(app)} "$code"`,
].join("\n")),
"printf '__UNIDESK_SECTION_END__ deleteApplications exit=0\\n'",
"printf '__UNIDESK_SECTION_BEGIN__ deleteNamespaces\\n'",
...legacyNamespaces.map((namespace) => [
`kubectl delete namespace ${shellQuote(namespace)} --ignore-not-found=true --wait=false`,
"code=$?",
"[ \"$code\" -eq 0 ] || overall=1",
`printf 'namespace\\t%s\\texit=%s\\n' ${shellQuote(namespace)} "$code"`,
].join("\n")),
"printf '__UNIDESK_SECTION_END__ deleteNamespaces exit=0\\n'",
"exit \"$overall\"",
].join("\n");
}
function waitForLegacyG14Retirement(timeoutSeconds: number): Record<string, unknown> {
const startedAtMs = Date.now();
let lastStatus: Record<string, unknown> = {};
let attempts = 0;
while (Date.now() - startedAtMs <= timeoutSeconds * 1000) {
attempts += 1;
lastStatus = legacyG14RetirementStatus();
if (lastStatus.retired === true) {
return { ok: true, attempts, elapsedMs: Date.now() - startedAtMs, status: lastStatus };
}
const wait = runCommand(["sleep", "5"], repoRoot, { timeoutMs: 7_000 });
if (wait.exitCode !== 0) break;
}
return { ok: false, attempts, elapsedMs: Date.now() - startedAtMs, status: lastStatus, degradedReason: "legacy-g14-retirement-not-complete-before-timeout" };
}
function runLegacyG14Retirement(options: G14LegacyRetirementOptions): Record<string, unknown> {
const before = legacyG14RetirementStatus();
if (options.action === "status") return before;
if (options.action === "plan" || options.dryRun) return { ...legacyG14RetirementPlan(before), command: `hwlab g14 retirement ${options.action}`, reason: options.reason };
const markerBeforeDelete = writeLegacyG14RetirementMarker("retiring", options.reason, { command: "hwlab g14 retirement execute --confirm" });
const localJobs = cancelLegacyG14MonitorJobs(false);
const deleteResult = g14K3s(["script", "--", legacyG14RetirementDeleteScript()], Math.max(60_000, options.timeoutSeconds * 1000));
const deleteSections = parseShellSections(statusText(deleteResult));
const afterDelete = legacyG14RetirementStatus();
const wait = options.wait ? waitForLegacyG14Retirement(options.timeoutSeconds) : null;
const finalStatus = wait === null ? afterDelete : record(wait.status);
const liveMutationOk = isCommandSuccess(deleteResult);
const finalRetired = finalStatus.retired === true;
const marker = writeLegacyG14RetirementMarker(finalRetired ? "retired" : liveMutationOk ? "retiring" : "failed", options.reason, {
command: "hwlab g14 retirement execute --confirm",
deleteExitCode: deleteResult.exitCode,
finalRetirementState: finalStatus.retirementState ?? null,
});
return {
ok: liveMutationOk && (options.wait ? finalRetired : finalStatus.retirementState !== "active"),
command: "hwlab g14 retirement execute",
mode: "confirmed-retirement",
mutation: true,
reason: options.reason,
before,
markerBeforeDelete,
localJobs,
delete: {
ok: liveMutationOk,
applications: String(deleteSections.deleteApplications?.stdout ?? "").split(/\r?\n/u).filter(Boolean),
namespaces: String(deleteSections.deleteNamespaces?.stdout ?? "").split(/\r?\n/u).filter(Boolean),
result: compactCommandResult(deleteResult),
},
afterDelete,
wait,
finalStatus,
marker,
next: finalRetired
? { verifyV02: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02", verifyV03: "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03" }
: { status: "bun scripts/cli.ts hwlab g14 retirement status" },
};
}
function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult {
return g14K3s([
"kubectl",
"delete",
"cronjob",
"-n",
GIT_MIRROR_NAMESPACE,
GIT_MIRROR_LEGACY_CRONJOB,
"--ignore-not-found=true",
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
], 60_000);
}
function applyGitMirrorManifestFile(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
return g14K3s([
"kubectl",
"apply",
"--server-side",
"--force-conflicts",
`--field-manager=${GIT_MIRROR_MANIFEST_FIELD_MANAGER}`,
...(dryRun ? ["--dry-run=server"] : []),
"-f",
`${renderDir}/devops-infra/git-mirror.yaml`,
], timeoutSeconds * 1000);
}
function gitMirrorSyncJobName(): string {
return `${GIT_MIRROR_SYNC_JOB_PREFIX}-${Date.now().toString(36)}`.slice(0, 63);
}
function gitMirrorFlushJobName(): string {
return `git-mirror-hwlab-flush-manual-${Date.now().toString(36)}`.slice(0, 63);
}
export function gitMirrorSyncJobManifest(name: string): Record<string, unknown> {
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name,
namespace: GIT_MIRROR_NAMESPACE,
labels: {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "devops-infra",
"app.kubernetes.io/component": "sync-controller",
"hwlab.pikastech.local/trigger": "manual-cli",
},
},
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 300,
ttlSecondsAfterFinished: 3600,
template: {
metadata: {
labels: {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "devops-infra",
"app.kubernetes.io/component": "sync-controller",
"hwlab.pikastech.local/trigger": "manual-cli",
},
},
spec: {
restartPolicy: "Never",
hostNetwork: true,
dnsPolicy: "ClusterFirstWithHostNet",
volumes: [
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } },
],
containers: [{
name: "sync",
image: "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1",
imagePullPolicy: "IfNotPresent",
command: ["/script/sync.sh"],
volumeMounts: [
{ name: "cache", mountPath: "/cache" },
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
{ name: "script", mountPath: "/script", readOnly: true },
],
}],
},
},
},
};
}
export function gitMirrorFlushJobManifest(name: string): Record<string, unknown> {
const manifest = gitMirrorSyncJobManifest(name);
record(record(manifest.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
const template = record(record(manifest.spec).template);
record(record(template.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
const podSpec = record(template.spec);
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
const first = record(containers[0]);
first.name = "flush";
first.command = ["/script/flush.sh"];
podSpec.containers = [first];
return manifest;
}
export function parseGitMirrorStatusRefs(raw: string): { refs: Record<string, string | null>; pendingFlush: boolean | null } {
const line = raw.split(/\r?\n/u).find((item) => item.startsWith("refs="));
if (line === undefined) return { refs: {}, pendingFlush: null };
try {
const parsed = record(JSON.parse(line.slice("refs=".length)) as unknown);
const refs = record(parsed.refs);
const parsedRefs: Record<string, string | null> = {};
for (const [key, value] of Object.entries(refs)) parsedRefs[key] = stringOrNull(value);
return {
refs: {
...parsedRefs,
localV02: stringOrNull(refs.localV02),
githubV02: stringOrNull(refs.githubV02),
localG14: stringOrNull(refs.localG14),
githubG14: stringOrNull(refs.githubG14),
localGitops: stringOrNull(refs.localGitops),
githubGitops: stringOrNull(refs.githubGitops),
localV03: stringOrNull(refs.localV03),
githubV03: stringOrNull(refs.githubV03),
localV03Gitops: stringOrNull(refs.localV03Gitops),
githubV03Gitops: stringOrNull(refs.githubV03Gitops),
},
pendingFlush: typeof parsed.pendingFlush === "boolean" ? parsed.pendingFlush : null,
};
} catch {
return { refs: {}, pendingFlush: null };
}
}
function parseLabeledJsonLine(raw: string, label: string): Record<string, unknown> {
const line = raw.split(/\r?\n/u).find((item) => item.startsWith(`${label}=`));
if (line === undefined) return {};
const value = line.slice(label.length + 1).trim();
if (value.length === 0) return {};
try {
return record(JSON.parse(value) as unknown);
} catch {
return {};
}
}
export function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
const refs = parseGitMirrorStatusRefs(raw);
const lastSync = parseLabeledJsonLine(raw, "lastSync");
const lastWrite = parseLabeledJsonLine(raw, "lastWrite");
const lastFlush = parseLabeledJsonLine(raw, "lastFlush");
const localV02 = refs.refs.localV02 ?? null;
const githubV02 = refs.refs.githubV02 ?? null;
const localGitops = refs.refs.localGitops ?? null;
const githubGitops = refs.refs.githubGitops ?? null;
const localV03 = refs.refs.localV03 ?? null;
const githubV03 = refs.refs.githubV03 ?? null;
const localV03Gitops = refs.refs.localV03Gitops ?? null;
const githubV03Gitops = refs.refs.githubV03Gitops ?? null;
const sourceInSync = Boolean(localV02 && githubV02 && localV02 === githubV02);
const gitopsInSync = Boolean(localGitops && githubGitops && localGitops === githubGitops);
return {
localV02,
githubV02,
localV03,
githubV03,
localG14: refs.refs.localG14 ?? null,
githubG14: refs.refs.githubG14 ?? null,
localGitops,
githubGitops,
localV03Gitops,
githubV03Gitops,
pendingFlush: refs.pendingFlush,
lastSync: {
status: stringOrNull(lastSync.status) ?? null,
at: stringOrNull(lastSync.publishedAt) ?? stringOrNull(lastSync.syncedAt) ?? null,
pendingFlush: typeof lastSync.pendingFlush === "boolean" ? lastSync.pendingFlush : null,
},
lastWrite: {
status: stringOrNull(lastWrite.status) ?? null,
at: stringOrNull(lastWrite.writtenAt) ?? null,
pendingFlush: typeof lastWrite.pendingFlush === "boolean" ? lastWrite.pendingFlush : null,
},
lastFlush: {
status: stringOrNull(lastFlush.status) ?? null,
at: stringOrNull(lastFlush.flushedAt) ?? null,
pendingFlush: typeof lastFlush.pendingFlush === "boolean" ? lastFlush.pendingFlush : null,
},
flushNeeded: refs.pendingFlush === true,
flushCommand: refs.pendingFlush === true ? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" : null,
sourceInSync,
gitopsInSync,
v03SourceInSync: Boolean(localV03 && githubV03 && localV03 === githubV03),
v03GitopsInSync: Boolean(localV03Gitops && githubV03Gitops && localV03Gitops === githubV03Gitops),
githubInSync: sourceInSync && gitopsInSync,
};
}
export function gitMirrorV02SyncRequirement(sourceCommit: string, rawStatus: string): Record<string, unknown> {
const parsed = parseGitMirrorStatusRefs(rawStatus);
const localV02 = parsed.refs.localV02 ?? null;
return {
required: localV02 !== sourceCommit,
sourceCommit,
localV02,
pendingFlush: parsed.pendingFlush,
refs: parsed.refs,
reason: localV02 === sourceCommit ? "local-v02-current" : localV02 === null ? "local-v02-unresolved" : "local-v02-stale",
};
}
export function runtimeLaneGitMirrorSourceInSyncForTest(lane: string, sourceCommit: string, summary: Record<string, unknown>): boolean {
const localKey = lane === "v02" ? "localV02" : lane === "v03" ? "localV03" : null;
return localKey !== null && typeof summary[localKey] === "string" && summary[localKey] === sourceCommit;
}
function gitMirrorStatusCacheRaw(status: Record<string, unknown>): string {
return String(nested(status, ["cache", "raw"]) ?? "");
}
function runtimeLaneGitMirrorSourceInSync(spec: HwlabRuntimeLaneSpec, sourceCommit: string, status: Record<string, unknown>): boolean {
return runtimeLaneGitMirrorSourceInSyncForTest(spec.lane, sourceCommit, record(nested(status, ["cache", "summary"])));
}
function compactGitMirrorStatus(status: Record<string, unknown>, sourceCommit: string): Record<string, unknown> {
const requirement = gitMirrorV02SyncRequirement(sourceCommit, gitMirrorStatusCacheRaw(status));
return {
ok: status.ok === true,
readUrl: status.readUrl ?? V02_GIT_READ_URL,
writeUrl: status.writeUrl ?? V02_GIT_WRITE_URL,
elapsedMs: status.elapsedMs ?? null,
legacyCronJobExists: nested(status, ["legacyCronJob", "exists"]) === true,
required: requirement.required,
sourceCommit,
localV02: requirement.localV02 ?? null,
githubV02: requirement.refs.githubV02 ?? null,
pendingFlush: requirement.pendingFlush ?? null,
sourceInSync: Boolean(requirement.refs.localV02 && requirement.refs.githubV02 && requirement.refs.localV02 === requirement.refs.githubV02),
reason: requirement.reason,
cacheOk: nested(status, ["cache", "ok"]) === true,
cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000),
resourcesOk: nested(status, ["resources", "ok"]) === true,
};
}
function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unknown> {
return {
ok: sync.ok === true,
command: sync.command ?? "hwlab g14 git-mirror sync",
mode: sync.mode ?? null,
skipped: sync.skipped === true,
reason: sync.reason ?? null,
namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE,
jobName: sync.jobName ?? null,
elapsedMs: sync.elapsedMs ?? null,
exitCode: nested(sync, ["result", "exitCode"]) ?? null,
stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600),
stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200),
stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null,
stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null,
};
}
interface V02GitMirrorPreSyncMarker {
ok: boolean;
sourceCommit: string;
syncedAt: string;
localV02?: string | null;
githubV02?: string | null;
reason?: string | null;
}
function v02GitMirrorPreSyncStateDir(): string {
return rootPath(".state", "hwlab-g14", "v02-git-mirror-presync");
}
function v02GitMirrorPreSyncMarkerPath(sourceCommit: string): string {
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.json`);
}
function v02GitMirrorPreSyncLockDir(sourceCommit: string): string {
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.lock`);
}
export function v02GitMirrorPreSyncWaitMs(timeoutSeconds: number): number {
const requestedMs = Math.max(0, Math.floor(timeoutSeconds * 1000));
if (requestedMs === 0) return V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS;
return Math.min(V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, requestedMs);
}
export function v02ReusableGitMirrorPreSyncMarker(marker: unknown, sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
const candidate = record(marker) as V02GitMirrorPreSyncMarker;
const syncedAtMs = timestampMs(candidate.syncedAt);
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit) return null;
if (syncedAtMs === null || nowMs - syncedAtMs > V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS) return null;
if (syncedAtMs < minSyncedAtMs) return null;
return candidate;
}
function readV02GitMirrorPreSyncMarker(sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
const path = v02GitMirrorPreSyncMarkerPath(sourceCommit);
if (!existsSync(path)) return null;
try {
return v02ReusableGitMirrorPreSyncMarker(JSON.parse(readFileSync(path, "utf8")) as unknown, sourceCommit, nowMs, minSyncedAtMs);
} catch {
return null;
}
}
function writeV02GitMirrorPreSyncMarker(sourceCommit: string, summary: Record<string, unknown>): V02GitMirrorPreSyncMarker {
const marker = {
ok: true,
sourceCommit,
syncedAt: new Date().toISOString(),
localV02: stringOrNull(summary.localV02),
githubV02: stringOrNull(summary.githubV02),
reason: stringOrNull(summary.reason),
};
const dir = v02GitMirrorPreSyncStateDir();
mkdirSync(dir, { recursive: true });
writeFileSync(v02GitMirrorPreSyncMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
return marker;
}
function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number, minSyncedAtMs = 0): { acquired: boolean; lockDir: string; waitedMs: number; marker?: V02GitMirrorPreSyncMarker } {
const stateDir = v02GitMirrorPreSyncStateDir();
mkdirSync(stateDir, { recursive: true });
const lockDir = v02GitMirrorPreSyncLockDir(sourceCommit);
const startedAtMs = Date.now();
for (;;) {
const marker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), minSyncedAtMs);
if (marker !== null) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs, marker };
try {
mkdirSync(lockDir);
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
} catch {
const ageMs = (() => {
try {
return Date.now() - statSync(lockDir).mtimeMs;
} catch {
return 0;
}
})();
if (ageMs > V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS) {
try {
rmSync(lockDir, { recursive: true, force: true });
continue;
} catch {
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
}
}
if (Date.now() - startedAtMs >= waitMs) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
}
}
}
function releaseV02GitMirrorPreSyncLock(lockDir: string): void {
try {
rmSync(lockDir, { recursive: true, force: true });
} catch {
// Best-effort cleanup; stale lock expiry handles interrupted workers.
}
}
function waitForV02GitMirrorPreSync(sourceCommit: string, waitMs: number): Record<string, unknown> {
const startedAtMs = Date.now();
const observations: Record<string, unknown>[] = [];
let attempt = 0;
for (;;) {
attempt += 1;
const statusStartMs = Date.now();
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync-wait",
status: "polling",
sourceCommit,
attempt,
elapsedMs: statusStartMs - startedAtMs,
});
const status = runGitMirrorStatus();
const summary = compactGitMirrorStatus(status, sourceCommit);
observations.push({
attempt,
elapsedMs: Date.now() - startedAtMs,
statusElapsedMs: Date.now() - statusStartMs,
ok: summary.ok,
required: summary.required,
localV02: summary.localV02,
githubV02: summary.githubV02,
pendingFlush: summary.pendingFlush,
reason: summary.reason,
});
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync-wait",
status: summary.required === false ? "succeeded" : "waiting",
sourceCommit,
attempt,
elapsedMs: Date.now() - startedAtMs,
statusElapsedMs: Date.now() - statusStartMs,
required: summary.required,
localV02: summary.localV02,
githubV02: summary.githubV02,
pendingFlush: summary.pendingFlush,
reason: summary.reason,
});
if (summary.ok === true && summary.required === false) {
return {
ok: true,
sourceCommit,
attempts: attempt,
elapsedMs: Date.now() - startedAtMs,
final: summary,
observations: observations.slice(-6),
};
}
if (Date.now() - startedAtMs >= waitMs) {
return {
ok: false,
sourceCommit,
attempts: attempt,
elapsedMs: Date.now() - startedAtMs,
final: summary,
observations: observations.slice(-6),
degradedReason: "git-mirror-local-v02-not-current-after-wait",
};
}
const remainingMs = waitMs - (Date.now() - startedAtMs);
const sleepMs = Math.max(500, Math.min(V02_GIT_MIRROR_PRESYNC_POLL_MS, remainingMs));
const wait = runCommand(["sleep", String(Math.ceil(sleepMs / 1000))], repoRoot, { timeoutMs: sleepMs + 2_000 });
if (wait.exitCode !== 0 && wait.timedOut) {
return {
ok: false,
sourceCommit,
attempts: attempt,
elapsedMs: Date.now() - startedAtMs,
final: summary,
observations: observations.slice(-6),
degradedReason: "git-mirror-pre-sync-wait-sleep-timeout",
};
}
}
}
function gitMirrorCacheProbeScript(): string {
const runtimeLanesJson = JSON.stringify(hwlabRuntimeLaneIds().map((lane) => {
const spec = hwlabRuntimeLaneSpec(lane);
return { lane, sourceBranch: spec.sourceBranch, gitopsBranch: spec.gitopsBranch };
}));
return [
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
"printf 'refs='",
"node - <<'NODE' 2>/dev/null || true",
"const { execFileSync } = require('node:child_process');",
"const repo = '/cache/pikasTech/HWLAB.git';",
"function rev(ref) {",
" try {",
" return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim();",
" } catch {",
" return null;",
" }",
"}",
`const runtimeLanes = ${runtimeLanesJson};`,
"const refs = {};",
"for (const spec of runtimeLanes) {",
" const key = spec.lane.toUpperCase();",
" const normalized = key.slice(0, 1) + key.slice(1).toLowerCase();",
" const localSource = rev('refs/heads/' + spec.sourceBranch);",
" const githubSource = rev('refs/mirror-stage/heads/' + spec.sourceBranch);",
" const localGitops = rev('refs/heads/' + spec.gitopsBranch);",
" const githubGitops = rev('refs/mirror-stage/heads/' + spec.gitopsBranch);",
" refs['local' + normalized] = localSource;",
" refs['github' + normalized] = githubSource;",
" refs['local' + normalized + 'Gitops'] = localGitops;",
" refs['github' + normalized + 'Gitops'] = githubGitops;",
" if (spec.lane === 'v02') {",
" refs.localV02 = localSource;",
" refs.githubV02 = githubSource;",
" refs.localGitops = localGitops;",
" refs.githubGitops = githubGitops;",
" }",
"}",
"refs.localG14 = rev('refs/heads/G14');",
"refs.githubG14 = rev('refs/mirror-stage/heads/G14');",
"const pendingFlush = runtimeLanes.some((spec) => {",
" const localGitops = rev('refs/heads/' + spec.gitopsBranch);",
" const githubGitops = rev('refs/mirror-stage/heads/' + spec.gitopsBranch);",
" return Boolean(localGitops && githubGitops && localGitops !== githubGitops);",
"});",
"console.log(JSON.stringify({ refs, pendingFlush }));",
"NODE",
].join("\n");
}
function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
const waitMs = v02GitMirrorPreSyncWaitMs(options.timeoutSeconds);
const statusStartMs = Date.now();
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
const before = runGitMirrorStatus();
const beforeSummary = compactGitMirrorStatus(before, sourceCommit);
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync-status",
status: beforeSummary.ok === true ? "succeeded" : "failed",
sourceCommit,
durationMs: Date.now() - statusStartMs,
required: beforeSummary.required,
localV02: beforeSummary.localV02,
pendingFlush: beforeSummary.pendingFlush,
reason: beforeSummary.reason,
});
if (options.dryRun) {
return {
ok: true,
mode: "dry-run",
sourceCommit,
before: beforeSummary,
action: beforeSummary.required === true ? "would-sync-before-trigger" : "already-current",
next: beforeSummary.required === true ? { syncAndTrigger: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" } : undefined,
};
}
if (beforeSummary.required !== true) {
return {
ok: true,
mode: "already-current",
sourceCommit,
before: beforeSummary,
};
}
const existingMarker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
if (existingMarker !== null) {
return {
ok: true,
mode: "reused-recent-git-mirror-presync-marker",
sourceCommit,
before: beforeSummary,
marker: existingMarker,
waitMs,
};
}
const lock = acquireV02GitMirrorPreSyncLock(sourceCommit, waitMs, statusStartMs);
if (!lock.acquired) {
const markerAfterLockTimeout = lock.marker ?? readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
if (markerAfterLockTimeout !== null) {
return {
ok: true,
mode: "waited-for-recent-git-mirror-presync-marker",
sourceCommit,
before: beforeSummary,
marker: markerAfterLockTimeout,
waitedMs: lock.waitedMs,
waitMs,
};
}
return {
ok: false,
mode: "local-git-mirror-presync-lock",
sourceCommit,
before: beforeSummary,
lockDir: lock.lockDir,
waitedMs: lock.waitedMs,
waitMs,
degradedReason: "git-mirror-pre-sync-lock-timeout",
};
}
try {
const markerAfterWait = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
if (markerAfterWait !== null) {
return {
ok: true,
mode: "waited-for-recent-git-mirror-presync-marker",
sourceCommit,
before: beforeSummary,
marker: markerAfterWait,
waitedMs: lock.waitedMs,
waitMs,
};
}
const recheckStartMs = Date.now();
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync-recheck",
status: "started",
sourceCommit,
waitedMs: lock.waitedMs,
});
const recheck = runGitMirrorStatus();
const recheckSummary = compactGitMirrorStatus(recheck, sourceCommit);
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync-recheck",
status: recheckSummary.ok === true ? "succeeded" : "failed",
sourceCommit,
durationMs: Date.now() - recheckStartMs,
required: recheckSummary.required,
localV02: recheckSummary.localV02,
githubV02: recheckSummary.githubV02,
pendingFlush: recheckSummary.pendingFlush,
reason: recheckSummary.reason,
});
if (recheckSummary.ok === true && recheckSummary.required === false) {
const marker = writeV02GitMirrorPreSyncMarker(sourceCommit, recheckSummary);
return {
ok: true,
mode: "became-current-after-lock-wait",
sourceCommit,
before: beforeSummary,
recheck: recheckSummary,
marker,
waitedMs: lock.waitedMs,
waitMs,
};
}
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync-sync",
status: "started",
sourceCommit,
reason: recheckSummary.reason,
localV02: recheckSummary.localV02,
pendingFlush: recheckSummary.pendingFlush,
});
const sync = runGitMirrorSync({
action: "sync",
lane: "v02",
confirm: true,
dryRun: false,
wait: true,
timeoutSeconds: options.timeoutSeconds,
});
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "git-mirror-pre-sync-sync",
status: sync.ok === true ? "succeeded" : "failed",
sourceCommit,
durationMs: sync.elapsedMs ?? null,
jobName: sync.jobName ?? null,
});
const syncStatus = record(sync.status);
const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
const afterSummary = compactGitMirrorStatus(after, sourceCommit);
const wait = sync.ok === true && afterSummary.required === true ? waitForV02GitMirrorPreSync(sourceCommit, waitMs) : null;
const waitFinal = wait === null ? null : record(record(wait).final);
const finalSummary = wait !== null && wait.ok === true ? waitFinal : afterSummary;
const ok = sync.ok === true && finalSummary.required === false;
const marker = ok ? writeV02GitMirrorPreSyncMarker(sourceCommit, finalSummary) : null;
return {
ok,
mode: "auto-sync-before-trigger",
sourceCommit,
before: beforeSummary,
recheck: recheckSummary,
sync: compactGitMirrorSync(sync),
after: afterSummary,
wait,
final: finalSummary,
marker,
waitedMs: lock.waitedMs,
waitMs,
degradedReason: ok ? undefined : stringOrNull(record(wait).degradedReason) ?? "git-mirror-local-v02-not-current-after-sync",
};
} finally {
releaseV02GitMirrorPreSyncLock(lock.lockDir);
}
}
function runGitMirrorStatus(): Record<string, unknown> {
const startedAtMs = Date.now();
const script = [
"set +e",
"section() {",
" name=\"$1\"",
" shift",
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
" \"$@\"",
" code=$?",
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
[
"section resources kubectl get deploy,svc,pvc,cm",
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
"-l app.kubernetes.io/name=git-mirror",
"-o name",
].join(" "),
[
"section legacyCronJob kubectl get cronjob",
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
shellQuote(GIT_MIRROR_LEGACY_CRONJOB),
"--ignore-not-found",
"-o name",
].join(" "),
[
`section jobs kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
"-l app.kubernetes.io/name=git-mirror",
"--sort-by=.metadata.creationTimestamp",
"-o custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime",
"--no-headers",
].join(" "),
[
"section cache kubectl exec",
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
"deploy/git-mirror-http",
"-- sh -lc",
shellQuote(gitMirrorCacheProbeScript()),
].join(" "),
].join("\n");
const bundle = g14K3s(["script", "--", script], 60_000);
const sections = parseShellSections(statusText(bundle));
const resources = sections.resources;
const legacyCronJob = sections.legacyCronJob;
const jobs = sections.jobs;
const cache = sections.cache;
const bundleStderr = bundle.stderr.trim().slice(0, 2000);
return {
ok: isCommandSuccess(bundle) && shellSectionOk(resources),
command: "hwlab g14 git-mirror status",
namespace: GIT_MIRROR_NAMESPACE,
readUrl: V02_GIT_READ_URL,
writeUrl: V02_GIT_WRITE_URL,
elapsedMs: Date.now() - startedAtMs,
resources: {
ok: shellSectionOk(resources),
names: String(resources?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
stderr: shellSectionOk(resources) ? "" : bundleStderr,
},
legacyCronJob: {
ok: shellSectionOk(legacyCronJob),
exists: String(legacyCronJob?.stdout ?? "").trim().length > 0,
name: String(legacyCronJob?.stdout ?? "").trim() || null,
cleanupCommand: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
},
jobs: {
ok: shellSectionOk(jobs),
raw: String(jobs?.stdout ?? "").trim(),
stderr: shellSectionOk(jobs) ? "" : bundleStderr,
},
cache: {
ok: shellSectionOk(cache),
summary: gitMirrorStatusSummary(String(cache?.stdout ?? "").trim()),
raw: String(cache?.stdout ?? "").trim(),
stderr: shellSectionOk(cache) ? "" : bundleStderr,
},
next: {
apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
},
};
}
function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
const spec = hwlabRuntimeLaneSpec(options.lane);
const head = resolveRuntimeLaneHead(spec);
const sourceCommit = head.sourceCommit;
if (sourceCommit === null) {
return { ok: false, command: `hwlab g14 git-mirror apply --lane ${spec.lane}`, lane: spec.lane, degradedReason: `${spec.lane}-head-unresolved`, sourceRepo: spec.cicdRepo, workspace: spec.workspace, headProbe: compactCommandResult(head.result) };
}
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
if (!isCommandSuccess(render.result)) {
return {
ok: false,
command: `hwlab g14 git-mirror apply --lane ${spec.lane}`,
lane: spec.lane,
phase: "source-render",
sourceCommit,
renderDir: render.renderDir,
render: compactCommandResult(render.result),
};
}
const apply = applyGitMirrorManifestFile(render.renderDir, options.dryRun, options.timeoutSeconds);
const cleanupLegacyCronJob = isCommandSuccess(apply)
? deleteLegacyGitMirrorCronJob(options.dryRun)
: {
ok: false,
command: [],
exitCode: null,
stdout: "",
stderr: "skipped because apply failed",
parsed: null,
};
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
return {
ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob),
command: `hwlab g14 git-mirror apply --lane ${spec.lane}`,
lane: spec.lane,
mode: options.dryRun ? "dry-run" : "confirmed-apply",
sourceCommit,
manifest: `${render.renderDir}/devops-infra/git-mirror.yaml`,
render: compactCommandResult(render.result),
apply: compactCommandResult(apply),
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
status: runGitMirrorStatus(),
next: options.dryRun
? { apply: `bun scripts/cli.ts hwlab g14 git-mirror apply --lane ${spec.lane} --confirm` }
: { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" },
};
}
function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown> {
const startedAtMs = Date.now();
const spec = hwlabRuntimeLaneSpec(options.lane);
const jobName = gitMirrorSyncJobName();
const manifest = gitMirrorSyncJobManifest(jobName);
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
const script = [
"set -eu",
`job=${shellQuote(jobName)}`,
`manifest_b64=${shellQuote(manifestB64)}`,
"manifest_path=\"/tmp/$job.json\"",
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
options.dryRun
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
: [
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
"kubectl create -f \"$manifest_path\"",
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
"while :; do",
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
" exit 44",
" fi",
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
" exit 45",
" fi",
" sleep 2",
"done",
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2 refs/heads/v0.2-gitops refs/heads/G14 2>/dev/null || true'`,
].join("\n"),
].join("\n");
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
const syncCommand = spec.lane === "v02"
? "bun scripts/cli.ts hwlab g14 git-mirror sync --lane v02 --confirm"
: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`;
const triggerCommand = spec.lane === "v02"
? "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm"
: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`;
return {
ok: isCommandSuccess(result),
command: spec.lane === "v02" ? "hwlab g14 git-mirror sync" : "hwlab nodes git-mirror sync",
mode: options.dryRun ? "dry-run" : "confirmed-sync",
namespace: GIT_MIRROR_NAMESPACE,
jobName,
elapsedMs: Date.now() - startedAtMs,
manifest: options.dryRun ? manifest : undefined,
result: compactCommandResult(result),
status: options.dryRun ? undefined : runGitMirrorStatus(),
next: options.dryRun ? { sync: syncCommand } : { triggerCurrent: triggerCommand },
};
}
function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown> {
const jobName = gitMirrorFlushJobName();
const manifest = gitMirrorFlushJobManifest(jobName);
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
const script = [
"set -eu",
`job=${shellQuote(jobName)}`,
`manifest_b64=${shellQuote(manifestB64)}`,
"manifest_path=\"/tmp/$job.json\"",
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
options.dryRun
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
: [
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
"kubectl create -f \"$manifest_path\"",
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
"while :; do",
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
" succeeded=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
" failed=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
" exit 54",
" fi",
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
" exit 55",
" fi",
" sleep 2",
"done",
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc 'cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/v0.2-gitops refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true'`,
].join("\n"),
].join("\n");
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 30_000);
return {
ok: isCommandSuccess(result),
command: "hwlab g14 git-mirror flush",
mode: options.dryRun ? "dry-run" : "confirmed-flush",
namespace: GIT_MIRROR_NAMESPACE,
jobName,
manifest: options.dryRun ? manifest : undefined,
result: compactCommandResult(result),
status: options.dryRun ? undefined : runGitMirrorStatus(),
next: options.dryRun ? { flush: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" } : { status: "bun scripts/cli.ts hwlab g14 git-mirror status" },
};
}
function runG14GitMirror(options: G14GitMirrorOptions): Record<string, unknown> {
if (options.action === "status") return runGitMirrorStatus();
if (options.action === "apply") return runGitMirrorApply(options);
if (options.action === "flush") return runGitMirrorFlush(options);
return runGitMirrorSync(options);
}
function observabilityLabels(component: string): Record<string, string> {
return {
"app.kubernetes.io/part-of": "devops-infra",
"app.kubernetes.io/component": "observability",
"g14.pikastech.local/observability-component": component,
};
}
function observabilityNamespaceLabel(): Record<string, string> {
return {
"g14.pikastech.local/observability-discovery": "enabled",
};
}
function g14PrometheusManifest(): Record<string, unknown> {
const namespaceSelector = { matchLabels: observabilityNamespaceLabel() };
const monitorSelector = { matchLabels: { "hwlab.pikastech.local/monitoring": "enabled" } };
return {
apiVersion: "v1",
kind: "List",
items: [
{
apiVersion: "v1",
kind: "Namespace",
metadata: {
name: G14_OBSERVABILITY_NAMESPACE,
labels: {
...observabilityNamespaceLabel(),
...observabilityLabels("namespace"),
},
},
},
{
apiVersion: "v1",
kind: "Namespace",
metadata: {
name: V02_RUNTIME_NAMESPACE,
labels: observabilityNamespaceLabel(),
},
},
{
apiVersion: "v1",
kind: "ServiceAccount",
metadata: {
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
namespace: G14_OBSERVABILITY_NAMESPACE,
labels: observabilityLabels("prometheus"),
},
},
{
apiVersion: "rbac.authorization.k8s.io/v1",
kind: "ClusterRole",
metadata: {
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
labels: observabilityLabels("prometheus"),
},
rules: [
{
apiGroups: [""],
resources: ["nodes", "nodes/metrics", "services", "endpoints", "pods"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["discovery.k8s.io"],
resources: ["endpointslices"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["networking.k8s.io"],
resources: ["ingresses"],
verbs: ["get", "list", "watch"],
},
{
nonResourceURLs: ["/metrics"],
verbs: ["get"],
},
],
},
{
apiVersion: "rbac.authorization.k8s.io/v1",
kind: "ClusterRoleBinding",
metadata: {
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
labels: observabilityLabels("prometheus"),
},
roleRef: {
apiGroup: "rbac.authorization.k8s.io",
kind: "ClusterRole",
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
},
subjects: [{
kind: "ServiceAccount",
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
namespace: G14_OBSERVABILITY_NAMESPACE,
}],
},
{
apiVersion: "monitoring.coreos.com/v1",
kind: "Prometheus",
metadata: {
name: G14_PROMETHEUS_NAME,
namespace: G14_OBSERVABILITY_NAMESPACE,
labels: observabilityLabels("prometheus"),
},
spec: {
replicas: 1,
version: G14_PROMETHEUS_VERSION,
serviceAccountName: G14_PROMETHEUS_SERVICE_ACCOUNT,
scrapeInterval: "30s",
evaluationInterval: "30s",
retention: "7d",
resources: {
requests: { cpu: "100m", memory: "256Mi" },
limits: { cpu: "500m", memory: "1Gi" },
},
storage: {
volumeClaimTemplate: {
spec: {
storageClassName: "local-path",
accessModes: ["ReadWriteOnce"],
resources: { requests: { storage: "10Gi" } },
},
},
},
serviceMonitorSelector: monitorSelector,
serviceMonitorNamespaceSelector: namespaceSelector,
podMonitorSelector: monitorSelector,
podMonitorNamespaceSelector: namespaceSelector,
ruleSelector: monitorSelector,
ruleNamespaceSelector: namespaceSelector,
probeSelector: monitorSelector,
probeNamespaceSelector: namespaceSelector,
},
},
{
apiVersion: "v1",
kind: "Service",
metadata: {
name: G14_PROMETHEUS_SERVICE,
namespace: G14_OBSERVABILITY_NAMESPACE,
labels: observabilityLabels("query"),
},
spec: {
type: "ClusterIP",
selector: {
prometheus: G14_PROMETHEUS_NAME,
},
ports: [{
name: "web",
port: 9090,
targetPort: "web",
}],
},
},
],
};
}
function parseSectionJson(section: ShellSection | undefined): Record<string, unknown> {
const text = String(section?.stdout ?? "").trim();
if (text.length === 0) return {};
try {
return record(JSON.parse(text) as unknown);
} catch {
return {};
}
}
function parseSectionJsonArray(section: ShellSection | undefined): Record<string, unknown>[] {
const parsed = parseSectionJson(section);
const items = parsed.kind === "List" && Array.isArray(parsed.items)
? parsed.items
: Array.isArray(parsed.items)
? parsed.items
: [];
return items.map((item) => record(item));
}
function sectionLines(section: ShellSection | undefined): string[] {
return String(section?.stdout ?? "")
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean);
}
function podRows(section: ShellSection | undefined): Record<string, unknown>[] {
return sectionLines(section).map((line) => {
const [name = "", phase = "", readyRaw = ""] = line.split("\t");
const readyValues = readyRaw.split(",").map((item) => item.trim()).filter(Boolean);
return {
name,
phase: phase || null,
ready: readyValues.length > 0 ? readyValues.every((item) => item === "true") : null,
};
}).filter((item) => String(item.name).length > 0);
}
function conditionStatus(items: Record<string, unknown>[], type: string): string | null {
for (const item of items) {
if (item.type === type) return typeof item.status === "string" ? item.status : null;
}
return null;
}
function deploymentReady(deployment: Record<string, unknown>): boolean {
const spec = record(deployment.spec);
const status = record(deployment.status);
const desired = numericValue(spec.replicas) ?? 1;
const ready = numericValue(status.readyReplicas) ?? 0;
const available = numericValue(status.availableReplicas) ?? 0;
return ready >= desired && available >= desired;
}
function prometheusReady(prometheus: Record<string, unknown>): boolean {
const conditions = Array.isArray(record(prometheus.status).conditions)
? record(prometheus.status).conditions.map((item) => record(item))
: [];
const available = conditionStatus(conditions, "Available");
const reconciled = conditionStatus(conditions, "Reconciled");
return available === "True" || reconciled === "True";
}
function observabilityQueryPath(promql: string): string {
return `/api/v1/namespaces/${G14_OBSERVABILITY_NAMESPACE}/services/http:${G14_PROMETHEUS_SERVICE}:9090/proxy/api/v1/query?query=${encodeURIComponent(promql)}`;
}
function arrayRecords(value: unknown): Record<string, unknown>[] {
return Array.isArray(value) ? value.map((item) => record(item)) : [];
}
function prometheusResultItems(parsed: unknown): Record<string, unknown>[] {
return arrayRecords(nested(parsed, ["data", "result"]));
}
function prometheusSeriesValue(series: Record<string, unknown>): string | null {
const value = series.value;
if (Array.isArray(value) && value.length >= 2) return String(value[1]);
if (typeof value === "string" || typeof value === "number") return String(value);
return null;
}
function prometheusSeriesTimestamp(series: Record<string, unknown>): number | null {
const value = series.value;
if (!Array.isArray(value) || value.length === 0) return null;
return numericValue(value[0]);
}
export function parseK8sCpuMillicores(value: unknown): number | null {
const raw = stringOrNull(value);
if (raw === null) return null;
const text = raw.trim();
if (text.length === 0) return null;
const match = text.match(/^([0-9]+(?:\.[0-9]+)?)(n|u|m)?$/u);
if (match === null) return null;
const amount = Number(match[1]);
if (!Number.isFinite(amount)) return null;
const unit = match[2] ?? "";
if (unit === "n") return amount / 1_000_000;
if (unit === "u") return amount / 1_000;
if (unit === "m") return amount;
return amount * 1000;
}
export function parseK8sMemoryMiB(value: unknown): number | null {
const raw = stringOrNull(value);
if (raw === null) return null;
const text = raw.trim();
if (text.length === 0) return null;
const match = text.match(/^([0-9]+(?:\.[0-9]+)?)(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)?$/u);
if (match === null) return null;
const amount = Number(match[1]);
if (!Number.isFinite(amount)) return null;
const unit = match[2] ?? "";
const binaryFactors: Record<string, number> = {
Ki: 1 / 1024,
Mi: 1,
Gi: 1024,
Ti: 1024 * 1024,
Pi: 1024 * 1024 * 1024,
Ei: 1024 * 1024 * 1024 * 1024,
};
const decimalByteFactors: Record<string, number> = {
"": 1,
K: 1000,
M: 1000 * 1000,
G: 1000 * 1000 * 1000,
T: 1000 * 1000 * 1000 * 1000,
P: 1000 * 1000 * 1000 * 1000 * 1000,
E: 1000 * 1000 * 1000 * 1000 * 1000 * 1000,
};
if (binaryFactors[unit] !== undefined) return amount * binaryFactors[unit];
const byteFactor = decimalByteFactors[unit];
return byteFactor === undefined ? null : (amount * byteFactor) / (1024 * 1024);
}
function prometheusSeriesSummary(series: Record<string, unknown>): Record<string, unknown> {
const metric = record(series.metric);
return {
metricName: stringOrNull(metric.__name__),
serviceId: stringOrNull(metric.hwlab_pikastech_local_service_id),
service: stringOrNull(metric.service),
job: stringOrNull(metric.job),
pod: stringOrNull(metric.pod),
namespace: stringOrNull(metric.namespace),
container: stringOrNull(metric.container),
instance: stringOrNull(metric.instance),
value: prometheusSeriesValue(series),
timestamp: prometheusSeriesTimestamp(series),
};
}
function prometheusSeriesSummaries(parsed: unknown, limit = 30): Record<string, unknown>[] {
return prometheusResultItems(parsed).slice(0, limit).map(prometheusSeriesSummary);
}
function metricValueMatches(actual: string | null, expected: string): boolean {
if (actual === expected) return true;
const actualNumber = actual === null ? Number.NaN : Number(actual);
const expectedNumber = Number(expected);
return Number.isFinite(actualNumber) && Number.isFinite(expectedNumber) && actualNumber === expectedNumber;
}
export function g14ObservabilityQueryAssertion(parsed: unknown, expectedCount?: number, expectedValue?: string): Record<string, unknown> | null {
if (expectedCount === undefined && expectedValue === undefined) return null;
const series = prometheusResultItems(parsed);
const actualCount = series.length;
const countOk = expectedCount === undefined || actualCount === expectedCount;
const badValues = expectedValue === undefined
? []
: series
.filter((item) => !metricValueMatches(prometheusSeriesValue(item), expectedValue))
.map(prometheusSeriesSummary);
const missingCount = expectedCount === undefined ? 0 : Math.max(0, expectedCount - actualCount);
const extraCount = expectedCount === undefined ? 0 : Math.max(0, actualCount - expectedCount);
const queryStatus = stringOrNull(record(parsed).status);
return {
ok: queryStatus === "success" && countOk && badValues.length === 0,
queryStatus,
expectedCount: expectedCount ?? null,
actualCount,
expectedValue: expectedValue ?? null,
countOk,
valueOk: expectedValue === undefined ? null : badValues.length === 0,
badValueCount: badValues.length,
badValues: badValues.slice(0, 20),
missingSeries: missingCount > 0 ? [{ expectedCount, actualCount, missingCount }] : [],
extraSeries: extraCount > 0 ? prometheusSeriesSummaries(parsed, Math.min(extraCount, 20)) : [],
};
}
function observabilitySemanticName(promql: string): string | null {
if (promql === V02_OBSERVABILITY_QUERIES.scrapeReachable) return "scrapeReachable";
if (promql === V02_OBSERVABILITY_QUERIES.sidecarServing) return "sidecarServing";
if (promql === V02_OBSERVABILITY_QUERIES.businessHealthProbe) return "businessHealthProbe";
return null;
}
function k8sItems(parsed: unknown): Record<string, unknown>[] {
const root = record(parsed);
return Array.isArray(root.items) ? root.items.map((item) => record(item)) : [];
}
function k8sMetadataName(item: Record<string, unknown>): string | null {
return stringOrNull(record(item.metadata).name);
}
function k8sLabels(item: Record<string, unknown>): Record<string, unknown> {
return record(record(item.metadata).labels);
}
function podReadyCondition(pod: Record<string, unknown>): boolean | null {
const conditions = arrayRecords(record(pod.status).conditions);
const ready = conditions.find((item) => item.type === "Ready");
return ready === undefined ? null : ready.status === "True";
}
function podMetricsSidecarSummary(pod: Record<string, unknown>): Record<string, unknown> {
const statuses = arrayRecords(record(pod.status).containerStatuses);
const metrics = statuses.find((item) => item.name === "hwlab-metrics")
?? statuses.find((item) => String(item.name ?? "").includes("metrics"));
return {
pod: k8sMetadataName(pod),
namespace: stringOrNull(record(pod.metadata).namespace),
serviceId: stringOrNull(k8sLabels(pod)["hwlab.pikastech.local/service-id"])
?? stringOrNull(k8sLabels(pod)["app.kubernetes.io/name"]),
phase: stringOrNull(record(pod.status).phase),
podReady: podReadyCondition(pod),
sidecarFound: metrics !== undefined,
sidecarName: metrics === undefined ? null : stringOrNull(metrics.name),
sidecarReady: metrics === undefined ? null : metrics.ready === true,
restartCount: metrics === undefined ? null : numericValue(metrics.restartCount),
};
}
function resourceUsageSummary(containers: Record<string, unknown>[]): Record<string, unknown> {
const rows = containers.map((container) => {
const usage = record(container.usage);
return {
name: stringOrNull(container.name),
cpuMillicores: parseK8sCpuMillicores(usage.cpu),
memoryMiB: parseK8sMemoryMiB(usage.memory),
};
});
const sum = (key: "cpuMillicores" | "memoryMiB"): number | null => {
const values = rows.map((item) => item[key]).filter((item): item is number => item !== null);
return values.length === 0 ? null : values.reduce((acc, item) => acc + item, 0);
};
return {
cpuMillicores: sum("cpuMillicores"),
memoryMiB: sum("memoryMiB"),
containers: rows,
};
}
function podResourceMetricSummary(item: Record<string, unknown>): Record<string, unknown> {
const containers = arrayRecords(item.containers);
const metricsSidecars = containers.filter((container) => stringOrNull(container.name) === "hwlab-metrics");
const businessContainers = containers.filter((container) => stringOrNull(container.name) !== "hwlab-metrics");
return {
pod: k8sMetadataName(item),
namespace: stringOrNull(record(item.metadata).namespace),
serviceId: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/service-id"])
?? stringOrNull(k8sLabels(item)["app.kubernetes.io/name"]),
timestamp: stringOrNull(item.timestamp),
window: stringOrNull(item.window),
total: resourceUsageSummary(containers),
business: resourceUsageSummary(businessContainers),
metricsSidecar: resourceUsageSummary(metricsSidecars),
};
}
function monitorSummaries(items: Record<string, unknown>[]): Record<string, unknown>[] {
return items.map((item) => ({
kind: stringOrNull(item.kind),
name: k8sMetadataName(item),
labels: {
monitoring: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/monitoring"]),
serviceId: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/service-id"]),
gitopsTarget: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/gitops-target"]),
},
}));
}
function monitoredServiceIds(monitors: Record<string, unknown>[], metricSeries: Record<string, Record<string, unknown>[]>): string[] {
const ids = new Set<string>();
for (const item of monitorSummaries(monitors)) {
const kind = stringOrNull(item.kind);
const serviceId = stringOrNull(record(item.labels).serviceId);
if ((kind === "ServiceMonitor" || kind === "PodMonitor") && serviceId !== null) ids.add(serviceId);
}
for (const seriesItems of Object.values(metricSeries)) {
for (const item of seriesItems.map(prometheusSeriesSummary)) {
const serviceId = stringOrNull(item.serviceId) ?? stringOrNull(item.service) ?? stringOrNull(item.job);
if (serviceId !== null) ids.add(serviceId);
}
}
return [...ids].sort();
}
function attachResourceMetricsToTargets(targets: Record<string, unknown>[], resources: Record<string, unknown>[]): Record<string, unknown>[] {
const byPod = new Map(resources.map((item) => [String(item.pod ?? ""), item]));
const byServiceId = new Map(resources.map((item) => [String(item.serviceId ?? ""), item]));
return targets.map((target) => {
const resource = byPod.get(String(target.pod ?? "")) ?? byServiceId.get(String(target.serviceId ?? ""));
return resource === undefined ? target : { ...target, resource };
});
}
function targetMetricValue(target: Record<string, unknown>, metricName: string): number | null {
const metric = record(record(target.metrics)[metricName]);
return numericValue(metric.value);
}
function targetMetricString(target: Record<string, unknown>, metricName: string): string | null {
return stringOrNull(record(record(target.metrics)[metricName]).value);
}
function targetMetricTimestamp(target: Record<string, unknown>, metricName: string): number | null {
return numericValue(record(record(target.metrics)[metricName]).timestamp);
}
function sortedNumeric(values: number[]): number[] {
return [...values].filter((item) => Number.isFinite(item)).sort((a, b) => a - b);
}
function percentile(values: number[], p: number): number | null {
const sorted = sortedNumeric(values);
if (sorted.length === 0) return null;
if (sorted.length === 1) return sorted[0];
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
return sorted[index];
}
function secondsStats(values: number[]): Record<string, unknown> {
const sorted = sortedNumeric(values);
if (sorted.length === 0) return { count: 0, minSeconds: null, maxSeconds: null, avgSeconds: null, p50Seconds: null, p95Seconds: null, p99Seconds: null };
const sum = sorted.reduce((acc, item) => acc + item, 0);
return {
count: sorted.length,
minSeconds: sorted[0],
maxSeconds: sorted[sorted.length - 1],
avgSeconds: sum / sorted.length,
p50Seconds: percentile(sorted, 50),
p95Seconds: percentile(sorted, 95),
p99Seconds: percentile(sorted, 99),
};
}
function numberStats(values: number[]): Record<string, unknown> {
const sorted = sortedNumeric(values);
if (sorted.length === 0) return { count: 0, min: null, max: null, avg: null, p50: null, p95: null, p99: null };
const sum = sorted.reduce((acc, item) => acc + item, 0);
return {
count: sorted.length,
min: sorted[0],
max: sorted[sorted.length - 1],
avg: sum / sorted.length,
p50: percentile(sorted, 50),
p95: percentile(sorted, 95),
p99: percentile(sorted, 99),
};
}
function buildObservabilityLevelSummary(targets: Record<string, unknown>[]): Record<string, unknown> {
const observedTargets = targets.filter((target) => Object.keys(record(target.metrics)).length > 0);
const healthDurations = observedTargets.map((target) => targetMetricValue(target, "healthProbeDurationSeconds")).filter((item): item is number => item !== null);
const scrapeDurations = observedTargets.map((target) => targetMetricValue(target, "scrapeDurationSeconds")).filter((item): item is number => item !== null);
const samplesScraped = observedTargets.map((target) => targetMetricValue(target, "scrapeSamplesScraped")).filter((item): item is number => item !== null);
const resourceSnapshots = observedTargets.map((target) => record(target.resource)).filter((item) => Object.keys(item).length > 0);
const resourceVisible = resourceSnapshots.length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT;
const resourceValue = (snapshot: Record<string, unknown>, path: string[]): number | null => numericValue(nested(snapshot, path));
const resourceTotals = {
totalCpuMillicores: resourceSnapshots.map((item) => resourceValue(item, ["total", "cpuMillicores"])).filter((item): item is number => item !== null),
totalMemoryMiB: resourceSnapshots.map((item) => resourceValue(item, ["total", "memoryMiB"])).filter((item): item is number => item !== null),
businessCpuMillicores: resourceSnapshots.map((item) => resourceValue(item, ["business", "cpuMillicores"])).filter((item): item is number => item !== null),
businessMemoryMiB: resourceSnapshots.map((item) => resourceValue(item, ["business", "memoryMiB"])).filter((item): item is number => item !== null),
metricsSidecarCpuMillicores: resourceSnapshots.map((item) => resourceValue(item, ["metricsSidecar", "cpuMillicores"])).filter((item): item is number => item !== null),
metricsSidecarMemoryMiB: resourceSnapshots.map((item) => resourceValue(item, ["metricsSidecar", "memoryMiB"])).filter((item): item is number => item !== null),
};
const healthRows = observedTargets.map((target) => ({
serviceId: target.serviceId ?? target.service ?? target.job ?? target.key,
scrapeReachable: targetMetricString(target, "scrapeReachable"),
sidecarServing: targetMetricString(target, "sidecarServing"),
businessHealthProbe: targetMetricString(target, "businessHealthProbe"),
statusCode: targetMetricValue(target, "healthProbeStatusCode"),
healthProbeDurationSeconds: targetMetricValue(target, "healthProbeDurationSeconds"),
scrapeDurationSeconds: targetMetricValue(target, "scrapeDurationSeconds"),
scrapeSamplesScraped: targetMetricValue(target, "scrapeSamplesScraped"),
processUptimeSeconds: targetMetricValue(target, "processUptimeSeconds"),
totalCpuMillicores: resourceValue(record(target.resource), ["total", "cpuMillicores"]),
totalMemoryMiB: resourceValue(record(target.resource), ["total", "memoryMiB"]),
businessCpuMillicores: resourceValue(record(target.resource), ["business", "cpuMillicores"]),
businessMemoryMiB: resourceValue(record(target.resource), ["business", "memoryMiB"]),
metricsSidecarCpuMillicores: resourceValue(record(target.resource), ["metricsSidecar", "cpuMillicores"]),
metricsSidecarMemoryMiB: resourceValue(record(target.resource), ["metricsSidecar", "memoryMiB"]),
lastSampleTimestamp: targetMetricTimestamp(target, "scrapeReachable") ?? targetMetricTimestamp(target, "businessHealthProbe"),
})).sort((a, b) => String(a.serviceId).localeCompare(String(b.serviceId)));
const unsupportedSignals = [
"request_rate",
"error_rate",
"route_latency_p95",
"route_latency_p99",
"business_operation_latency",
...(resourceVisible ? [] : ["cpu_memory_node_pod_usage"]),
];
return {
observedTargetCount: observedTargets.length,
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
healthProbeDuration: secondsStats(healthDurations),
scrapeDuration: secondsStats(scrapeDurations),
scrapeSamplesScraped: {
count: samplesScraped.length,
min: sortedNumeric(samplesScraped)[0] ?? null,
max: sortedNumeric(samplesScraped)[sortedNumeric(samplesScraped).length - 1] ?? null,
values: samplesScraped,
},
resourceUsage: {
source: "metrics.k8s.io/v1beta1",
observedTargetCount: resourceSnapshots.length,
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
totalCpuMillicores: numberStats(resourceTotals.totalCpuMillicores),
totalMemoryMiB: numberStats(resourceTotals.totalMemoryMiB),
businessCpuMillicores: numberStats(resourceTotals.businessCpuMillicores),
businessMemoryMiB: numberStats(resourceTotals.businessMemoryMiB),
metricsSidecarCpuMillicores: numberStats(resourceTotals.metricsSidecarCpuMillicores),
metricsSidecarMemoryMiB: numberStats(resourceTotals.metricsSidecarMemoryMiB),
},
services: healthRows,
interpretation: {
visible: [
"scrape availability",
"metrics sidecar availability",
"business health endpoint success",
"business health endpoint status code",
"business health probe duration",
"Prometheus scrape duration",
"metrics sidecar uptime",
...(resourceVisible ? ["current pod/container CPU and memory snapshot from metrics.k8s.io"] : []),
],
notYetVisible: unsupportedSignals,
conclusion: unsupportedSignals.length > 0
? "current metrics are sufficient for availability, synthetic health latency, and current resource snapshot when metrics.k8s.io is available, but not yet sufficient for real request throughput, route latency, or error rate analysis"
: "current metrics cover the requested performance surface",
},
};
}
function compactObservabilityTarget(target: Record<string, unknown>): Record<string, unknown> {
const sidecar = record(target.sidecar);
return {
serviceId: target.serviceId ?? target.service ?? target.job ?? target.key,
pod: target.pod ?? sidecar.pod ?? null,
serviceMonitors: target.serviceMonitors ?? [],
sidecar: {
found: sidecar.sidecarFound ?? null,
ready: sidecar.sidecarReady ?? null,
podReady: sidecar.podReady ?? null,
restartCount: sidecar.restartCount ?? null,
},
metrics: {
scrapeReachable: targetMetricString(target, "scrapeReachable"),
sidecarServing: targetMetricString(target, "sidecarServing"),
businessHealthProbe: targetMetricString(target, "businessHealthProbe"),
healthProbeConfigured: targetMetricString(target, "healthProbeConfigured"),
healthProbeStatusCode: targetMetricValue(target, "healthProbeStatusCode"),
healthProbeDurationSeconds: targetMetricValue(target, "healthProbeDurationSeconds"),
scrapeDurationSeconds: targetMetricValue(target, "scrapeDurationSeconds"),
scrapeSamplesScraped: targetMetricValue(target, "scrapeSamplesScraped"),
processUptimeSeconds: targetMetricValue(target, "processUptimeSeconds"),
lastSampleTimestamp: targetMetricTimestamp(target, "scrapeReachable") ?? targetMetricTimestamp(target, "businessHealthProbe"),
},
resource: target.resource ?? null,
};
}
function mergeObservabilityTargets(
pods: Record<string, unknown>[],
monitors: Record<string, unknown>[],
metricSeries: Record<string, Record<string, unknown>[]>,
): Record<string, unknown>[] {
const byKey = new Map<string, Record<string, unknown>>();
const ensure = (key: string): Record<string, unknown> => {
const existing = byKey.get(key);
if (existing !== undefined) return existing;
const next = { key, serviceId: null, service: null, job: null, pod: null, sidecar: null, metrics: {} as Record<string, unknown> };
byKey.set(key, next);
return next;
};
for (const pod of pods.map(podMetricsSidecarSummary)) {
const key = String(pod.serviceId ?? pod.pod ?? "unknown");
const target = ensure(key);
target.serviceId = target.serviceId ?? pod.serviceId;
target.pod = target.pod ?? pod.pod;
target.sidecar = pod;
}
for (const [metricName, seriesItems] of Object.entries(metricSeries)) {
for (const item of seriesItems.map(prometheusSeriesSummary)) {
const key = String(item.serviceId ?? item.service ?? item.job ?? item.pod ?? "unknown");
const target = ensure(key);
target.serviceId = target.serviceId ?? item.serviceId;
target.service = target.service ?? item.service;
target.job = target.job ?? item.job;
target.pod = target.pod ?? item.pod;
record(target.metrics)[metricName] = {
value: item.value,
timestamp: item.timestamp,
instance: item.instance,
container: item.container,
};
}
}
const serviceIds = new Set(monitoredServiceIds(monitors, metricSeries));
const monitorNames = monitorSummaries(monitors);
return [...byKey.values()]
.filter((target) => {
const key = String(target.serviceId ?? target.service ?? target.job ?? target.key ?? "");
return serviceIds.has(key);
})
.map((target) => ({
...target,
serviceMonitors: monitorNames
.filter((item) => stringOrNull(record(item.labels).serviceId) === stringOrNull(target.serviceId))
.map((item) => stringOrNull(item.name))
.filter((name): name is string => name !== null)
.slice(0, 5),
}))
.sort((a, b) => String(a.serviceId ?? a.service ?? a.job ?? a.pod).localeCompare(String(b.serviceId ?? b.service ?? b.job ?? b.pod)));
}
function publicMetricsProbe(name: string, url: string): Record<string, unknown> {
const result = commandJson([
"curl",
"-sS",
"-L",
"--max-time",
"20",
"--range",
"0-4095",
"-w",
"\n__UNIDESK_HTTP_CODE__:%{http_code}\n__UNIDESK_CONTENT_TYPE__:%{content_type}\n",
url,
], 30_000);
const text = result.stdout;
const codeMatch = text.match(/\n__UNIDESK_HTTP_CODE__:(\d{3})\n/u);
const contentTypeMatch = text.match(/\n__UNIDESK_CONTENT_TYPE__:(.*)\n?$/u);
const body = codeMatch === null ? text : text.slice(0, codeMatch.index).trim();
const httpCode = codeMatch === null ? null : Number(codeMatch[1]);
const contentType = contentTypeMatch === null ? "" : contentTypeMatch[1].trim();
const prometheusText = looksLikePrometheusText(body, contentType);
const exposed = httpCode !== null && httpCode >= 200 && httpCode < 300 && prometheusText;
const denied = httpCode !== null && !exposed && (httpCode >= 400 || !prometheusText);
return {
ok: result.ok && denied,
name,
url,
httpCode,
contentType,
exposed,
denied,
reason: result.ok
? exposed
? "prometheus-text-exposed"
: httpCode !== null && httpCode >= 400
? "http-error-denied"
: "non-prometheus-body"
: "curl-failed",
bodyPreview: body.slice(0, 240),
commandResult: compactCommandMetadata(result, !result.ok || exposed),
};
}
function looksLikePrometheusText(body: string, contentType: string): boolean {
const type = contentType.toLowerCase();
if (type.includes("openmetrics")) return true;
const hasMetricLine = /(?:^|\n)(?:#\s+(?:HELP|TYPE)\s+[A-Za-z_:][A-Za-z0-9_:]*|[A-Za-z_:][A-Za-z0-9_:]*(?:\{[^}\n]*\})?\s+[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?)\b/u.test(body);
if (!hasMetricLine) return false;
return type.includes("text/plain") || type.length === 0;
}
function g14ObservabilityStatus(): Record<string, unknown> {
const startedAtMs = Date.now();
const queryPath = `/api/v1/namespaces/${G14_OBSERVABILITY_NAMESPACE}/services/http:${G14_PROMETHEUS_SERVICE}:9090/proxy/api/v1/query?query=${encodeURIComponent("up")}`;
const crds = [
"servicemonitors.monitoring.coreos.com",
"podmonitors.monitoring.coreos.com",
"prometheusrules.monitoring.coreos.com",
"prometheuses.monitoring.coreos.com",
"alertmanagers.monitoring.coreos.com",
];
const script = [
"set +e",
"section() {",
" name=\"$1\"",
" shift",
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
" \"$@\"",
" code=$?",
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
`section namespace kubectl get namespace ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -o json`,
`section discoveryNamespace kubectl get namespace ${shellQuote(V02_RUNTIME_NAMESPACE)} -o json`,
`section crds kubectl get crd ${crds.map(shellQuote).join(" ")} -o 'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}'`,
`section operator kubectl get deploy -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} prometheus-operator -o json`,
`section operatorPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l app.kubernetes.io/name=prometheus-operator -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .status.containerStatuses[*]}{.ready}{","}{end}{"\\n"}{end}'`,
`section prometheus kubectl get prometheus -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} ${shellQuote(G14_PROMETHEUS_NAME)} -o json`,
`section prometheusPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l prometheus=${shellQuote(G14_PROMETHEUS_NAME)} -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .status.containerStatuses[*]}{.ready}{","}{end}{"\\n"}{end}'`,
`section prometheusService kubectl get service -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} ${shellQuote(G14_PROMETHEUS_SERVICE)} -o json`,
`section workloadMonitors kubectl get servicemonitor,prometheusrule -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -l hwlab.pikastech.local/monitoring=enabled -o json`,
`section query kubectl get --raw ${shellQuote(queryPath)}`,
].join("\n");
const bundle = g14K3s(["script", "--", script], 120_000);
const sections = parseShellSections(statusText(bundle));
const namespace = parseSectionJson(sections.namespace);
const discoveryNamespace = parseSectionJson(sections.discoveryNamespace);
const crdNames = sectionLines(sections.crds);
const operator = parseSectionJson(sections.operator);
const operatorPods = podRows(sections.operatorPods);
const prometheus = parseSectionJson(sections.prometheus);
const prometheusPods = podRows(sections.prometheusPods);
const prometheusService = parseSectionJson(sections.prometheusService);
const workloadMonitorItems = parseSectionJsonArray(sections.workloadMonitors);
const query = parseSectionJson(sections.query);
const requiredCrdsPresent = crds.every((name) => crdNames.includes(name));
const namespaceLabel = stringOrNull(record(record(namespace.metadata).labels)["g14.pikastech.local/observability-discovery"]);
const workloadNamespaceLabel = stringOrNull(record(record(discoveryNamespace.metadata).labels)["g14.pikastech.local/observability-discovery"]);
const operatorIsReady = Object.keys(operator).length > 0 && deploymentReady(operator);
const prometheusExists = Object.keys(prometheus).length > 0;
const prometheusIsReady = prometheusExists && (
prometheusReady(prometheus)
|| prometheusPods.some((pod) => pod.ready === true)
);
const queryOk = sections.query?.exitCode === 0 && query.status === "success";
const ok = isCommandSuccess(bundle) && requiredCrdsPresent && operatorIsReady && prometheusExists && prometheusIsReady && queryOk;
return {
ok,
command: "hwlab g14 observability status",
namespace: G14_OBSERVABILITY_NAMESPACE,
mode: "status",
elapsedMs: Date.now() - startedAtMs,
versions: {
prometheusOperator: G14_PROMETHEUS_OPERATOR_VERSION,
prometheus: G14_PROMETHEUS_VERSION,
operatorBundle: G14_PROMETHEUS_OPERATOR_RELEASE_ASSET,
},
discovery: {
namespaceLabel,
workloadNamespace: V02_RUNTIME_NAMESPACE,
workloadNamespaceLabel,
selectorLabel: "hwlab.pikastech.local/monitoring=enabled",
},
crds: {
ok: requiredCrdsPresent,
required: crds,
present: crdNames,
missing: crds.filter((name) => !crdNames.includes(name)),
sectionOk: shellSectionOk(sections.crds),
},
operator: {
ok: operatorIsReady,
deployment: stringOrNull(record(operator.metadata).name),
desiredReplicas: numericValue(record(operator.spec).replicas) ?? 1,
readyReplicas: numericValue(record(operator.status).readyReplicas) ?? 0,
availableReplicas: numericValue(record(operator.status).availableReplicas) ?? 0,
pods: operatorPods.map((pod) => ({
name: stringOrNull(pod.name),
phase: stringOrNull(pod.phase),
ready: pod.ready ?? null,
})),
sectionOk: shellSectionOk(sections.operator),
},
prometheus: {
ok: prometheusExists && prometheusIsReady,
name: G14_PROMETHEUS_NAME,
service: G14_PROMETHEUS_SERVICE,
serviceExists: Object.keys(prometheusService).length > 0,
ready: prometheusIsReady,
conditions: Array.isArray(record(prometheus.status).conditions) ? record(prometheus.status).conditions : [],
pods: prometheusPods.map((pod) => ({
name: stringOrNull(pod.name),
phase: stringOrNull(pod.phase),
ready: pod.ready ?? null,
})),
sectionOk: shellSectionOk(sections.prometheus),
},
workloadMonitors: {
ok: shellSectionOk(sections.workloadMonitors),
namespace: V02_RUNTIME_NAMESPACE,
count: workloadMonitorItems.length,
items: workloadMonitorItems.map((item) => ({
kind: item.kind ?? null,
name: stringOrNull(record(item.metadata).name),
})),
stderr: shellSectionOk(sections.workloadMonitors) ? "" : commandErrorSummary(bundle),
},
query: {
ok: queryOk,
promql: "up",
serviceProxyPath: queryPath,
resultType: nested(query, ["data", "resultType"]) ?? null,
resultCount: Array.isArray(nested(query, ["data", "result"])) ? (nested(query, ["data", "result"]) as unknown[]).length : null,
status: query.status ?? null,
sectionOk: shellSectionOk(sections.query),
stderr: shellSectionOk(sections.query) ? "" : commandErrorSummary(bundle),
},
result: compactCommandMetadata(bundle, !ok),
next: requiredCrdsPresent && operatorIsReady && prometheusExists
? { query: 'bun scripts/cli.ts hwlab g14 observability query --promql \'up{namespace="hwlab-v02"}\'' }
: { apply: "bun scripts/cli.ts hwlab g14 observability apply --confirm" },
};
}
function g14ObservabilityApplyScript(options: G14ObservabilityOptions, manifestB64: string): string {
const dryRunArg = options.dryRun ? "--dry-run=server" : "";
const stackDryRunCommand = options.dryRun
? [
"core_stack_path=\"$tmpdir/g14-prometheus-core-stack.json\"",
"node - \"$stack_path\" \"$core_stack_path\" <<'NODE'",
"const fs = require('node:fs');",
"const input = process.argv[2];",
"const output = process.argv[3];",
"const stack = JSON.parse(fs.readFileSync(input, 'utf8'));",
"stack.items = (stack.items || []).filter((item) => item.kind !== 'Prometheus');",
"fs.writeFileSync(output, JSON.stringify(stack));",
"NODE",
"kubectl apply --dry-run=client --validate=false -f \"$core_stack_path\"",
"echo prometheus_cr_dry_run=skipped_until_monitoring_crds_are_installed",
].join("\n")
: `kubectl apply --server-side --force-conflicts --field-manager=${shellQuote(G14_OBSERVABILITY_FIELD_MANAGER)} -f "$stack_path"`;
const preStackWaitCommands = options.dryRun
? "echo observability_wait=skipped_dry_run"
: [
"kubectl wait --for=condition=Established --timeout=45s crd/servicemonitors.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com crd/prometheuses.monitoring.coreos.com",
`kubectl -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} get deploy/prometheus-operator -o name`,
].join("\n");
const postStackWaitCommands = options.dryRun
? "echo prometheus_wait=skipped_dry_run"
: "echo prometheus_wait=deferred_to_status_command";
return [
"set -eu",
`namespace=${shellQuote(G14_OBSERVABILITY_NAMESPACE)}`,
`bundle_url=${shellQuote(G14_PROMETHEUS_OPERATOR_RELEASE_ASSET)}`,
`operator_version=${shellQuote(G14_PROMETHEUS_OPERATOR_VERSION)}`,
`prometheus_version=${shellQuote(G14_PROMETHEUS_VERSION)}`,
`stack_b64=${shellQuote(manifestB64)}`,
"tmpdir=$(mktemp -d /tmp/g14-observability-XXXXXX)",
"cleanup() { rm -rf \"$tmpdir\"; }",
"trap cleanup EXIT",
"bundle_path=\"$tmpdir/operator-bundle.yaml\"",
"operator_path=\"$tmpdir/operator-rendered.yaml\"",
"stack_path=\"$tmpdir/g14-prometheus-stack.json\"",
"printf '%s' \"$stack_b64\" | base64 -d > \"$stack_path\"",
"export HTTP_PROXY=${HTTP_PROXY:-http://127.0.0.1:10808}",
"export HTTPS_PROXY=${HTTPS_PROXY:-http://127.0.0.1:10808}",
"export http_proxy=$HTTP_PROXY",
"export https_proxy=$HTTPS_PROXY",
"export NO_PROXY=${NO_PROXY:-localhost,127.0.0.1,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc}",
"export no_proxy=$NO_PROXY",
"curl -fsSL --connect-timeout 20 --retry 3 --retry-delay 2 -o \"$bundle_path\" \"$bundle_url\"",
"cat > \"$tmpdir/kustomization.yaml\" <<'YAML'",
"apiVersion: kustomize.config.k8s.io/v1beta1",
"kind: Kustomization",
`namespace: ${G14_OBSERVABILITY_NAMESPACE}`,
"resources:",
"- operator-bundle.yaml",
"YAML",
"kubectl kustomize \"$tmpdir\" > \"$operator_path\"",
"grep -q 'namespace: devops-infra' \"$operator_path\"",
`kubectl create namespace ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${shellQuote(G14_OBSERVABILITY_FIELD_MANAGER)} ${dryRunArg} -f -`,
`kubectl apply --server-side --force-conflicts --field-manager=${shellQuote(G14_OBSERVABILITY_FIELD_MANAGER)} ${dryRunArg} -f "$operator_path"`,
preStackWaitCommands,
stackDryRunCommand,
postStackWaitCommands,
`printf 'observability_apply=ok namespace=%s operator=%s prometheus=%s dryRun=%s\\n' "$namespace" "$operator_version" "$prometheus_version" ${shellQuote(String(options.dryRun))}`,
].join("\n");
}
function runG14ObservabilityApply(options: G14ObservabilityOptions): Record<string, unknown> {
const startedAtMs = Date.now();
const manifest = g14PrometheusManifest();
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
const script = g14ObservabilityApplyScript(options, manifestB64);
const result = g14K3s(["script", "--", script], options.timeoutSeconds * 1000 + 90_000);
const ok = isCommandSuccess(result);
return {
ok,
command: "hwlab g14 observability apply",
mode: options.dryRun ? "dry-run" : "confirmed-apply",
namespace: G14_OBSERVABILITY_NAMESPACE,
versions: {
prometheusOperator: G14_PROMETHEUS_OPERATOR_VERSION,
prometheus: G14_PROMETHEUS_VERSION,
operatorBundle: G14_PROMETHEUS_OPERATOR_RELEASE_ASSET,
},
manifest: options.dryRun ? manifest : undefined,
elapsedMs: Date.now() - startedAtMs,
result: compactCommandResult(result),
status: ok && !options.dryRun ? g14ObservabilityStatus() : undefined,
next: options.dryRun
? { apply: "bun scripts/cli.ts hwlab g14 observability apply --confirm" }
: { status: "bun scripts/cli.ts hwlab g14 observability status", query: 'bun scripts/cli.ts hwlab g14 observability query --promql \'up{namespace="hwlab-v02"}\'' },
};
}
function g14PrometheusQuery(promql: string, timeoutMs: number): { serviceProxyPath: string; result: CommandJsonResult; parsed: Record<string, unknown>; ok: boolean } {
const serviceProxyPath = observabilityQueryPath(promql);
const result = g14K3s(["kubectl", "get", "--raw", serviceProxyPath], timeoutMs);
const parsed = (() => {
try {
return record(JSON.parse(statusText(result)) as unknown);
} catch {
return {};
}
})();
return { serviceProxyPath, result, parsed, ok: isCommandSuccess(result) && parsed.status === "success" };
}
function runG14ObservabilityQuery(options: G14ObservabilityOptions): Record<string, unknown> {
const query = g14PrometheusQuery(options.query, options.timeoutSeconds * 1000);
const assertion = g14ObservabilityQueryAssertion(query.parsed, options.expectCount, options.expectValue);
const semantic = observabilitySemanticName(options.query);
const resultCount = prometheusResultItems(query.parsed).length;
const hasAssertion = options.expectCount !== undefined || options.expectValue !== undefined;
const includeData = !hasAssertion && resultCount <= 20;
const ok = query.ok && (assertion === null || assertion.ok === true);
return {
ok,
command: "hwlab g14 observability query",
namespace: G14_OBSERVABILITY_NAMESPACE,
service: G14_PROMETHEUS_SERVICE,
promql: options.query,
semantic,
serviceProxyPath: query.serviceProxyPath,
status: query.parsed.status ?? null,
resultType: nested(query.parsed, ["data", "resultType"]) ?? null,
resultCount,
series: prometheusSeriesSummaries(query.parsed, 50),
assertion,
data: includeData ? query.parsed.data ?? null : undefined,
dataOmitted: includeData ? undefined : {
reason: hasAssertion ? "assertion-mode-uses-series-summary" : "result-count-exceeds-default-inline-limit",
resultCount,
inlineLimit: 20,
summaryField: "series",
queryCommand: `bun scripts/cli.ts hwlab g14 observability query --promql ${shellQuote(options.query)}`,
},
raw: Object.keys(query.parsed).length === 0 ? tailText(statusText(query.result), 4000) : undefined,
commandResult: compactCommandMetadata(query.result, !query.ok, !query.ok),
degradedReason: ok ? undefined : assertion !== null && assertion.ok !== true ? "assertion-failed" : "query-failed",
next: assertion !== null && assertion.ok !== true
? { targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02", closeout: "bun scripts/cli.ts hwlab g14 observability closeout --lane v02" }
: undefined,
};
}
function runG14ObservabilityTargets(options: G14ObservabilityOptions): Record<string, unknown> {
const startedAtMs = Date.now();
const queryCommands = Object.entries(V02_OBSERVABILITY_QUERIES).map(([name, promql]) => `section query_${name} kubectl get --raw ${shellQuote(observabilityQueryPath(promql))}`);
const script = [
"set +e",
"section() {",
" name=\"$1\"",
" shift",
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
" \"$@\"",
" code=$?",
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
`section pods kubectl get pods -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -o json`,
`section monitors kubectl get servicemonitor,podmonitor,prometheusrule -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -l hwlab.pikastech.local/monitoring=enabled -o json`,
`section podResourceMetrics kubectl get --raw ${shellQuote(`/apis/metrics.k8s.io/v1beta1/namespaces/${V02_RUNTIME_NAMESPACE}/pods`)}`,
...queryCommands,
].join("\n");
const bundle = g14K3s(["script", "--", script], options.timeoutSeconds * 1000);
const sections = parseShellSections(statusText(bundle));
const pods = k8sItems(parseSectionJson(sections.pods));
const monitors = k8sItems(parseSectionJson(sections.monitors));
const podResourceMetrics = k8sItems(parseSectionJson(sections.podResourceMetrics)).map(podResourceMetricSummary);
const parsedQueries = Object.fromEntries(Object.entries(V02_OBSERVABILITY_QUERIES).map(([name]) => [name, parseSectionJson(sections[`query_${name}`])]));
const metricSeries = Object.fromEntries(Object.entries(parsedQueries).map(([name, parsed]) => [name, prometheusResultItems(parsed)]));
const booleanQueryNames = new Set<string>(V02_OBSERVABILITY_BOOLEAN_QUERY_NAMES);
const closeoutQueryNames = new Set<string>(V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES);
const assertions = Object.fromEntries(Object.entries(parsedQueries).map(([name, parsed]) => [
name,
g14ObservabilityQueryAssertion(parsed, V02_OBSERVABILITY_EXPECTED_TARGET_COUNT, booleanQueryNames.has(name) ? "1" : undefined),
]));
const targets = attachResourceMetricsToTargets(mergeObservabilityTargets(pods, monitors, metricSeries), podResourceMetrics);
const compactTargets = targets.map(compactObservabilityTarget);
const levelSummary = buildObservabilityLevelSummary(targets);
const sidecars = targets
.map((target) => record(target.sidecar))
.filter((sidecar) => Object.keys(sidecar).length > 0);
const notReadySidecars = sidecars.filter((sidecar) => sidecar.sidecarReady !== true || sidecar.podReady !== true);
const restartProblems = sidecars.filter((sidecar) => (numericValue(sidecar.restartCount) ?? 0) > 0);
const queryOk = V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES.every((name) => record(assertions[name]).ok === true);
const sidecarOk = targets.length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT
&& sidecars.length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT
&& notReadySidecars.length === 0;
const monitorsOk = shellSectionOk(sections.monitors) && monitors.length > 0;
const resourceOk = shellSectionOk(sections.podResourceMetrics)
&& podResourceMetrics.filter((item) => {
const serviceId = stringOrNull(item.serviceId);
return serviceId !== null && V02_OBSERVABILITY_SERVICE_IDS.includes(serviceId);
}).length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT;
const ok = isCommandSuccess(bundle) && shellSectionOk(sections.pods) && monitorsOk && queryOk && sidecarOk && resourceOk;
return {
ok,
command: "hwlab g14 observability targets",
lane: options.lane,
namespace: V02_RUNTIME_NAMESPACE,
elapsedMs: Date.now() - startedAtMs,
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
targetCount: targets.length,
levelSummary,
targets: compactTargets,
sidecars: {
ok: sidecarOk,
observedCount: sidecars.length,
readyCount: sidecars.filter((sidecar) => sidecar.sidecarReady === true && sidecar.podReady === true).length,
notReady: notReadySidecars,
restartProblems,
},
monitors: {
ok: monitorsOk,
count: monitors.length,
items: monitorSummaries(monitors),
sectionOk: shellSectionOk(sections.monitors),
},
resourceSnapshot: {
ok: resourceOk,
source: "metrics.k8s.io/v1beta1",
observedTargetCount: podResourceMetrics.filter((item) => {
const serviceId = stringOrNull(item.serviceId);
return serviceId !== null && V02_OBSERVABILITY_SERVICE_IDS.includes(serviceId);
}).length,
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
sectionOk: shellSectionOk(sections.podResourceMetrics),
stderr: shellSectionOk(sections.podResourceMetrics) ? "" : commandErrorSummary(bundle),
},
queries: Object.fromEntries(Object.entries(parsedQueries).map(([name, parsed]) => [name, {
promql: record(V02_OBSERVABILITY_QUERIES)[name],
resultType: nested(parsed, ["data", "resultType"]) ?? null,
resultCount: prometheusResultItems(parsed).length,
assertion: assertions[name],
seriesOmitted: {
reason: closeoutQueryNames.has(name) ? "target-summary-uses-assertion-and-target-rows" : "auxiliary-metric-summarized-in-levelSummary",
summaryField: closeoutQueryNames.has(name) ? "targets" : "levelSummary",
drillDownCommand: closeoutQueryNames.has(name)
? `bun scripts/cli.ts hwlab g14 observability query --promql ${shellQuote(String(record(V02_OBSERVABILITY_QUERIES)[name]))} --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`
: undefined,
},
sectionOk: shellSectionOk(sections[`query_${name}`]),
}])),
result: compactCommandMetadata(bundle, !ok),
degradedReason: ok ? undefined : "target-health-failed",
next: ok ? { closeout: "bun scripts/cli.ts hwlab g14 observability closeout --lane v02" } : { boundary: "bun scripts/cli.ts hwlab g14 observability boundary --lane v02" },
};
}
function runG14ObservabilityBoundary(options: G14ObservabilityOptions): Record<string, unknown> {
const startedAtMs = Date.now();
const script = [
"set +e",
"section() {",
" name=\"$1\"",
" shift",
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
" \"$@\"",
" code=$?",
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
`section workloadMonitoring kubectl get servicemonitor,podmonitor,prometheusrule,prometheus,alertmanager -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -o json`,
`section infraControlPlane kubectl get prometheus,alertmanager -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -o json`,
].join("\n");
const bundle = g14K3s(["script", "--", script], options.timeoutSeconds * 1000);
const sections = parseShellSections(statusText(bundle));
const workloadItems = k8sItems(parseSectionJson(sections.workloadMonitoring));
const infraItems = k8sItems(parseSectionJson(sections.infraControlPlane));
const forbiddenWorkloadControlPlane = workloadItems.filter((item) => item.kind === "Prometheus" || item.kind === "Alertmanager");
const allowedWorkloadDeclarations = workloadItems.filter((item) => item.kind === "ServiceMonitor" || item.kind === "PodMonitor" || item.kind === "PrometheusRule");
const namespaceBoundaryOk = shellSectionOk(sections.workloadMonitoring) && forbiddenWorkloadControlPlane.length === 0;
const infraControlPlaneOk = shellSectionOk(sections.infraControlPlane) && infraItems.some((item) => item.kind === "Prometheus" && k8sMetadataName(item) === G14_PROMETHEUS_NAME);
const publicProbes = [
publicMetricsProbe("hwlab-v02-cloud-web", `${V02_CLOUD_WEB_URL}/metrics`),
publicMetricsProbe("hwlab-v02-cloud-api", `${V02_CLOUD_API_URL}/metrics`),
];
const publicMetricsExposureOk = publicProbes.every((probe) => probe.ok === true && probe.exposed !== true);
const ok = isCommandSuccess(bundle) && namespaceBoundaryOk && infraControlPlaneOk && publicMetricsExposureOk;
return {
ok,
command: "hwlab g14 observability boundary",
lane: options.lane,
elapsedMs: Date.now() - startedAtMs,
namespaceBoundary: {
ok: namespaceBoundaryOk,
workloadNamespace: V02_RUNTIME_NAMESPACE,
allowedKinds: ["ServiceMonitor", "PodMonitor", "PrometheusRule"],
allowedDeclarationCount: allowedWorkloadDeclarations.length,
allowedDeclarations: monitorSummaries(allowedWorkloadDeclarations),
forbiddenKinds: ["Prometheus", "Alertmanager"],
forbiddenControlPlaneCount: forbiddenWorkloadControlPlane.length,
forbiddenControlPlane: forbiddenWorkloadControlPlane.map((item) => ({ kind: item.kind, name: k8sMetadataName(item) })),
sectionOk: shellSectionOk(sections.workloadMonitoring),
},
infraControlPlane: {
ok: infraControlPlaneOk,
namespace: G14_OBSERVABILITY_NAMESPACE,
items: infraItems.map((item) => ({ kind: item.kind, name: k8sMetadataName(item) })),
sectionOk: shellSectionOk(sections.infraControlPlane),
},
publicMetricsExposure: {
ok: publicMetricsExposureOk,
expected: "denied",
probes: publicProbes,
},
result: compactCommandMetadata(bundle, !ok),
degradedReason: ok ? undefined : "boundary-failed",
next: ok ? { closeout: "bun scripts/cli.ts hwlab g14 observability closeout --lane v02" } : { targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02" },
};
}
function passFail(value: boolean): "pass" | "fail" {
return value ? "pass" : "fail";
}
function closeoutAdvice(summary: Record<string, unknown>): string[] {
const advice: string[] = [];
if (summary.platformReady !== "pass") advice.push("platformReady failed -> run observability status/apply and inspect Prometheus Operator plus Prometheus readiness in devops-infra");
if (summary.scrapeReachable !== "pass") advice.push("scrapeReachable failed -> check ServiceMonitor labels, metrics sidecar port name, and Prometheus target discovery");
if (summary.sidecarServing !== "pass") advice.push("sidecarServing failed -> check hwlab-metrics sidecar readiness, restartCount, and metrics script/container logs");
if (summary.businessHealthProbe !== "pass") advice.push("businessHealthProbe failed -> up=1 but health_probe=0 usually means sidecar can be scraped but cannot reach the business health endpoint");
if (summary.resourceSnapshot !== "pass") advice.push("resourceSnapshot failed -> check metrics.k8s.io APIService and metrics-server availability on G14 k3s");
if (summary.namespaceControlPlaneBoundary !== "pass") advice.push("namespaceControlPlaneBoundary failed -> remove Prometheus/Alertmanager from workload namespace; shared control plane belongs in devops-infra");
if (summary.publicMetricsExposure !== "pass") advice.push("publicMetricsExposure failed -> public /metrics returned Prometheus text; remove FRP/edge exposure or add an authenticated internal-only route");
return advice;
}
function runG14ObservabilityCloseout(options: G14ObservabilityOptions): Record<string, unknown> {
const startedAtMs = Date.now();
const status = g14ObservabilityStatus();
const targets = runG14ObservabilityTargets(options);
const boundary = runG14ObservabilityBoundary(options);
const targetQueries = record(targets.queries);
const queryOk = (name: string): boolean => record(record(targetQueries[name]).assertion).ok === true;
const sidecarsOk = record(record(targets.sidecars)).ok === true;
const platformReady = record(status.crds).ok === true && record(status.operator).ok === true && record(status.prometheus).ok === true && record(status.query).ok === true;
const namespaceBoundaryOk = record(record(boundary.namespaceBoundary)).ok === true && record(record(boundary.infraControlPlane)).ok === true;
const publicDenied = record(record(boundary.publicMetricsExposure)).ok === true;
const resourceOk = record(record(targets.resourceSnapshot)).ok === true;
const summary = {
platformReady: passFail(platformReady),
workloadMonitorCount: numericValue(record(status.workloadMonitors).count) ?? numericValue(record(targets.monitors).count) ?? 0,
observedTargetCount: numericValue(targets.targetCount) ?? 0,
scrapeReachable: passFail(queryOk("scrapeReachable")),
sidecarServing: passFail(queryOk("sidecarServing") && sidecarsOk),
businessHealthProbe: passFail(queryOk("businessHealthProbe")),
sidecarReady: passFail(sidecarsOk),
resourceSnapshot: passFail(resourceOk),
namespaceControlPlaneBoundary: passFail(namespaceBoundaryOk),
publicMetricsExposure: passFail(publicDenied),
publicMetricsExposureState: publicDenied ? "denied" : "exposed-or-unknown",
};
const ok = Object.entries(summary).every(([key, value]) => key === "workloadMonitorCount"
? numericValue(value) !== null && Number(value) > 0
: key === "observedTargetCount"
? numericValue(value) === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT
: key === "publicMetricsExposureState"
? value === "denied"
: value === "pass");
return {
ok,
command: "hwlab g14 observability closeout",
lane: options.lane,
elapsedMs: Date.now() - startedAtMs,
summary,
advice: closeoutAdvice(summary),
evidence: {
platform: {
ok: status.ok,
crdsOk: record(status.crds).ok === true,
operatorReady: record(status.operator).ok === true,
prometheusReady: record(status.prometheus).ok === true,
workloadMonitorCount: summary.workloadMonitorCount,
upResultCount: record(status.query).resultCount ?? null,
},
targets: {
ok: targets.ok,
expectedTargetCount: targets.expectedTargetCount,
observedTargetCount: targets.targetCount,
readySidecarCount: record(targets.sidecars).readyCount ?? null,
healthProbeDuration: record(record(targets.levelSummary).healthProbeDuration),
scrapeDuration: record(record(targets.levelSummary).scrapeDuration),
resourceSnapshot: record(targets.resourceSnapshot),
resourceUsage: record(record(targets.levelSummary).resourceUsage),
services: arrayRecords(record(targets.levelSummary).services).map((service) => ({
serviceId: service.serviceId ?? null,
scrapeReachable: service.scrapeReachable ?? null,
sidecarServing: service.sidecarServing ?? null,
businessHealthProbe: service.businessHealthProbe ?? null,
statusCode: service.statusCode ?? null,
totalCpuMillicores: service.totalCpuMillicores ?? null,
totalMemoryMiB: service.totalMemoryMiB ?? null,
businessCpuMillicores: service.businessCpuMillicores ?? null,
businessMemoryMiB: service.businessMemoryMiB ?? null,
})),
},
boundary: {
ok: boundary.ok,
forbiddenControlPlaneCount: numericValue(record(boundary.namespaceBoundary).forbiddenControlPlaneCount) ?? 0,
publicMetricsExposure: record(boundary.publicMetricsExposure),
},
drillDown: {
status: "bun scripts/cli.ts hwlab g14 observability status",
targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02",
boundary: "bun scripts/cli.ts hwlab g14 observability boundary --lane v02",
scrapeReachable: `bun scripts/cli.ts hwlab g14 observability query --promql 'up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
sidecarServing: `bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
businessHealthProbe: `bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_health_probe_success{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
},
},
degradedReason: ok ? undefined : "observability-closeout-failed",
next: {
targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02",
boundary: "bun scripts/cli.ts hwlab g14 observability boundary --lane v02",
controlPlaneCloseout: "bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit <full-sha>",
},
};
}
function runG14Observability(options: G14ObservabilityOptions): Record<string, unknown> {
if (options.action === "status") return g14ObservabilityStatus();
if (options.action === "query") return runG14ObservabilityQuery(options);
if (options.action === "targets") return runG14ObservabilityTargets(options);
if (options.action === "boundary") return runG14ObservabilityBoundary(options);
if (options.action === "closeout") return runG14ObservabilityCloseout(options);
return runG14ObservabilityApply(options);
}
function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record<string, unknown> {
const job = startJob(name, command, note);
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
return {
ok: true,
mode: "async-job",
job,
statusCommand,
tailCommand: `tail -f ${job.stdoutFile}`,
next: {
status: statusCommand,
tail: `tail -f ${job.stdoutFile}`,
},
};
}
function startControlPlaneTriggerJob(options: G14ControlPlaneOptions): Record<string, unknown> {
const command = [
"bun",
"scripts/cli.ts",
"hwlab",
"g14",
"control-plane",
"trigger-current",
"--lane",
options.lane,
"--confirm",
"--timeout-seconds",
String(options.timeoutSeconds),
"--wait",
];
return {
command: `hwlab g14 control-plane trigger-current --lane ${options.lane}`,
lane: options.lane,
reason: "confirmed trigger can spend tens of seconds syncing git mirror and creating PipelineRun; default is fire-and-forget to avoid silent blocking",
waitCommand: command.join(" "),
...startAsyncHwlabG14Job(
`hwlab_g14_${options.lane}_trigger_current`,
command,
`Trigger HWLAB ${options.lane} current commit PipelineRun with git mirror pre-sync through G14 control-plane`,
),
};
}
function startGitMirrorJob(options: G14GitMirrorOptions): Record<string, unknown> {
const command = [
"bun",
"scripts/cli.ts",
"hwlab",
"g14",
"git-mirror",
options.action,
"--lane",
options.lane,
"--confirm",
"--timeout-seconds",
String(options.timeoutSeconds),
"--wait",
];
return {
command: `hwlab g14 git-mirror ${options.action} --lane ${options.lane}`,
lane: options.lane,
namespace: GIT_MIRROR_NAMESPACE,
reason: "manual git mirror sync/flush waits for a Kubernetes Job; default is fire-and-forget to keep CLI output immediately visible",
waitCommand: command.join(" "),
...startAsyncHwlabG14Job(
`hwlab_g14_git_mirror_${options.action}`,
command,
`Run HWLAB devops-infra git mirror ${options.action} through a bounded manual Kubernetes Job`,
),
};
}
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
}
function g14CiToolsImage(tag: string): string {
return `${G14_CI_TOOLS_IMAGE_REPO}:${tag}`;
}
function runG14ToolsImageStatus(options: G14ToolsImageOptions): Record<string, unknown> {
const image = g14CiToolsImage(options.tag);
const script = [
"set -u",
`image=${shellQuote(image)}`,
"exists=false",
"image_id=",
"created=",
"size=",
"node_version=",
"npm_version=",
"bun_version=",
"git_version=",
"docker_version=",
"python_version=",
"registry_tags=",
"if docker image inspect \"$image\" >/tmp/hwlab-tools-image-inspect.json 2>/tmp/hwlab-tools-image-inspect.err; then",
" exists=true",
" image_id=$(docker image inspect \"$image\" --format '{{.Id}}' 2>/dev/null || true)",
" created=$(docker image inspect \"$image\" --format '{{.Created}}' 2>/dev/null || true)",
" size=$(docker image inspect \"$image\" --format '{{.Size}}' 2>/dev/null || true)",
" node_version=$(docker run --rm \"$image\" node --version 2>/dev/null || true)",
" npm_version=$(docker run --rm \"$image\" npm --version 2>/dev/null || true)",
" bun_version=$(docker run --rm \"$image\" bun --version 2>/dev/null || true)",
" git_version=$(docker run --rm \"$image\" git --version 2>/dev/null || true)",
" docker_version=$(docker run --rm \"$image\" docker --version 2>/dev/null || true)",
" python_version=$(docker run --rm \"$image\" python3 --version 2>/dev/null || true)",
"fi",
`registry_tags=$(curl -fsS --max-time 10 http://127.0.0.1:5000/v2/hwlab/hwlab-ci-node-tools/tags/list 2>/dev/null || true)`,
"export image exists image_id created size node_version npm_version bun_version git_version docker_version python_version registry_tags",
"node - <<'NODE'",
"const fs = require('node:fs');",
"const env = process.env;",
"const payload = {",
" image: env.image,",
" exists: env.exists === 'true',",
" imageId: env.image_id || null,",
" created: env.created || null,",
" sizeBytes: env.size ? Number(env.size) : null,",
" tools: {",
" node: env.node_version || null,",
" npm: env.npm_version || null,",
" bun: env.bun_version || null,",
" git: env.git_version || null,",
" docker: env.docker_version || null,",
" python: env.python_version || null",
" },",
" registryTagsRaw: env.registry_tags || null",
"};",
"console.log(JSON.stringify(payload, null, 2));",
"NODE",
].join("\n");
const result = g14HostScript(script, 180_000);
let parsedStatus: unknown = null;
const text = statusText(result);
if (text.length > 0) {
try {
parsedStatus = JSON.parse(text) as unknown;
} catch {
parsedStatus = null;
}
}
return {
ok: isCommandSuccess(result) && record(parsedStatus).exists === true,
command: "hwlab g14 tools-image status --name ci-node-tools",
name: options.name,
image,
status: parsedStatus ?? text,
result,
};
}
function runG14ToolsImageBuild(options: G14ToolsImageOptions): Record<string, unknown> {
const image = g14CiToolsImage(options.tag);
const script = [
"set -eu",
`cd ${shellQuote(V02_WORKSPACE)}`,
"git fetch origin v0.2 --prune",
"git checkout v0.2 >/dev/null 2>&1 || true",
"git merge --ff-only origin/v0.2",
`test -f ${shellQuote(options.dockerfile)}`,
`image=${shellQuote(image)}`,
`dockerfile=${shellQuote(options.dockerfile)}`,
"echo \"{\\\"phase\\\":\\\"preflight\\\",\\\"image\\\":\\\"$image\\\",\\\"dockerfile\\\":\\\"$dockerfile\\\",\\\"commit\\\":\\\"$(git rev-parse HEAD)\\\"}\"",
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
"export no_proxy=$NO_PROXY",
"docker build --pull --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg http_proxy --build-arg https_proxy --build-arg NO_PROXY --build-arg no_proxy -f \"$dockerfile\" -t \"$image\" .",
"docker run --rm \"$image\" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V'",
"docker push \"$image\"",
"digest=$(docker image inspect \"$image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"image\\\":\\\"$image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
].join("\n");
if (options.dryRun) {
return {
ok: true,
command: "hwlab g14 tools-image build --name ci-node-tools",
mode: "dry-run",
image,
workspace: V02_WORKSPACE,
dockerfile: options.dockerfile,
buildScriptPreview: script.split(/\n/u),
next: { build: `bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag ${options.tag} --confirm` },
};
}
const result = g14HostScript(script, options.timeoutSeconds * 1000);
return {
ok: isCommandSuccess(result),
command: "hwlab g14 tools-image build --name ci-node-tools",
mode: "confirmed-build",
image,
workspace: V02_WORKSPACE,
dockerfile: options.dockerfile,
result,
status: runG14ToolsImageStatus({ ...options, action: "status", dryRun: true, confirm: false }),
};
}
function runG14ToolsImage(options: G14ToolsImageOptions): Record<string, unknown> {
if (options.action === "status") return runG14ToolsImageStatus(options);
return runG14ToolsImageBuild(options);
}
function g14UpstreamImageTarget(options: G14UpstreamImageOptions): string {
return `${V02_REGISTRY_PREFIX}/${options.name}:${options.tag}`;
}
function g14UpstreamImageSource(options: G14UpstreamImageOptions): string {
if (options.name === "openfga") return `docker.io/openfga/openfga:${options.tag}`;
return `${options.name}:${options.tag}`;
}
function runG14UpstreamImageStatus(options: G14UpstreamImageOptions): Record<string, unknown> {
const sourceImage = g14UpstreamImageSource(options);
const targetImage = g14UpstreamImageTarget(options);
const script = [
"set +e",
`source_image=${shellQuote(sourceImage)}`,
`target_image=${shellQuote(targetImage)}`,
`repo_path=${shellQuote(`hwlab/${options.name}`)}`,
`tag=${shellQuote(options.tag)}`,
"local_id=$(docker image inspect \"$target_image\" --format '{{.Id}}' 2>/dev/null || true)",
"local_created=$(docker image inspect \"$target_image\" --format '{{.Created}}' 2>/dev/null || true)",
"registry_tags=$(curl -fsS --max-time 10 \"http://127.0.0.1:5000/v2/$repo_path/tags/list\" 2>/dev/null || true)",
"registry_has_tag=false",
"if printf '%s' \"$registry_tags\" | grep -F '\"'$tag'\"' >/dev/null 2>&1; then registry_has_tag=true; fi",
"export source_image target_image local_id local_created registry_tags registry_has_tag",
"node <<'NODE'",
"const env = process.env;",
"console.log(JSON.stringify({",
" sourceImage: env.source_image,",
" targetImage: env.target_image,",
" localExists: Boolean(env.local_id),",
" localImageId: env.local_id || null,",
" localCreated: env.local_created || null,",
" registryHasTag: env.registry_has_tag === 'true',",
" registryTagsRaw: env.registry_tags || null",
"}, null, 2));",
"NODE",
].join("\n");
const result = g14HostScript(script, 120_000);
let parsedStatus: unknown = null;
const text = statusText(result);
if (text.length > 0) {
try {
parsedStatus = JSON.parse(text) as unknown;
} catch {
parsedStatus = null;
}
}
return {
ok: isCommandSuccess(result) && record(parsedStatus).registryHasTag === true,
command: "hwlab g14 upstream-image status --name openfga",
name: options.name,
tag: options.tag,
sourceImage,
targetImage,
status: parsedStatus ?? text,
result,
};
}
function runG14UpstreamImageEnsure(options: G14UpstreamImageOptions): Record<string, unknown> {
const sourceImage = g14UpstreamImageSource(options);
const targetImage = g14UpstreamImageTarget(options);
if (options.dryRun) {
return {
ok: true,
command: "hwlab g14 upstream-image ensure --name openfga",
mode: "dry-run",
name: options.name,
tag: options.tag,
sourceImage,
targetImage,
mutation: false,
next: { confirm: `bun scripts/cli.ts hwlab g14 upstream-image ensure --name ${options.name} --tag ${options.tag} --confirm` },
};
}
const script = [
"set -eu",
`source_image=${shellQuote(sourceImage)}`,
`target_image=${shellQuote(targetImage)}`,
"export HTTP_PROXY=http://127.0.0.1:10808 HTTPS_PROXY=http://127.0.0.1:10808 http_proxy=http://127.0.0.1:10808 https_proxy=http://127.0.0.1:10808",
"export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000",
"export no_proxy=$NO_PROXY",
"echo \"{\\\"phase\\\":\\\"pull\\\",\\\"sourceImage\\\":\\\"$source_image\\\"}\"",
"docker pull \"$source_image\"",
"docker tag \"$source_image\" \"$target_image\"",
"echo \"{\\\"phase\\\":\\\"push\\\",\\\"targetImage\\\":\\\"$target_image\\\"}\"",
"docker push \"$target_image\"",
"digest=$(docker image inspect \"$target_image\" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)",
"echo \"{\\\"phase\\\":\\\"published\\\",\\\"targetImage\\\":\\\"$target_image\\\",\\\"digest\\\":\\\"$digest\\\"}\"",
].join("\n");
const result = g14HostScript(script, options.timeoutSeconds * 1000);
const status = runG14UpstreamImageStatus(options);
return {
ok: isCommandSuccess(result) && status.ok === true,
command: "hwlab g14 upstream-image ensure --name openfga",
mode: "confirmed-ensure",
name: options.name,
tag: options.tag,
sourceImage,
targetImage,
mutation: true,
result,
status,
};
}
function runG14UpstreamImage(options: G14UpstreamImageOptions): Record<string, unknown> {
if (options.action === "status") return runG14UpstreamImageStatus(options);
return runG14UpstreamImageEnsure(options);
}
function listOpenG14PullRequests(): CommandJsonResult {
return cliJson(["gh", "pr", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "30", "--json", "number,title,state,url,head,base,draft,headRefName,baseRefName"], 60_000);
}
function preflightPullRequest(number: number): CommandJsonResult {
return cliJson(["gh", "pr", "preflight", String(number), "--repo", HWLAB_REPO], 80_000);
}
function mergePullRequest(number: number, dryRun: boolean): CommandJsonResult {
return cliJson(["gh", "pr", "merge", String(number), "--repo", HWLAB_REPO, "--merge", "--delete-branch", ...(dryRun ? ["--dry-run"] : [])], 100_000);
}
function getG14Head(): string | null {
const result = cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "git fetch origin G14 --prune >/dev/null 2>&1; git rev-parse origin/G14"], 120_000);
if (!isCommandSuccess(result)) return null;
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
const match = /[0-9a-f]{40}/iu.exec(output);
return match?.[0] ?? null;
}
function refreshArgoDev(): void {
cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "annotate", "application", "-n", ARGO_NAMESPACE, DEV_APP, "argocd.argoproj.io/refresh=hard", "--overwrite"], 60_000);
}
function getPipelineStatus(sourceCommit: string): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "pipelinerun", "-n", CI_NAMESPACE, `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}"], 60_000);
}
function getArgoStatus(): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "application", "-n", ARGO_NAMESPACE, DEV_APP, "-o", "jsonpath={.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.operationState.phase}{\"\\n\"}{.status.operationState.syncResult.revision}{\"\\n\"}"], 60_000);
}
function getDevWorkloads(): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, "kubectl", "get", "deploy,statefulset,pod", "-n", DEV_NAMESPACE, "-o", "wide"], 60_000);
}
function getLiveHealth(): CommandJsonResult {
return commandJson(["curl", "-fsS", "--max-time", "20", "http://74.48.78.17:17667/health/live"], 30_000);
}
function statusText(result: CommandJsonResult): string {
return String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
}
function workloadReadiness(workloadsText: string, commandOk: boolean): Record<string, unknown> {
const blockers: string[] = [];
const ignoredPods: string[] = [];
const readyWorkloads: string[] = [];
const allowedZeroDeployments = new Set(["deployment.apps/hwlab-gateway", "deployment.apps/hwlab-tunnel-client"]);
for (const line of workloadsText.split(/\r?\n/u)) {
const trimmed = line.trim();
if (trimmed.length === 0 || trimmed.startsWith("NAME ")) continue;
const fields = trimmed.split(/\s+/u);
const name = fields[0] ?? "";
const ready = fields[1] ?? "";
if (name.startsWith("deployment.apps/") || name.startsWith("statefulset.apps/")) {
if (ready === "0/0" && allowedZeroDeployments.has(name)) {
readyWorkloads.push(`${name}:${ready}:scaled-zero`);
continue;
}
const match = /^(\d+)\/(\d+)$/u.exec(ready);
if (match === null) {
blockers.push(`${name}:unparseable-ready:${ready}`);
continue;
}
const readyCount = Number(match[1]);
const desiredCount = Number(match[2]);
if (desiredCount > 0 && readyCount < desiredCount) blockers.push(`${name}:not-ready:${ready}`);
else readyWorkloads.push(`${name}:${ready}`);
continue;
}
if (name.startsWith("pod/")) {
const status = fields[2] ?? "";
if (status === "Completed" || status === "Succeeded") ignoredPods.push(`${name}:${status}`);
if (/^(CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError|RunContainerError)$/u.test(status)) {
blockers.push(`${name}:pod-status:${status}`);
}
}
}
return {
ready: commandOk && blockers.length === 0,
blockers,
readyWorkloads: readyWorkloads.slice(0, 20),
ignoredPods: ignoredPods.slice(0, 20),
};
}
function beijingParts(date = new Date()): { date: string; time: string; iso: string } {
const shifted = new Date(date.getTime() + BEIJING_OFFSET_MS).toISOString();
return { date: shifted.slice(0, 10), time: shifted.slice(11, 19), iso: shifted };
}
function durationSeconds(start?: string | null, end?: string | null): number | null {
if (start === undefined || start === null || end === undefined || end === null) return null;
const startMs = Date.parse(start);
const endMs = Date.parse(end);
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) return null;
return Math.round((endMs - startMs) / 1000);
}
function formatDuration(seconds: number | null): string {
if (seconds === null) return "n/a";
const minutes = Math.floor(seconds / 60);
const rest = seconds % 60;
if (minutes === 0) return `${rest}s`;
return `${minutes}m${String(rest).padStart(2, "0")}s`;
}
function pipelineTaskRunResultsJsonPath(): string {
return [
"jsonpath=",
"{range .items[*]}",
"{.metadata.name}{\"\\t\"}",
"{.metadata.labels.tekton\\.dev/pipelineTask}{\"\\t\"}",
"{.status.startTime}{\"\\t\"}",
"{.status.completionTime}{\"\\t\"}",
"{range .status.results[*]}{.name}{\"=\"}{.value}{\";\"}{end}",
"{\"\\n\"}",
"{end}",
].join("");
}
function getPipelineTaskRunResults(pipelineRun: string): CommandJsonResult {
return cliJson([
"ssh",
`${G14_PROVIDER}:k3s`,
"kubectl",
"get",
"taskrun",
"-n",
CI_NAMESPACE,
"-l",
`tekton.dev/pipelineRun=${pipelineRun}`,
"-o",
pipelineTaskRunResultsJsonPath(),
], 80_000);
}
function parseResultMap(raw: string): Record<string, string> {
const values: Record<string, string> = {};
for (const item of raw.split(";")) {
if (item.length === 0) continue;
const separator = item.indexOf("=");
if (separator <= 0) continue;
values[item.slice(0, separator)] = item.slice(separator + 1);
}
return values;
}
function serviceMetricSort(left: CiServiceMetric, right: CiServiceMetric): number {
return left.serviceId.localeCompare(right.serviceId);
}
export function parsePipelineTaskRunMetrics(pipelineRun: string, text: string): CiPipelineMetrics {
const services: CiServiceMetric[] = [];
for (const line of text.split(/\r?\n/u)) {
const trimmed = line.trimEnd();
if (trimmed.length === 0) continue;
const [taskRunName = "", pipelineTask = "", startTime = "", completionTime = "", resultsRaw = ""] = trimmed.split("\t");
if (!pipelineTask.startsWith("build-")) continue;
const results = parseResultMap(resultsRaw);
const serviceId = results["service-id"] ?? pipelineTask.replace(/^build-/u, "");
if (serviceId.length === 0) continue;
services.push({
taskRun: taskRunName,
serviceId,
status: results.status ?? "unknown",
durationSeconds: durationSeconds(startTime, completionTime),
buildBackend: results["build-backend"] ?? null,
reusedFrom: results["reused-from"] ?? null,
imageTag: results["image-tag"] ?? null,
sourceCommitId: results["source-commit-id"] ?? null,
});
}
services.sort(serviceMetricSort);
const reusedServices = services.filter((service) => service.status === "reused").map((service) => service.serviceId);
const rebuildServices = services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
return {
ok: services.length > 0,
pipelineRun,
serviceTotal: services.length,
reusedCount: reusedServices.length,
rebuildCount: rebuildServices.length,
reusedServices,
rebuildServices,
services,
...(services.length === 0 ? { degradedReason: "no-build-task-results", rawText: text.slice(0, 2000) } : {}),
};
}
function ciMetricsFromValue(value: unknown): CiPipelineMetrics | null {
const data = record(value);
if (data.ok !== true || typeof data.pipelineRun !== "string") return null;
const services = Array.isArray(data.services) ? data.services.map((item) => {
const service = record(item);
return {
taskRun: String(service.taskRun ?? ""),
serviceId: String(service.serviceId ?? ""),
status: String(service.status ?? "unknown"),
durationSeconds: typeof service.durationSeconds === "number" ? service.durationSeconds : null,
buildBackend: typeof service.buildBackend === "string" ? service.buildBackend : null,
reusedFrom: typeof service.reusedFrom === "string" ? service.reusedFrom : null,
imageTag: typeof service.imageTag === "string" ? service.imageTag : null,
sourceCommitId: typeof service.sourceCommitId === "string" ? service.sourceCommitId : null,
};
}).filter((service) => service.serviceId.length > 0).sort(serviceMetricSort) : [];
const reusedServices = Array.isArray(data.reusedServices) ? data.reusedServices.map(String) : services.filter((service) => service.status === "reused").map((service) => service.serviceId);
const rebuildServices = Array.isArray(data.rebuildServices) ? data.rebuildServices.map(String) : services.filter((service) => service.status !== "reused").map((service) => service.serviceId);
return {
ok: true,
pipelineRun: data.pipelineRun,
serviceTotal: typeof data.serviceTotal === "number" ? data.serviceTotal : services.length,
reusedCount: typeof data.reusedCount === "number" ? data.reusedCount : reusedServices.length,
rebuildCount: typeof data.rebuildCount === "number" ? data.rebuildCount : rebuildServices.length,
reusedServices,
rebuildServices,
services,
};
}
function collectPipelineMetrics(pipelineRun: string): CiPipelineMetrics {
const result = getPipelineTaskRunResults(pipelineRun);
if (!isCommandSuccess(result)) {
return {
ok: false,
pipelineRun,
serviceTotal: 0,
reusedCount: 0,
rebuildCount: 0,
reusedServices: [],
rebuildServices: [],
services: [],
degradedReason: "taskrun-query-failed",
rawText: `${result.stderr}\n${result.stdout}`.trim().slice(0, 2000),
};
}
return parsePipelineTaskRunMetrics(pipelineRun, statusText(result));
}
function ciMetricLines(metrics: CiPipelineMetrics | null): string[] {
if (metrics === null) {
return [
"- CI/CD 关键指标:",
" - lazy build reused: n/a",
" - reused services: n/a",
" - rebuild services: n/a",
" - service durations: n/a",
];
}
if (metrics.ok !== true) {
return [
"- CI/CD 关键指标:",
` - lazy build reused: unavailable (${metrics.degradedReason ?? "unknown"})`,
" - reused services: n/a",
" - rebuild services: n/a",
" - service durations: n/a",
];
}
const lines = [
"- CI/CD 关键指标:",
` - lazy build reused: ${metrics.reusedCount}/${metrics.serviceTotal}`,
` - reused services: ${metrics.reusedServices.length > 0 ? metrics.reusedServices.join(", ") : "none"}`,
` - rebuild services: ${metrics.rebuildServices.length > 0 ? metrics.rebuildServices.join(", ") : "none"}`,
" - service durations:",
];
for (const service of metrics.services) {
const backend = service.buildBackend === null ? "" : `, backend=${service.buildBackend}`;
lines.push(` - ${service.serviceId}: ${service.status}, ${formatDuration(service.durationSeconds)}${backend}`);
}
return lines;
}
function stripMarkdownInline(value: string): string {
return value
.replace(/\[([^\]]+)\]\([^)]+\)/gu, "$1")
.replace(/\*\*([^*]+)\*\*/gu, "$1")
.replace(/\*([^*]+)\*/gu, "$1")
.replace(/__([^_]+)__/gu, "$1")
.replace(/_([^_]+)_/gu, "$1")
.replace(/\s+/gu, " ")
.trim();
}
function markdownSection(body: string, headings: string[]): string {
const normalizedHeadings = new Set(headings.map((heading) => heading.toLowerCase()));
const lines = body.split(/\r?\n/u);
let capturing = false;
const captured: string[] = [];
for (const line of lines) {
const heading = /^(#{1,6})\s+(.+?)\s*$/u.exec(line);
if (heading !== null) {
const title = stripMarkdownInline(heading[2] ?? "").replace(/[：:]\s*$/u, "").toLowerCase();
if (capturing) break;
capturing = normalizedHeadings.has(title);
continue;
}
if (capturing) captured.push(line);
}
return captured.join("\n").trim();
}
function firstParagraph(section: string): string | null {
for (const paragraph of section.split(/\n\s*\n/u)) {
const trimmed = stripMarkdownInline(paragraph.replace(/^[-*]\s+/gmu, ""));
if (trimmed.length > 0) return trimmed;
}
return null;
}
function topLevelBullets(section: string, limit = 8): string[] {
const bullets: string[] = [];
for (const line of section.split(/\r?\n/u)) {
const match = /^[-*]\s+(.+)$/u.exec(line);
if (match === null) continue;
const text = stripMarkdownInline(match[1] ?? "");
if (text.length > 0) bullets.push(text);
if (bullets.length >= limit) break;
}
return bullets;
}
function conventionalTitleSummary(title: string): string {
const match = /^(feat|fix|docs|test|refactor|perf|ci|build|chore)(?:\([^)]+\))?:\s*(.+)$/iu.exec(title.trim());
if (match === null) return stripMarkdownInline(title);
const kind = String(match[1] ?? "").toLowerCase();
const subject = stripMarkdownInline(match[2] ?? title);
const labels: Record<string, string> = {
feat: "功能",
fix: "修复",
docs: "文档",
test: "测试",
refactor: "重构",
perf: "性能",
ci: "CI/CD",
build: "构建",
chore: "维护",
};
return `${labels[kind] ?? "变更"}：${subject}`;
}
function fileHeuristicBullets(fileSummary: Record<string, unknown>): string[] {
const keyFiles = Array.isArray(fileSummary.keyFiles) ? fileSummary.keyFiles.map(String) : [];
const bullets: string[] = [];
if (keyFiles.some((file) => file.includes("web/hwlab-cloud-web/"))) {
bullets.push("前端：更新 HWLAB Cloud Web / Workbench 交互行为。");
}
if (keyFiles.some((file) => /\.(test|spec)\./u.test(file) || file.includes("/test"))) {
bullets.push("验证：补充或更新回归测试，覆盖本次变更路径。");
}
if (keyFiles.some((file) => file.includes("deploy/") || file.includes("k8s") || file.includes("gitops"))) {
bullets.push("部署：更新 G14 Kubernetes/GitOps 发布声明或发布计划。");
}
if (keyFiles.some((file) => file.includes("internal/cloud/") || file.includes("code-agent"))) {
bullets.push("运行时：调整 Cloud API / Code Agent 运行链路。");
}
return bullets;
}
export function semanticChangelogBullets(title: string, body: string, fileSummary: Record<string, unknown>): string[] {
const bullets: string[] = [];
const background = firstParagraph(markdownSection(body, ["背景", "问题", "problem", "background", "context"]));
if (background !== null && /^(fix|perf)(?:\([^)]+\))?:/iu.test(title)) {
bullets.push(`修复目标：${background}`);
}
const changeSection = markdownSection(body, ["上线 changelog", "changelog", "摘要", "概述", "总结", "修改", "变更", "改动", "changes", "what changed", "summary"]);
bullets.push(...topLevelBullets(changeSection));
if (bullets.length === 0) bullets.push(conventionalTitleSummary(title));
for (const bullet of fileHeuristicBullets(fileSummary)) {
if (bullets.length >= 8) break;
if (!bullets.includes(bullet)) bullets.push(bullet);
}
return bullets.slice(0, 8);
}
function readIssue(issueNumber: number): CommandJsonResult {
return cliJson(["gh", "issue", "read", String(issueNumber), "--repo", HWLAB_REPO, "--json", "title,body,state,updatedAt", "--raw"], 80_000);
}
function issueFromRead(result: CommandJsonResult): Record<string, unknown> {
return record(commandData(result).issue);
}
function parseBriefIssueFromIndex(indexBody: string, date: string): { number: number; url: string } | null {
const escapedDate = date.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const linePattern = new RegExp(`^\\|\\s*${escapedDate}\\s*\\|([^\\n]+)$`, "mu");
const line = linePattern.exec(indexBody)?.[0];
if (line === undefined) return null;
const match = /\[#(\d+)\]\((https:\/\/github\.com\/pikasTech\/HWLAB\/issues\/\d+)\)/u.exec(line);
if (match === null) return null;
return { number: Number(match[1]), url: match[2] };
}
function dailyBriefTitle(date: string): string {
return `${date} 指挥简报（北京时间）`;
}
function writeStateFile(name: string, content: string): string {
const dir = rootPath(".state", "hwlab-g14");
mkdirSync(dir, { recursive: true });
const path = join(dir, name);
writeFileSync(path, content, "utf8");
return path;
}
function listIssues(): CommandJsonResult {
return cliJson(["gh", "issue", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "100", "--json", "number,title,state,url"], 80_000);
}
function findBriefIssueInList(date: string): { number: number; url: string } | null {
const listed = listIssues();
if (!isCommandSuccess(listed)) return null;
const issues = commandData(listed).issues;
if (!Array.isArray(issues)) return null;
const title = dailyBriefTitle(date);
const match = issues.map((item) => record(item)).find((issue) => issue.title === title);
if (match === undefined) return null;
return { number: Number(match.number), url: String(match.url) };
}
function newDailyBriefBody(date: string): string {
return [
`# ${dailyBriefTitle(date)}`,
"",
"## 当日更新",
"",
"## 常驻观察与长期建议",
"",
"- Legacy G14 DEV rollout 已退役；当前 HWLAB runtime 证据必须来自 active runtime lane。",
"- GitHub issue/PR 写操作必须走 UniDesk `gh` 子命令；G14 k3s 操作必须走 UniDesk `trans G14:k3s`。",
].join("\n");
}
function insertBriefIndexRow(indexBody: string, date: string, brief: { number: number; url: string }): string {
if (parseBriefIssueFromIndex(indexBody, date) !== null) return indexBody;
const row = `| ${date} | [#${brief.number}](${brief.url}) | legacy G14 DEV rollout 自动简报入口已退役；当前证据写入 active runtime lane closeout。 | 使用 active runtime lane 的受控 CI/CD 入口和 issue 评论记录。 |`;
const marker = "### 指挥简报索引";
const markerIndex = indexBody.indexOf(marker);
if (markerIndex === -1) return `${indexBody.trimEnd()}\n\n${marker}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n`;
const afterMarker = indexBody.slice(markerIndex);
const separator = "| --- | --- | --- | --- |";
const separatorInBody = afterMarker.indexOf(separator);
if (separatorInBody === -1) return `${indexBody.slice(0, markerIndex + marker.length)}\n\n| 日期（北京时间） | 指挥简报 issue | 当日推进 | 下一步计划 |\n| --- | --- | --- | --- |\n${row}\n${indexBody.slice(markerIndex + marker.length)}`;
const insertAt = markerIndex + separatorInBody + separator.length;
return `${indexBody.slice(0, insertAt)}\n${row}${indexBody.slice(insertAt)}`;
}
function ensureDailyBriefIssue(date: string, dryRun: boolean): Record<string, unknown> {
const indexRead = readIssue(G14_BRIEF_INDEX_ISSUE);
if (!isCommandSuccess(indexRead)) return { ok: false, phase: "read-index", indexRead };
const indexIssue = issueFromRead(indexRead);
const indexBody = String(indexIssue.body ?? "");
let brief = parseBriefIssueFromIndex(indexBody, date) ?? findBriefIssueInList(date);
const actions: Record<string, unknown>[] = [];
if (brief === null) {
const bodyPath = writeStateFile(`daily-brief-${date}.md`, `${newDailyBriefBody(date)}\n`);
if (dryRun) {
brief = { number: 0, url: `https://github.com/pikasTech/HWLAB/issues/new?title=${encodeURIComponent(dailyBriefTitle(date))}` };
actions.push({ action: "would-create-daily-brief", title: dailyBriefTitle(date), bodyPath });
} else {
const created = cliJson(["gh", "issue", "create", "--repo", HWLAB_REPO, "--title", dailyBriefTitle(date), "--body-file", bodyPath], 80_000);
if (!isCommandSuccess(created)) return { ok: false, phase: "create-daily-brief", created };
const createdIssue = record(commandData(created).issue);
brief = { number: Number(createdIssue.number), url: String(createdIssue.url) };
actions.push({ action: "created-daily-brief", issue: brief, bodyPath });
}
}
if (brief.number > 0 && parseBriefIssueFromIndex(indexBody, date) === null) {
const nextBody = insertBriefIndexRow(indexBody, date, brief);
const bodyPath = writeStateFile(`issue-${G14_BRIEF_INDEX_ISSUE}-brief-index-${date}.md`, `${nextBody.trimEnd()}\n`);
if (dryRun) {
actions.push({ action: "would-update-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
} else {
const bodySha = String(indexIssue.bodySha ?? "");
const update = cliJson(["gh", "issue", "update", String(G14_BRIEF_INDEX_ISSUE), "--repo", HWLAB_REPO, "--mode", "replace", "--body-file", bodyPath, ...(bodySha.length > 0 ? ["--expect-body-sha", bodySha] : [])], 100_000);
if (!isCommandSuccess(update)) return { ok: false, phase: "update-brief-index", update, bodyPath };
actions.push({ action: "updated-brief-index", issue: G14_BRIEF_INDEX_ISSUE, bodyPath });
}
}
return { ok: true, date, brief, actions };
}
function readPullRequest(number: number): CommandJsonResult {
return cliJson(["gh", "pr", "read", String(number), "--repo", HWLAB_REPO, "--json", "title,body,url,stateDetail,merged,mergedAt,mergeCommit,headRefName,baseRefName"], 80_000);
}
function readPullRequestFiles(number: number): CommandJsonResult {
return cliJson(["gh", "pr", "files", String(number), "--repo", HWLAB_REPO, "--limit", "30"], 80_000);
}
function summarizePrFiles(filesResult: CommandJsonResult): Record<string, unknown> {
if (!isCommandSuccess(filesResult)) return { ok: false, filesResult };
const data = commandData(filesResult);
const files = Array.isArray(data.files) ? data.files.map((item) => record(item)) : [];
return {
ok: true,
summary: data.summary ?? null,
keyFiles: files.slice(0, 8).map((file) => `${file.status ?? "changed"} ${file.filename ?? ""}`),
};
}
function mergeCommitFromPr(prData: Record<string, unknown>): string | null {
const mergeCommit = record(record(prData.json).mergeCommit);
const fromJson = mergeCommit.oid ?? mergeCommit.sha;
if (typeof fromJson === "string") return fromJson;
const fromSummary = record(record(prData.pullRequest).mergeCommit).oid ?? record(record(prData.pullRequest).mergeCommit).sha;
return typeof fromSummary === "string" ? fromSummary : null;
}
function sourceCommitFromMergeResult(merge: CommandJsonResult, prNumber: number): string | null {
const fromMerge = mergeCommitFromPr(commandData(merge));
if (fromMerge !== null) return fromMerge;
const prRead = readPullRequest(prNumber);
if (!isCommandSuccess(prRead)) return null;
return mergeCommitFromPr(commandData(prRead));
}
async function waitForV02Head(expectedCommit: string | null, timeoutSeconds: number): Promise<Record<string, unknown>> {
const started = Date.now();
let head: string | null = null;
while (Date.now() - started < timeoutSeconds * 1000) {
head = getV02Head();
if (expectedCommit === null) {
if (head !== null) return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
} else if (head === expectedCommit) {
return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
}
printEvent("v02.source.wait", { expectedCommit, head, waitedSeconds: Math.round((Date.now() - started) / 1000) });
await sleep(5_000);
}
return { ok: false, sourceCommit: head, expectedCommit, matched: false, waitedSeconds: Math.round((Date.now() - started) / 1000), degradedReason: "v02-source-head-not-aligned-after-merge" };
}
function currentGitopsRevision(): string | null {
const argo = getArgoStatus();
if (!isCommandSuccess(argo)) return null;
return statusText(argo).split(/\r?\n/u)[0] ?? null;
}
export function rolloutRecordBody(input: {
pr: OpenPullRequest;
prData: Record<string, unknown>;
fileSummary: Record<string, unknown>;
sourceCommit: string;
pipelineRun: string;
gitopsRevision: string | null;
mergedAt: string | null;
pipelineSucceededAt: string | null;
finishedAt: string | null;
rollout: Record<string, unknown>;
ciMetrics?: CiPipelineMetrics | null;
}): string {
const now = beijingParts();
const title = String(record(input.prData.json).title ?? input.pr.title ?? `PR #${input.pr.number}`);
const prBody = String(record(input.prData.json).body ?? record(input.prData.pullRequest).body ?? "");
const url = String(record(input.prData.json).url ?? input.pr.url ?? `https://github.com/pikasTech/HWLAB/pull/${input.pr.number}`);
const summary = record(input.fileSummary.summary);
const keyFiles = Array.isArray(input.fileSummary.keyFiles) ? input.fileSummary.keyFiles.map(String) : [];
const semanticBullets = semanticChangelogBullets(title, prBody, input.fileSummary);
const mergeToPipeline = durationSeconds(input.mergedAt, input.pipelineSucceededAt);
const pipelineToDev = durationSeconds(input.pipelineSucceededAt, input.finishedAt);
const mergeToDev = durationSeconds(input.mergedAt, input.finishedAt);
const metrics = input.ciMetrics ?? ciMetricsFromValue(input.rollout.ciMetrics);
return [
"",
"",
`## 更新 ${now.date} ${now.time.slice(0, 5)} 北京时间`,
"",
`### Retired legacy G14 DEV rollout record：PR #${input.pr.number}`,
"",
`- PR: [#${input.pr.number} ${title}](${url})`,
`- 合并 commit: \`${input.sourceCommit}\``,
`- GitOps revision: \`${input.gitopsRevision ?? "unknown"}\``,
`- PipelineRun: \`${input.pipelineRun}\``,
"- CI/CD 耗时:",
` - merge -> pipeline succeeded: ${formatDuration(mergeToPipeline)}`,
` - pipeline succeeded -> DEV Healthy: ${formatDuration(pipelineToDev)}`,
` - merge -> DEV Healthy: ${formatDuration(mergeToDev)}`,
...ciMetricLines(metrics),
"- 上线 changelog（语义化）:",
...semanticBullets.map((bullet) => ` - ${bullet}`),
"- 自动 diff 摘要:",
` - changed files: ${String(summary.files ?? "n/a")}; +${String(summary.additions ?? "n/a")} / -${String(summary.deletions ?? "n/a")}; commits: ${String(summary.commits ?? "n/a")}`,
...keyFiles.map((file) => ` - ${file}`),
"- DEV 验证:",
` - Tekton: ${String(input.rollout.pipelineText ?? "Succeeded").split(/\r?\n/u).join(" / ")}`,
` - Argo: ${String(input.rollout.argoText ?? "Synced / Healthy").split(/\r?\n/u).slice(0, 5).join(" / ")}`,
` - health/live: ${input.rollout.healthOk === false ? "not-ok" : "ok"}`,
" - workload readiness: Deployment/StatefulSet ready；历史 Completed smoke/debug pod 不作为 DEV rollout blocker。",
].join("\n");
}
function appendRolloutBrief(options: G14RecordRolloutOptions, rollout: Record<string, unknown> = {}): Record<string, unknown> {
const now = beijingParts();
const ensured = ensureDailyBriefIssue(now.date, options.dryRun);
if (record(ensured).ok !== true) return ensured;
const brief = record(record(ensured).brief);
const prRead = readPullRequest(options.prNumber);
if (!isCommandSuccess(prRead)) return { ok: false, phase: "read-pr", prRead };
const prData = commandData(prRead);
const sourceCommit = options.sourceCommit ?? mergeCommitFromPr(prData);
if (sourceCommit === null || sourceCommit === undefined) return { ok: false, phase: "source-commit", message: "source commit unavailable", prData };
const pipelineRun = options.pipelineRun ?? `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
const files = summarizePrFiles(readPullRequestFiles(options.prNumber));
const rolloutGitopsRevision = typeof rollout.gitopsRevision === "string" && rollout.gitopsRevision.length > 0 ? rollout.gitopsRevision : null;
const gitopsRevision = options.gitopsRevision ?? rolloutGitopsRevision ?? currentGitopsRevision();
const prJson = record(prData.json);
const prSummary = record(prData.pullRequest);
const prMergedAt = typeof prJson.mergedAt === "string" && prJson.mergedAt.length > 0
? prJson.mergedAt
: typeof prSummary.mergedAt === "string" && prSummary.mergedAt.length > 0
? prSummary.mergedAt
: null;
const rolloutPipelineSucceededAt = typeof rollout.pipelineSucceededAt === "string" && rollout.pipelineSucceededAt.length > 0 ? rollout.pipelineSucceededAt : null;
const rolloutFinishedAt = typeof rollout.finishedAt === "string" && rollout.finishedAt.length > 0 ? rollout.finishedAt : null;
const mergedAt = options.mergedAt ?? prMergedAt;
const pipelineSucceededAt = options.pipelineSucceededAt ?? rolloutPipelineSucceededAt;
const finishedAt = options.finishedAt ?? rolloutFinishedAt;
const ciMetrics = ciMetricsFromValue(rollout.ciMetrics) ?? collectPipelineMetrics(pipelineRun);
const pr: OpenPullRequest = {
number: options.prNumber,
title: String(record(prData.json).title ?? ""),
url: String(record(prData.json).url ?? ""),
baseRefName: String(record(prData.json).baseRefName ?? ""),
headRefName: String(record(prData.json).headRefName ?? ""),
};
const body = rolloutRecordBody({ pr, prData, fileSummary: files, sourceCommit, pipelineRun, gitopsRevision, mergedAt, pipelineSucceededAt, finishedAt, rollout, ciMetrics });
if (Number(brief.number) <= 0 || options.dryRun) {
return { ok: true, dryRun: true, date: now.date, brief, wouldAppend: { bodyPreview: body.slice(0, 1000), bodyChars: body.length }, ensured };
}
const currentBrief = readIssue(Number(brief.number));
if (!isCommandSuccess(currentBrief)) return { ok: false, phase: "read-daily-brief", currentBrief, brief };
const currentBody = String(issueFromRead(currentBrief).body ?? "");
if (currentBody.includes(`PR #${options.prNumber}`) && currentBody.includes(sourceCommit)) {
return { ok: true, skipped: true, reason: "rollout-brief-already-recorded", brief, sourceCommit, ensured };
}
const bodyPath = writeStateFile(`rollout-pr-${options.prNumber}-${shortSha(sourceCommit)}.md`, `${body}\n`);
const update = cliJson(["gh", "issue", "update", String(brief.number), "--repo", HWLAB_REPO, "--mode", "append", "--body-file", bodyPath, "--body-profile", "commander-brief"], 100_000);
if (!isCommandSuccess(update)) return { ok: false, phase: "append-rollout-brief", update, bodyPath, brief, ensured };
return { ok: true, date: now.date, brief, sourceCommit, pipelineRun, gitopsRevision, ciMetrics, bodyPath, update: commandData(update), ensured };
}
function v02PrCommentStatePath(): string {
return rootPath(".state", "hwlab-g14", "v02-pr-comment-signatures.json");
}
function readV02PrCommentState(): Record<string, string> {
const path = v02PrCommentStatePath();
if (!existsSync(path)) return {};
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
const state: Record<string, string> = {};
for (const [key, value] of Object.entries(record(parsed))) {
if (typeof value === "string") state[key] = value;
}
return state;
} catch {
return {};
}
}
function writeV02PrCommentState(state: Record<string, string>): string {
const path = v02PrCommentStatePath();
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`, "utf8");
return path;
}
function preflightSummary(preflight: CommandJsonResult): Record<string, unknown> {
const data = commandData(preflight);
const mergeability = record(data.mergeability);
return {
ok: isCommandSuccess(preflight),
conclusion: stringOrNull(mergeability.conclusion) ?? (isCommandSuccess(preflight) ? "unknown" : "preflight-command-failed"),
readyForCommanderMerge: mergeability.readyForCommanderMerge === true,
blockers: Array.isArray(mergeability.blockers) ? mergeability.blockers.map(String).slice(0, 20) : [],
pending: Array.isArray(mergeability.pending) ? mergeability.pending.map(String).slice(0, 20) : [],
mergeable: mergeability.mergeable ?? nested(data, ["pullRequest", "mergeable"]) ?? null,
mergeStateStatus: mergeability.mergeStateStatus ?? nested(data, ["pullRequest", "mergeStateStatus"]) ?? null,
headRefName: nested(data, ["pullRequest", "headRefName"]) ?? null,
baseRefName: nested(data, ["pullRequest", "baseRefName"]) ?? null,
statusChecks: record(data.statusChecks).summary ?? data.statusChecks ?? null,
};
}
function v02PrConflictState(summary: Record<string, unknown>): string {
const mergeable = String(summary.mergeable ?? "").toUpperCase();
const mergeStateStatus = String(summary.mergeStateStatus ?? "").toUpperCase();
const blockers = Array.isArray(summary.blockers) ? summary.blockers.map(String).join("\n") : "";
if (mergeable.includes("CONFLICT") || mergeStateStatus.includes("DIRTY") || /conflict/i.test(blockers)) return "conflict";
return "clear-or-unknown";
}
export function summarizeV02CdStatus(status: Record<string, unknown>): Record<string, unknown> {
const pipelineRun = record(status.pipelineRun);
const targetValidation = record(status.targetValidation);
const gitMirror = record(status.gitMirror);
const gitMirrorSummary = record(gitMirror.summary);
const planArtifacts = record(status.planArtifacts);
const planSummary = record(planArtifacts.summary);
const argo = record(status.argo);
const webAssets = record(status.webAssets);
return {
ok: status.ok === true,
sourceCommit: status.sourceCommit ?? nested(status, ["statusTarget", "sourceCommit"]) ?? null,
pipelineRun: pipelineRun.pipelineRun ?? pipelineRun.name ?? nested(status, ["statusTarget", "pipelineRun"]) ?? null,
pipelineStatus: pipelineRun.status ?? null,
pipelineReason: pipelineRun.reason ?? null,
targetValidationState: targetValidation.state ?? null,
targetValidationOk: targetValidation.ok ?? null,
targetValidationFailures: Array.isArray(targetValidation.failures) ? targetValidation.failures.slice(0, 10) : [],
targetValidationWarnings: Array.isArray(targetValidation.warnings) ? targetValidation.warnings.slice(0, 10) : [],
argoSync: nested(argo, ["fields", "syncStatus"]) ?? argo.syncStatus ?? nested(argo, ["summary", "syncStatus"]) ?? null,
argoHealth: nested(argo, ["fields", "health"]) ?? argo.healthStatus ?? nested(argo, ["summary", "healthStatus"]) ?? null,
webAssetsOk: webAssets.ok ?? null,
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
githubInSync: gitMirrorSummary.githubInSync ?? null,
buildServices: planSummary.buildServices ?? planArtifacts.buildServices ?? null,
reusedServices: planSummary.reusedServices ?? planArtifacts.reusedServices ?? null,
rolloutServices: planSummary.rolloutServices ?? planArtifacts.rolloutServices ?? null,
};
}
function v02CdPassed(status: Record<string, unknown>): boolean {
const state = String(nested(status, ["targetValidation", "state"]) ?? "");
return state === "passed" || state === "superseded";
}
function v02ControlPlaneCloseout(status: Record<string, unknown>): Record<string, unknown> {
const closeout = record(status.closeout);
const sourceCommit = stringOrNull(status.sourceCommit);
const pipelineRun = stringOrNull(nested(status, ["statusTarget", "pipelineRun"])) ?? stringOrNull(nested(status, ["pipelineRun", "pipelineRun"]));
const command = sourceCommit !== null
? `hwlab g14 control-plane closeout --lane v02 --source-commit ${sourceCommit}`
: pipelineRun !== null
? `hwlab g14 control-plane closeout --lane v02 --pipeline-run ${pipelineRun}`
: "hwlab g14 control-plane closeout --lane v02";
return {
ok: closeout.closeable === true,
command,
lane: "v02",
closeout,
target: status.statusTarget ?? null,
sourceCommit: status.sourceCommit ?? null,
pipelineRun: status.pipelineRun ?? null,
targetValidation: status.targetValidation ?? null,
latestHead: nested(status, ["commitAlignment", "originHead"]) ?? nested(status, ["sourceHeads", "originHead"]) ?? null,
gitMirror: record(record(status.gitMirror).summary),
argo: record(status.argo).fields ?? null,
activePipelineRuns: status.activePipelineRuns ?? [],
recommendedNext: closeout.recommendedNext ?? null,
issueCommentMarkdown: closeout.issueCommentMarkdown ?? null,
};
}
function v02PipelineSucceeded(status: Record<string, unknown>): boolean {
return String(nested(status, ["pipelineRun", "status"]) ?? "") === "True";
}
function v02CdFailed(status: Record<string, unknown>): boolean {
const pipelineStatus = String(nested(status, ["pipelineRun", "status"]) ?? "");
return pipelineStatus === "False";
}
export function activeV02PipelineRuns(status: Record<string, unknown>): Record<string, unknown>[] {
const items = Array.isArray(status.activePipelineRuns)
? status.activePipelineRuns
: Array.isArray(record(status.activePipelineRuns).items)
? record(status.activePipelineRuns).items
: [];
return items.map((item) => record(item)).filter((item) => String(item.status ?? "") === "Unknown");
}
export function v02StatusHistoryPolicy(status: Record<string, unknown>): Record<string, unknown> {
const history = record(status.history);
return {
included: history.included === true,
recentPipelineRunsVisible: Object.hasOwn(status, "recentPipelineRuns"),
activePipelineRunsVisible: Object.hasOwn(status, "activePipelineRuns"),
note: history.note ?? null,
command: history.command ?? null,
};
}
function v02PrCommentSignature(input: V02PrCommentInput): string {
const summary = input.preflight ?? {};
const cd = input.cd ?? {};
const flush = input.flush ?? {};
return textHash(JSON.stringify({
pr: input.pr.number,
state: input.state,
phase: input.phase,
conclusion: summary.conclusion ?? null,
readyForCommanderMerge: summary.readyForCommanderMerge ?? null,
conflict: v02PrConflictState(summary),
blockers: summary.blockers ?? [],
pending: summary.pending ?? [],
sourceCommit: input.sourceCommit ?? null,
pipelineRun: input.pipelineRun ?? null,
pipelineStatus: cd.pipelineStatus ?? null,
pipelineReason: cd.pipelineReason ?? null,
targetValidationState: cd.targetValidationState ?? null,
pendingFlush: cd.pendingFlush ?? null,
flushOk: flush.ok ?? null,
dryRun: input.dryRun === true,
}));
}
function listValue(value: unknown): string {
if (!Array.isArray(value) || value.length === 0) return "none";
return value.map(String).slice(0, 8).join("; ");
}
function scalarValue(value: unknown): string {
if (value === null || value === undefined || value === "") return "n/a";
if (typeof value === "object") return JSON.stringify(value).slice(0, 240);
return String(value);
}
export function v02PrAutomationCommentBody(input: V02PrCommentInput): string {
const preflight = input.preflight ?? {};
const cd = input.cd ?? {};
const flush = input.flush ?? {};
const title = input.pr.title ?? `PR #${input.pr.number}`;
const url = input.pr.url ?? `https://github.com/${HWLAB_REPO}/pull/${input.pr.number}`;
const elapsed = formatDuration(input.elapsedSeconds);
const dryRunPrefix = input.dryRun === true ? "[dry-run] " : "";
return [
`### ${dryRunPrefix}v0.2 自动 CI/CD 状态`,
"",
`${input.message ?? "UniDesk monitor 已记录本轮 PR 自动化状态。"}`,
"",
"- PR:",
` - [#${input.pr.number} ${title}](${url})`,
` - base: \`${input.pr.baseRefName ?? V02_SOURCE_BRANCH}\`; head: \`${input.pr.headRefName ?? "unknown"}\``,
"- 自动化状态:",
` - state: \`${input.state}\`; phase: \`${input.phase}\``,
` - startedAt: \`${input.startedAt}\`; observedAt: \`${input.observedAt}\`; elapsed: ${elapsed}`,
"- CI / mergeability:",
` - conclusion: \`${scalarValue(preflight.conclusion)}\`; readyForCommanderMerge: \`${String(preflight.readyForCommanderMerge ?? "n/a")}\`; conflict: \`${v02PrConflictState(preflight)}\``,
` - blockers: ${listValue(preflight.blockers)}`,
` - pending: ${listValue(preflight.pending)}`,
"- CD / runtime:",
` - sourceCommit: \`${input.sourceCommit ?? "n/a"}\``,
` - PipelineRun: \`${input.pipelineRun ?? scalarValue(cd.pipelineRun)}\``,
` - pipeline: \`${scalarValue(cd.pipelineStatus)} / ${scalarValue(cd.pipelineReason)}\`; targetValidation: \`${scalarValue(cd.targetValidationState)}\``,
` - Argo: \`${scalarValue(cd.argoSync)} / ${scalarValue(cd.argoHealth)}\`; webAssets.ok: \`${String(cd.webAssetsOk ?? "n/a")}\``,
` - Git mirror: pendingFlush=\`${String(cd.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(cd.githubInSync ?? "n/a")}\`, flush.ok=\`${String(flush.ok ?? "n/a")}\``,
` - rolloutServices: ${listValue(cd.rolloutServices)}; buildServices: ${listValue(cd.buildServices)}; reusedServices: ${listValue(cd.reusedServices)}`,
"",
"后续动作由 `bun scripts/cli.ts hwlab g14 monitor-prs --lane v02` 继续驱动；同一状态签名不会重复刷评论。",
].join("\n");
}
function commentV02PullRequest(input: V02PrCommentInput): Record<string, unknown> {
const body = v02PrAutomationCommentBody(input);
const signature = v02PrCommentSignature(input);
const cacheKey = `pr-${input.pr.number}`;
if (input.dryRun === true) {
return { ok: true, dryRun: true, skipped: true, signature, wouldComment: { bodyPreview: body.slice(0, 1200), bodyChars: body.length } };
}
const state = readV02PrCommentState();
if (state[cacheKey] === signature) {
return { ok: true, skipped: true, reason: "same-pr-comment-signature", signature, stateFile: v02PrCommentStatePath() };
}
const bodyPath = writeStateFile(`v02-pr-${input.pr.number}-${input.state}-${signature}.md`, `${body}\n`);
const create = cliJson(["gh", "pr", "comment", "create", String(input.pr.number), "--repo", HWLAB_REPO, "--body-file", bodyPath], 100_000);
if (!isCommandSuccess(create)) return { ok: false, phase: "pr-comment", signature, bodyPath, create };
state[cacheKey] = signature;
const stateFile = writeV02PrCommentState(state);
return { ok: true, signature, bodyPath, stateFile, create: commandData(create) };
}
function runtimeLanePrCommentStatePath(spec: HwlabRuntimeLaneSpec): string {
return rootPath(".state", "hwlab-g14", `${spec.lane}-pr-comment-signatures.json`);
}
function readRuntimeLanePrCommentState(spec: HwlabRuntimeLaneSpec): Record<string, string> {
const path = runtimeLanePrCommentStatePath(spec);
if (!existsSync(path)) return {};
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
const state: Record<string, string> = {};
for (const [key, value] of Object.entries(record(parsed))) {
if (typeof value === "string") state[key] = value;
}
return state;
} catch {
return {};
}
}
function writeRuntimeLanePrCommentState(spec: HwlabRuntimeLaneSpec, state: Record<string, string>): string {
const path = runtimeLanePrCommentStatePath(spec);
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`, "utf8");
return path;
}
function summarizeRuntimeLaneCdStatus(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>): Record<string, unknown> {
const pipelineRun = record(status.pipelineRun);
const argo = record(status.argo);
const runtimeWorkloads = record(status.runtimeWorkloads);
const publicProbes = record(status.publicProbes);
const gitMirror = record(status.gitMirror);
const gitMirrorSummary = record(gitMirror.summary);
return {
ok: status.ok === true,
sourceCommit: status.sourceCommit ?? nested(status, ["statusTarget", "sourceCommit"]) ?? null,
pipelineRun: pipelineRun.pipelineRun ?? pipelineRun.name ?? nested(status, ["statusTarget", "pipelineRun"]) ?? null,
pipelineStatus: pipelineRun.status ?? null,
pipelineReason: pipelineRun.reason ?? null,
argoSync: nested(argo, ["fields", "syncStatus"]) ?? null,
argoHealth: nested(argo, ["fields", "health"]) ?? null,
runtimeNamespace: spec.runtimeNamespace,
runtimeWorkloadsOk: runtimeWorkloads.ok ?? null,
runtimeWorkloadCount: runtimeWorkloads.count ?? null,
publicProbesOk: publicProbes.ok ?? null,
publicApiUrl: publicProbes.apiUrl ?? `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`,
publicWebUrl: publicProbes.webUrl ?? spec.publicWebUrl,
publicApiStatusOk: publicProbes.apiStatusOk ?? null,
publicWebBytes: publicProbes.webBytes ?? null,
pendingFlush: gitMirrorSummary.pendingFlush ?? null,
githubInSync: spec.lane === "v03" ? gitMirrorSummary.v03GitopsInSync ?? null : gitMirrorSummary.githubInSync ?? null,
};
}
function runtimeLaneCdPassed(status: Record<string, unknown>): boolean {
return status.ok === true
&& String(nested(status, ["pipelineRun", "status"]) ?? "") === "True"
&& String(nested(status, ["argo", "fields", "syncStatus"]) ?? "") === "Synced"
&& String(nested(status, ["argo", "fields", "health"]) ?? "") === "Healthy"
&& nested(status, ["runtimeWorkloads", "ok"]) === true
&& nested(status, ["publicProbes", "ok"]) === true;
}
function runtimeLaneCdFailed(status: Record<string, unknown>): boolean {
return String(nested(status, ["pipelineRun", "status"]) ?? "") === "False";
}
function runtimeLanePrCommentSignature(spec: HwlabRuntimeLaneSpec, input: V02PrCommentInput): string {
const summary = input.preflight ?? {};
const cd = input.cd ?? {};
const flush = input.flush ?? {};
return textHash(JSON.stringify({
lane: spec.lane,
pr: input.pr.number,
state: input.state,
phase: input.phase,
conclusion: summary.conclusion ?? null,
readyForCommanderMerge: summary.readyForCommanderMerge ?? null,
conflict: v02PrConflictState(summary),
blockers: summary.blockers ?? [],
pending: summary.pending ?? [],
sourceCommit: input.sourceCommit ?? null,
pipelineRun: input.pipelineRun ?? null,
pipelineStatus: cd.pipelineStatus ?? null,
pipelineReason: cd.pipelineReason ?? null,
argoSync: cd.argoSync ?? null,
argoHealth: cd.argoHealth ?? null,
publicProbesOk: cd.publicProbesOk ?? null,
flushOk: flush.ok ?? null,
dryRun: input.dryRun === true,
}));
}
function runtimeLanePrAutomationCommentBody(spec: HwlabRuntimeLaneSpec, input: V02PrCommentInput): string {
const preflight = input.preflight ?? {};
const cd = input.cd ?? {};
const flush = input.flush ?? {};
const title = input.pr.title ?? `PR #${input.pr.number}`;
const url = input.pr.url ?? `https://github.com/${HWLAB_REPO}/pull/${input.pr.number}`;
const elapsed = formatDuration(input.elapsedSeconds);
const dryRunPrefix = input.dryRun === true ? "[dry-run] " : "";
return [
`### ${dryRunPrefix}${spec.version} 自动 CI/CD 状态`,
"",
`${input.message ?? "UniDesk monitor 已记录本轮 PR 自动化状态。"}`,
"",
"- PR:",
` - [#${input.pr.number} ${title}](${url})`,
` - base: \`${input.pr.baseRefName ?? spec.sourceBranch}\`; head: \`${input.pr.headRefName ?? "unknown"}\``,
"- 自动化状态:",
` - lane: \`${spec.lane}\`; state: \`${input.state}\`; phase: \`${input.phase}\``,
` - startedAt: \`${input.startedAt}\`; observedAt: \`${input.observedAt}\`; elapsed: ${elapsed}`,
"- CI / mergeability:",
` - conclusion: \`${scalarValue(preflight.conclusion)}\`; readyForCommanderMerge: \`${String(preflight.readyForCommanderMerge ?? "n/a")}\`; conflict: \`${v02PrConflictState(preflight)}\``,
` - blockers: ${listValue(preflight.blockers)}`,
` - pending: ${listValue(preflight.pending)}`,
"- CD / runtime:",
` - sourceCommit: \`${input.sourceCommit ?? "n/a"}\``,
` - PipelineRun: \`${input.pipelineRun ?? scalarValue(cd.pipelineRun)}\``,
` - pipeline: \`${scalarValue(cd.pipelineStatus)} / ${scalarValue(cd.pipelineReason)}\``,
` - Argo: \`${scalarValue(cd.argoSync)} / ${scalarValue(cd.argoHealth)}\`; runtimeWorkloads.ok: \`${String(cd.runtimeWorkloadsOk ?? "n/a")}\``,
` - public probes: ok=\`${String(cd.publicProbesOk ?? "n/a")}\`, api=\`${scalarValue(cd.publicApiUrl)}\`, webBytes=\`${scalarValue(cd.publicWebBytes)}\``,
` - Git mirror: pendingFlush=\`${String(cd.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(cd.githubInSync ?? "n/a")}\`, flush.ok=\`${String(flush.ok ?? "n/a")}\``,
"",
`后续动作由 \`bun scripts/cli.ts hwlab g14 monitor-prs --lane ${spec.lane}\` 继续驱动；同一状态签名不会重复刷评论。`,
].join("\n");
}
function commentRuntimeLanePullRequest(spec: HwlabRuntimeLaneSpec, input: V02PrCommentInput): Record<string, unknown> {
const body = runtimeLanePrAutomationCommentBody(spec, input);
const signature = runtimeLanePrCommentSignature(spec, input);
const cacheKey = `pr-${input.pr.number}`;
if (input.dryRun === true) {
return { ok: true, dryRun: true, skipped: true, signature, wouldComment: { bodyPreview: body.slice(0, 1200), bodyChars: body.length } };
}
const state = readRuntimeLanePrCommentState(spec);
if (state[cacheKey] === signature) {
return { ok: true, skipped: true, reason: "same-pr-comment-signature", signature, stateFile: runtimeLanePrCommentStatePath(spec) };
}
const bodyPath = writeStateFile(`${spec.lane}-pr-${input.pr.number}-${input.state}-${signature}.md`, `${body}\n`);
const create = cliJson(["gh", "pr", "comment", "create", String(input.pr.number), "--repo", HWLAB_REPO, "--body-file", bodyPath], 100_000);
if (!isCommandSuccess(create)) return { ok: false, phase: "pr-comment", signature, bodyPath, create };
state[cacheKey] = signature;
const stateFile = writeRuntimeLanePrCommentState(spec, state);
return { ok: true, signature, bodyPath, stateFile, create: commandData(create) };
}
function runtimeLaneAutomationIssueTitle(spec: HwlabRuntimeLaneSpec, pr: OpenPullRequest, state: string, phase: string): string {
return `[${spec.version} 自动 CI/CD] PR #${pr.number} ${state}: ${phase}`;
}
function runtimeLaneAutomationIssueBody(input: {
spec: HwlabRuntimeLaneSpec;
pr: OpenPullRequest;
state: string;
phase: string;
startedAt: string;
observedAt: string;
preflight?: Record<string, unknown>;
sourceCommit?: string | null;
pipelineRun?: string | null;
cd?: Record<string, unknown>;
details?: Record<string, unknown>;
message?: string;
}): string {
const preflight = input.preflight ?? {};
const cd = input.cd ?? {};
const details = input.details ?? {};
const title = input.pr.title ?? `PR #${input.pr.number}`;
const url = input.pr.url ?? `https://github.com/${HWLAB_REPO}/pull/${input.pr.number}`;
const dedupeKey = `${input.spec.lane}/pr-${input.pr.number}/${input.state}/${input.phase}/${input.sourceCommit ?? "no-source"}/${input.pipelineRun ?? "no-run"}`;
return [
`## ${input.spec.version} 自动 CI/CD 阻塞`,
"",
input.message ?? "UniDesk v0.3 PR monitor 发现自动 CI/CD 阻塞，已按 #1072 的失败沉淀规则创建或更新 issue。",
"",
"- 来源:",
` - PR: [#${input.pr.number} ${title}](${url})`,
` - base: \`${input.pr.baseRefName ?? input.spec.sourceBranch}\`; head: \`${input.pr.headRefName ?? "unknown"}\``,
` - lane: \`${input.spec.lane}\`; runtime namespace: \`${input.spec.runtimeNamespace}\``,
` - tracking issue: #1072`,
"- 阶段:",
` - state: \`${input.state}\`; phase: \`${input.phase}\``,
` - startedAt: \`${input.startedAt}\`; observedAt: \`${input.observedAt}\``,
` - dedupeKey: \`${dedupeKey}\``,
"- CI / mergeability:",
` - conclusion: \`${scalarValue(preflight.conclusion)}\`; readyForCommanderMerge: \`${String(preflight.readyForCommanderMerge ?? "n/a")}\`; conflict: \`${v02PrConflictState(preflight)}\``,
` - blockers: ${listValue(preflight.blockers)}`,
` - pending: ${listValue(preflight.pending)}`,
"- CD / runtime:",
` - sourceCommit: \`${input.sourceCommit ?? "n/a"}\``,
` - PipelineRun: \`${input.pipelineRun ?? scalarValue(cd.pipelineRun)}\``,
` - pipeline: \`${scalarValue(cd.pipelineStatus)} / ${scalarValue(cd.pipelineReason)}\``,
` - Argo: \`${scalarValue(cd.argoSync)} / ${scalarValue(cd.argoHealth)}\`; publicProbes.ok: \`${String(cd.publicProbesOk ?? "n/a")}\``,
"- 下一步:",
` - PR preflight: \`bun scripts/cli.ts gh pr preflight ${input.pr.number} --repo ${HWLAB_REPO}\``,
` - monitor once: \`bun scripts/cli.ts hwlab g14 monitor-prs --lane ${input.spec.lane} --once\``,
input.sourceCommit !== null && input.sourceCommit !== undefined
? ` - lane status: \`bun scripts/cli.ts hwlab g14 control-plane status --lane ${input.spec.lane} --source-commit ${input.sourceCommit}\``
: ` - lane status: \`bun scripts/cli.ts hwlab g14 control-plane status --lane ${input.spec.lane}\``,
"",
"## 细节摘要",
"",
"```json",
JSON.stringify({ preflight, cd, details }, null, 2).slice(0, 5000),
"```",
].join("\n");
}
function reportRuntimeLaneAutomationIssue(input: {
spec: HwlabRuntimeLaneSpec;
pr: OpenPullRequest;
state: string;
phase: string;
startedAt: string;
observedAt: string;
preflight?: Record<string, unknown>;
sourceCommit?: string | null;
pipelineRun?: string | null;
cd?: Record<string, unknown>;
details?: Record<string, unknown>;
message?: string;
dryRun?: boolean;
}): Record<string, unknown> {
const title = runtimeLaneAutomationIssueTitle(input.spec, input.pr, input.state, input.phase);
const body = runtimeLaneAutomationIssueBody(input);
const signature = textHash(body);
if (input.dryRun === true) {
return { ok: true, dryRun: true, title, signature, wouldIssue: { bodyPreview: body.slice(0, 1200), bodyChars: body.length } };
}
const listed = cliJson(["gh", "issue", "list", "--repo", HWLAB_REPO, "--state", "open", "--limit", "10", "--search", title, "--json", "number,title,state,url"], 80_000);
if (!isCommandSuccess(listed)) return { ok: false, phase: "issue-search", title, listed };
const issues = Array.isArray(commandData(listed).issues) ? commandData(listed).issues.map((item) => record(item)) : [];
const existing = issues.find((issue) => issue.title === title);
const bodyPath = writeStateFile(`${input.spec.lane}-automation-issue-pr-${input.pr.number}-${input.state}-${input.phase}-${signature}.md`, `${body}\n`);
if (existing !== undefined) {
const comment = cliJson(["gh", "issue", "comment", "create", String(existing.number), "--repo", HWLAB_REPO, "--body-file", bodyPath], 100_000);
if (!isCommandSuccess(comment)) return { ok: false, phase: "issue-comment", title, issue: existing, bodyPath, comment };
return { ok: true, action: "commented-existing-issue", title, issue: existing, bodyPath, comment: commandData(comment) };
}
const created = cliJson(["gh", "issue", "create", "--repo", HWLAB_REPO, "--title", title, "--body-file", bodyPath], 100_000);
if (!isCommandSuccess(created)) return { ok: false, phase: "issue-create", title, bodyPath, created };
return { ok: true, action: "created-issue", title, bodyPath, issue: commandData(created).issue, created: commandData(created) };
}
function triggerV02Current(timeoutSeconds: number): Record<string, unknown> {
return runV02ControlPlane({
action: "trigger-current",
lane: "v02",
dryRun: false,
confirm: true,
wait: true,
allowLiveDbRead: false,
timeoutSeconds,
minAgeMinutes: 30,
limit: 20,
});
}
function flushV02GitMirrorIfNeeded(status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
const pendingFlush = nested(status, ["gitMirror", "summary", "pendingFlush"]);
if (pendingFlush !== true) return { ok: true, skipped: true, reason: "git-mirror-already-flushed" };
return runG14GitMirror({
action: "flush",
lane: "v02",
confirm: true,
dryRun: false,
wait: true,
timeoutSeconds,
});
}
function triggerRuntimeLaneCurrent(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
return runRuntimeLaneControlPlane(spec, {
action: "trigger-current",
lane: spec.lane,
dryRun: false,
confirm: true,
wait: true,
allowLiveDbRead: false,
timeoutSeconds,
minAgeMinutes: 30,
limit: 20,
history: false,
});
}
function runtimeLaneStatusWithGitMirror(spec: HwlabRuntimeLaneSpec, sourceCommit: string): Record<string, unknown> {
const status = runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "source-commit" });
const gitMirrorStatus = runGitMirrorStatus();
return {
...status,
gitMirror: record(gitMirrorStatus.cache),
gitMirrorStatus,
};
}
function runtimeLaneGitopsSynced(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>): boolean {
const summary = record(nested(status, ["gitMirror", "summary"]));
const inSync = spec.lane === "v03" ? summary.v03GitopsInSync : summary.githubInSync;
return summary.pendingFlush !== true && inSync === true;
}
function runtimeLaneCdPassedWithMirror(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>): boolean {
return runtimeLaneCdPassed(status) && runtimeLaneGitopsSynced(spec, status);
}
function flushRuntimeLaneGitMirrorIfNeeded(spec: HwlabRuntimeLaneSpec, status: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> {
if (runtimeLaneGitopsSynced(spec, status)) return { ok: true, skipped: true, reason: `${spec.lane}-git-mirror-already-flushed` };
return runG14GitMirror({
action: "flush",
lane: spec.lane,
confirm: true,
dryRun: false,
wait: true,
timeoutSeconds,
});
}
function pullRequestFromMergeCommand(merge: CommandJsonResult): Record<string, unknown> {
const data = commandData(merge);
const direct = record(data.pullRequest);
if (Object.keys(direct).length > 0) return direct;
return record(nested(data, ["details", "details", "pullRequest"]));
}
function mergeCommandRaceState(merge: CommandJsonResult): "merged" | "closed" | null {
const pullRequest = pullRequestFromMergeCommand(merge);
if (pullRequest.merged === true || pullRequest.stateDetail === "merged") return "merged";
if (pullRequest.closed === true || pullRequest.state === "closed") return "closed";
return null;
}
async function waitForRuntimeLaneHead(spec: HwlabRuntimeLaneSpec, expectedCommit: string | null, timeoutSeconds: number): Promise<Record<string, unknown>> {
const started = Date.now();
let head: string | null = null;
while (Date.now() - started < timeoutSeconds * 1000) {
head = resolveRuntimeLaneHead(spec).sourceCommit;
if (expectedCommit === null) {
if (head !== null) return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
} else if (head === expectedCommit) {
return { ok: true, sourceCommit: head, expectedCommit, matched: true, waitedSeconds: Math.round((Date.now() - started) / 1000) };
}
printEvent(`${spec.lane}.source.wait`, { lane: spec.lane, expectedCommit, head, waitedSeconds: Math.round((Date.now() - started) / 1000) });
await sleep(5_000);
}
return { ok: false, sourceCommit: head, expectedCommit, matched: false, waitedSeconds: Math.round((Date.now() - started) / 1000), degradedReason: `${spec.lane}-source-head-not-aligned-after-merge` };
}
async function waitForV02Cd(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
const started = Date.now();
const startedAt = new Date(started).toISOString();
const pipelineRun = v02PipelineRunName(sourceCommit);
let lastStatus: Record<string, unknown> = {};
let firstPassedAt: string | null = null;
while (Date.now() - started < timeoutSeconds * 1000) {
lastStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
const summary = summarizeV02CdStatus(lastStatus);
printEvent("v02.cd.status", { sourceCommit, pipelineRun, summary });
printV02PrMonitorProgress({ stage: "cd-status", status: "running", sourceCommit, pipelineRun, targetValidationState: summary.targetValidationState ?? null, pipelineStatus: summary.pipelineStatus ?? null, pendingFlush: summary.pendingFlush ?? null });
if (v02PipelineSucceeded(lastStatus) && summary.pendingFlush === true) {
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
if (record(flush).ok !== true) {
return {
ok: false,
phase: "git-mirror-flush",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
status: summary,
rawStatus: lastStatus,
flush,
};
}
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
const finalSummary = summarizeV02CdStatus(finalStatus);
if (v02CdPassed(finalStatus)) {
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: "succeeded", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
return {
ok: true,
phase: "cd-passed",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
status: finalSummary,
rawStatus: finalStatus,
flush,
};
}
lastStatus = finalStatus;
printEvent("v02.cd.after-flush", { sourceCommit, pipelineRun, flushOk: record(flush).ok, summary: finalSummary });
printV02PrMonitorProgress({ stage: "git-mirror-flush", status: record(flush).ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun, pendingFlush: finalSummary.pendingFlush ?? null });
}
if (v02CdPassed(lastStatus)) {
firstPassedAt ??= new Date().toISOString();
const flush = flushV02GitMirrorIfNeeded(lastStatus, Math.min(timeoutSeconds, 600));
const finalStatus = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
printV02PrMonitorProgress({ stage: "cd-status", status: v02CdPassed(finalStatus) ? "succeeded" : "failed", sourceCommit, pipelineRun, targetValidationState: nested(finalStatus, ["targetValidation", "state"]) ?? null });
return {
ok: v02CdPassed(finalStatus),
phase: v02CdPassed(finalStatus) ? "cd-passed" : "git-mirror-flush",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
status: summarizeV02CdStatus(finalStatus),
rawStatus: finalStatus,
flush,
};
}
if (v02CdFailed(lastStatus)) {
printV02PrMonitorProgress({ stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
return {
ok: false,
phase: "cd-failed",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
status: summary,
rawStatus: lastStatus,
};
}
await sleep(30_000);
}
return {
ok: false,
phase: "timeout",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
timeoutSeconds,
firstPassedAt,
status: summarizeV02CdStatus(lastStatus),
rawStatus: lastStatus,
};
}
async function waitForRuntimeLaneCd(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
const started = Date.now();
const startedAt = new Date(started).toISOString();
let pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit);
let lastStatus: Record<string, unknown> = {};
let firstPassedAt: string | null = null;
while (Date.now() - started < timeoutSeconds * 1000) {
lastStatus = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
const summary = summarizeRuntimeLaneCdStatus(spec, lastStatus);
pipelineRun = stringOrNull(summary.pipelineRun) ?? pipelineRun;
printEvent(`${spec.lane}.cd.status`, { lane: spec.lane, sourceCommit, pipelineRun, summary });
printRuntimeLanePrMonitorProgress(spec, {
stage: "cd-status",
status: "running",
sourceCommit,
pipelineRun,
pipelineStatus: summary.pipelineStatus ?? null,
pipelineReason: summary.pipelineReason ?? null,
argoSync: summary.argoSync ?? null,
argoHealth: summary.argoHealth ?? null,
publicProbesOk: summary.publicProbesOk ?? null,
pendingFlush: summary.pendingFlush ?? null,
});
if (runtimeLaneCdPassed(lastStatus)) {
firstPassedAt ??= new Date().toISOString();
const flush = flushRuntimeLaneGitMirrorIfNeeded(spec, lastStatus, Math.min(timeoutSeconds, 600));
if (record(flush).ok !== true) {
return {
ok: false,
phase: "git-mirror-flush",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
status: summary,
rawStatus: lastStatus,
flush,
};
}
const finalStatus = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
const finalSummary = summarizeRuntimeLaneCdStatus(spec, finalStatus);
printRuntimeLanePrMonitorProgress(spec, {
stage: "git-mirror-flush",
status: runtimeLaneCdPassedWithMirror(spec, finalStatus) ? "succeeded" : "failed",
sourceCommit,
pipelineRun,
pendingFlush: finalSummary.pendingFlush ?? null,
});
return {
ok: runtimeLaneCdPassedWithMirror(spec, finalStatus),
phase: runtimeLaneCdPassedWithMirror(spec, finalStatus) ? "cd-passed" : "git-mirror-flush",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
status: finalSummary,
rawStatus: finalStatus,
flush,
};
}
if (runtimeLaneCdFailed(lastStatus)) {
printRuntimeLanePrMonitorProgress(spec, { stage: "cd-status", status: "failed", sourceCommit, pipelineRun, pipelineStatus: summary.pipelineStatus ?? null, pipelineReason: summary.pipelineReason ?? null });
return {
ok: false,
phase: "cd-failed",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
status: summary,
rawStatus: lastStatus,
};
}
await sleep(30_000);
}
return {
ok: false,
phase: "timeout",
startedAt,
finishedAt: new Date().toISOString(),
sourceCommit,
pipelineRun,
timeoutSeconds,
firstPassedAt,
status: summarizeRuntimeLaneCdStatus(spec, lastStatus),
rawStatus: lastStatus,
};
}
async function runV02PrAutoCd(pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
printV02PrMonitorProgress({ stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
const comment = commentV02PullRequest({
pr,
phase: "merge",
state: "merge-failed",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: compactCommandResult(merge),
dryRun: options.dryRun,
message: mergeRaceState === "closed"
? "PR preflight 曾通过，但自动合并前 PR 已被关闭且未确认 merged；本轮不会触发 CD。"
: "PR preflight 已通过，但自动合并失败；需要处理 GitHub 返回的 merge blocker 后重试。",
});
return { ok: mergeRaceState === "closed", phase: "merge", action: "merge-race-closed", pr, merge, comment };
}
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
if (!options.dryRun && expectedCommit === null) {
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
const comment = commentV02PullRequest({
pr,
phase: "merge-commit",
state: "cd-failed",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit: null,
pipelineRun: null,
dryRun: false,
message: "PR 已合并，但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit；为避免触发错误 source head，本轮未启动 CD。",
});
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, degradedReason: "merge-commit-unresolved" };
}
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForV02Head(expectedCommit, 120);
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
printV02PrMonitorProgress({ stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
const startedComment = commentV02PullRequest({
pr,
phase: options.dryRun ? "dry-run" : "cd-trigger",
state: "cd-started",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
dryRun: options.dryRun,
message: options.dryRun
? "dry-run：PR 已达到自动合并条件；本轮不会写 GitHub merge 或 CD。"
: mergeRaceState === "merged"
? "PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 v0.2 CD。后续成功、失败或超时会继续在本 PR 下回复。"
: "PR 已自动合并，v0.2 CD 准备开始；其他 commit 的运行中 PipelineRun 不会阻塞本轮 CI，旧 run 若发现 source head 已推进会以 superseded/no-op 收口。后续成功、superseded、失败或超时会继续在本 PR 下回复。",
});
if (!options.dryRun && record(startedComment).ok !== true) {
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
}
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
if (sourceCommit === null || record(headWait).ok !== true) {
printV02PrMonitorProgress({ stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
const comment = commentV02PullRequest({
pr,
phase: "v02-source-head",
state: "cd-timeout",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
dryRun: false,
message: "PR 已合并，但 G14 v0.2 CI/CD source repo 没有在等待窗口内对齐 merge commit；为避免触发旧 head，本轮未启动 CD。",
});
return { ok: false, phase: "v02-head", pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, degradedReason: "v02-head-unresolved-after-merge" };
}
const before = v02ControlPlaneStatus({ sourceCommit, mode: "source-commit" });
const beforeSummary = summarizeV02CdStatus(before);
const activeRuns = activeV02PipelineRuns(before);
if (activeRuns.length > 0) {
printEvent("v02.cd.active-runs-observed", { pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, activeRuns: activeRuns.slice(0, 5), latestOnlyPolicy: "do-not-wait-or-cancel-old-runs" });
}
const trigger = v02CdPassed(before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerV02Current(Math.min(options.timeoutSeconds, 600));
printEvent("v02.cd.trigger", { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
printV02PrMonitorProgress({ stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, activeCount: activeRuns.length, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
if (record(trigger).ok !== true) {
const comment = commentV02PullRequest({
pr,
phase: "cd-trigger",
state: "cd-failed",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
cd: beforeSummary,
dryRun: false,
message: "PR 已合并，但 v0.2 CD 触发失败；需要查看 trigger 阶段的 degradedReason。",
});
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment };
}
const cd = await waitForV02Cd(sourceCommit, options.timeoutSeconds);
const cdStatus = record(cd.status);
const flush = record(cd.flush);
const cdOk = cd.ok === true;
const cdPhase = String(cd.phase ?? "");
const cdSuperseded = cdOk && cdStatus.targetValidationState === "superseded";
const finalComment = commentV02PullRequest({
pr,
phase: cdPhase,
state: cdOk ? (cdSuperseded ? "cd-superseded" : "cd-succeeded") : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
cd: cdStatus,
flush,
dryRun: false,
message: cdOk
? cdSuperseded
? "v0.2 自动 CD 已收口：本 merge head 的 PipelineRun 已完成，但 source head 已被后续提交推进，本轮按 latest-only 语义 superseded/no-op，未回写旧 GitOps revision。"
: "v0.2 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。"
: cdPhase === "timeout"
? "v0.2 自动 CD 超时未收口；本评论保留最后一次 targetValidation / PipelineRun / Git mirror 状态用于接续排障。"
: "v0.2 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。",
});
return {
ok: cdOk && record(finalComment).ok === true,
action: cdOk ? (cdSuperseded ? "merged-and-superseded-v02" : "merged-and-rolled-v02") : "v02-cd-failed",
phase: cdOk ? "cd-passed" : cdPhase,
pr,
sourceCommit,
pipelineRun,
trigger,
cd,
comment: finalComment,
};
}
async function runRuntimeLanePrAutoCd(spec: HwlabRuntimeLaneSpec, pr: OpenPullRequest, preflight: Record<string, unknown>, merge: CommandJsonResult, options: G14MonitorOptions, startedAt: string): Promise<Record<string, unknown>> {
const mergeRaceState = isCommandSuccess(merge) ? null : mergeCommandRaceState(merge);
if (!isCommandSuccess(merge) && mergeRaceState !== "merged") {
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: "failed", pr: pr.number, mergeRaceState });
const observedAt = new Date().toISOString();
const comment = commentRuntimeLanePullRequest(spec, {
pr,
phase: "merge",
state: "merge-failed",
startedAt,
observedAt,
elapsedSeconds: durationSeconds(startedAt, observedAt),
preflight,
merge: compactCommandResult(merge),
dryRun: options.dryRun,
message: mergeRaceState === "closed"
? `PR preflight 曾通过，但自动合并前 PR 已被关闭且未确认 merged；本轮不会触发 ${spec.version} CD。`
: "PR preflight 已通过，但自动合并失败；需要处理 GitHub 返回的 merge blocker 后重试。",
});
const issue = reportRuntimeLaneAutomationIssue({
spec,
pr,
state: "merge-failed",
phase: "merge",
startedAt,
observedAt,
preflight,
details: { merge: compactCommandResult(merge), mergeRaceState },
message: "自动合并失败，v0.3 monitor 已暂停 CD 触发。",
dryRun: options.dryRun,
});
return { ok: mergeRaceState === "closed" && record(comment).ok === true && record(issue).ok === true, phase: "merge", action: "merge-race-closed", pr, merge, comment, issue };
}
const expectedCommit = options.dryRun ? null : sourceCommitFromMergeResult(merge, pr.number);
if (!options.dryRun && expectedCommit === null) {
const observedAt = new Date().toISOString();
printRuntimeLanePrMonitorProgress(spec, { stage: "source-head", status: "failed", pr: pr.number, sourceCommit: null, pipelineRun: null, degradedReason: "merge-commit-unresolved" });
const comment = commentRuntimeLanePullRequest(spec, {
pr,
phase: "merge-commit",
state: "cd-failed",
startedAt,
observedAt,
elapsedSeconds: durationSeconds(startedAt, observedAt),
preflight,
merge: commandData(merge),
sourceCommit: null,
pipelineRun: null,
dryRun: false,
message: "PR 已合并，但 UniDesk 未能从 GitHub merge/read 结果解析 merge commit；为避免触发错误 source head，本轮未启动 CD。",
});
const issue = reportRuntimeLaneAutomationIssue({
spec,
pr,
state: "cd-failed",
phase: "merge-commit",
startedAt,
observedAt,
preflight,
details: { merge: commandData(merge) },
message: "PR 已合并但 merge commit 不可解析，v0.3 CD 未启动。",
dryRun: false,
});
return { ok: false, phase: "merge-commit", pr, merge: commandData(merge), comment, issue, degradedReason: "merge-commit-unresolved" };
}
const headWait = options.dryRun ? { ok: true, sourceCommit: null, expectedCommit, matched: true } : await waitForRuntimeLaneHead(spec, expectedCommit, 120);
const sourceCommit = typeof record(headWait).sourceCommit === "string" ? String(record(headWait).sourceCommit) : null;
const pipelineRun = sourceCommit === null ? null : runtimeLanePipelineRunName(spec, sourceCommit);
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: "succeeded", pr: pr.number, sourceCommit, pipelineRun, mergeRaceState });
const startedComment = commentRuntimeLanePullRequest(spec, {
pr,
phase: options.dryRun ? "dry-run" : "cd-trigger",
state: "cd-started",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
dryRun: options.dryRun,
message: options.dryRun
? `dry-run：PR 已达到 ${spec.version} 自动合并条件；本轮不会写 GitHub merge 或 CD。`
: mergeRaceState === "merged"
? `PR 在本轮 merge 前已被合并；worker 继续对齐 merge commit 并驱动 ${spec.version} CD。后续成功、失败或超时会继续在本 PR 下回复。`
: `PR 已自动合并，${spec.version} CD 准备开始；后续成功、失败或超时会继续在本 PR 下回复。`,
});
if (!options.dryRun && record(startedComment).ok !== true) {
return { ok: false, phase: "pr-comment", pr, sourceCommit, pipelineRun, comment: startedComment };
}
if (options.dryRun) return { ok: true, action: "dry-run-merge", pr, merge: commandData(merge), comment: startedComment };
if (sourceCommit === null || record(headWait).ok !== true) {
const observedAt = new Date().toISOString();
printRuntimeLanePrMonitorProgress(spec, { stage: "source-head", status: "failed", pr: pr.number, sourceCommit, pipelineRun, expectedCommit });
const comment = commentRuntimeLanePullRequest(spec, {
pr,
phase: `${spec.lane}-source-head`,
state: "cd-timeout",
startedAt,
observedAt,
elapsedSeconds: durationSeconds(startedAt, observedAt),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
dryRun: false,
message: `PR 已合并，但 G14 ${spec.version} CI/CD source repo 没有在等待窗口内对齐 merge commit；为避免触发旧 head，本轮未启动 CD。`,
});
const issue = reportRuntimeLaneAutomationIssue({
spec,
pr,
state: "cd-timeout",
phase: `${spec.lane}-source-head`,
startedAt,
observedAt,
preflight,
sourceCommit,
pipelineRun,
details: { expectedCommit, headWait },
message: `${spec.version} source head 未在等待窗口内对齐 merge commit，CD 未启动。`,
dryRun: false,
});
return { ok: false, phase: `${spec.lane}-head`, pr, merge: commandData(merge), expectedCommit, headWait, comment: record(comment).ok === true ? comment : startedComment, issue, degradedReason: `${spec.lane}-head-unresolved-after-merge` };
}
const before = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
const beforeSummary = summarizeRuntimeLaneCdStatus(spec, before);
const trigger = runtimeLaneCdPassedWithMirror(spec, before) ? { ok: true, skipped: true, reason: "source-commit-already-deployed" } : triggerRuntimeLaneCurrent(spec, Math.min(options.timeoutSeconds, 600));
printEvent(`${spec.lane}.cd.trigger`, { pr: pr.number, sourceCommit, pipelineRun, ok: record(trigger).ok, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
printRuntimeLanePrMonitorProgress(spec, { stage: "cd-trigger", status: record(trigger).ok === true ? "succeeded" : "failed", pr: pr.number, sourceCommit, pipelineRun, skipped: record(trigger).skipped ?? false, degradedReason: record(trigger).degradedReason ?? null });
if (record(trigger).ok !== true) {
const observedAt = new Date().toISOString();
const comment = commentRuntimeLanePullRequest(spec, {
pr,
phase: "cd-trigger",
state: "cd-failed",
startedAt,
observedAt,
elapsedSeconds: durationSeconds(startedAt, observedAt),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun,
cd: beforeSummary,
dryRun: false,
message: `PR 已合并，但 ${spec.version} CD 触发失败；需要查看 trigger 阶段的 degradedReason。`,
});
const issue = reportRuntimeLaneAutomationIssue({
spec,
pr,
state: "cd-failed",
phase: "cd-trigger",
startedAt,
observedAt,
preflight,
sourceCommit,
pipelineRun,
cd: beforeSummary,
details: { trigger },
message: `${spec.version} CD 触发失败，PipelineRun 未进入可等待状态。`,
dryRun: false,
});
return { ok: false, phase: "cd-trigger", pr, sourceCommit, pipelineRun, trigger, comment, issue };
}
const cd = await waitForRuntimeLaneCd(spec, sourceCommit, options.timeoutSeconds);
const cdStatus = record(cd.status);
const flush = record(cd.flush);
const cdOk = cd.ok === true;
const cdPhase = String(cd.phase ?? "");
const cdPipelineRun = stringOrNull(cdStatus.pipelineRun) ?? pipelineRun;
const observedAt = new Date().toISOString();
const finalComment = commentRuntimeLanePullRequest(spec, {
pr,
phase: cdPhase,
state: cdOk ? "cd-succeeded" : cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
startedAt,
observedAt,
elapsedSeconds: durationSeconds(startedAt, observedAt),
preflight,
merge: commandData(merge),
sourceCommit,
pipelineRun: cdPipelineRun,
cd: cdStatus,
flush,
dryRun: false,
message: cdOk
? `${spec.version} 自动 CD 已完成：PipelineRun、Argo/runtime、公开探针和 Git mirror flush 收口均已检查。`
: cdPhase === "timeout"
? `${spec.version} 自动 CD 超时未收口；本评论保留最后一次 PipelineRun / Argo / public probe / Git mirror 状态用于接续排障。`
: `${spec.version} 自动 CD 失败；本评论保留失败阶段和最后一次状态用于接续排障。`,
});
const issue = cdOk ? { ok: true, skipped: true, reason: "cd-succeeded" } : reportRuntimeLaneAutomationIssue({
spec,
pr,
state: cdPhase === "timeout" ? "cd-timeout" : "cd-failed",
phase: cdPhase || "cd",
startedAt,
observedAt,
preflight,
sourceCommit,
pipelineRun: cdPipelineRun,
cd: cdStatus,
details: { cd },
message: cdPhase === "timeout" ? `${spec.version} 自动 CD 超时。` : `${spec.version} 自动 CD 失败。`,
dryRun: false,
});
return {
ok: cdOk && record(finalComment).ok === true && record(issue).ok === true,
action: cdOk ? `merged-and-rolled-${spec.lane}` : `${spec.lane}-cd-failed`,
phase: cdOk ? "cd-passed" : cdPhase,
pr,
sourceCommit,
pipelineRun: cdPipelineRun,
trigger,
cd,
comment: finalComment,
issue,
};
}
async function waitForG14Dev(sourceCommit: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
const started = Date.now();
const startedAt = new Date(started).toISOString();
let pipelineText = "";
let argoText = "";
let healthOk = false;
let workloadsReady: Record<string, unknown> = { ready: false };
let pipelineSucceededAt: string | null = null;
while (Date.now() - started < timeoutSeconds * 1000) {
const pipeline = getPipelineStatus(sourceCommit);
pipelineText = statusText(pipeline);
printEvent("g14.pipeline.status", { sourceCommit, pipelineRun: `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`, text: pipelineText.slice(0, 500) });
if (!pipelineText.startsWith("True\nSucceeded")) {
if (pipelineText.startsWith("False\n")) return { ok: false, phase: "pipeline", sourceCommit, pipelineText };
await sleep(30_000);
continue;
}
pipelineSucceededAt ??= new Date().toISOString();
refreshArgoDev();
const argo = getArgoStatus();
argoText = statusText(argo);
printEvent("g14.argo.status", { text: argoText.slice(0, 500) });
const argoLines = argoText.split(/\r?\n/u);
const synced = argoLines[1] === "Synced";
const healthy = argoLines[2] === "Healthy";
const gitopsRevision = argoLines[0] ?? null;
const workloads = getDevWorkloads();
const workloadsText = statusText(workloads);
workloadsReady = workloadReadiness(workloadsText, isCommandSuccess(workloads));
const health = getLiveHealth();
healthOk = health.ok && /"status"\s*:\s*"ok"/u.test(health.stdout);
if (synced && healthy && record(workloadsReady).ready === true && healthOk) {
const finishedAt = new Date().toISOString();
const pipelineRun = `hwlab-g14-ci-poll-${shortSha(sourceCommit)}`;
const ciMetrics = collectPipelineMetrics(pipelineRun);
return { ok: true, sourceCommit, gitopsRevision, pipelineRun, pipelineText, argoText, startedAt, pipelineSucceededAt, finishedAt, workloadsReady, healthOk, ciMetrics };
}
await sleep(30_000);
}
return { ok: false, phase: "timeout", sourceCommit, pipelineText, argoText, startedAt, pipelineSucceededAt, workloadsReady, healthOk, timeoutSeconds };
}
async function monitorV02Cycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
printEvent("v02.monitor.cycle.start", { cycle, dryRun: options.dryRun });
printV02PrMonitorProgress({ stage: "cycle", status: "started", cycle, dryRun: options.dryRun });
const listed = listOpenG14PullRequests();
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
const prs = extractPullRequests(listed, V02_SOURCE_BRANCH);
printEvent("v02.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
if (prs.length === 0) {
printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, pullRequests: 0, intervalSeconds: options.intervalSeconds });
return { ok: true, cycle, action: "none", lane: "v02", pullRequests: [] };
}
const observations: unknown[] = [];
for (const pr of prs) {
const startedAt = new Date().toISOString();
const preflightResult = preflightPullRequest(pr.number);
const preflight = preflightSummary(preflightResult);
printEvent("v02.pr.preflight", {
cycle,
number: pr.number,
ok: isCommandSuccess(preflightResult),
conclusion: preflight.conclusion,
readyForCommanderMerge: preflight.readyForCommanderMerge,
conflict: v02PrConflictState(preflight),
});
printV02PrMonitorProgress({
stage: "preflight",
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
pr: pr.number,
conclusion: preflight.conclusion,
conflict: v02PrConflictState(preflight),
});
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
const conclusion = String(preflight.conclusion ?? "unknown");
const blocked = conclusion === "blocked" || v02PrConflictState(preflight) === "conflict" || !isCommandSuccess(preflightResult);
const comment = commentV02PullRequest({
pr,
phase: "preflight",
state: blocked ? "blocked" : "waiting-ci",
startedAt,
observedAt: new Date().toISOString(),
elapsedSeconds: durationSeconds(startedAt, new Date().toISOString()),
preflight,
dryRun: options.dryRun,
message: blocked
? "v0.2 自动化已暂停在 PR preflight：存在 blocker 或冲突，需要先修复后再继续。"
: "v0.2 自动化正在等待 GitHub CI / mergeability 收敛；CI 通过且无冲突后会自动合并并触发 CD。",
});
observations.push({ pullRequest: pr, preflight, comment });
printV02PrMonitorProgress({ stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
if (record(comment).ok !== true) return { ok: false, cycle, lane: "v02", phase: "pr-comment", pullRequest: pr, preflight, comment, observations };
continue;
}
const merge = mergePullRequest(pr.number, options.dryRun);
printEvent("v02.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
printV02PrMonitorProgress({ stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
const result = await runV02PrAutoCd(pr, preflight, merge, options, startedAt);
observations.push(result);
if (record(result).ok !== true) return { ok: false, cycle, lane: "v02", phase: record(result).phase ?? "v02-auto-cd", pullRequest: pr, result, observations };
return { ok: true, cycle, lane: "v02", action: record(result).action ?? (options.dryRun ? "dry-run-merge" : "merged-and-rolled-v02"), result, observations };
}
return { ok: true, cycle, lane: "v02", action: "none", observations };
}
function runtimeLanePreflightBlocked(preflightResult: CommandJsonResult, preflight: Record<string, unknown>): boolean {
const blockers = Array.isArray(preflight.blockers) ? preflight.blockers : [];
const pending = Array.isArray(preflight.pending) ? preflight.pending : [];
const conclusion = String(preflight.conclusion ?? "unknown").toLowerCase();
return !isCommandSuccess(preflightResult)
|| v02PrConflictState(preflight) === "conflict"
|| conclusion === "blocked"
|| conclusion === "failed"
|| conclusion === "failure"
|| conclusion === "error"
|| (blockers.length > 0 && pending.length === 0);
}
async function monitorRuntimeLaneCycle(spec: HwlabRuntimeLaneSpec, options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
printEvent(`${spec.lane}.monitor.cycle.start`, { cycle, dryRun: options.dryRun });
printRuntimeLanePrMonitorProgress(spec, { stage: "cycle", status: "started", cycle, dryRun: options.dryRun });
const listed = listOpenG14PullRequests();
if (!isCommandSuccess(listed)) return { ok: false, cycle, lane: spec.lane, phase: "list-prs", listed };
const prs = extractPullRequests(listed, spec.sourceBranch);
printEvent(`${spec.lane}.monitor.prs`, { cycle, count: prs.length, pullRequests: prs });
if (prs.length === 0) {
printRuntimeLanePrMonitorProgress(spec, { stage: "idle", status: "waiting", cycle, pullRequests: 0, intervalSeconds: options.intervalSeconds });
return { ok: true, cycle, action: "none", lane: spec.lane, pullRequests: [] };
}
const observations: unknown[] = [];
for (const pr of prs) {
const startedAt = new Date().toISOString();
const preflightResult = preflightPullRequest(pr.number);
const preflight = preflightSummary(preflightResult);
printEvent(`${spec.lane}.pr.preflight`, {
cycle,
number: pr.number,
ok: isCommandSuccess(preflightResult),
conclusion: preflight.conclusion,
readyForCommanderMerge: preflight.readyForCommanderMerge,
conflict: v02PrConflictState(preflight),
});
printRuntimeLanePrMonitorProgress(spec, {
stage: "preflight",
status: preflight.readyForCommanderMerge === true ? "succeeded" : "running",
pr: pr.number,
conclusion: preflight.conclusion,
conflict: v02PrConflictState(preflight),
});
if (!isCommandSuccess(preflightResult) || preflight.readyForCommanderMerge !== true) {
const blocked = runtimeLanePreflightBlocked(preflightResult, preflight);
const observedAt = new Date().toISOString();
const comment = commentRuntimeLanePullRequest(spec, {
pr,
phase: "preflight",
state: blocked ? "blocked" : "waiting-ci",
startedAt,
observedAt,
elapsedSeconds: durationSeconds(startedAt, observedAt),
preflight,
dryRun: options.dryRun,
message: blocked
? `${spec.version} 自动化已暂停在 PR preflight：存在 blocker、失败 check 或冲突，需要先修复后再继续。`
: `${spec.version} 自动化正在等待 GitHub CI / mergeability 收敛；CI 通过且无冲突后会自动合并并触发 CD。`,
});
const issue = blocked ? reportRuntimeLaneAutomationIssue({
spec,
pr,
state: "blocked",
phase: "preflight",
startedAt,
observedAt,
preflight,
details: { preflightResult: compactCommandResult(preflightResult) },
message: `${spec.version} PR preflight 未通过或存在冲突，自动合并/CD 已暂停。`,
dryRun: options.dryRun,
}) : { ok: true, skipped: true, reason: "waiting-ci-without-blocker" };
observations.push({ pullRequest: pr, preflight, comment, issue });
printRuntimeLanePrMonitorProgress(spec, { stage: "pr-comment", status: record(comment).ok === true ? "succeeded" : "failed", pr: pr.number, conclusion: preflight.conclusion });
if (record(comment).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: "pr-comment", pullRequest: pr, preflight, comment, issue, observations };
if (record(issue).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: "failure-issue", pullRequest: pr, preflight, comment, issue, observations };
continue;
}
const merge = mergePullRequest(pr.number, options.dryRun);
printEvent(`${spec.lane}.pr.merge`, { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
printRuntimeLanePrMonitorProgress(spec, { stage: "merge", status: isCommandSuccess(merge) ? "succeeded" : "running", pr: pr.number, dryRun: options.dryRun });
const result = await runRuntimeLanePrAutoCd(spec, pr, preflight, merge, options, startedAt);
observations.push(result);
if (record(result).ok !== true) return { ok: false, cycle, lane: spec.lane, phase: record(result).phase ?? `${spec.lane}-auto-cd`, pullRequest: pr, result, observations };
return { ok: true, cycle, lane: spec.lane, action: record(result).action ?? (options.dryRun ? "dry-run-merge" : `merged-and-rolled-${spec.lane}`), result, observations };
}
return { ok: true, cycle, lane: spec.lane, action: "none", observations };
}
async function monitorCycle(options: G14MonitorOptions, cycle: number): Promise<Record<string, unknown>> {
if (options.lane === "v02") return monitorV02Cycle(options, cycle);
if (options.lane !== "g14") return monitorRuntimeLaneCycle(hwlabRuntimeLaneSpec(options.lane), options, cycle);
const retirement = legacyG14RetirementBlocksMonitor();
if (retirement !== null) {
return {
ok: false,
cycle,
phase: "retired",
degradedReason: "legacy-g14-dev-prod-retired",
retirement,
next: { status: "bun scripts/cli.ts hwlab g14 retirement status", v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02" },
};
}
printEvent("g14.monitor.cycle.start", { cycle, dryRun: options.dryRun });
const precheck = precheckWorkspace();
if (!isCommandSuccess(precheck)) return { ok: false, cycle, phase: "workspace-precheck", precheck };
const listed = listOpenG14PullRequests();
if (!isCommandSuccess(listed)) return { ok: false, cycle, phase: "list-prs", listed };
const prs = extractPullRequests(listed);
printEvent("g14.monitor.prs", { cycle, count: prs.length, pullRequests: prs });
if (prs.length === 0) return { ok: true, cycle, action: "none", pullRequests: [] };
const merged: unknown[] = [];
for (const pr of prs) {
const preflight = preflightPullRequest(pr.number);
printEvent("g14.pr.preflight", { cycle, number: pr.number, ok: isCommandSuccess(preflight) });
if (!isCommandSuccess(preflight) || nested(preflight.parsed, ["data", "mergeability", "readyForCommanderMerge"]) !== true) {
return { ok: false, cycle, phase: "pr-preflight", pullRequest: pr, preflight };
}
const merge = mergePullRequest(pr.number, options.dryRun);
printEvent("g14.pr.merge", { cycle, number: pr.number, dryRun: options.dryRun, ok: isCommandSuccess(merge) });
if (!isCommandSuccess(merge)) return { ok: false, cycle, phase: "pr-merge", pullRequest: pr, merge };
const sourceCommit = getG14Head();
const rollout = options.dryRun || sourceCommit === null ? { skipped: true, dryRun: options.dryRun, sourceCommit } : await waitForG14Dev(sourceCommit, options.timeoutSeconds);
const brief = options.dryRun || sourceCommit === null || record(rollout).ok !== true
? { skipped: true, dryRun: options.dryRun }
: appendRolloutBrief({ prNumber: pr.number, sourceCommit, dryRun: false }, rollout);
printEvent("g14.rollout.brief", { cycle, number: pr.number, ok: record(brief).ok, skipped: record(brief).skipped ?? false, brief: record(brief).brief ?? null });
merged.push({ pullRequest: pr, merge: commandData(merge), sourceCommit, rollout, brief });
if (record(rollout).ok === false) return { ok: false, cycle, phase: "rollout", pullRequest: pr, sourceCommit, rollout, merged };
if (record(brief).ok === false) return { ok: false, cycle, phase: "rollout-brief", pullRequest: pr, sourceCommit, rollout, brief, merged };
}
return { ok: true, cycle, action: options.dryRun ? "dry-run-merge" : "merged-and-rolled-dev", merged };
}
async function runMonitorWorker(options: G14MonitorOptions): Promise<Record<string, unknown>> {
const maxCycles = options.once ? 1 : options.maxCycles;
let cycle = 0;
const results: unknown[] = [];
while (maxCycles === 0 || cycle < maxCycles) {
cycle += 1;
const result = await monitorCycle(options, cycle);
results.push(result);
printEvent(`${options.lane}.monitor.cycle.done`, { cycle, lane: options.lane, ok: record(result).ok, action: record(result).action ?? null, phase: record(result).phase ?? null });
if (record(result).ok !== true) return { ok: false, cycles: cycle, lastResult: result, results };
if (options.once || (options.lane === "g14" && record(result).action !== "none")) return { ok: true, cycles: cycle, lastResult: result, results };
printEvent(`${options.lane}.monitor.sleep`, { cycle, lane: options.lane, intervalSeconds: options.intervalSeconds });
if (options.lane === "v02") printV02PrMonitorProgress({ stage: "idle", status: "waiting", cycle, intervalSeconds: options.intervalSeconds });
else if (options.lane !== "g14") printRuntimeLanePrMonitorProgress(hwlabRuntimeLaneSpec(options.lane), { stage: "idle", status: "waiting", cycle, intervalSeconds: options.intervalSeconds });
await sleep(options.intervalSeconds * 1000);
}
return { ok: true, cycles: cycle, results };
}
export function hwlabG14Help(): Record<string, unknown> {
return {
command: "hwlab g14",
output: "json",
usage: [
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --status",
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03 --status",
"bun scripts/cli.ts hwlab g14 retirement status",
"bun scripts/cli.ts hwlab g14 retirement plan",
"bun scripts/cli.ts hwlab g14 retirement execute --confirm",
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v02 --once --dry-run",
"bun scripts/cli.ts hwlab g14 monitor-prs --lane v03 --once --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --history",
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
"bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit <full-sha>",
"bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha>",
"bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit <full-sha>",
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait",
"bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm --wait",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha> --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --source-commit <full-sha> --confirm",
"bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run hwlab-v03-ci-poll-<short-sha> --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --source-commit <full-sha> --confirm",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --allow-live-db-read --dry-run",
"bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --confirm",
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-openfga",
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --dry-run",
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
"bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-master-server-admin-api-key",
"bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --dry-run",
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror status",
"bun scripts/cli.ts hwlab g14 git-mirror apply --lane v02 --confirm",
"bun scripts/cli.ts hwlab nodes git-mirror apply --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
"bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait",
"bun scripts/cli.ts hwlab g14 git-mirror flush --confirm --wait",
"bun scripts/cli.ts hwlab g14 observability status",
"bun scripts/cli.ts hwlab g14 observability apply --dry-run",
"bun scripts/cli.ts hwlab g14 observability apply --confirm",
`bun scripts/cli.ts hwlab g14 observability query --promql 'up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
`bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
`bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_health_probe_success{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
"bun scripts/cli.ts hwlab g14 observability targets --lane v02",
"bun scripts/cli.ts hwlab g14 observability boundary --lane v02",
"bun scripts/cli.ts hwlab g14 observability closeout --lane v02",
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
"bun scripts/cli.ts hwlab g14 upstream-image status --name openfga --tag v1.17.0",
"bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --dry-run",
"bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --confirm",
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
],
description: "G14 HWLAB PR monitor, legacy DEV/PROD retirement status/plan/execute command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, node-scoped runtime lane v03 control-plane apply/status/refresh/trigger entry, runtime lane SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The legacy base=G14 monitor is blocked by the retirement contract; the local retirement marker records live execution evidence. `--lane v02` monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. `--lane v03` monitors base=v0.3 PRs through the runtime lane control-plane, waits for GitHub preflight/CI readiness, auto-merges ready non-conflicting PRs, triggers v0.3 CD, verifies PipelineRun/Argo/runtime public probes/Git mirror flush, posts deduplicated PR comments, and creates or updates failure issues for failed checks, conflicts, merge blockers, CD failures, and timeouts. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ is advertised through `hwlab nodes ... --node <node-id> --lane vNN` so node identity remains configuration data instead of a command family. secret status/ensure is the standard runtime lane SecretRef bootstrap path for OpenFGA, cloud-api DB, master admin API key, and code-agent provider refs; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.",
defaults: {
repo: HWLAB_REPO,
legacyBase: G14_SOURCE_BRANCH,
v02Base: V02_SOURCE_BRANCH,
runtimeLaneConfig: hwlabRuntimeLaneConfigPath(),
runtimeLanes: hwlabRuntimeLaneIds(),
provider: G14_PROVIDER,
legacyWorkspace: G14_WORKSPACE,
v02Workspace: V02_WORKSPACE,
v03Workspace: hwlabRuntimeLaneSpec("v03").workspace,
ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO,
intervalSeconds: DEFAULT_INTERVAL_SECONDS,
devApplication: DEV_APP,
prodApplication: PROD_APP,
legacyRetirementState: legacyG14RetirementStatePath(),
v02Application: V02_APP,
v03Application: hwlabRuntimeLaneSpec("v03").app,
v03Node: hwlabRuntimeLaneSpec("v03").nodeId,
v03NetworkProfile: hwlabRuntimeLaneSpec("v03").networkProfileId,
v03DownloadProfile: hwlabRuntimeLaneSpec("v03").downloadProfileId,
requiredNoProxy: hwlabRequiredNoProxyEntries(),
briefIndexIssue: G14_BRIEF_INDEX_ISSUE,
observabilityNamespace: G14_OBSERVABILITY_NAMESPACE,
prometheusOperatorVersion: G14_PROMETHEUS_OPERATOR_VERSION,
prometheusVersion: G14_PROMETHEUS_VERSION,
},
stateFiles: {
monitor: ".state/hwlab-g14/latest-monitor-job.json",
once: ".state/hwlab-g14/latest-once-job.json",
dryRun: ".state/hwlab-g14/latest-dry-run-job.json",
onceDryRun: ".state/hwlab-g14/latest-once-dry-run-job.json",
v02Monitor: ".state/hwlab-g14/latest-v02-monitor-job.json",
v02Once: ".state/hwlab-g14/latest-v02-once-job.json",
v02DryRun: ".state/hwlab-g14/latest-v02-dry-run-job.json",
v02OnceDryRun: ".state/hwlab-g14/latest-v02-once-dry-run-job.json",
v02PrCommentSignatures: ".state/hwlab-g14/v02-pr-comment-signatures.json",
v03Monitor: ".state/hwlab-g14/latest-v03-monitor-job.json",
v03Once: ".state/hwlab-g14/latest-v03-once-job.json",
v03DryRun: ".state/hwlab-g14/latest-v03-dry-run-job.json",
v03OnceDryRun: ".state/hwlab-g14/latest-v03-once-dry-run-job.json",
v03PrCommentSignatures: ".state/hwlab-g14/v03-pr-comment-signatures.json",
},
};
}
function monitorStatus(options: G14MonitorOptions): Record<string, unknown> {
const stateDir = rootPath(".state", "hwlab-g14");
const stateFileName = hwlabG14MonitorStateFileName(options);
const stateFileRole = hwlabG14MonitorStateRole(options);
const latestPath = join(stateDir, stateFileName);
const retirement = options.lane === "g14" ? legacyG14RetirementBlocksMonitor() : null;
const exists = existsSync(latestPath);
let latest: Record<string, unknown> | null = null;
let job: Record<string, unknown> | null = null;
let statusCommand: string | null = null;
let degradedReason: string | null = null;
if (exists) {
try {
latest = record(JSON.parse(readFileSync(latestPath, "utf8")) as unknown);
const jobId = typeof latest.jobId === "string" ? latest.jobId : null;
if (jobId !== null) {
statusCommand = `bun scripts/cli.ts job status ${jobId} --tail-bytes 30000`;
try {
const current = readJob(jobId);
job = {
id: current.id,
name: current.name,
status: current.status,
createdAt: current.createdAt,
startedAt: current.startedAt,
finishedAt: current.finishedAt,
exitCode: current.exitCode,
runnerPid: current.runnerPid ?? null,
stdoutFile: current.stdoutFile,
stderrFile: current.stderrFile,
note: current.note,
};
} catch (error) {
degradedReason = error instanceof Error ? error.message : String(error);
}
}
} catch (error) {
degradedReason = `failed to parse ${latestPath}: ${error instanceof Error ? error.message : String(error)}`;
}
}
return {
ok: degradedReason === null,
command: "hwlab g14 monitor-prs --status",
lane: options.lane,
baseBranch: monitorBaseBranch(options.lane),
mode: "status",
latestPath,
stateFileName,
stateFileRole,
exists,
latest,
job,
statusCommand,
degradedReason,
retirement,
next: retirement !== null ? {
retirementStatus: "bun scripts/cli.ts hwlab g14 retirement status",
v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
v03Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
} : statusCommand === null ? {
start: `bun scripts/cli.ts hwlab g14 monitor-prs --lane ${options.lane}`,
} : {
status: statusCommand,
},
};
}
export async function runHwlabG14Command(_config: Config, args: string[]): Promise<Record<string, unknown>> {
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return { ok: true, ...hwlabG14Help() };
const [action] = args;
if (action === "record-rollout") {
return legacyG14RecordRolloutRetired();
}
if (action === "control-plane") {
const options = parseControlPlaneOptions(args.slice(1));
if (options.action === "trigger-current" && options.confirm && !options.dryRun && !options.wait) {
return startControlPlaneTriggerJob(options);
}
if (isHwlabRuntimeLane(options.lane) && options.lane !== "v02") {
return runRuntimeLaneControlPlane(hwlabRuntimeLaneSpec(options.lane), options);
}
return runV02ControlPlane(options);
}
if (action === "retirement") {
const options = parseLegacyRetirementOptions(args.slice(1));
return runLegacyG14Retirement(options);
}
if (action === "secret") {
const options = parseSecretOptions(args.slice(1));
return runG14Secret(options);
}
if (action === "tools-image") {
const options = parseToolsImageOptions(args.slice(1));
return runG14ToolsImage(options);
}
if (action === "upstream-image") {
const options = parseUpstreamImageOptions(args.slice(1));
return runG14UpstreamImage(options);
}
if (action === "git-mirror") {
const options = parseGitMirrorOptions(args.slice(1));
if ((options.action === "sync" || options.action === "flush") && options.confirm && !options.dryRun && !options.wait) {
return startGitMirrorJob(options);
}
return runG14GitMirror(options);
}
if (action === "observability") {
const options = parseObservabilityOptions(args.slice(1));
return runG14Observability(options);
}
if (action !== "monitor-prs") {
return { ok: false, command: `hwlab g14 ${action ?? ""}`.trim(), degradedReason: "unsupported-command", message: "supported commands: hwlab g14 monitor-prs, hwlab g14 retirement, hwlab g14 control-plane, hwlab g14 secret, hwlab g14 git-mirror, hwlab g14 observability, hwlab g14 tools-image, hwlab g14 upstream-image" };
}
const options = parseOptions(args.slice(1));
if (options.worker) return runMonitorWorker(options);
if (args.includes("--status")) return monitorStatus(options);
const retirement = options.lane === "g14" ? legacyG14RetirementBlocksMonitor() : null;
if (retirement !== null) {
return {
ok: false,
command: "hwlab g14 monitor-prs",
lane: options.lane,
baseBranch: monitorBaseBranch(options.lane),
degradedReason: "legacy-g14-dev-prod-retired",
message: "Legacy G14 DEV/PROD has a retirement marker; start a runtime lane monitor with --lane v02/v03 or inspect retirement status.",
retirement,
next: {
retirementStatus: "bun scripts/cli.ts hwlab g14 retirement status",
v02Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v02",
v03Monitor: "bun scripts/cli.ts hwlab g14 monitor-prs --lane v03",
},
};
}
const command = ["bun", "scripts/cli.ts", "hwlab", "g14", "monitor-prs", "--worker", "--lane", options.lane, "--interval-seconds", String(options.intervalSeconds), "--timeout-seconds", String(options.timeoutSeconds), ...(options.once ? ["--once"] : []), ...(options.dryRun ? ["--dry-run"] : []), ...(options.maxCycles > 0 ? ["--max-cycles", String(options.maxCycles)] : [])];
const jobName = options.lane === "g14" ? "hwlab_g14_pr_monitor" : `hwlab_g14_${options.lane}_pr_monitor`;
const jobNote = options.lane === "g14"
? `Monitor ${HWLAB_REPO} PRs targeting ${G14_SOURCE_BRANCH} and roll merged changes to G14 DEV`
: `Monitor ${HWLAB_REPO} PRs targeting ${monitorBaseBranch(options.lane)}, merge ready PRs, trigger ${hwlabRuntimeLaneSpec(options.lane).version} CD, comment PR progress, and file failure issues`;
const job = startJob(jobName, command, jobNote);
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`;
const stateDir = rootPath(".state", "hwlab-g14");
mkdirSync(stateDir, { recursive: true });
const stateFileName = hwlabG14MonitorStateFileName(options);
const stateFileRole = hwlabG14MonitorStateRole(options);
const latestPath = join(stateDir, stateFileName);
const previousLatest = existsSync(latestPath) ? readFileSync(latestPath, "utf8") : null;
writeFileSync(latestPath, `${JSON.stringify({ jobId: job.id, createdAt: job.createdAt, statusCommand, role: stateFileRole, lane: options.lane, baseBranch: monitorBaseBranch(options.lane) }, null, 2)}\n`, "utf8");
return {
ok: true,
command: "hwlab g14 monitor-prs",
lane: options.lane,
baseBranch: monitorBaseBranch(options.lane),
mode: "async-job",
job,
statusCommand,
latestPath,
stateFileName,
stateFileRole,
previousLatest,
next: {
status: statusCommand,
tail: `tail -f ${job.stdoutFile}`,
},
};
}
Reference in New Issue View Git Blame Copy Permalink