fix: expose HWLAB CI retention reclaim estimates
This commit is contained in:
Codex
+291
-18
@@ -100,7 +100,7 @@ const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000;
|
||||
const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000;
|
||||
const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000;
|
||||
|
||||
export type G14MonitorLane = "g14" | "v02";
|
||||
export type G14MonitorLane = "g14" | HwlabRuntimeLane;
|
||||
|
||||
interface G14MonitorOptions {
|
||||
lane: G14MonitorLane;
|
||||
@@ -197,13 +197,13 @@ interface G14ObservabilityOptions {
|
||||
}
|
||||
|
||||
interface G14SecretOptions {
|
||||
action: "status" | "ensure" | "delete";
|
||||
action: "status" | "ensure" | "delete" | "cleanup-owned-postgres";
|
||||
lane: HwlabRuntimeLane;
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
name: string;
|
||||
key?: string;
|
||||
preset: "openfga" | "master-server-admin-api-key" | "generic-delete";
|
||||
preset: "openfga" | "master-server-admin-api-key" | "generic-delete" | "owned-postgres-cleanup";
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
@@ -282,16 +282,15 @@ export interface V02PrCommentInput {
|
||||
}
|
||||
|
||||
export function hwlabG14MonitorStateFileName(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||||
const prefix = options.lane === "v02" ? "latest-v02-" : "latest-";
|
||||
if (options.once && options.dryRun) return options.lane === "v02" ? `${prefix}once-dry-run-job.json` : "latest-once-dry-run-job.json";
|
||||
if (options.once && options.lane === "v02") return `${prefix}once-job.json`;
|
||||
if (options.once) return "latest-once-job.json";
|
||||
const prefix = options.lane !== undefined && options.lane !== "g14" ? `latest-${options.lane}-` : "latest-";
|
||||
if (options.once && options.dryRun) return `${prefix}once-dry-run-job.json`;
|
||||
if (options.once) return `${prefix}once-job.json`;
|
||||
if (options.dryRun) return `${prefix}dry-run-job.json`;
|
||||
return `${prefix}monitor-job.json`;
|
||||
}
|
||||
|
||||
function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dryRun"> & { lane?: G14MonitorLane }): string {
|
||||
const lanePrefix = options.lane === "v02" ? "v02-" : "";
|
||||
const lanePrefix = options.lane !== undefined && options.lane !== "g14" ? `${options.lane}-` : "";
|
||||
if (options.once && options.dryRun) return `${lanePrefix}once-dry-run`;
|
||||
if (options.once) return `${lanePrefix}once`;
|
||||
if (options.dryRun) return `${lanePrefix}dry-run-monitor`;
|
||||
@@ -300,7 +299,7 @@ function hwlabG14MonitorStateRole(options: Pick<G14MonitorOptions, "once" | "dry
|
||||
|
||||
function parseMonitorLane(args: string[]): G14MonitorLane {
|
||||
const lane = optionValue(args, "--lane") ?? "g14";
|
||||
if (lane !== "g14" && lane !== "v02") throw new Error("monitor-prs --lane must be g14 or v02");
|
||||
if (lane !== "g14" && !isHwlabRuntimeLane(lane)) throw new Error(`monitor-prs --lane must be g14 or ${hwlabRuntimeLaneIds().join("|")}`);
|
||||
return lane;
|
||||
}
|
||||
|
||||
@@ -570,8 +569,8 @@ function parseObservabilityOptions(args: string[]): G14ObservabilityOptions {
|
||||
|
||||
function parseSecretOptions(args: string[]): G14SecretOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "delete") {
|
||||
throw new Error("secret usage: status|ensure --lane v02|v03 --name hwlab-v0x-openfga|hwlab-v0x-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name <obsolete-secret> [--dry-run|--confirm]");
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "delete" && actionRaw !== "cleanup-owned-postgres") {
|
||||
throw new Error("secret usage: status|ensure --lane v02|v03 --name hwlab-v0x-openfga|hwlab-v0x-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name <obsolete-secret> [--dry-run|--confirm] | cleanup-owned-postgres --lane v03 [--dry-run|--confirm]");
|
||||
}
|
||||
const laneRaw = optionValue(args, "--lane") ?? "v02";
|
||||
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`secret --lane must be one of ${hwlabRuntimeLaneIds().join(", ")}`);
|
||||
@@ -585,6 +584,21 @@ function parseSecretOptions(args: string[]): G14SecretOptions {
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
||||
if (actionRaw === "cleanup-owned-postgres") {
|
||||
if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
|
||||
if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
|
||||
const expectedName = postgresSecret;
|
||||
if (name !== openFgaSecret && name !== expectedName) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${expectedName}; omit --name or pass --name ${expectedName}`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane,
|
||||
confirm,
|
||||
dryRun: explicitDryRun || !confirm,
|
||||
name: expectedName,
|
||||
preset: "owned-postgres-cleanup",
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
};
|
||||
}
|
||||
if (actionRaw === "delete") {
|
||||
if (lane !== "v02") throw new Error("secret delete is currently limited to --lane v02 obsolete cleanup");
|
||||
if (key !== undefined) throw new Error("secret delete does not accept --key; it deletes a whole obsolete Secret object");
|
||||
@@ -913,7 +927,7 @@ function isCommandSuccess(result: CommandJsonResult): boolean {
|
||||
}
|
||||
|
||||
function monitorBaseBranch(lane: G14MonitorLane): string {
|
||||
return lane === "v02" ? V02_SOURCE_BRANCH : G14_SOURCE_BRANCH;
|
||||
return lane === "g14" ? G14_SOURCE_BRANCH : hwlabRuntimeLaneSpec(lane).sourceBranch;
|
||||
}
|
||||
|
||||
function extractPullRequests(result: CommandJsonResult, baseBranch = G14_SOURCE_BRANCH): OpenPullRequest[] {
|
||||
@@ -951,6 +965,14 @@ function printV02PrMonitorProgress(data: Record<string, unknown> = {}): void {
|
||||
printProgressEvent("hwlab.v02.pr-monitor.progress", data);
|
||||
}
|
||||
|
||||
function printRuntimeLanePrMonitorProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
||||
printProgressEvent("hwlab.runtime-lane.pr-monitor.progress", {
|
||||
lane: spec.lane,
|
||||
node: spec.nodeId,
|
||||
...data,
|
||||
});
|
||||
}
|
||||
|
||||
function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
||||
printProgressEvent("hwlab.runtime-lane.trigger.progress", {
|
||||
lane: spec.lane,
|
||||
@@ -2649,6 +2671,59 @@ function parseCleanupPipelineRunLine(line: string, nowMs: number): CleanupPipeli
|
||||
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
|
||||
}
|
||||
|
||||
function formatRetentionBytes(bytes: number): string | null {
|
||||
if (!Number.isFinite(bytes)) return null;
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`;
|
||||
}
|
||||
|
||||
function storageHostPathFromClaim(volume: string | null, namespace: string, claimName: string | null): string | null {
|
||||
if (volume === null || claimName === null) return null;
|
||||
if (!/^pvc-[a-z0-9-]+$/iu.test(volume) || !/^pvc-[a-z0-9]+$/iu.test(claimName)) return null;
|
||||
return `/var/lib/rancher/k3s/storage/${volume}_${namespace}_${claimName}`;
|
||||
}
|
||||
|
||||
function safeStorageHostPath(value: unknown): string | null {
|
||||
const path = stringOrNull(value);
|
||||
return path !== null && path.startsWith("/var/lib/rancher/k3s/storage/") ? path : null;
|
||||
}
|
||||
|
||||
function remoteStoragePathEstimates(paths: Array<string | null | undefined>): Map<string, number | null> {
|
||||
const uniquePaths = [...new Set(paths.map((item) => safeStorageHostPath(item)).filter((item): item is string => item !== null))];
|
||||
const estimates = new Map<string, number | null>(uniquePaths.map((path) => [path, null]));
|
||||
if (uniquePaths.length === 0) return estimates;
|
||||
const script = [
|
||||
"set -eu",
|
||||
`for path in ${uniquePaths.map((path) => shellQuote(path)).join(" ")}; do`,
|
||||
" bytes=\"\"",
|
||||
" if [ -e \"$path\" ]; then bytes=$(du -sB1 \"$path\" 2>/dev/null | awk '{print $1}' || true); fi",
|
||||
" printf '%s\\t%s\\n' \"$path\" \"$bytes\"",
|
||||
"done",
|
||||
].join("\n");
|
||||
const result = g14HostScript(script, 120_000);
|
||||
if (!isCommandSuccess(result)) return estimates;
|
||||
for (const line of statusText(result).split(/\r?\n/u)) {
|
||||
const [path = "", rawBytes = ""] = line.trim().split("\t");
|
||||
if (!estimates.has(path)) continue;
|
||||
const bytes = /^\d+$/u.test(rawBytes) ? Number(rawBytes) : null;
|
||||
estimates.set(path, Number.isFinite(bytes) ? bytes : null);
|
||||
}
|
||||
return estimates;
|
||||
}
|
||||
|
||||
function sumEstimatedBytes(items: Record<string, unknown>[]): number {
|
||||
return items.reduce((sum, item) => {
|
||||
const bytes = item.estimatedBytes;
|
||||
return sum + (typeof bytes === "number" && Number.isFinite(bytes) ? bytes : 0);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
function cleanupPipelineRunTargetCandidate(input: {
|
||||
targetPipelineRun: string;
|
||||
text: string;
|
||||
@@ -2810,22 +2885,75 @@ function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unkn
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const pvResult = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pv",
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(pvResult)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRun PV backing paths: ${commandErrorSummary(pvResult)}`);
|
||||
}
|
||||
const podResult = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pod",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{","}{end}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(podResult)) {
|
||||
throw new Error(`failed to list hwlab-ci active pod PVC mounts: ${commandErrorSummary(podResult)}`);
|
||||
}
|
||||
const pvByName = new Map<string, Record<string, unknown>>();
|
||||
for (const line of statusText(pvResult).split(/\r?\n/u)) {
|
||||
const [name = "", storageClass = "", reclaimPolicy = "", localPath = "", hostPath = ""] = line.trim().split("\t");
|
||||
if (name.length === 0) continue;
|
||||
pvByName.set(name, {
|
||||
storageClass: storageClass || null,
|
||||
reclaimPolicy: reclaimPolicy || null,
|
||||
hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath),
|
||||
});
|
||||
}
|
||||
const activeClaimPods = new Map<string, string[]>();
|
||||
for (const line of statusText(podResult).split(/\r?\n/u)) {
|
||||
const [podName = "", phase = "", claims = ""] = line.trim().split("\t");
|
||||
if (podName.length === 0 || phase === "Succeeded" || phase === "Failed") continue;
|
||||
for (const claimName of claims.split(",").map((item) => item.trim()).filter(Boolean)) {
|
||||
const entry = activeClaimPods.get(claimName) ?? [];
|
||||
entry.push(podName);
|
||||
activeClaimPods.set(claimName, entry);
|
||||
}
|
||||
}
|
||||
const wanted = new Set(pipelineRunNames);
|
||||
return statusText(result)
|
||||
const ownedPvcs = statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
|
||||
const pv = volume.length > 0 ? pvByName.get(volume) : undefined;
|
||||
const hostPath = safeStorageHostPath(pv?.hostPath) ?? storageHostPathFromClaim(volume || null, CI_NAMESPACE, name || null);
|
||||
return {
|
||||
name,
|
||||
volume: volume || null,
|
||||
phase: phase || null,
|
||||
ownerKind: ownerKind || null,
|
||||
owner: owner || null,
|
||||
storageClass: pv?.storageClass ?? null,
|
||||
reclaimPolicy: pv?.reclaimPolicy ?? null,
|
||||
hostPath,
|
||||
activeMountPods: activeClaimPods.get(name) ?? [],
|
||||
};
|
||||
})
|
||||
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
|
||||
const estimates = remoteStoragePathEstimates(ownedPvcs.map((item) => stringOrNull(item.hostPath)));
|
||||
return ownedPvcs.map((item) => ({
|
||||
...item,
|
||||
estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
@@ -2848,17 +2976,17 @@ function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<str
|
||||
"get",
|
||||
"pv",
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\n"}{end}',
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
return statusText(result)
|
||||
const candidates = statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t");
|
||||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = "", localPath = "", hostPath = ""] = line.split("\t");
|
||||
return {
|
||||
name,
|
||||
createdAt,
|
||||
@@ -2868,7 +2996,7 @@ function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<str
|
||||
claimNamespace,
|
||||
claimName,
|
||||
capacity,
|
||||
hostPath: claimNamespace === CI_NAMESPACE && claimName.length > 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null,
|
||||
hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath) ?? (claimNamespace === CI_NAMESPACE ? storageHostPathFromClaim(name || null, claimNamespace, claimName || null) : null),
|
||||
};
|
||||
})
|
||||
.filter((item) => item.phase === "Released")
|
||||
@@ -2876,6 +3004,11 @@ function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<str
|
||||
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, options.limit);
|
||||
const estimates = remoteStoragePathEstimates(candidates.map((item) => stringOrNull(item.hostPath)));
|
||||
return candidates.map((item) => ({
|
||||
...item,
|
||||
estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
@@ -2895,6 +3028,7 @@ function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJso
|
||||
function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listReleasedCiWorkspacePvs(options);
|
||||
const candidateNames = candidates.map((item) => String(item.name));
|
||||
const estimatedReclaimBytes = sumEstimatedBytes(candidates);
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
@@ -2904,6 +3038,10 @@ function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Reco
|
||||
limit: options.limit,
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
selectedPersistentVolumes: candidateNames,
|
||||
selectedPersistentVolumeCount: candidateNames.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
mutation: false,
|
||||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
|
||||
};
|
||||
@@ -2918,6 +3056,8 @@ function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Reco
|
||||
deletedPersistentVolumes: candidateNames,
|
||||
deletedPersistentVolumeCount: candidateNames.length,
|
||||
candidatesBefore: candidates,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
deletion,
|
||||
followUp: {
|
||||
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
@@ -2985,6 +3125,7 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string,
|
||||
.filter((item) => item.selected !== false)
|
||||
.map((item) => String(item.name));
|
||||
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
|
||||
const estimatedReclaimBytes = sumEstimatedBytes(ownedPvcs);
|
||||
const followUpStatusLane = isHwlabRuntimeLane(options.lane) ? options.lane : "v02";
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
@@ -3002,6 +3143,8 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string,
|
||||
selectedPipelineRunCount: candidateNames.length,
|
||||
ownedPvcs,
|
||||
ownedPvcCount: ownedPvcs.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
mutation: false,
|
||||
next: { confirm: [
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs",
|
||||
@@ -3028,6 +3171,8 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string,
|
||||
deletedPipelineRunCount: candidateNames.length,
|
||||
ownedPvcsBefore: ownedPvcs,
|
||||
ownedPvcCountBefore: ownedPvcs.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes),
|
||||
deletion,
|
||||
followUp: {
|
||||
status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${followUpStatusLane}`,
|
||||
@@ -3783,6 +3928,7 @@ function runtimeLaneControlPlaneStatusBundle(spec: HwlabRuntimeLaneSpec, target:
|
||||
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(spec.app)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
||||
`section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'`,
|
||||
`section runtimeWorkloads kubectl get deploy,statefulset,svc,ingress,configmap -n ${shellQuote(spec.runtimeNamespace)} -l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)} -o name`,
|
||||
`section publicProbes sh -lc ${shellQuote(runtimeLanePublicProbeScript(spec))}`,
|
||||
[
|
||||
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)}`,
|
||||
`-l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)}`,
|
||||
@@ -3794,6 +3940,40 @@ function runtimeLaneControlPlaneStatusBundle(spec: HwlabRuntimeLaneSpec, target:
|
||||
return g14K3s(["script", "--", script], 120_000);
|
||||
}
|
||||
|
||||
function runtimeLanePublicProbeScript(spec: HwlabRuntimeLaneSpec): string {
|
||||
const apiUrl = `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`;
|
||||
const webUrl = spec.publicWebUrl.replace(/\/+$/u, "/");
|
||||
return [
|
||||
"set +e",
|
||||
`api_url=${shellQuote(apiUrl)}`,
|
||||
`web_url=${shellQuote(webUrl)}`,
|
||||
"api_output=$(curl -fsS --connect-timeout 3 --max-time 15 \"$api_url\" 2>&1)",
|
||||
"api_exit=$?",
|
||||
"api_status_ok=no",
|
||||
"if [ \"$api_exit\" = 0 ] && printf '%s' \"$api_output\" | grep -Eq '\"status\"[[:space:]]*:[[:space:]]*\"ok\"'; then api_status_ok=yes; fi",
|
||||
"web_tmp=$(mktemp /tmp/hwlab-lane-web-probe.XXXXXX)",
|
||||
"web_output=$(curl -fsS --connect-timeout 3 --max-time 15 -o \"$web_tmp\" \"$web_url\" 2>&1)",
|
||||
"web_exit=$?",
|
||||
"web_bytes=0",
|
||||
"web_has_html=no",
|
||||
"if [ -f \"$web_tmp\" ]; then",
|
||||
" web_bytes=$(wc -c < \"$web_tmp\" | tr -d ' ')",
|
||||
" if grep -Eiq '<html|<script|<div' \"$web_tmp\"; then web_has_html=yes; fi",
|
||||
"fi",
|
||||
"printf 'apiUrl\\t%s\\n' \"$api_url\"",
|
||||
"printf 'apiExitCode\\t%s\\n' \"$api_exit\"",
|
||||
"printf 'apiStatusOk\\t%s\\n' \"$api_status_ok\"",
|
||||
"printf 'apiPreview\\t%s\\n' \"$(printf '%s' \"$api_output\" | tr '\\n\\t' ' ' | cut -c1-240)\"",
|
||||
"printf 'webUrl\\t%s\\n' \"$web_url\"",
|
||||
"printf 'webExitCode\\t%s\\n' \"$web_exit\"",
|
||||
"printf 'webBytes\\t%s\\n' \"$web_bytes\"",
|
||||
"printf 'webHasHtml\\t%s\\n' \"$web_has_html\"",
|
||||
"printf 'webErrorPreview\\t%s\\n' \"$(printf '%s' \"$web_output\" | tr '\\n\\t' ' ' | cut -c1-240)\"",
|
||||
"rm -f \"$web_tmp\"",
|
||||
"exit 0",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02ControlPlaneStatusTarget = {}): Record<string, unknown> {
|
||||
const targetMode: V02StatusTargetMode = target.mode
|
||||
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
||||
@@ -3814,8 +3994,14 @@ function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02Co
|
||||
bundle.stderr,
|
||||
);
|
||||
const runtimeWorkloadNames = String(sections.runtimeWorkloads?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
const publicProbeFields = keyValueLinesFromText(sections.publicProbes?.stdout ?? "");
|
||||
const publicProbesOk = shellSectionOk(sections.publicProbes)
|
||||
&& publicProbeFields.apiExitCode === "0"
|
||||
&& publicProbeFields.apiStatusOk === "yes"
|
||||
&& publicProbeFields.webExitCode === "0"
|
||||
&& Number(publicProbeFields.webBytes ?? 0) > 0;
|
||||
const recentPipelineRuns = parsePipelineRunRows(sections.recentPipelineRuns?.stdout ?? "", 8, Date.now(), spec.pipelineRunPrefix);
|
||||
const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && shellSectionOk(sections.argo);
|
||||
const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && shellSectionOk(sections.argo) && publicProbesOk;
|
||||
return {
|
||||
ok: baseOk,
|
||||
command: `hwlab g14 control-plane status --lane ${spec.lane}`,
|
||||
@@ -3875,6 +4061,19 @@ function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02Co
|
||||
count: runtimeWorkloadNames.length,
|
||||
exitCode: sections.runtimeWorkloads?.exitCode ?? null,
|
||||
},
|
||||
publicProbes: {
|
||||
ok: publicProbesOk,
|
||||
apiUrl: publicProbeFields.apiUrl || `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`,
|
||||
apiExitCode: numericField(publicProbeFields.apiExitCode),
|
||||
apiStatusOk: publicProbeFields.apiStatusOk === "yes",
|
||||
apiPreview: publicProbeFields.apiPreview || null,
|
||||
webUrl: publicProbeFields.webUrl || spec.publicWebUrl,
|
||||
webExitCode: numericField(publicProbeFields.webExitCode),
|
||||
webBytes: numericField(publicProbeFields.webBytes),
|
||||
webHasHtml: publicProbeFields.webHasHtml === "yes",
|
||||
webErrorPreview: publicProbeFields.webErrorPreview || null,
|
||||
exitCode: sections.publicProbes?.exitCode ?? null,
|
||||
},
|
||||
recentPipelineRuns,
|
||||
query: {
|
||||
ok: isCommandSuccess(bundle),
|
||||
@@ -4610,6 +4809,7 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
|
||||
|
||||
function v02SecretScript(options: G14SecretOptions): string {
|
||||
const spec = hwlabRuntimeLaneSpec(options.lane);
|
||||
if (options.preset === "owned-postgres-cleanup") return runtimeLaneOwnedPostgresCleanupScript(options, spec);
|
||||
if (options.preset === "generic-delete") return v02DeleteSecretScript(options);
|
||||
if (options.preset === "master-server-admin-api-key") return v02MasterAdminApiKeySecretScript(options, spec);
|
||||
return [
|
||||
@@ -4617,6 +4817,79 @@ function v02SecretScript(options: G14SecretOptions): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function runtimeLaneOwnedPostgresCleanupScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string {
|
||||
const namespace = spec.runtimeNamespace;
|
||||
const name = runtimeLanePostgresSecretName(spec);
|
||||
const pvc = `data-${name}-0`;
|
||||
const oldStatefulSet = name;
|
||||
const platformService = "g14-platform-postgres";
|
||||
return [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`name=${shellQuote(name)}`,
|
||||
`pvc=${shellQuote(pvc)}`,
|
||||
`old_statefulset=${shellQuote(oldStatefulSet)}`,
|
||||
`platform_service=${shellQuote(platformService)}`,
|
||||
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
||||
"preset=owned-postgres-cleanup",
|
||||
"exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||||
"pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }",
|
||||
"phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }",
|
||||
"before_secret_exists=$(exists_flag secret \"$name\")",
|
||||
"before_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
||||
"before_pvc_phase=$(phase_of_pvc)",
|
||||
"before_pv=$(pv_name)",
|
||||
"before_statefulset_exists=$(exists_flag statefulset \"$old_statefulset\")",
|
||||
"platform_service_exists=$(exists_flag service \"$platform_service\")",
|
||||
"action=observed",
|
||||
"mutation=false",
|
||||
"delete_secret_exit=",
|
||||
"delete_pvc_exit=",
|
||||
"if [ \"$dry_run\" = true ]; then",
|
||||
" if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
|
||||
"else",
|
||||
" kubectl -n \"$namespace\" delete secret \"$name\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err",
|
||||
" delete_secret_exit=$?",
|
||||
" kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err",
|
||||
" delete_pvc_exit=$?",
|
||||
" if [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then",
|
||||
" if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
|
||||
" else",
|
||||
" action=delete-failed",
|
||||
" fi",
|
||||
"fi",
|
||||
"after_secret_exists=$(exists_flag secret \"$name\")",
|
||||
"after_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
||||
"after_pvc_phase=$(phase_of_pvc)",
|
||||
"after_pv=$(pv_name)",
|
||||
"after_statefulset_exists=$(exists_flag statefulset \"$old_statefulset\")",
|
||||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||||
"printf 'secret\\t%s\\n' \"$name\"",
|
||||
"printf 'pvc\\t%s\\n' \"$pvc\"",
|
||||
"printf 'preset\\t%s\\n' \"$preset\"",
|
||||
"printf 'action\\t%s\\n' \"$action\"",
|
||||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||||
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
||||
"printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"",
|
||||
"printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"",
|
||||
"printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"",
|
||||
"printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"",
|
||||
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
|
||||
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
||||
"printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"",
|
||||
"printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"",
|
||||
"printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"",
|
||||
"printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"",
|
||||
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
|
||||
"printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"",
|
||||
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
|
||||
"if [ \"$before_statefulset_exists\" = yes ] || [ \"$after_statefulset_exists\" = yes ]; then exit 45; fi",
|
||||
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
|
||||
"if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function v02DeleteSecretScript(options: G14SecretOptions): string {
|
||||
return [
|
||||
"set +e",
|
||||
|
||||
Reference in New Issue
Block a user