diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 9e4ccad4..e7a056d6 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -633,6 +633,14 @@ assertCondition( && jobsSourceText.includes("hwlab nodes control-plane status --node"), "v0.3 runtime lane retry and trigger visibility must be represented by controlled cleanup/help/progress paths", ); +assertCondition( + sourceText.includes("remoteStoragePathEstimates") + && sourceText.includes("activeMountPods") + && sourceText.includes("estimatedReclaimBytes") + && sourceText.includes("selectedPersistentVolumes") + && sourceText.includes("storageHostPathFromClaim"), + "HWLAB CI workspace retention dry-run must expose owner-aware PVC/PV hostPath, active mount, and reclaim estimate visibility", +); const staleSuccessAlignment = v02CommitAlignment({ expectedSourceHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @@ -1048,6 +1056,7 @@ console.log(JSON.stringify({ "v0.2 PipelineRun service matrix excludes hwlab-cli", "v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance", "v0.2 status warns on slow build TaskRuns", + "HWLAB CI workspace retention dry-run exposes PVC/PV hostPath and reclaim estimates", "rollout brief includes natural-language changelog before automatic diff summary", "semantic changelog extracts Chinese summary sections", "rollout brief includes lazy-build reused/rebuild metrics and service durations", diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 9e362844..f59370eb 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -100,7 +100,7 @@ const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000; const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000; const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000; -export type G14MonitorLane = "g14" | "v02"; +export type G14MonitorLane = "g14" | HwlabRuntimeLane; interface G14MonitorOptions { lane: G14MonitorLane; @@ -197,13 +197,13 @@ interface G14ObservabilityOptions { } interface G14SecretOptions { - action: "status" | "ensure" | "delete"; + action: "status" | "ensure" | "delete" | "cleanup-owned-postgres"; lane: HwlabRuntimeLane; dryRun: boolean; confirm: boolean; name: string; key?: string; - preset: "openfga" | "master-server-admin-api-key" | "generic-delete"; + preset: "openfga" | "master-server-admin-api-key" | "generic-delete" | "owned-postgres-cleanup"; timeoutSeconds: number; } @@ -282,16 +282,15 @@ export interface V02PrCommentInput { } export function hwlabG14MonitorStateFileName(options: Pick & { lane?: G14MonitorLane }): string { - const prefix = options.lane === "v02" ? "latest-v02-" : "latest-"; - if (options.once && options.dryRun) return options.lane === "v02" ? `${prefix}once-dry-run-job.json` : "latest-once-dry-run-job.json"; - if (options.once && options.lane === "v02") return `${prefix}once-job.json`; - if (options.once) return "latest-once-job.json"; + const prefix = options.lane !== undefined && options.lane !== "g14" ? `latest-${options.lane}-` : "latest-"; + if (options.once && options.dryRun) return `${prefix}once-dry-run-job.json`; + if (options.once) return `${prefix}once-job.json`; if (options.dryRun) return `${prefix}dry-run-job.json`; return `${prefix}monitor-job.json`; } function hwlabG14MonitorStateRole(options: Pick & { lane?: G14MonitorLane }): string { - const lanePrefix = options.lane === "v02" ? "v02-" : ""; + const lanePrefix = options.lane !== undefined && options.lane !== "g14" ? `${options.lane}-` : ""; if (options.once && options.dryRun) return `${lanePrefix}once-dry-run`; if (options.once) return `${lanePrefix}once`; if (options.dryRun) return `${lanePrefix}dry-run-monitor`; @@ -300,7 +299,7 @@ function hwlabG14MonitorStateRole(options: Pick [--dry-run|--confirm]"); + if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "delete" && actionRaw !== "cleanup-owned-postgres") { + throw new Error("secret usage: status|ensure --lane v02|v03 --name hwlab-v0x-openfga|hwlab-v0x-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name [--dry-run|--confirm] | cleanup-owned-postgres --lane v03 [--dry-run|--confirm]"); } const laneRaw = optionValue(args, "--lane") ?? "v02"; if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`secret --lane must be one of ${hwlabRuntimeLaneIds().join(", ")}`); @@ -585,6 +584,21 @@ function parseSecretOptions(args: string[]): G14SecretOptions { const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run"); + if (actionRaw === "cleanup-owned-postgres") { + if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres"); + if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key"); + const expectedName = postgresSecret; + if (name !== openFgaSecret && name !== expectedName) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${expectedName}; omit --name or pass --name ${expectedName}`); + return { + action: actionRaw, + lane, + confirm, + dryRun: explicitDryRun || !confirm, + name: expectedName, + preset: "owned-postgres-cleanup", + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), + }; + } if (actionRaw === "delete") { if (lane !== "v02") throw new Error("secret delete is currently limited to --lane v02 obsolete cleanup"); if (key !== undefined) throw new Error("secret delete does not accept --key; it deletes a whole obsolete Secret object"); @@ -913,7 +927,7 @@ function isCommandSuccess(result: CommandJsonResult): boolean { } function monitorBaseBranch(lane: G14MonitorLane): string { - return lane === "v02" ? V02_SOURCE_BRANCH : G14_SOURCE_BRANCH; + return lane === "g14" ? G14_SOURCE_BRANCH : hwlabRuntimeLaneSpec(lane).sourceBranch; } function extractPullRequests(result: CommandJsonResult, baseBranch = G14_SOURCE_BRANCH): OpenPullRequest[] { @@ -951,6 +965,14 @@ function printV02PrMonitorProgress(data: Record = {}): void { printProgressEvent("hwlab.v02.pr-monitor.progress", data); } +function printRuntimeLanePrMonitorProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { + printProgressEvent("hwlab.runtime-lane.pr-monitor.progress", { + lane: spec.lane, + node: spec.nodeId, + ...data, + }); +} + function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { printProgressEvent("hwlab.runtime-lane.trigger.progress", { lane: spec.lane, @@ -2649,6 +2671,59 @@ function parseCleanupPipelineRunLine(line: string, nowMs: number): CleanupPipeli return { name, createdAt, ageMinutes, status: status || null, reason: reason || null }; } +function formatRetentionBytes(bytes: number): string | null { + if (!Number.isFinite(bytes)) return null; + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`; +} + +function storageHostPathFromClaim(volume: string | null, namespace: string, claimName: string | null): string | null { + if (volume === null || claimName === null) return null; + if (!/^pvc-[a-z0-9-]+$/iu.test(volume) || !/^pvc-[a-z0-9]+$/iu.test(claimName)) return null; + return `/var/lib/rancher/k3s/storage/${volume}_${namespace}_${claimName}`; +} + +function safeStorageHostPath(value: unknown): string | null { + const path = stringOrNull(value); + return path !== null && path.startsWith("/var/lib/rancher/k3s/storage/") ? path : null; +} + +function remoteStoragePathEstimates(paths: Array): Map { + const uniquePaths = [...new Set(paths.map((item) => safeStorageHostPath(item)).filter((item): item is string => item !== null))]; + const estimates = new Map(uniquePaths.map((path) => [path, null])); + if (uniquePaths.length === 0) return estimates; + const script = [ + "set -eu", + `for path in ${uniquePaths.map((path) => shellQuote(path)).join(" ")}; do`, + " bytes=\"\"", + " if [ -e \"$path\" ]; then bytes=$(du -sB1 \"$path\" 2>/dev/null | awk '{print $1}' || true); fi", + " printf '%s\\t%s\\n' \"$path\" \"$bytes\"", + "done", + ].join("\n"); + const result = g14HostScript(script, 120_000); + if (!isCommandSuccess(result)) return estimates; + for (const line of statusText(result).split(/\r?\n/u)) { + const [path = "", rawBytes = ""] = line.trim().split("\t"); + if (!estimates.has(path)) continue; + const bytes = /^\d+$/u.test(rawBytes) ? Number(rawBytes) : null; + estimates.set(path, Number.isFinite(bytes) ? bytes : null); + } + return estimates; +} + +function sumEstimatedBytes(items: Record[]): number { + return items.reduce((sum, item) => { + const bytes = item.estimatedBytes; + return sum + (typeof bytes === "number" && Number.isFinite(bytes) ? bytes : 0); + }, 0); +} + function cleanupPipelineRunTargetCandidate(input: { targetPipelineRun: string; text: string; @@ -2810,22 +2885,75 @@ function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record>(); + for (const line of statusText(pvResult).split(/\r?\n/u)) { + const [name = "", storageClass = "", reclaimPolicy = "", localPath = "", hostPath = ""] = line.trim().split("\t"); + if (name.length === 0) continue; + pvByName.set(name, { + storageClass: storageClass || null, + reclaimPolicy: reclaimPolicy || null, + hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath), + }); + } + const activeClaimPods = new Map(); + for (const line of statusText(podResult).split(/\r?\n/u)) { + const [podName = "", phase = "", claims = ""] = line.trim().split("\t"); + if (podName.length === 0 || phase === "Succeeded" || phase === "Failed") continue; + for (const claimName of claims.split(",").map((item) => item.trim()).filter(Boolean)) { + const entry = activeClaimPods.get(claimName) ?? []; + entry.push(podName); + activeClaimPods.set(claimName, entry); + } + } const wanted = new Set(pipelineRunNames); - return statusText(result) + const ownedPvcs = statusText(result) .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t"); + const pv = volume.length > 0 ? pvByName.get(volume) : undefined; + const hostPath = safeStorageHostPath(pv?.hostPath) ?? storageHostPathFromClaim(volume || null, CI_NAMESPACE, name || null); return { name, volume: volume || null, phase: phase || null, ownerKind: ownerKind || null, owner: owner || null, + storageClass: pv?.storageClass ?? null, + reclaimPolicy: pv?.reclaimPolicy ?? null, + hostPath, + activeMountPods: activeClaimPods.get(name) ?? [], }; }) .filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner)); + const estimates = remoteStoragePathEstimates(ownedPvcs.map((item) => stringOrNull(item.hostPath))); + return ownedPvcs.map((item) => ({ + ...item, + estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null, + })); } function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult { @@ -2848,17 +2976,17 @@ function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record line.trim()) .filter(Boolean) .map((line) => { - const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t"); + const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = "", localPath = "", hostPath = ""] = line.split("\t"); return { name, createdAt, @@ -2868,7 +2996,7 @@ function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null, + hostPath: safeStorageHostPath(localPath) ?? safeStorageHostPath(hostPath) ?? (claimNamespace === CI_NAMESPACE ? storageHostPathFromClaim(name || null, claimNamespace, claimName || null) : null), }; }) .filter((item) => item.phase === "Released") @@ -2876,6 +3004,11 @@ function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName)) .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))) .slice(0, options.limit); + const estimates = remoteStoragePathEstimates(candidates.map((item) => stringOrNull(item.hostPath))); + return candidates.map((item) => ({ + ...item, + estimatedBytes: item.hostPath === null ? null : estimates.get(String(item.hostPath)) ?? null, + })); } function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult { @@ -2895,6 +3028,7 @@ function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJso function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record { const candidates = listReleasedCiWorkspacePvs(options); const candidateNames = candidates.map((item) => String(item.name)); + const estimatedReclaimBytes = sumEstimatedBytes(candidates); if (options.dryRun) { return { ok: true, @@ -2904,6 +3038,10 @@ function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Reco limit: options.limit, candidates, candidateCount: candidates.length, + selectedPersistentVolumes: candidateNames, + selectedPersistentVolumeCount: candidateNames.length, + estimatedReclaimBytes, + estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes), mutation: false, next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` }, }; @@ -2918,6 +3056,8 @@ function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Reco deletedPersistentVolumes: candidateNames, deletedPersistentVolumeCount: candidateNames.length, candidatesBefore: candidates, + estimatedReclaimBytes, + estimatedReclaimHuman: formatRetentionBytes(estimatedReclaimBytes), deletion, followUp: { diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'", @@ -2985,6 +3125,7 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record item.selected !== false) .map((item) => String(item.name)); const ownedPvcs = listOwnedWorkspacePvcs(candidateNames); + const estimatedReclaimBytes = sumEstimatedBytes(ownedPvcs); const followUpStatusLane = isHwlabRuntimeLane(options.lane) ? options.lane : "v02"; if (options.dryRun) { return { @@ -3002,6 +3143,8 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record&1)", + "api_exit=$?", + "api_status_ok=no", + "if [ \"$api_exit\" = 0 ] && printf '%s' \"$api_output\" | grep -Eq '\"status\"[[:space:]]*:[[:space:]]*\"ok\"'; then api_status_ok=yes; fi", + "web_tmp=$(mktemp /tmp/hwlab-lane-web-probe.XXXXXX)", + "web_output=$(curl -fsS --connect-timeout 3 --max-time 15 -o \"$web_tmp\" \"$web_url\" 2>&1)", + "web_exit=$?", + "web_bytes=0", + "web_has_html=no", + "if [ -f \"$web_tmp\" ]; then", + " web_bytes=$(wc -c < \"$web_tmp\" | tr -d ' ')", + " if grep -Eiq ' { const targetMode: V02StatusTargetMode = target.mode ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); @@ -3814,8 +3994,14 @@ function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02Co bundle.stderr, ); const runtimeWorkloadNames = String(sections.runtimeWorkloads?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + const publicProbeFields = keyValueLinesFromText(sections.publicProbes?.stdout ?? ""); + const publicProbesOk = shellSectionOk(sections.publicProbes) + && publicProbeFields.apiExitCode === "0" + && publicProbeFields.apiStatusOk === "yes" + && publicProbeFields.webExitCode === "0" + && Number(publicProbeFields.webBytes ?? 0) > 0; const recentPipelineRuns = parsePipelineRunRows(sections.recentPipelineRuns?.stdout ?? "", 8, Date.now(), spec.pipelineRunPrefix); - const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && shellSectionOk(sections.argo); + const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && shellSectionOk(sections.argo) && publicProbesOk; return { ok: baseOk, command: `hwlab g14 control-plane status --lane ${spec.lane}`, @@ -3875,6 +4061,19 @@ function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02Co count: runtimeWorkloadNames.length, exitCode: sections.runtimeWorkloads?.exitCode ?? null, }, + publicProbes: { + ok: publicProbesOk, + apiUrl: publicProbeFields.apiUrl || `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`, + apiExitCode: numericField(publicProbeFields.apiExitCode), + apiStatusOk: publicProbeFields.apiStatusOk === "yes", + apiPreview: publicProbeFields.apiPreview || null, + webUrl: publicProbeFields.webUrl || spec.publicWebUrl, + webExitCode: numericField(publicProbeFields.webExitCode), + webBytes: numericField(publicProbeFields.webBytes), + webHasHtml: publicProbeFields.webHasHtml === "yes", + webErrorPreview: publicProbeFields.webErrorPreview || null, + exitCode: sections.publicProbes?.exitCode ?? null, + }, recentPipelineRuns, query: { ok: isCommandSuccess(bundle), @@ -4610,6 +4809,7 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record/dev/null 2>&1 && printf yes || printf no; }", + "pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }", + "phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }", + "before_secret_exists=$(exists_flag secret \"$name\")", + "before_pvc_exists=$(exists_flag pvc \"$pvc\")", + "before_pvc_phase=$(phase_of_pvc)", + "before_pv=$(pv_name)", + "before_statefulset_exists=$(exists_flag statefulset \"$old_statefulset\")", + "platform_service_exists=$(exists_flag service \"$platform_service\")", + "action=observed", + "mutation=false", + "delete_secret_exit=", + "delete_pvc_exit=", + "if [ \"$dry_run\" = true ]; then", + " if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=would-delete; else action=already-absent; fi", + "else", + " kubectl -n \"$namespace\" delete secret \"$name\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err", + " delete_secret_exit=$?", + " kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err", + " delete_pvc_exit=$?", + " if [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then", + " if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi", + " else", + " action=delete-failed", + " fi", + "fi", + "after_secret_exists=$(exists_flag secret \"$name\")", + "after_pvc_exists=$(exists_flag pvc \"$pvc\")", + "after_pvc_phase=$(phase_of_pvc)", + "after_pv=$(pv_name)", + "after_statefulset_exists=$(exists_flag statefulset \"$old_statefulset\")", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$name\"", + "printf 'pvc\\t%s\\n' \"$pvc\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", + "printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"", + "printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"", + "printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"", + "printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"", + "printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"", + "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", + "printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"", + "printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"", + "printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"", + "printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"", + "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", + "printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"", + "if [ \"$platform_service_exists\" != yes ]; then exit 44; fi", + "if [ \"$before_statefulset_exists\" = yes ] || [ \"$after_statefulset_exists\" = yes ]; then exit 45; fi", + "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", + "if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi", + ].join("\n"); +} + function v02DeleteSecretScript(options: G14SecretOptions): string { return [ "set +e",