// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. cleanup module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:1886-2201 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { NodeRuntimeCleanupOptions, NodeRuntimeCleanupPipelineRunRow } from "./entry"; import { deleteNodeRuntimeCleanupRuns, nodeRuntimeApply, nodeRuntimeCiObjectCounts, nodeRuntimeCleanupOwnedPodsFromText, nodeRuntimeCleanupOwnedPvcsFromText, nodeRuntimeCleanupOwnedTaskRunsFromText, nodeRuntimeMigration, nodeRuntimeRefresh, nodeRuntimeSync } from "./control-actions"; import { HWLAB_CI_NAMESPACE } from "./entry"; import { nodeRuntimeTriggerCurrentOutput } from "./git-mirror"; import { parseNodeScopedDelegatedOptions } from "./plan"; import { compactRuntimeCommand, compactRuntimeCommandStats, nodeRuntimeUnsupportedAction, runNodeHostScript, transPath } from "./runtime-common"; import { optionValue, positiveIntegerOption, shellQuote, statusText } from "./utils"; export function parseJsonObject(text: string): Record { try { const parsed = JSON.parse(text) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; } catch { return {}; } } export function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record, statusPath: string, timedOut: boolean): CommandResult { return { command: [transPath(), spec.nodeRoute, "sh", "--", ""], cwd: repoRoot, exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1, stdout: typeof payload.stdout === "string" ? payload.stdout : "", stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`, signal: null, timedOut, }; } export function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult { return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 }); } export function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { return runCommand([transPath(), spec.nodeKubeRoute, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } export function isCommandSuccess(result: CommandResult): boolean { return result.exitCode === 0 && !result.timedOut; } export function shortSha(value: string): string { return value.slice(0, 12).toLowerCase(); } export function shortValue(value: unknown): string { if (typeof value !== "string" || value.length === 0) return "-"; if (/^[0-9a-f]{40}$/iu.test(value)) return shortSha(value); return value.length > 30 ? `${value.slice(0, 27)}~` : value; } export function formatElapsedMs(value: unknown): string { const ms = Number(value); if (!Number.isFinite(ms) || ms < 0) return "-"; const seconds = Math.round(ms / 1000); const minutes = Math.floor(seconds / 60); const rest = seconds % 60; return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`; } export function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`; } export function nodeRuntimeRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { return `${nodeRuntimePipelineRunName(spec, sourceCommit)}-r${Date.now().toString(36)}`.slice(0, 63); } export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string { const suffix = `/${spec.runtimeRenderDir}`; if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length); return spec.gitopsRoot; } export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 }); if (!isCommandSuccess(result)) return { sourceCommit: null, result }; const match = /[0-9a-f]{40}/iu.exec(statusText(result)); return { sourceCommit: match?.[0].toLowerCase() ?? null, result }; } export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { const gitRetries = Math.max(1, spec.downloadProfile.git.retries); const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); return [ `cicd_repo=${shellQuote(spec.cicdRepo)}`, `cicd_url=${shellQuote(spec.gitUrl)}`, `cicd_branch=${shellQuote(spec.sourceBranch)}`, `cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`, `git_retries=${shellQuote(String(gitRetries))}`, `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", "retry_git() {", " op=$1", " shift", " attempt=1", " while [ \"$attempt\" -le \"$git_retries\" ]; do", " echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2", " run_git \"$@\"", " code=$?", " if [ \"$code\" -eq 0 ]; then return 0; fi", " echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2", " attempt=$((attempt + 1))", " done", " return 1", "}", "mkdir -p \"$(dirname \"$cicd_repo\")\"", "if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", " :", "elif [ -e \"$cicd_repo\" ]; then", " echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", " exit 41", "else", " retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", "fi", "git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"", "git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", "if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then", " rm -rf \"$cicd_repo\"", " retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", " git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", " retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune", "fi", ].join("\n"); } export function nodeRuntimeControlPlaneRun(scoped: ReturnType): Record | RenderedCliResult { if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped); if (scoped.action === "sync") return nodeRuntimeSync(scoped); if (scoped.action === "apply") return nodeRuntimeApply(scoped); if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrentOutput(scoped); if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped); if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped); if (scoped.action === "cleanup-released-pvs") return nodeRuntimeCleanupReleasedPvs(scoped); if (scoped.action === "cleanup-legacy-docker-images") return nodeRuntimeCleanupLegacyDockerImages(scoped); if (scoped.action === "cleanup-legacy-docker-registry-volume") return nodeRuntimeCleanupLegacyDockerRegistryVolume(scoped); return nodeRuntimeUnsupportedAction(scoped); } export function parseNodeRuntimeCleanupOptions(scoped: ReturnType): NodeRuntimeCleanupOptions { const sourceCommitRaw = optionValue(scoped.originalArgs, "--source-commit"); const pipelineRunRaw = optionValue(scoped.originalArgs, "--pipeline-run"); if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) { throw new Error("control-plane cleanup-runs accepts only one of --source-commit or --pipeline-run"); } const sourceCommit = sourceCommitRaw?.toLowerCase(); if (sourceCommit !== undefined && !/^[0-9a-f]{40}$/u.test(sourceCommit)) { throw new Error("--source-commit must be a full 40-character git sha for cleanup-runs"); } const pipelineRun = pipelineRunRaw === undefined ? undefined : validateNodeRuntimePipelineRunName(scoped.spec, pipelineRunRaw); return { minAgeMinutes: positiveIntegerOption(scoped.originalArgs, "--min-age-minutes", 60, 10080), limit: positiveIntegerOption(scoped.originalArgs, "--limit", 20, 200), sourceCommit, pipelineRun, targetPipelineRun: pipelineRun ?? (sourceCommit === undefined ? undefined : nodeRuntimePipelineRunName(scoped.spec, sourceCommit)), includeActive: scoped.originalArgs.includes("--include-active"), dryRun: scoped.dryRun || !scoped.confirm, }; } export function validateNodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, value: string): string { const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) { throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-[-rerun] PipelineRun name for --lane ${spec.lane}`); } return value.toLowerCase(); } export function nodeRuntimeCleanupRuns(scoped: ReturnType): Record { const options = parseNodeRuntimeCleanupOptions(scoped); const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec); const candidates = listNodeRuntimeCleanupPipelineRuns(scoped.spec, options); const selectedPipelineRuns = candidates .filter((item) => item.selected !== false) .map((item) => item.name); const ownedResources = listNodeRuntimeCleanupOwnedResources(scoped.spec, selectedPipelineRuns); const command = `hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane}`; if (options.dryRun) { return { ok: true, command, mode: "dry-run", node: scoped.node, lane: scoped.lane, namespace: HWLAB_CI_NAMESPACE, minAgeMinutes: options.minAgeMinutes, limit: options.limit, sourceCommit: options.sourceCommit, pipelineRun: options.pipelineRun, includeActive: options.includeActive, candidates, candidateCount: candidates.length, selectedPipelineRuns, selectedPipelineRunCount: selectedPipelineRuns.length, ownedResources, ciObjectCounts: beforeCounts, mutation: false, next: { confirm: [ command, `--min-age-minutes ${options.minAgeMinutes}`, `--limit ${options.limit}`, options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`, options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`, options.includeActive ? "--include-active" : "", "--confirm", "--wait", ].filter(Boolean).join(" "), }, }; } const deletion = deleteNodeRuntimeCleanupRuns(scoped.spec, selectedPipelineRuns, scoped.timeoutSeconds); const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec); return { ok: isCommandSuccess(deletion), command, mode: "confirmed-cleanup", node: scoped.node, lane: scoped.lane, namespace: HWLAB_CI_NAMESPACE, minAgeMinutes: options.minAgeMinutes, limit: options.limit, sourceCommit: options.sourceCommit, pipelineRun: options.pipelineRun, includeActive: options.includeActive, deletedPipelineRuns: selectedPipelineRuns, deletedPipelineRunCount: selectedPipelineRuns.length, ownedResourcesBefore: ownedResources, ciObjectCountsBefore: beforeCounts, ciObjectCountsAfter: afterCounts, deletion: compactRuntimeCommand(deletion), mutation: isCommandSuccess(deletion), degradedReason: isCommandSuccess(deletion) ? undefined : "node-runtime-ci-cleanup-delete-failed", next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, rerunStatus: options.targetPipelineRun === undefined ? undefined : `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${options.targetPipelineRun}`, }, }; } export function listNodeRuntimeCleanupPipelineRuns(spec: HwlabRuntimeLaneSpec, options: NodeRuntimeCleanupOptions): NodeRuntimeCleanupPipelineRunRow[] { if (options.targetPipelineRun !== undefined) { const target = getNodeRuntimeCleanupPipelineRun(spec, options.targetPipelineRun); if (target === null) { return [{ name: options.targetPipelineRun, createdAt: null, ageMinutes: null, status: null, reason: "target-pipelinerun-not-found", selected: false, }]; } if (target.status !== "True" && target.status !== "False" && !options.includeActive) { return [{ ...target, selected: false, selectedReason: "target-pipelinerun-not-terminal" }]; } if ((target.status === "True" || target.status === "False") && (target.ageMinutes === null || target.ageMinutes < options.minAgeMinutes)) { return [{ ...target, selected: false, selectedReason: target.ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }]; } return [target]; } const result = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pipelinerun", "-o", 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}', ], 60); if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`); const rows = nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout); const prefix = `${spec.pipelineRunPrefix}-`; const terminalRuns = rows .filter((item) => item.name.startsWith(prefix)) .filter((item) => item.status === "True" || item.status === "False") .sort((left, right) => String(left.createdAt ?? "").localeCompare(String(right.createdAt ?? ""))); const protectedLatest = terminalRuns .slice() .sort((left, right) => String(right.createdAt ?? "").localeCompare(String(left.createdAt ?? "")))[0]?.name ?? null; return terminalRuns .filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes) .map((item) => item.name === protectedLatest ? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" } : item) .slice(0, options.limit); } export function getNodeRuntimeCleanupPipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): NodeRuntimeCleanupPipelineRunRow | null { const result = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pipelinerun", pipelineRun, "-o", 'jsonpath={.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}', ], 60); if (result.exitCode !== 0) return null; return nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout)[0] ?? null; } export function nodeRuntimeCleanupPipelineRunRowsFromText(text: string): NodeRuntimeCleanupPipelineRunRow[] { const now = Date.now(); return text.split(/\r?\n/u).map((line) => { const [name = "", createdAtRaw = "", statusRaw = "", reasonRaw = ""] = line.trim().split("\t"); const createdAt = createdAtRaw.length > 0 ? createdAtRaw : null; const createdMs = createdAt === null ? NaN : Date.parse(createdAt); return { name, createdAt, ageMinutes: Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null, status: statusRaw || null, reason: reasonRaw || null, }; }).filter((item) => item.name.length > 0); } export function listNodeRuntimeCleanupOwnedResources(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[]): Record { const previewLimit = 24; if (pipelineRunNames.length === 0) { return { taskRunPreview: [], podPreview: [], pvcPreview: [], previewLimit, truncated: false, taskRunCount: 0, podCount: 0, pvcCount: 0, }; } const wanted = new Set(pipelineRunNames); const taskRunsResult = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "Succeeded"}}{{.status}}{{"\\t"}}{{.reason}}{{end}}{{end}}{{"\\n"}}{{end}}', ], 60); const podsResult = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\n"}}{{end}}', ], 60); const pvcsResult = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pvc", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}', ], 60); const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimeCleanupOwnedTaskRunsFromText(taskRunsResult.stdout, wanted) : []; const pods = isCommandSuccess(podsResult) ? nodeRuntimeCleanupOwnedPodsFromText(podsResult.stdout, wanted) : []; const pvcs = isCommandSuccess(pvcsResult) ? nodeRuntimeCleanupOwnedPvcsFromText(pvcsResult.stdout, wanted) : []; return { taskRunPreview: taskRuns.slice(0, previewLimit), podPreview: pods.slice(0, previewLimit), pvcPreview: pvcs.slice(0, previewLimit), previewLimit, truncated: taskRuns.length > previewLimit || pods.length > previewLimit || pvcs.length > previewLimit, taskRunCount: taskRuns.length, podCount: pods.length, pvcCount: pvcs.length, query: { taskRuns: compactRuntimeCommandStats(taskRunsResult), pods: compactRuntimeCommandStats(podsResult), pvcs: compactRuntimeCommandStats(pvcsResult), }, }; } interface NodeRuntimeReleasedPvCleanupOptions { limit: number; dryRun: boolean; } interface NodeRuntimeCiPvcRow { name: string; volumeName: string; phase: string; ownerKind: string; ownerName: string; storage: string; } interface NodeRuntimeCiPodClaimRow { name: string; phase: string; claims: string[]; } interface NodeRuntimeReleasedPvRow { name: string; createdAt: string; phase: string; storageClass: string; reclaimPolicy: string; claimNamespace: string; claimName: string; capacity: string; hostPath: string | null; } interface NodeRuntimeLegacyDockerImageCleanupPolicy { enabled: boolean; minAgeHours: number; keepPerRepository: number; repositories: string[]; source: string; } interface NodeRuntimeLegacyDockerImageCleanupOptions { minAgeHours: number; keepPerRepository: number; limit: number; dryRun: boolean; } interface NodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy { enabled: boolean; requireK8sRegistryReady: boolean; source: string; } interface NodeRuntimeLegacyDockerRegistryVolumeTarget { dockerContainerName: string; dockerImage: string; mountDestination: string; k8sRegistry: { mode: string; endpoint: string; namespace: string; deploymentName: string; pvcName: string; }; source: string; } function parseNodeRuntimeReleasedPvCleanupOptions(scoped: ReturnType): NodeRuntimeReleasedPvCleanupOptions { return { limit: positiveIntegerOption(scoped.originalArgs, "--limit", 200, 500), dryRun: scoped.dryRun || !scoped.confirm, }; } function parseNodeRuntimeLegacyDockerImageCleanupOptions(scoped: ReturnType, policy: NodeRuntimeLegacyDockerImageCleanupPolicy): NodeRuntimeLegacyDockerImageCleanupOptions { return { minAgeHours: positiveIntegerOption(scoped.originalArgs, "--min-age-hours", policy.minAgeHours, 8760), keepPerRepository: positiveIntegerOption(scoped.originalArgs, "--keep-per-repository", policy.keepPerRepository, 100), limit: positiveIntegerOption(scoped.originalArgs, "--limit", 50, 500), dryRun: scoped.dryRun || !scoped.confirm, }; } export function nodeRuntimeCleanupReleasedPvs(scoped: ReturnType): Record { const options = parseNodeRuntimeReleasedPvCleanupOptions(scoped); const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec); const orphanedPvcs = listNodeRuntimeOrphanedCiPvcs(scoped.spec, options.limit); const releasedPvsBefore = listNodeRuntimeReleasedCiPvs(scoped.spec, options.limit); const command = `hwlab nodes control-plane cleanup-released-pvs --node ${scoped.node} --lane ${scoped.lane}`; if (options.dryRun) { return { ok: true, command, mode: "dry-run", node: scoped.node, lane: scoped.lane, namespace: HWLAB_CI_NAMESPACE, limit: options.limit, orphanedPvcs, orphanedPvcCount: orphanedPvcs.length, releasedPvs: releasedPvsBefore, releasedPvCount: releasedPvsBefore.length, mutation: false, policy: { orphanedPvc: "ownerRef.kind=PipelineRun, owner PipelineRun absent, pvc name matches pvc-*, and no non-terminal pod mounts the claim", releasedPv: "phase=Released, storageClass=local-path, reclaimPolicy=Delete, claimRef.namespace=hwlab-ci", }, next: { confirm: `${command} --limit ${options.limit} --confirm` }, }; } const pvcDeletion = deleteNodeRuntimeCiPvcs(scoped.spec, orphanedPvcs.map((item) => item.name), scoped.timeoutSeconds); const releasedPvsAfterPvc = listNodeRuntimeReleasedCiPvs(scoped.spec, options.limit); const pvNames = releasedPvsAfterPvc.map((item) => item.name); const pvDeletion = deleteNodeRuntimePersistentVolumes(scoped.spec, pvNames, scoped.timeoutSeconds); const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec); const ok = isCommandSuccess(pvcDeletion) && isCommandSuccess(pvDeletion); return { ok, command, mode: "confirmed-cleanup", node: scoped.node, lane: scoped.lane, namespace: HWLAB_CI_NAMESPACE, limit: options.limit, deletedPvcs: orphanedPvcs.map((item) => item.name), deletedPvcCount: orphanedPvcs.length, deletedPersistentVolumes: pvNames, deletedPersistentVolumeCount: pvNames.length, orphanedPvcsBefore: orphanedPvcs.slice(0, 50), releasedPvsBefore, releasedPvsAfterPvcDelete: releasedPvsAfterPvc.slice(0, 50), ciObjectCountsBefore: beforeCounts, ciObjectCountsAfter: afterCounts, pvcDeletion: compactRuntimeCommand(pvcDeletion), pvDeletion: compactRuntimeCommand(pvDeletion), mutation: ok, degradedReason: ok ? undefined : "node-runtime-ci-pvc-pv-cleanup-failed", next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, }; } export function nodeRuntimeCleanupLegacyDockerImages(scoped: ReturnType): Record { const policy = nodeRuntimeLegacyDockerImageCleanupPolicy(scoped.node); const options = parseNodeRuntimeLegacyDockerImageCleanupOptions(scoped, policy); const command = `hwlab nodes control-plane cleanup-legacy-docker-images --node ${scoped.node} --lane ${scoped.lane}`; if (!policy.enabled) { return { ok: false, command, node: scoped.node, lane: scoped.lane, mode: "blocked", mutation: false, config: policy.source, degradedReason: "legacy-docker-image-cleanup-disabled", message: `${policy.source} is disabled`, valuesRedacted: true, }; } if (policy.repositories.length === 0) { return { ok: false, command, node: scoped.node, lane: scoped.lane, mode: "blocked", mutation: false, config: policy.source, degradedReason: "legacy-docker-image-cleanup-repositories-empty", message: `${policy.source}.repositories must include at least one repository`, valuesRedacted: true, }; } const remoteOptions = { mode: options.dryRun ? "plan" : "run", minAgeHours: options.minAgeHours, keepPerRepository: options.keepPerRepository, limit: options.limit, repositories: policy.repositories, node: scoped.node, lane: scoped.lane, valuesRedacted: true, }; const script = [ "set -eu", `node - ${shellQuote(JSON.stringify(remoteOptions))} <<'UNIDESK_LEGACY_DOCKER_IMAGE_GC'`, nodeRuntimeLegacyDockerImageCleanupNodeScript(), "UNIDESK_LEGACY_DOCKER_IMAGE_GC", ].join("\n"); const result = runNodeHostScript(scoped.spec, script, scoped.timeoutSeconds); const payload = parseJsonObject(result.stdout); const ok = isCommandSuccess(result) && payload.ok !== false; return { ok, command, node: scoped.node, lane: scoped.lane, mode: options.dryRun ? "dry-run" : "confirmed-cleanup", mutation: !options.dryRun && ok, config: policy.source, policy: { repositories: policy.repositories, minAgeHours: options.minAgeHours, keepPerRepository: options.keepPerRepository, limit: options.limit, dockerPruneUsed: false, dockerVolumesTouched: false, protectedContainerImages: true, legacyOnly: true, deploymentDependency: false, valuesRedacted: true, }, cleanup: options.dryRun || ok ? compactNodeRuntimeLegacyDockerImageCleanupPayload(payload) : payload, result: ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result), degradedReason: ok ? undefined : "legacy-docker-image-cleanup-failed", next: options.dryRun ? { confirm: `${command} --min-age-hours ${options.minAgeHours} --keep-per-repository ${options.keepPerRepository} --limit ${options.limit} --confirm` } : { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, valuesRedacted: true, }; } export function nodeRuntimeCleanupLegacyDockerRegistryVolume(scoped: ReturnType): Record { const policy = nodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy(); const target = nodeRuntimeLegacyDockerRegistryVolumeTarget(scoped.node); const dryRun = scoped.dryRun || !scoped.confirm; const command = `hwlab nodes control-plane cleanup-legacy-docker-registry-volume --node ${scoped.node} --lane ${scoped.lane}`; if (!policy.enabled) { return { ok: false, command, node: scoped.node, lane: scoped.lane, mode: "blocked", mutation: false, config: policy.source, degradedReason: "legacy-docker-registry-volume-cleanup-disabled", message: `${policy.source} is disabled`, valuesRedacted: true, }; } if (target.k8sRegistry.mode !== "k8s-workload") { return { ok: false, command, node: scoped.node, lane: scoped.lane, mode: "blocked", mutation: false, config: target.source, degradedReason: "node-local-registry-is-not-k8s-workload", message: "legacy Docker registry volume cleanup is allowed only after node-local registry is declared as a k8s workload", target, valuesRedacted: true, }; } const remoteOptions = { mode: dryRun ? "plan" : "run", target, requireK8sRegistryReady: policy.requireK8sRegistryReady, node: scoped.node, lane: scoped.lane, valuesRedacted: true, }; const script = [ "set -eu", `node - ${shellQuote(JSON.stringify(remoteOptions))} <<'UNIDESK_LEGACY_DOCKER_REGISTRY_VOLUME_GC'`, nodeRuntimeLegacyDockerRegistryVolumeCleanupNodeScript(), "UNIDESK_LEGACY_DOCKER_REGISTRY_VOLUME_GC", ].join("\n"); const result = runNodeHostScript(scoped.spec, script, scoped.timeoutSeconds); const payload = parseJsonObject(result.stdout); const ok = isCommandSuccess(result) && payload.ok !== false; return { ok, command, node: scoped.node, lane: scoped.lane, mode: dryRun ? "dry-run" : "confirmed-cleanup", mutation: !dryRun && ok && payload.mutation === true, config: { policy: policy.source, target: target.source, }, target, cleanup: compactNodeRuntimeLegacyDockerRegistryVolumeCleanupPayload(payload), result: ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result), degradedReason: ok ? undefined : "legacy-docker-registry-volume-cleanup-failed", next: dryRun ? { confirm: `${command} --confirm --wait` } : { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, valuesRedacted: true, }; } function nodeRuntimeLegacyDockerImageCleanupPolicy(node: string): NodeRuntimeLegacyDockerImageCleanupPolicy { const source = "config/unidesk-cli.yaml#gc.legacyDockerImages"; const parsed = objectRecord(Bun.YAML.parse(readFileSync(rootPath("config/unidesk-cli.yaml"), "utf8"))); const raw = objectRecord(objectRecord(parsed.gc).legacyDockerImages); const nodeLower = node.toLowerCase(); const repositories = Array.isArray(raw.repositories) ? [...new Set(raw.repositories .filter((item): item is string => typeof item === "string") .map((item) => item .replace(/\$\{nodeLower\}/gu, nodeLower) .replace(/\$\{node\}/gu, node) .replace(/\$\{NODE\}/gu, node.toUpperCase())) .map((item) => item.trim()) .filter(Boolean))] : []; return { enabled: raw.enabled === true, minAgeHours: nonNegativeInteger(raw.minAgeHours, `${source}.minAgeHours`), keepPerRepository: nonNegativeInteger(raw.keepPerRepository, `${source}.keepPerRepository`), repositories, source, }; } function nodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy(): NodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy { const source = "config/unidesk-cli.yaml#gc.legacyDockerRegistryVolumes"; const parsed = objectRecord(Bun.YAML.parse(readFileSync(rootPath("config/unidesk-cli.yaml"), "utf8"))); const raw = objectRecord(objectRecord(parsed.gc).legacyDockerRegistryVolumes); return { enabled: raw.enabled === true, requireK8sRegistryReady: raw.requireK8sRegistryReady !== false, source, }; } function nodeRuntimeLegacyDockerRegistryVolumeTarget(node: string): NodeRuntimeLegacyDockerRegistryVolumeTarget { const source = `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}#nodes.${node}`; const parsed = objectRecord(Bun.YAML.parse(readFileSync(rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH), "utf8"))); const nodeConfig = objectRecord(objectRecord(parsed.nodes)[node]); const localRegistry = objectRecord(objectRecord(objectRecord(nodeConfig.k3s).install).localRegistry); const registry = objectRecord(nodeConfig.registry); return { dockerContainerName: requiredConfigString(localRegistry.containerName, `${source}.k3s.install.localRegistry.containerName`), dockerImage: requiredConfigString(localRegistry.canonicalImage ?? localRegistry.image, `${source}.k3s.install.localRegistry.canonicalImage`), mountDestination: "/var/lib/registry", k8sRegistry: { mode: requiredConfigString(registry.mode, `${source}.registry.mode`), endpoint: requiredConfigString(registry.endpoint, `${source}.registry.endpoint`), namespace: requiredConfigString(registry.namespace, `${source}.registry.namespace`), deploymentName: requiredConfigString(registry.deploymentName, `${source}.registry.deploymentName`), pvcName: requiredConfigString(registry.pvcName, `${source}.registry.pvcName`), }, source, }; } function objectRecord(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function nonNegativeInteger(value: unknown, label: string): number { if (!Number.isInteger(value) || value < 0) throw new Error(`${label} must be a non-negative integer`); return value; } function requiredConfigString(value: unknown, label: string): string { if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${label} must be a non-empty string`); return value.trim(); } function compactNodeRuntimeLegacyDockerImageCleanupPayload(payload: Record): Record { const candidates = Array.isArray(payload.candidates) ? payload.candidates.map(objectRecord) : []; const selected = Array.isArray(payload.selected) ? payload.selected.map(objectRecord) : []; const protectedImages = Array.isArray(payload.protected) ? payload.protected.map(objectRecord) : []; const results = Array.isArray(payload.results) ? payload.results.map(objectRecord) : []; return { ok: payload.ok ?? null, mode: payload.mode ?? null, mutation: payload.mutation ?? null, diskBefore: payload.diskBefore ?? null, diskAfter: payload.diskAfter ?? null, actualDiskReclaimBytes: payload.actualDiskReclaimBytes ?? null, repositoryCount: payload.repositoryCount ?? null, imageCount: payload.imageCount ?? null, containerCount: payload.containerCount ?? null, candidateCount: payload.candidateCount ?? null, selectedCount: payload.selectedCount ?? null, deferredCount: payload.deferredCount ?? null, protectedCount: payload.protectedCount ?? null, estimatedReclaimBytes: payload.estimatedReclaimBytes ?? null, estimatedReclaimHuman: payload.estimatedReclaimHuman ?? null, candidates: candidates.slice(0, 5).map(compactLegacyDockerImageRow), selected: selected.slice(0, 10).map(compactLegacyDockerImageRow), protected: protectedImages.slice(0, 5).map((item) => ({ ...compactLegacyDockerImageRow(item), reasons: item.reasons ?? null, })), results: results.slice(0, 10).map((item) => ({ shortId: item.shortId ?? null, tags: compactLegacyDockerTags(item.refs), status: item.status ?? null, exitCode: item.exitCode ?? null, stderrTail: item.status === "failed" ? item.stderrTail ?? null : undefined, })), policy: payload.policy ?? null, disclosure: { candidateRowsShown: Math.min(5, candidates.length), selectedRowsShown: Math.min(10, selected.length), protectedRowsShown: Math.min(5, protectedImages.length), resultRowsShown: Math.min(10, results.length), }, valuesRedacted: true, }; } function compactLegacyDockerImageRow(item: Record): Record { return { shortId: item.shortId ?? null, tags: compactLegacyDockerTags(item.repoTags), createdAt: item.createdAt ?? null, ageHours: item.ageHours ?? null, sizeHuman: item.sizeHuman ?? null, }; } function compactLegacyDockerTags(value: unknown): string[] { if (!Array.isArray(value)) return []; return value .filter((item): item is string => typeof item === "string") .slice(0, 3) .map((item) => { const index = item.lastIndexOf(":"); return index > 0 ? item.slice(index + 1) : item; }); } function compactNodeRuntimeLegacyDockerRegistryVolumeCleanupPayload(payload: Record): Record { const candidate = objectRecord(payload.candidate); return { ok: payload.ok ?? null, mode: payload.mode ?? null, mutation: payload.mutation ?? null, diskBefore: payload.diskBefore ?? null, diskAfter: payload.diskAfter ?? null, actualDiskReclaimBytes: payload.actualDiskReclaimBytes ?? null, candidate: Object.keys(candidate).length === 0 ? null : { containerName: candidate.containerName ?? null, containerStatus: candidate.containerStatus ?? null, image: candidate.image ?? null, volumeName: candidate.volumeName ?? null, mountDestination: candidate.mountDestination ?? null, sizeBytes: candidate.sizeBytes ?? null, sizeHuman: candidate.sizeHuman ?? null, }, k8sRegistry: payload.k8sRegistry ?? null, blockers: Array.isArray(payload.blockers) ? payload.blockers.slice(0, 20) : [], actions: payload.actions ?? null, policy: payload.policy ?? null, valuesRedacted: true, }; } function nodeRuntimeLegacyDockerImageCleanupNodeScript(): string { return String.raw` const childProcess = require("child_process"); const options = JSON.parse(process.argv[2] || "{}"); const mode = options.mode === "run" ? "run" : "plan"; const repositories = Array.isArray(options.repositories) ? options.repositories.filter((item) => typeof item === "string" && item.length > 0) : []; const repositorySet = new Set(repositories); const minAgeHours = Number(options.minAgeHours); const keepPerRepository = Number(options.keepPerRepository); const limit = Number(options.limit); const nowMs = Date.now(); function run(args) { const result = childProcess.spawnSync(args[0], args.slice(1), { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); return { command: args, exitCode: typeof result.status === "number" ? result.status : null, signal: result.signal || null, stdout: result.stdout || "", stderr: result.stderr || "", ok: result.status === 0, }; } function parseJsonArray(text) { try { const parsed = JSON.parse(text); return Array.isArray(parsed) ? parsed : []; } catch { return []; } } function diskSnapshot() { const result = run(["df", "-B1", "/"]); const line = result.stdout.trim().split(/\r?\n/)[1] || ""; const cols = line.trim().split(/\s+/); const usePercent = Number(String(cols[4] || "0").replace(/%$/, "")); return { filesystem: cols[0] || null, sizeBytes: Number(cols[1] || 0), usedBytes: Number(cols[2] || 0), availableBytes: Number(cols[3] || 0), usePercent: Number.isFinite(usePercent) ? usePercent : null, mount: cols[5] || "/", }; } function repoFromTag(tag) { const value = String(tag || ""); const atless = value.split("@")[0]; const colon = atless.lastIndexOf(":"); const slash = atless.lastIndexOf("/"); return colon > slash ? atless.slice(0, colon) : atless; } function ageHours(createdAt) { const ms = Date.parse(String(createdAt || "")); if (!Number.isFinite(ms)) return null; return Math.max(0, Math.round(((nowMs - ms) / 3600000) * 10) / 10); } function human(bytes) { const value = Number(bytes || 0); if (!Number.isFinite(value) || value <= 0) return "0B"; const units = ["B", "KiB", "MiB", "GiB", "TiB"]; let scaled = value; let index = 0; while (scaled >= 1024 && index < units.length - 1) { scaled /= 1024; index += 1; } return String(Math.round(scaled * 10) / 10) + units[index]; } function shortId(id) { return String(id || "").replace(/^sha256:/, "").slice(0, 12); } function dockerImagePresent(id) { return run(["docker", "image", "inspect", id]).ok; } const errors = []; const dockerProbe = run(["docker", "version", "--format", "{{json .Server.Version}}"]); if (!dockerProbe.ok) { console.log(JSON.stringify({ ok: false, mode, mutation: false, error: "docker-daemon-unavailable", stderrTail: dockerProbe.stderr.slice(-2000), valuesRedacted: true })); process.exit(0); } const diskBefore = diskSnapshot(); const imageList = run(["docker", "image", "ls", "-q", "--no-trunc"]); if (!imageList.ok) errors.push({ command: imageList.command, exitCode: imageList.exitCode, stderrTail: imageList.stderr.slice(-1000) }); const imageIds = [...new Set(imageList.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean))]; const imageInspect = imageIds.length === 0 ? { ok: true, stdout: "[]", stderr: "", exitCode: 0, command: [] } : run(["docker", "image", "inspect", ...imageIds]); if (!imageInspect.ok) errors.push({ command: ["docker", "image", "inspect", ""], exitCode: imageInspect.exitCode, stderrTail: imageInspect.stderr.slice(-1000) }); const images = parseJsonArray(imageInspect.stdout).map((image) => { const repoTags = Array.isArray(image.RepoTags) ? image.RepoTags.filter((tag) => typeof tag === "string" && tag !== ":") : []; const matchedTags = repoTags.filter((tag) => repositorySet.has(repoFromTag(tag))); const matchedRepositories = [...new Set(matchedTags.map(repoFromTag))]; const createdAt = typeof image.Created === "string" ? image.Created : null; return { id: String(image.Id || ""), shortId: shortId(image.Id), repoTags, matchedTags, matchedRepositories, createdAt, createdMs: createdAt === null ? 0 : Date.parse(createdAt) || 0, ageHours: ageHours(createdAt), sizeBytes: Number(image.Size || 0), sizeHuman: human(Number(image.Size || 0)), }; }).filter((image) => image.id.length > 0); const containerList = run(["docker", "ps", "-a", "--no-trunc", "--format", "{{.ID}}"]); if (!containerList.ok) errors.push({ command: containerList.command, exitCode: containerList.exitCode, stderrTail: containerList.stderr.slice(-1000) }); const containerIds = [...new Set(containerList.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean))]; const containerInspect = containerIds.length === 0 ? { ok: true, stdout: "[]", stderr: "", exitCode: 0, command: [] } : run(["docker", "container", "inspect", ...containerIds]); if (!containerInspect.ok) errors.push({ command: ["docker", "container", "inspect", ""], exitCode: containerInspect.exitCode, stderrTail: containerInspect.stderr.slice(-1000) }); const containers = parseJsonArray(containerInspect.stdout).map((container) => ({ id: String(container.Id || ""), name: String(container.Name || "").replace(/^\//, ""), imageId: String(container.Image || ""), imageRef: String((container.Config && container.Config.Image) || ""), state: String((container.State && container.State.Status) || ""), })); const protectedByContainer = new Set(containers.map((container) => container.imageId).filter(Boolean)); const latestProtected = new Set(); for (const repository of repositories) { images .filter((image) => image.matchedRepositories.includes(repository)) .sort((a, b) => (b.createdMs - a.createdMs) || a.id.localeCompare(b.id)) .slice(0, keepPerRepository) .forEach((image) => latestProtected.add(image.id)); } const candidates = []; const protectedImages = []; for (const image of images.filter((item) => item.matchedRepositories.length > 0)) { const reasons = []; if (protectedByContainer.has(image.id)) reasons.push("container-referenced"); if (latestProtected.has(image.id)) reasons.push("latest-retention"); if (image.ageHours === null) reasons.push("created-at-unknown"); if (image.ageHours !== null && image.ageHours < minAgeHours) reasons.push("under-min-age"); if (reasons.length > 0) { protectedImages.push({ ...image, reasons }); } else { candidates.push({ ...image, reasons: ["repository-allowlisted", "unreferenced-by-container", "older-than-min-age", "outside-latest-retention"] }); } } candidates.sort((a, b) => (a.createdMs - b.createdMs) || a.id.localeCompare(b.id)); const selected = candidates.slice(0, limit); const results = []; if (mode === "run") { for (const image of selected) { const refs = image.matchedTags.length > 0 ? image.matchedTags : [image.id]; const remove = run(["docker", "image", "rm", ...refs]); results.push({ id: image.id, shortId: image.shortId, refs, status: remove.ok && !dockerImagePresent(image.id) ? "deleted" : remove.ok ? "untagged-or-shared" : "failed", exitCode: remove.exitCode, stdoutTail: remove.stdout.slice(-1000), stderrTail: remove.stderr.slice(-1000), }); } } const diskAfter = mode === "run" ? diskSnapshot() : null; const actualDiskReclaimBytes = diskAfter === null ? null : Math.max(0, Number(diskBefore.usedBytes || 0) - Number(diskAfter.usedBytes || 0)); const ok = errors.length === 0 && results.every((item) => item.status !== "failed"); console.log(JSON.stringify({ ok, mode, mutation: mode === "run", node: options.node || null, lane: options.lane || null, repositories, repositoryCount: repositories.length, diskBefore, diskAfter, actualDiskReclaimBytes, imageCount: images.length, containerCount: containers.length, activeContainers: containers.filter((container) => container.state === "running").slice(0, 20), candidateCount: candidates.length, selectedCount: selected.length, deferredCount: Math.max(0, candidates.length - selected.length), protectedCount: protectedImages.length, estimatedReclaimBytes: selected.reduce((sum, image) => sum + Number(image.sizeBytes || 0), 0), estimatedReclaimHuman: human(selected.reduce((sum, image) => sum + Number(image.sizeBytes || 0), 0)), candidates, selected, protected: protectedImages, results, errors, policy: { minAgeHours, keepPerRepository, limit, dockerPruneUsed: false, dockerVolumesTouched: false, matchedRepositoriesOnly: true, protectedContainerImages: true, valuesRedacted: true, }, valuesRedacted: true, })); `; } function nodeRuntimeLegacyDockerRegistryVolumeCleanupNodeScript(): string { return String.raw` const childProcess = require("child_process"); const options = JSON.parse(process.argv[2] || "{}"); const mode = options.mode === "run" ? "run" : "plan"; const target = options.target || {}; const requireK8sRegistryReady = options.requireK8sRegistryReady !== false; function run(args, extraEnv) { const result = childProcess.spawnSync(args[0], args.slice(1), { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, env: { ...process.env, ...(extraEnv || {}) }, }); return { command: args, exitCode: typeof result.status === "number" ? result.status : null, signal: result.signal || null, stdout: result.stdout || "", stderr: result.stderr || "", ok: result.status === 0, }; } function parseJson(text) { try { return JSON.parse(text); } catch { return null; } } function diskSnapshot() { const result = run(["df", "-B1", "/"]); const line = result.stdout.trim().split(/\r?\n/)[1] || ""; const cols = line.trim().split(/\s+/); const usePercent = Number(String(cols[4] || "0").replace(/%$/, "")); return { filesystem: cols[0] || null, sizeBytes: Number(cols[1] || 0), usedBytes: Number(cols[2] || 0), availableBytes: Number(cols[3] || 0), usePercent: Number.isFinite(usePercent) ? usePercent : null, mount: cols[5] || "/", }; } function duBytes(path) { const bytes = run(["du", "-sb", path]); if (bytes.ok) { const value = Number((bytes.stdout.trim().split(/\s+/)[0] || "0")); if (Number.isFinite(value)) return value; } const kib = run(["du", "-sk", path]); const value = Number((kib.stdout.trim().split(/\s+/)[0] || "0")); return Number.isFinite(value) ? value * 1024 : null; } function human(bytes) { const value = Number(bytes || 0); if (!Number.isFinite(value) || value <= 0) return "0B"; const units = ["B", "KiB", "MiB", "GiB", "TiB"]; let scaled = value; let index = 0; while (scaled >= 1024 && index < units.length - 1) { scaled /= 1024; index += 1; } return String(Math.round(scaled * 10) / 10) + units[index]; } function inspectContainer(name) { const result = run(["docker", "container", "inspect", name]); if (!result.ok) return { ok: false, result, container: null }; const parsed = parseJson(result.stdout); return { ok: Array.isArray(parsed) && parsed.length > 0, result, container: Array.isArray(parsed) ? parsed[0] : null }; } function listContainers() { const list = run(["docker", "ps", "-a", "--no-trunc", "--format", "{{.ID}}"]); if (!list.ok) return []; const ids = [...new Set(list.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean))]; if (ids.length === 0) return []; const inspect = run(["docker", "container", "inspect", ...ids]); const parsed = parseJson(inspect.stdout); return Array.isArray(parsed) ? parsed : []; } function k8sRegistryStatus(registry) { const env = { KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" }; const namespace = String(registry.namespace || ""); const deploymentName = String(registry.deploymentName || ""); const pvcName = String(registry.pvcName || ""); const deployResult = run(["kubectl", "-n", namespace, "get", "deployment", deploymentName, "-o", "json"], env); const pvcResult = run(["kubectl", "-n", namespace, "get", "pvc", pvcName, "-o", "json"], env); const endpoint = String(registry.endpoint || ""); const curlResult = endpoint.length > 0 ? run(["curl", "-fsS", "--max-time", "5", "http://" + endpoint + "/v2/"]) : { ok: false, exitCode: null, stderr: "endpoint missing" }; const deployment = parseJson(deployResult.stdout) || {}; const pvc = parseJson(pvcResult.stdout) || {}; const desired = Math.max(1, Number((deployment.spec && deployment.spec.replicas) || 1)); const readyReplicas = Number((deployment.status && deployment.status.readyReplicas) || 0); const updatedReplicas = Number((deployment.status && deployment.status.updatedReplicas) || 0); const pvcPhase = String((pvc.status && pvc.status.phase) || ""); return { ok: deployResult.ok && pvcResult.ok && curlResult.ok && readyReplicas >= desired && updatedReplicas >= desired && pvcPhase === "Bound", namespace, deploymentName, pvcName, desiredReplicas: desired, readyReplicas, updatedReplicas, pvcPhase, endpoint, endpointOk: curlResult.ok, deployExitCode: deployResult.exitCode, pvcExitCode: pvcResult.exitCode, curlExitCode: curlResult.exitCode, valuesRedacted: true, }; } const diskBefore = diskSnapshot(); const inspect = inspectContainer(String(target.dockerContainerName || "registry")); if (!inspect.ok || inspect.container === null) { console.log(JSON.stringify({ ok: true, mode, mutation: false, diskBefore, diskAfter: null, actualDiskReclaimBytes: null, candidate: null, k8sRegistry: k8sRegistryStatus(target.k8sRegistry || {}), blockers: [], actions: { containerRemoved: false, volumeRemoved: false }, policy: { noCandidateIsOk: true, dockerPruneUsed: false, dockerVolumeRmScoped: false, valuesRedacted: true }, valuesRedacted: true, })); process.exit(0); } const container = inspect.container; const state = container.State || {}; const config = container.Config || {}; const mounts = Array.isArray(container.Mounts) ? container.Mounts : []; const registryMount = mounts.find((mount) => mount && mount.Type === "volume" && mount.Destination === target.mountDestination); const allContainers = listContainers(); const otherUsers = registryMount === undefined ? [] : allContainers .filter((item) => item && item.Id !== container.Id) .filter((item) => Array.isArray(item.Mounts) && item.Mounts.some((mount) => mount && mount.Name === registryMount.Name)) .map((item) => ({ id: String(item.Id || "").slice(0, 12), name: String(item.Name || "").replace(/^\//, ""), state: String((item.State && item.State.Status) || "") })); const k8sRegistry = k8sRegistryStatus(target.k8sRegistry || {}); const volumeSource = registryMount && typeof registryMount.Source === "string" ? registryMount.Source : null; const volumeName = registryMount && typeof registryMount.Name === "string" ? registryMount.Name : null; const sizeBytes = volumeSource === null ? null : duBytes(volumeSource); const candidate = registryMount === undefined ? null : { containerName: String(container.Name || "").replace(/^\//, ""), containerStatus: String(state.Status || ""), running: state.Running === true, image: String(config.Image || ""), volumeName, volumeSource, mountDestination: String(registryMount.Destination || ""), sizeBytes, sizeHuman: human(sizeBytes), otherUsers, valuesRedacted: true, }; const blockers = []; if (registryMount === undefined) blockers.push("registry-volume-mount-missing"); if (state.Running === true || String(state.Status || "") === "running") blockers.push("docker-registry-container-running"); if (String(config.Image || "") !== String(target.dockerImage || "")) blockers.push("docker-registry-image-mismatch"); if (otherUsers.length > 0) blockers.push("docker-volume-has-other-container-users"); if (volumeName === null || volumeName.length === 0) blockers.push("docker-volume-name-missing"); if (volumeSource === null || !volumeSource.startsWith("/var/lib/docker/volumes/") || !volumeSource.endsWith("/_data")) blockers.push("docker-volume-source-not-under-docker-volumes"); if (requireK8sRegistryReady && k8sRegistry.ok !== true) blockers.push("replacement-k8s-registry-not-ready"); let actions = { containerRemoved: false, volumeRemoved: false, containerRmExitCode: null, volumeRmExitCode: null, stderrTail: "" }; let diskAfter = null; let mutation = false; if (mode === "run" && blockers.length === 0 && candidate !== null) { const removeContainer = run(["docker", "rm", String(target.dockerContainerName || "registry")]); const removeVolume = removeContainer.ok ? run(["docker", "volume", "rm", String(volumeName)]) : { ok: false, exitCode: null, stderr: "container removal failed" }; actions = { containerRemoved: removeContainer.ok, volumeRemoved: removeVolume.ok, containerRmExitCode: removeContainer.exitCode, volumeRmExitCode: removeVolume.exitCode, stderrTail: (removeContainer.stderr + "\n" + removeVolume.stderr).trim().slice(-1000), }; mutation = removeContainer.ok && removeVolume.ok; if (!removeContainer.ok) blockers.push("docker-container-rm-failed"); if (removeContainer.ok && !removeVolume.ok) blockers.push("docker-volume-rm-failed"); diskAfter = diskSnapshot(); } if (diskAfter === null) diskAfter = mode === "run" ? diskSnapshot() : null; const actualDiskReclaimBytes = diskAfter === null ? null : Math.max(0, Number(diskBefore.usedBytes || 0) - Number(diskAfter.usedBytes || 0)); console.log(JSON.stringify({ ok: blockers.length === 0, mode, mutation, diskBefore, diskAfter, actualDiskReclaimBytes, candidate, k8sRegistry, blockers, actions, policy: { requireK8sRegistryReady, dockerPruneUsed: false, dockerVolumeRmScoped: mode === "run" && mutation, dockerComposeTouched: false, databaseTouched: false, matchedExitedRegistryContainerOnly: true, valuesRedacted: true, }, valuesRedacted: true, })); `; } function listNodeRuntimePipelineRunNames(spec: HwlabRuntimeLaneSpec): Set { const result = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pipelinerun", "-o", 'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}', ], 60); if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`); return new Set(result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean)); } function listNodeRuntimeCiPodClaims(spec: HwlabRuntimeLaneSpec): NodeRuntimeCiPodClaimRow[] { const result = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .spec.volumes}}{{if .persistentVolumeClaim}}{{.persistentVolumeClaim.claimName}}{{","}}{{end}}{{end}}{{"\\n"}}{{end}}', ], 60); if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} Pod PVC mounts: ${result.stderr.trim().slice(0, 1000)}`); return result.stdout.split(/\r?\n/u).map((line) => { const [name = "", phase = "", claimsRaw = ""] = line.trim().split("\t"); return { name, phase, claims: claimsRaw.split(",").map((claim) => claim.trim()).filter(Boolean) }; }).filter((item) => item.name.length > 0); } function listNodeRuntimeCiPvcs(spec: HwlabRuntimeLaneSpec): NodeRuntimeCiPvcRow[] { const result = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pvc", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}', ], 60); if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PVCs: ${result.stderr.trim().slice(0, 1000)}`); return result.stdout.split(/\r?\n/u).map((line) => { const [name = "", volumeName = "", phase = "", ownerKind = "", ownerName = "", storage = ""] = line.trim().split("\t"); return { name, volumeName, phase, ownerKind, ownerName, storage }; }).filter((item) => item.name.length > 0); } export function listNodeRuntimeOrphanedCiPvcs(spec: HwlabRuntimeLaneSpec, limit: number): NodeRuntimeCiPvcRow[] { const pipelineRuns = listNodeRuntimePipelineRunNames(spec); const activeClaims = new Set(listNodeRuntimeCiPodClaims(spec) .filter((pod) => pod.phase !== "Succeeded" && pod.phase !== "Failed") .flatMap((pod) => pod.claims)); return listNodeRuntimeCiPvcs(spec) .filter((item) => /^pvc-[a-z0-9]+$/u.test(item.name)) .filter((item) => item.ownerKind === "PipelineRun" && item.ownerName.length > 0) .filter((item) => !pipelineRuns.has(item.ownerName)) .filter((item) => !activeClaims.has(item.name)) .sort((a, b) => a.name.localeCompare(b.name)) .slice(0, limit); } export function listNodeRuntimeReleasedCiPvs(spec: HwlabRuntimeLaneSpec, limit: number): NodeRuntimeReleasedPvRow[] { const result = runNodeK3sArgs(spec, [ "kubectl", "get", "pv", "-o", 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}', ], 60); if (!isCommandSuccess(result)) throw new Error(`failed to list released ${HWLAB_CI_NAMESPACE} PVs: ${result.stderr.trim().slice(0, 1000)}`); return result.stdout.split(/\r?\n/u).map((line) => { const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = "", localPath = "", hostPath = ""] = line.trim().split("\t"); return { name, createdAt, phase, storageClass, reclaimPolicy, claimNamespace, claimName, capacity, hostPath: safeNodeRuntimeStoragePath(localPath) ?? safeNodeRuntimeStoragePath(hostPath) }; }) .filter((item) => item.name.length > 0) .filter((item) => item.phase === "Released") .filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete") .filter((item) => item.claimNamespace === HWLAB_CI_NAMESPACE && /^pvc-[a-z0-9]+$/u.test(item.claimName)) .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))) .slice(0, limit); } function safeNodeRuntimeStoragePath(value: string): string | null { if (!value.startsWith("/var/lib/rancher/k3s/storage/")) return null; if (value.includes("\0") || value.split("/").some((part) => part === "..")) return null; return value; } function deleteNodeRuntimeCiPvcs(spec: HwlabRuntimeLaneSpec, names: string[], timeoutSeconds: number): CommandResult { if (names.length === 0) return { command: [], cwd: repoRoot, exitCode: 0, stdout: "no orphaned PVC candidates", stderr: "", signal: null, timedOut: false }; return runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "delete", "pvc", ...names, "--ignore-not-found=true", "--wait=true", `--timeout=${timeoutSeconds}s`], timeoutSeconds); } function deleteNodeRuntimePersistentVolumes(spec: HwlabRuntimeLaneSpec, names: string[], timeoutSeconds: number): CommandResult { if (names.length === 0) return { command: [], cwd: repoRoot, exitCode: 0, stdout: "no released PV candidates", stderr: "", signal: null, timedOut: false }; return runNodeK3sArgs(spec, ["kubectl", "delete", "pv", ...names, "--ignore-not-found=true", "--wait=true", `--timeout=${timeoutSeconds}s`], timeoutSeconds); }