Codex
1404 lines
62 KiB
TypeScript
1404 lines
62 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. cleanup module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:1886-2201 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { NodeRuntimeCleanupOptions, NodeRuntimeCleanupPipelineRunRow } from "./entry";
|
|
import { deleteNodeRuntimeCleanupRuns, nodeRuntimeApply, nodeRuntimeCiObjectCounts, nodeRuntimeCleanupOwnedPodsFromText, nodeRuntimeCleanupOwnedPvcsFromText, nodeRuntimeCleanupOwnedTaskRunsFromText, nodeRuntimeMigration, nodeRuntimeRefresh, nodeRuntimeSync } from "./control-actions";
|
|
import { HWLAB_CI_NAMESPACE } from "./entry";
|
|
import { nodeRuntimeTriggerCurrentOutput } from "./git-mirror";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { compactRuntimeCommand, compactRuntimeCommandStats, nodeRuntimeUnsupportedAction, runNodeHostScript, transPath } from "./runtime-common";
|
|
import { optionValue, positiveIntegerOption, shellQuote, statusText } from "./utils";
|
|
|
|
export function parseJsonObject(text: string): Record<string, unknown> {
|
|
try {
|
|
const parsed = JSON.parse(text) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
export function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record<string, unknown>, statusPath: string, timedOut: boolean): CommandResult {
|
|
return {
|
|
command: [transPath(), spec.nodeRoute, "sh", "--", "<remote async script>"],
|
|
cwd: repoRoot,
|
|
exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1,
|
|
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
|
|
stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`,
|
|
signal: null,
|
|
timedOut,
|
|
};
|
|
}
|
|
|
|
export function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
export function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult {
|
|
return runCommand([transPath(), spec.nodeKubeRoute, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
export function isCommandSuccess(result: CommandResult): boolean {
|
|
return result.exitCode === 0 && !result.timedOut;
|
|
}
|
|
|
|
export function shortSha(value: string): string {
|
|
return value.slice(0, 12).toLowerCase();
|
|
}
|
|
|
|
export function shortValue(value: unknown): string {
|
|
if (typeof value !== "string" || value.length === 0) return "-";
|
|
if (/^[0-9a-f]{40}$/iu.test(value)) return shortSha(value);
|
|
return value.length > 30 ? `${value.slice(0, 27)}~` : value;
|
|
}
|
|
|
|
export function formatElapsedMs(value: unknown): string {
|
|
const ms = Number(value);
|
|
if (!Number.isFinite(ms) || ms < 0) return "-";
|
|
const seconds = Math.round(ms / 1000);
|
|
const minutes = Math.floor(seconds / 60);
|
|
const rest = seconds % 60;
|
|
return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`;
|
|
}
|
|
|
|
export function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
|
|
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
|
|
}
|
|
|
|
export function nodeRuntimeRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
|
|
return `${nodeRuntimePipelineRunName(spec, sourceCommit)}-r${Date.now().toString(36)}`.slice(0, 63);
|
|
}
|
|
|
|
export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
|
|
const suffix = `/${spec.runtimeRenderDir}`;
|
|
if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length);
|
|
return spec.gitopsRoot;
|
|
}
|
|
|
|
export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
|
|
const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 });
|
|
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
|
const match = /[0-9a-f]{40}/iu.exec(statusText(result));
|
|
return { sourceCommit: match?.[0].toLowerCase() ?? null, result };
|
|
}
|
|
|
|
export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
|
|
const gitRetries = Math.max(1, spec.downloadProfile.git.retries);
|
|
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
|
|
return [
|
|
`cicd_repo=${shellQuote(spec.cicdRepo)}`,
|
|
`cicd_url=${shellQuote(spec.gitUrl)}`,
|
|
`cicd_branch=${shellQuote(spec.sourceBranch)}`,
|
|
`cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`,
|
|
`git_retries=${shellQuote(String(gitRetries))}`,
|
|
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
|
|
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
|
|
"retry_git() {",
|
|
" op=$1",
|
|
" shift",
|
|
" attempt=1",
|
|
" while [ \"$attempt\" -le \"$git_retries\" ]; do",
|
|
" echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2",
|
|
" run_git \"$@\"",
|
|
" code=$?",
|
|
" if [ \"$code\" -eq 0 ]; then return 0; fi",
|
|
" echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2",
|
|
" attempt=$((attempt + 1))",
|
|
" done",
|
|
" return 1",
|
|
"}",
|
|
"mkdir -p \"$(dirname \"$cicd_repo\")\"",
|
|
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
|
" :",
|
|
"elif [ -e \"$cicd_repo\" ]; then",
|
|
" echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
|
|
" exit 41",
|
|
"else",
|
|
" retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"",
|
|
"fi",
|
|
"git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"",
|
|
"git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
|
|
"if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then",
|
|
" rm -rf \"$cicd_repo\"",
|
|
" retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"",
|
|
" git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
|
|
" retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune",
|
|
"fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
|
|
if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped);
|
|
if (scoped.action === "sync") return nodeRuntimeSync(scoped);
|
|
if (scoped.action === "apply") return nodeRuntimeApply(scoped);
|
|
if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrentOutput(scoped);
|
|
if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped);
|
|
if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped);
|
|
if (scoped.action === "cleanup-released-pvs") return nodeRuntimeCleanupReleasedPvs(scoped);
|
|
if (scoped.action === "cleanup-legacy-docker-images") return nodeRuntimeCleanupLegacyDockerImages(scoped);
|
|
if (scoped.action === "cleanup-legacy-docker-registry-volume") return nodeRuntimeCleanupLegacyDockerRegistryVolume(scoped);
|
|
return nodeRuntimeUnsupportedAction(scoped);
|
|
}
|
|
|
|
export function parseNodeRuntimeCleanupOptions(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): NodeRuntimeCleanupOptions {
|
|
const sourceCommitRaw = optionValue(scoped.originalArgs, "--source-commit");
|
|
const pipelineRunRaw = optionValue(scoped.originalArgs, "--pipeline-run");
|
|
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) {
|
|
throw new Error("control-plane cleanup-runs accepts only one of --source-commit or --pipeline-run");
|
|
}
|
|
const sourceCommit = sourceCommitRaw?.toLowerCase();
|
|
if (sourceCommit !== undefined && !/^[0-9a-f]{40}$/u.test(sourceCommit)) {
|
|
throw new Error("--source-commit must be a full 40-character git sha for cleanup-runs");
|
|
}
|
|
const pipelineRun = pipelineRunRaw === undefined ? undefined : validateNodeRuntimePipelineRunName(scoped.spec, pipelineRunRaw);
|
|
return {
|
|
minAgeMinutes: positiveIntegerOption(scoped.originalArgs, "--min-age-minutes", 60, 10080),
|
|
limit: positiveIntegerOption(scoped.originalArgs, "--limit", 20, 200),
|
|
sourceCommit,
|
|
pipelineRun,
|
|
targetPipelineRun: pipelineRun ?? (sourceCommit === undefined ? undefined : nodeRuntimePipelineRunName(scoped.spec, sourceCommit)),
|
|
includeActive: scoped.originalArgs.includes("--include-active"),
|
|
dryRun: scoped.dryRun || !scoped.confirm,
|
|
};
|
|
}
|
|
|
|
export function validateNodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, value: string): string {
|
|
const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) {
|
|
throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-<sha>[-rerun] PipelineRun name for --lane ${spec.lane}`);
|
|
}
|
|
return value.toLowerCase();
|
|
}
|
|
|
|
export function nodeRuntimeCleanupRuns(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const options = parseNodeRuntimeCleanupOptions(scoped);
|
|
const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
const candidates = listNodeRuntimeCleanupPipelineRuns(scoped.spec, options);
|
|
const selectedPipelineRuns = candidates
|
|
.filter((item) => item.selected !== false)
|
|
.map((item) => item.name);
|
|
const ownedResources = listNodeRuntimeCleanupOwnedResources(scoped.spec, selectedPipelineRuns);
|
|
const command = `hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane}`;
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
command,
|
|
mode: "dry-run",
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
limit: options.limit,
|
|
sourceCommit: options.sourceCommit,
|
|
pipelineRun: options.pipelineRun,
|
|
includeActive: options.includeActive,
|
|
candidates,
|
|
candidateCount: candidates.length,
|
|
selectedPipelineRuns,
|
|
selectedPipelineRunCount: selectedPipelineRuns.length,
|
|
ownedResources,
|
|
ciObjectCounts: beforeCounts,
|
|
mutation: false,
|
|
next: {
|
|
confirm: [
|
|
command,
|
|
`--min-age-minutes ${options.minAgeMinutes}`,
|
|
`--limit ${options.limit}`,
|
|
options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`,
|
|
options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`,
|
|
options.includeActive ? "--include-active" : "",
|
|
"--confirm",
|
|
"--wait",
|
|
].filter(Boolean).join(" "),
|
|
},
|
|
};
|
|
}
|
|
const deletion = deleteNodeRuntimeCleanupRuns(scoped.spec, selectedPipelineRuns, scoped.timeoutSeconds);
|
|
const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
return {
|
|
ok: isCommandSuccess(deletion),
|
|
command,
|
|
mode: "confirmed-cleanup",
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
limit: options.limit,
|
|
sourceCommit: options.sourceCommit,
|
|
pipelineRun: options.pipelineRun,
|
|
includeActive: options.includeActive,
|
|
deletedPipelineRuns: selectedPipelineRuns,
|
|
deletedPipelineRunCount: selectedPipelineRuns.length,
|
|
ownedResourcesBefore: ownedResources,
|
|
ciObjectCountsBefore: beforeCounts,
|
|
ciObjectCountsAfter: afterCounts,
|
|
deletion: compactRuntimeCommand(deletion),
|
|
mutation: isCommandSuccess(deletion),
|
|
degradedReason: isCommandSuccess(deletion) ? undefined : "node-runtime-ci-cleanup-delete-failed",
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
rerunStatus: options.targetPipelineRun === undefined
|
|
? undefined
|
|
: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${options.targetPipelineRun}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function listNodeRuntimeCleanupPipelineRuns(spec: HwlabRuntimeLaneSpec, options: NodeRuntimeCleanupOptions): NodeRuntimeCleanupPipelineRunRow[] {
|
|
if (options.targetPipelineRun !== undefined) {
|
|
const target = getNodeRuntimeCleanupPipelineRun(spec, options.targetPipelineRun);
|
|
if (target === null) {
|
|
return [{
|
|
name: options.targetPipelineRun,
|
|
createdAt: null,
|
|
ageMinutes: null,
|
|
status: null,
|
|
reason: "target-pipelinerun-not-found",
|
|
selected: false,
|
|
}];
|
|
}
|
|
if (target.status !== "True" && target.status !== "False" && !options.includeActive) {
|
|
return [{ ...target, selected: false, selectedReason: "target-pipelinerun-not-terminal" }];
|
|
}
|
|
if ((target.status === "True" || target.status === "False") && (target.ageMinutes === null || target.ageMinutes < options.minAgeMinutes)) {
|
|
return [{ ...target, selected: false, selectedReason: target.ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }];
|
|
}
|
|
return [target];
|
|
}
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pipelinerun",
|
|
"-o",
|
|
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
|
], 60);
|
|
if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`);
|
|
const rows = nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout);
|
|
const prefix = `${spec.pipelineRunPrefix}-`;
|
|
const terminalRuns = rows
|
|
.filter((item) => item.name.startsWith(prefix))
|
|
.filter((item) => item.status === "True" || item.status === "False")
|
|
.sort((left, right) => String(left.createdAt ?? "").localeCompare(String(right.createdAt ?? "")));
|
|
const protectedLatest = terminalRuns
|
|
.slice()
|
|
.sort((left, right) => String(right.createdAt ?? "").localeCompare(String(left.createdAt ?? "")))[0]?.name ?? null;
|
|
return terminalRuns
|
|
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
|
.map((item) => item.name === protectedLatest ? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" } : item)
|
|
.slice(0, options.limit);
|
|
}
|
|
|
|
export function getNodeRuntimeCleanupPipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): NodeRuntimeCleanupPipelineRunRow | null {
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pipelinerun",
|
|
pipelineRun,
|
|
"-o",
|
|
'jsonpath={.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}',
|
|
], 60);
|
|
if (result.exitCode !== 0) return null;
|
|
return nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout)[0] ?? null;
|
|
}
|
|
|
|
export function nodeRuntimeCleanupPipelineRunRowsFromText(text: string): NodeRuntimeCleanupPipelineRunRow[] {
|
|
const now = Date.now();
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
const [name = "", createdAtRaw = "", statusRaw = "", reasonRaw = ""] = line.trim().split("\t");
|
|
const createdAt = createdAtRaw.length > 0 ? createdAtRaw : null;
|
|
const createdMs = createdAt === null ? NaN : Date.parse(createdAt);
|
|
return {
|
|
name,
|
|
createdAt,
|
|
ageMinutes: Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null,
|
|
status: statusRaw || null,
|
|
reason: reasonRaw || null,
|
|
};
|
|
}).filter((item) => item.name.length > 0);
|
|
}
|
|
|
|
export function listNodeRuntimeCleanupOwnedResources(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[]): Record<string, unknown> {
|
|
const previewLimit = 24;
|
|
if (pipelineRunNames.length === 0) {
|
|
return {
|
|
taskRunPreview: [],
|
|
podPreview: [],
|
|
pvcPreview: [],
|
|
previewLimit,
|
|
truncated: false,
|
|
taskRunCount: 0,
|
|
podCount: 0,
|
|
pvcCount: 0,
|
|
};
|
|
}
|
|
const wanted = new Set(pipelineRunNames);
|
|
const taskRunsResult = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"taskrun",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "Succeeded"}}{{.status}}{{"\\t"}}{{.reason}}{{end}}{{end}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
const podsResult = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pod",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
const pvcsResult = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pvc",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimeCleanupOwnedTaskRunsFromText(taskRunsResult.stdout, wanted) : [];
|
|
const pods = isCommandSuccess(podsResult) ? nodeRuntimeCleanupOwnedPodsFromText(podsResult.stdout, wanted) : [];
|
|
const pvcs = isCommandSuccess(pvcsResult) ? nodeRuntimeCleanupOwnedPvcsFromText(pvcsResult.stdout, wanted) : [];
|
|
return {
|
|
taskRunPreview: taskRuns.slice(0, previewLimit),
|
|
podPreview: pods.slice(0, previewLimit),
|
|
pvcPreview: pvcs.slice(0, previewLimit),
|
|
previewLimit,
|
|
truncated: taskRuns.length > previewLimit || pods.length > previewLimit || pvcs.length > previewLimit,
|
|
taskRunCount: taskRuns.length,
|
|
podCount: pods.length,
|
|
pvcCount: pvcs.length,
|
|
query: {
|
|
taskRuns: compactRuntimeCommandStats(taskRunsResult),
|
|
pods: compactRuntimeCommandStats(podsResult),
|
|
pvcs: compactRuntimeCommandStats(pvcsResult),
|
|
},
|
|
};
|
|
}
|
|
|
|
interface NodeRuntimeReleasedPvCleanupOptions {
|
|
limit: number;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface NodeRuntimeCiPvcRow {
|
|
name: string;
|
|
volumeName: string;
|
|
phase: string;
|
|
ownerKind: string;
|
|
ownerName: string;
|
|
storage: string;
|
|
}
|
|
|
|
interface NodeRuntimeCiPodClaimRow {
|
|
name: string;
|
|
phase: string;
|
|
claims: string[];
|
|
}
|
|
|
|
interface NodeRuntimeReleasedPvRow {
|
|
name: string;
|
|
createdAt: string;
|
|
phase: string;
|
|
storageClass: string;
|
|
reclaimPolicy: string;
|
|
claimNamespace: string;
|
|
claimName: string;
|
|
capacity: string;
|
|
hostPath: string | null;
|
|
}
|
|
|
|
interface NodeRuntimeLegacyDockerImageCleanupPolicy {
|
|
enabled: boolean;
|
|
minAgeHours: number;
|
|
keepPerRepository: number;
|
|
repositories: string[];
|
|
source: string;
|
|
}
|
|
|
|
interface NodeRuntimeLegacyDockerImageCleanupOptions {
|
|
minAgeHours: number;
|
|
keepPerRepository: number;
|
|
limit: number;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface NodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy {
|
|
enabled: boolean;
|
|
requireK8sRegistryReady: boolean;
|
|
source: string;
|
|
}
|
|
|
|
interface NodeRuntimeLegacyDockerRegistryVolumeTarget {
|
|
dockerContainerName: string;
|
|
dockerImage: string;
|
|
mountDestination: string;
|
|
k8sRegistry: {
|
|
mode: string;
|
|
endpoint: string;
|
|
namespace: string;
|
|
deploymentName: string;
|
|
pvcName: string;
|
|
};
|
|
source: string;
|
|
}
|
|
|
|
function parseNodeRuntimeReleasedPvCleanupOptions(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): NodeRuntimeReleasedPvCleanupOptions {
|
|
return {
|
|
limit: positiveIntegerOption(scoped.originalArgs, "--limit", 200, 500),
|
|
dryRun: scoped.dryRun || !scoped.confirm,
|
|
};
|
|
}
|
|
|
|
function parseNodeRuntimeLegacyDockerImageCleanupOptions(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, policy: NodeRuntimeLegacyDockerImageCleanupPolicy): NodeRuntimeLegacyDockerImageCleanupOptions {
|
|
return {
|
|
minAgeHours: positiveIntegerOption(scoped.originalArgs, "--min-age-hours", policy.minAgeHours, 8760),
|
|
keepPerRepository: positiveIntegerOption(scoped.originalArgs, "--keep-per-repository", policy.keepPerRepository, 100),
|
|
limit: positiveIntegerOption(scoped.originalArgs, "--limit", 50, 500),
|
|
dryRun: scoped.dryRun || !scoped.confirm,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeCleanupReleasedPvs(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const options = parseNodeRuntimeReleasedPvCleanupOptions(scoped);
|
|
const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
const orphanedPvcs = listNodeRuntimeOrphanedCiPvcs(scoped.spec, options.limit);
|
|
const releasedPvsBefore = listNodeRuntimeReleasedCiPvs(scoped.spec, options.limit);
|
|
const command = `hwlab nodes control-plane cleanup-released-pvs --node ${scoped.node} --lane ${scoped.lane}`;
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
command,
|
|
mode: "dry-run",
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
limit: options.limit,
|
|
orphanedPvcs,
|
|
orphanedPvcCount: orphanedPvcs.length,
|
|
releasedPvs: releasedPvsBefore,
|
|
releasedPvCount: releasedPvsBefore.length,
|
|
mutation: false,
|
|
policy: {
|
|
orphanedPvc: "ownerRef.kind=PipelineRun, owner PipelineRun absent, pvc name matches pvc-*, and no non-terminal pod mounts the claim",
|
|
releasedPv: "phase=Released, storageClass=local-path, reclaimPolicy=Delete, claimRef.namespace=hwlab-ci",
|
|
},
|
|
next: { confirm: `${command} --limit ${options.limit} --confirm` },
|
|
};
|
|
}
|
|
const pvcDeletion = deleteNodeRuntimeCiPvcs(scoped.spec, orphanedPvcs.map((item) => item.name), scoped.timeoutSeconds);
|
|
const releasedPvsAfterPvc = listNodeRuntimeReleasedCiPvs(scoped.spec, options.limit);
|
|
const pvNames = releasedPvsAfterPvc.map((item) => item.name);
|
|
const pvDeletion = deleteNodeRuntimePersistentVolumes(scoped.spec, pvNames, scoped.timeoutSeconds);
|
|
const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
const ok = isCommandSuccess(pvcDeletion) && isCommandSuccess(pvDeletion);
|
|
return {
|
|
ok,
|
|
command,
|
|
mode: "confirmed-cleanup",
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
limit: options.limit,
|
|
deletedPvcs: orphanedPvcs.map((item) => item.name),
|
|
deletedPvcCount: orphanedPvcs.length,
|
|
deletedPersistentVolumes: pvNames,
|
|
deletedPersistentVolumeCount: pvNames.length,
|
|
orphanedPvcsBefore: orphanedPvcs.slice(0, 50),
|
|
releasedPvsBefore,
|
|
releasedPvsAfterPvcDelete: releasedPvsAfterPvc.slice(0, 50),
|
|
ciObjectCountsBefore: beforeCounts,
|
|
ciObjectCountsAfter: afterCounts,
|
|
pvcDeletion: compactRuntimeCommand(pvcDeletion),
|
|
pvDeletion: compactRuntimeCommand(pvDeletion),
|
|
mutation: ok,
|
|
degradedReason: ok ? undefined : "node-runtime-ci-pvc-pv-cleanup-failed",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeCleanupLegacyDockerImages(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const policy = nodeRuntimeLegacyDockerImageCleanupPolicy(scoped.node);
|
|
const options = parseNodeRuntimeLegacyDockerImageCleanupOptions(scoped, policy);
|
|
const command = `hwlab nodes control-plane cleanup-legacy-docker-images --node ${scoped.node} --lane ${scoped.lane}`;
|
|
if (!policy.enabled) {
|
|
return {
|
|
ok: false,
|
|
command,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "blocked",
|
|
mutation: false,
|
|
config: policy.source,
|
|
degradedReason: "legacy-docker-image-cleanup-disabled",
|
|
message: `${policy.source} is disabled`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
if (policy.repositories.length === 0) {
|
|
return {
|
|
ok: false,
|
|
command,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "blocked",
|
|
mutation: false,
|
|
config: policy.source,
|
|
degradedReason: "legacy-docker-image-cleanup-repositories-empty",
|
|
message: `${policy.source}.repositories must include at least one repository`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
const remoteOptions = {
|
|
mode: options.dryRun ? "plan" : "run",
|
|
minAgeHours: options.minAgeHours,
|
|
keepPerRepository: options.keepPerRepository,
|
|
limit: options.limit,
|
|
repositories: policy.repositories,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
valuesRedacted: true,
|
|
};
|
|
const script = [
|
|
"set -eu",
|
|
`node - ${shellQuote(JSON.stringify(remoteOptions))} <<'UNIDESK_LEGACY_DOCKER_IMAGE_GC'`,
|
|
nodeRuntimeLegacyDockerImageCleanupNodeScript(),
|
|
"UNIDESK_LEGACY_DOCKER_IMAGE_GC",
|
|
].join("\n");
|
|
const result = runNodeHostScript(scoped.spec, script, scoped.timeoutSeconds);
|
|
const payload = parseJsonObject(result.stdout);
|
|
const ok = isCommandSuccess(result) && payload.ok !== false;
|
|
return {
|
|
ok,
|
|
command,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-cleanup",
|
|
mutation: !options.dryRun && ok,
|
|
config: policy.source,
|
|
policy: {
|
|
repositories: policy.repositories,
|
|
minAgeHours: options.minAgeHours,
|
|
keepPerRepository: options.keepPerRepository,
|
|
limit: options.limit,
|
|
dockerPruneUsed: false,
|
|
dockerVolumesTouched: false,
|
|
protectedContainerImages: true,
|
|
legacyOnly: true,
|
|
deploymentDependency: false,
|
|
valuesRedacted: true,
|
|
},
|
|
cleanup: options.dryRun || ok ? compactNodeRuntimeLegacyDockerImageCleanupPayload(payload) : payload,
|
|
result: ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result),
|
|
degradedReason: ok ? undefined : "legacy-docker-image-cleanup-failed",
|
|
next: options.dryRun
|
|
? { confirm: `${command} --min-age-hours ${options.minAgeHours} --keep-per-repository ${options.keepPerRepository} --limit ${options.limit} --confirm` }
|
|
: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeCleanupLegacyDockerRegistryVolume(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const policy = nodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy();
|
|
const target = nodeRuntimeLegacyDockerRegistryVolumeTarget(scoped.node);
|
|
const dryRun = scoped.dryRun || !scoped.confirm;
|
|
const command = `hwlab nodes control-plane cleanup-legacy-docker-registry-volume --node ${scoped.node} --lane ${scoped.lane}`;
|
|
if (!policy.enabled) {
|
|
return {
|
|
ok: false,
|
|
command,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "blocked",
|
|
mutation: false,
|
|
config: policy.source,
|
|
degradedReason: "legacy-docker-registry-volume-cleanup-disabled",
|
|
message: `${policy.source} is disabled`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
if (target.k8sRegistry.mode !== "k8s-workload") {
|
|
return {
|
|
ok: false,
|
|
command,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "blocked",
|
|
mutation: false,
|
|
config: target.source,
|
|
degradedReason: "node-local-registry-is-not-k8s-workload",
|
|
message: "legacy Docker registry volume cleanup is allowed only after node-local registry is declared as a k8s workload",
|
|
target,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
const remoteOptions = {
|
|
mode: dryRun ? "plan" : "run",
|
|
target,
|
|
requireK8sRegistryReady: policy.requireK8sRegistryReady,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
valuesRedacted: true,
|
|
};
|
|
const script = [
|
|
"set -eu",
|
|
`node - ${shellQuote(JSON.stringify(remoteOptions))} <<'UNIDESK_LEGACY_DOCKER_REGISTRY_VOLUME_GC'`,
|
|
nodeRuntimeLegacyDockerRegistryVolumeCleanupNodeScript(),
|
|
"UNIDESK_LEGACY_DOCKER_REGISTRY_VOLUME_GC",
|
|
].join("\n");
|
|
const result = runNodeHostScript(scoped.spec, script, scoped.timeoutSeconds);
|
|
const payload = parseJsonObject(result.stdout);
|
|
const ok = isCommandSuccess(result) && payload.ok !== false;
|
|
return {
|
|
ok,
|
|
command,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: dryRun ? "dry-run" : "confirmed-cleanup",
|
|
mutation: !dryRun && ok && payload.mutation === true,
|
|
config: {
|
|
policy: policy.source,
|
|
target: target.source,
|
|
},
|
|
target,
|
|
cleanup: compactNodeRuntimeLegacyDockerRegistryVolumeCleanupPayload(payload),
|
|
result: ok ? compactRuntimeCommandStats(result) : compactRuntimeCommand(result),
|
|
degradedReason: ok ? undefined : "legacy-docker-registry-volume-cleanup-failed",
|
|
next: dryRun
|
|
? { confirm: `${command} --confirm --wait` }
|
|
: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeLegacyDockerImageCleanupPolicy(node: string): NodeRuntimeLegacyDockerImageCleanupPolicy {
|
|
const source = "config/unidesk-cli.yaml#gc.legacyDockerImages";
|
|
const parsed = objectRecord(Bun.YAML.parse(readFileSync(rootPath("config/unidesk-cli.yaml"), "utf8")));
|
|
const raw = objectRecord(objectRecord(parsed.gc).legacyDockerImages);
|
|
const nodeLower = node.toLowerCase();
|
|
const repositories = Array.isArray(raw.repositories)
|
|
? [...new Set(raw.repositories
|
|
.filter((item): item is string => typeof item === "string")
|
|
.map((item) => item
|
|
.replace(/\$\{nodeLower\}/gu, nodeLower)
|
|
.replace(/\$\{node\}/gu, node)
|
|
.replace(/\$\{NODE\}/gu, node.toUpperCase()))
|
|
.map((item) => item.trim())
|
|
.filter(Boolean))]
|
|
: [];
|
|
return {
|
|
enabled: raw.enabled === true,
|
|
minAgeHours: nonNegativeInteger(raw.minAgeHours, `${source}.minAgeHours`),
|
|
keepPerRepository: nonNegativeInteger(raw.keepPerRepository, `${source}.keepPerRepository`),
|
|
repositories,
|
|
source,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy(): NodeRuntimeLegacyDockerRegistryVolumeCleanupPolicy {
|
|
const source = "config/unidesk-cli.yaml#gc.legacyDockerRegistryVolumes";
|
|
const parsed = objectRecord(Bun.YAML.parse(readFileSync(rootPath("config/unidesk-cli.yaml"), "utf8")));
|
|
const raw = objectRecord(objectRecord(parsed.gc).legacyDockerRegistryVolumes);
|
|
return {
|
|
enabled: raw.enabled === true,
|
|
requireK8sRegistryReady: raw.requireK8sRegistryReady !== false,
|
|
source,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeLegacyDockerRegistryVolumeTarget(node: string): NodeRuntimeLegacyDockerRegistryVolumeTarget {
|
|
const source = `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}#nodes.${node}`;
|
|
const parsed = objectRecord(Bun.YAML.parse(readFileSync(rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH), "utf8")));
|
|
const nodeConfig = objectRecord(objectRecord(parsed.nodes)[node]);
|
|
const localRegistry = objectRecord(objectRecord(objectRecord(nodeConfig.k3s).install).localRegistry);
|
|
const registry = objectRecord(nodeConfig.registry);
|
|
return {
|
|
dockerContainerName: requiredConfigString(localRegistry.containerName, `${source}.k3s.install.localRegistry.containerName`),
|
|
dockerImage: requiredConfigString(localRegistry.canonicalImage ?? localRegistry.image, `${source}.k3s.install.localRegistry.canonicalImage`),
|
|
mountDestination: "/var/lib/registry",
|
|
k8sRegistry: {
|
|
mode: requiredConfigString(registry.mode, `${source}.registry.mode`),
|
|
endpoint: requiredConfigString(registry.endpoint, `${source}.registry.endpoint`),
|
|
namespace: requiredConfigString(registry.namespace, `${source}.registry.namespace`),
|
|
deploymentName: requiredConfigString(registry.deploymentName, `${source}.registry.deploymentName`),
|
|
pvcName: requiredConfigString(registry.pvcName, `${source}.registry.pvcName`),
|
|
},
|
|
source,
|
|
};
|
|
}
|
|
|
|
function objectRecord(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function nonNegativeInteger(value: unknown, label: string): number {
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${label} must be a non-negative integer`);
|
|
return value;
|
|
}
|
|
|
|
function requiredConfigString(value: unknown, label: string): string {
|
|
if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${label} must be a non-empty string`);
|
|
return value.trim();
|
|
}
|
|
|
|
function compactNodeRuntimeLegacyDockerImageCleanupPayload(payload: Record<string, unknown>): Record<string, unknown> {
|
|
const candidates = Array.isArray(payload.candidates) ? payload.candidates.map(objectRecord) : [];
|
|
const selected = Array.isArray(payload.selected) ? payload.selected.map(objectRecord) : [];
|
|
const protectedImages = Array.isArray(payload.protected) ? payload.protected.map(objectRecord) : [];
|
|
const results = Array.isArray(payload.results) ? payload.results.map(objectRecord) : [];
|
|
return {
|
|
ok: payload.ok ?? null,
|
|
mode: payload.mode ?? null,
|
|
mutation: payload.mutation ?? null,
|
|
diskBefore: payload.diskBefore ?? null,
|
|
diskAfter: payload.diskAfter ?? null,
|
|
actualDiskReclaimBytes: payload.actualDiskReclaimBytes ?? null,
|
|
repositoryCount: payload.repositoryCount ?? null,
|
|
imageCount: payload.imageCount ?? null,
|
|
containerCount: payload.containerCount ?? null,
|
|
candidateCount: payload.candidateCount ?? null,
|
|
selectedCount: payload.selectedCount ?? null,
|
|
deferredCount: payload.deferredCount ?? null,
|
|
protectedCount: payload.protectedCount ?? null,
|
|
estimatedReclaimBytes: payload.estimatedReclaimBytes ?? null,
|
|
estimatedReclaimHuman: payload.estimatedReclaimHuman ?? null,
|
|
candidates: candidates.slice(0, 5).map(compactLegacyDockerImageRow),
|
|
selected: selected.slice(0, 10).map(compactLegacyDockerImageRow),
|
|
protected: protectedImages.slice(0, 5).map((item) => ({
|
|
...compactLegacyDockerImageRow(item),
|
|
reasons: item.reasons ?? null,
|
|
})),
|
|
results: results.slice(0, 10).map((item) => ({
|
|
shortId: item.shortId ?? null,
|
|
tags: compactLegacyDockerTags(item.refs),
|
|
status: item.status ?? null,
|
|
exitCode: item.exitCode ?? null,
|
|
stderrTail: item.status === "failed" ? item.stderrTail ?? null : undefined,
|
|
})),
|
|
policy: payload.policy ?? null,
|
|
disclosure: {
|
|
candidateRowsShown: Math.min(5, candidates.length),
|
|
selectedRowsShown: Math.min(10, selected.length),
|
|
protectedRowsShown: Math.min(5, protectedImages.length),
|
|
resultRowsShown: Math.min(10, results.length),
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactLegacyDockerImageRow(item: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
shortId: item.shortId ?? null,
|
|
tags: compactLegacyDockerTags(item.repoTags),
|
|
createdAt: item.createdAt ?? null,
|
|
ageHours: item.ageHours ?? null,
|
|
sizeHuman: item.sizeHuman ?? null,
|
|
};
|
|
}
|
|
|
|
function compactLegacyDockerTags(value: unknown): string[] {
|
|
if (!Array.isArray(value)) return [];
|
|
return value
|
|
.filter((item): item is string => typeof item === "string")
|
|
.slice(0, 3)
|
|
.map((item) => {
|
|
const index = item.lastIndexOf(":");
|
|
return index > 0 ? item.slice(index + 1) : item;
|
|
});
|
|
}
|
|
|
|
function compactNodeRuntimeLegacyDockerRegistryVolumeCleanupPayload(payload: Record<string, unknown>): Record<string, unknown> {
|
|
const candidate = objectRecord(payload.candidate);
|
|
return {
|
|
ok: payload.ok ?? null,
|
|
mode: payload.mode ?? null,
|
|
mutation: payload.mutation ?? null,
|
|
diskBefore: payload.diskBefore ?? null,
|
|
diskAfter: payload.diskAfter ?? null,
|
|
actualDiskReclaimBytes: payload.actualDiskReclaimBytes ?? null,
|
|
candidate: Object.keys(candidate).length === 0 ? null : {
|
|
containerName: candidate.containerName ?? null,
|
|
containerStatus: candidate.containerStatus ?? null,
|
|
image: candidate.image ?? null,
|
|
volumeName: candidate.volumeName ?? null,
|
|
mountDestination: candidate.mountDestination ?? null,
|
|
sizeBytes: candidate.sizeBytes ?? null,
|
|
sizeHuman: candidate.sizeHuman ?? null,
|
|
},
|
|
k8sRegistry: payload.k8sRegistry ?? null,
|
|
blockers: Array.isArray(payload.blockers) ? payload.blockers.slice(0, 20) : [],
|
|
actions: payload.actions ?? null,
|
|
policy: payload.policy ?? null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeLegacyDockerImageCleanupNodeScript(): string {
|
|
return String.raw`
|
|
const childProcess = require("child_process");
|
|
|
|
const options = JSON.parse(process.argv[2] || "{}");
|
|
const mode = options.mode === "run" ? "run" : "plan";
|
|
const repositories = Array.isArray(options.repositories) ? options.repositories.filter((item) => typeof item === "string" && item.length > 0) : [];
|
|
const repositorySet = new Set(repositories);
|
|
const minAgeHours = Number(options.minAgeHours);
|
|
const keepPerRepository = Number(options.keepPerRepository);
|
|
const limit = Number(options.limit);
|
|
const nowMs = Date.now();
|
|
|
|
function run(args) {
|
|
const result = childProcess.spawnSync(args[0], args.slice(1), { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
|
|
return {
|
|
command: args,
|
|
exitCode: typeof result.status === "number" ? result.status : null,
|
|
signal: result.signal || null,
|
|
stdout: result.stdout || "",
|
|
stderr: result.stderr || "",
|
|
ok: result.status === 0,
|
|
};
|
|
}
|
|
|
|
function parseJsonArray(text) {
|
|
try {
|
|
const parsed = JSON.parse(text);
|
|
return Array.isArray(parsed) ? parsed : [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function diskSnapshot() {
|
|
const result = run(["df", "-B1", "/"]);
|
|
const line = result.stdout.trim().split(/\r?\n/)[1] || "";
|
|
const cols = line.trim().split(/\s+/);
|
|
const usePercent = Number(String(cols[4] || "0").replace(/%$/, ""));
|
|
return {
|
|
filesystem: cols[0] || null,
|
|
sizeBytes: Number(cols[1] || 0),
|
|
usedBytes: Number(cols[2] || 0),
|
|
availableBytes: Number(cols[3] || 0),
|
|
usePercent: Number.isFinite(usePercent) ? usePercent : null,
|
|
mount: cols[5] || "/",
|
|
};
|
|
}
|
|
|
|
function repoFromTag(tag) {
|
|
const value = String(tag || "");
|
|
const atless = value.split("@")[0];
|
|
const colon = atless.lastIndexOf(":");
|
|
const slash = atless.lastIndexOf("/");
|
|
return colon > slash ? atless.slice(0, colon) : atless;
|
|
}
|
|
|
|
function ageHours(createdAt) {
|
|
const ms = Date.parse(String(createdAt || ""));
|
|
if (!Number.isFinite(ms)) return null;
|
|
return Math.max(0, Math.round(((nowMs - ms) / 3600000) * 10) / 10);
|
|
}
|
|
|
|
function human(bytes) {
|
|
const value = Number(bytes || 0);
|
|
if (!Number.isFinite(value) || value <= 0) return "0B";
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let scaled = value;
|
|
let index = 0;
|
|
while (scaled >= 1024 && index < units.length - 1) {
|
|
scaled /= 1024;
|
|
index += 1;
|
|
}
|
|
return String(Math.round(scaled * 10) / 10) + units[index];
|
|
}
|
|
|
|
function shortId(id) {
|
|
return String(id || "").replace(/^sha256:/, "").slice(0, 12);
|
|
}
|
|
|
|
function dockerImagePresent(id) {
|
|
return run(["docker", "image", "inspect", id]).ok;
|
|
}
|
|
|
|
const errors = [];
|
|
const dockerProbe = run(["docker", "version", "--format", "{{json .Server.Version}}"]);
|
|
if (!dockerProbe.ok) {
|
|
console.log(JSON.stringify({ ok: false, mode, mutation: false, error: "docker-daemon-unavailable", stderrTail: dockerProbe.stderr.slice(-2000), valuesRedacted: true }));
|
|
process.exit(0);
|
|
}
|
|
|
|
const diskBefore = diskSnapshot();
|
|
const imageList = run(["docker", "image", "ls", "-q", "--no-trunc"]);
|
|
if (!imageList.ok) errors.push({ command: imageList.command, exitCode: imageList.exitCode, stderrTail: imageList.stderr.slice(-1000) });
|
|
const imageIds = [...new Set(imageList.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean))];
|
|
const imageInspect = imageIds.length === 0 ? { ok: true, stdout: "[]", stderr: "", exitCode: 0, command: [] } : run(["docker", "image", "inspect", ...imageIds]);
|
|
if (!imageInspect.ok) errors.push({ command: ["docker", "image", "inspect", "<image ids>"], exitCode: imageInspect.exitCode, stderrTail: imageInspect.stderr.slice(-1000) });
|
|
const images = parseJsonArray(imageInspect.stdout).map((image) => {
|
|
const repoTags = Array.isArray(image.RepoTags) ? image.RepoTags.filter((tag) => typeof tag === "string" && tag !== "<none>:<none>") : [];
|
|
const matchedTags = repoTags.filter((tag) => repositorySet.has(repoFromTag(tag)));
|
|
const matchedRepositories = [...new Set(matchedTags.map(repoFromTag))];
|
|
const createdAt = typeof image.Created === "string" ? image.Created : null;
|
|
return {
|
|
id: String(image.Id || ""),
|
|
shortId: shortId(image.Id),
|
|
repoTags,
|
|
matchedTags,
|
|
matchedRepositories,
|
|
createdAt,
|
|
createdMs: createdAt === null ? 0 : Date.parse(createdAt) || 0,
|
|
ageHours: ageHours(createdAt),
|
|
sizeBytes: Number(image.Size || 0),
|
|
sizeHuman: human(Number(image.Size || 0)),
|
|
};
|
|
}).filter((image) => image.id.length > 0);
|
|
|
|
const containerList = run(["docker", "ps", "-a", "--no-trunc", "--format", "{{.ID}}"]);
|
|
if (!containerList.ok) errors.push({ command: containerList.command, exitCode: containerList.exitCode, stderrTail: containerList.stderr.slice(-1000) });
|
|
const containerIds = [...new Set(containerList.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean))];
|
|
const containerInspect = containerIds.length === 0 ? { ok: true, stdout: "[]", stderr: "", exitCode: 0, command: [] } : run(["docker", "container", "inspect", ...containerIds]);
|
|
if (!containerInspect.ok) errors.push({ command: ["docker", "container", "inspect", "<container ids>"], exitCode: containerInspect.exitCode, stderrTail: containerInspect.stderr.slice(-1000) });
|
|
const containers = parseJsonArray(containerInspect.stdout).map((container) => ({
|
|
id: String(container.Id || ""),
|
|
name: String(container.Name || "").replace(/^\//, ""),
|
|
imageId: String(container.Image || ""),
|
|
imageRef: String((container.Config && container.Config.Image) || ""),
|
|
state: String((container.State && container.State.Status) || ""),
|
|
}));
|
|
const protectedByContainer = new Set(containers.map((container) => container.imageId).filter(Boolean));
|
|
|
|
const latestProtected = new Set();
|
|
for (const repository of repositories) {
|
|
images
|
|
.filter((image) => image.matchedRepositories.includes(repository))
|
|
.sort((a, b) => (b.createdMs - a.createdMs) || a.id.localeCompare(b.id))
|
|
.slice(0, keepPerRepository)
|
|
.forEach((image) => latestProtected.add(image.id));
|
|
}
|
|
|
|
const candidates = [];
|
|
const protectedImages = [];
|
|
for (const image of images.filter((item) => item.matchedRepositories.length > 0)) {
|
|
const reasons = [];
|
|
if (protectedByContainer.has(image.id)) reasons.push("container-referenced");
|
|
if (latestProtected.has(image.id)) reasons.push("latest-retention");
|
|
if (image.ageHours === null) reasons.push("created-at-unknown");
|
|
if (image.ageHours !== null && image.ageHours < minAgeHours) reasons.push("under-min-age");
|
|
if (reasons.length > 0) {
|
|
protectedImages.push({ ...image, reasons });
|
|
} else {
|
|
candidates.push({ ...image, reasons: ["repository-allowlisted", "unreferenced-by-container", "older-than-min-age", "outside-latest-retention"] });
|
|
}
|
|
}
|
|
|
|
candidates.sort((a, b) => (a.createdMs - b.createdMs) || a.id.localeCompare(b.id));
|
|
const selected = candidates.slice(0, limit);
|
|
const results = [];
|
|
if (mode === "run") {
|
|
for (const image of selected) {
|
|
const refs = image.matchedTags.length > 0 ? image.matchedTags : [image.id];
|
|
const remove = run(["docker", "image", "rm", ...refs]);
|
|
results.push({
|
|
id: image.id,
|
|
shortId: image.shortId,
|
|
refs,
|
|
status: remove.ok && !dockerImagePresent(image.id) ? "deleted" : remove.ok ? "untagged-or-shared" : "failed",
|
|
exitCode: remove.exitCode,
|
|
stdoutTail: remove.stdout.slice(-1000),
|
|
stderrTail: remove.stderr.slice(-1000),
|
|
});
|
|
}
|
|
}
|
|
|
|
const diskAfter = mode === "run" ? diskSnapshot() : null;
|
|
const actualDiskReclaimBytes = diskAfter === null ? null : Math.max(0, Number(diskBefore.usedBytes || 0) - Number(diskAfter.usedBytes || 0));
|
|
const ok = errors.length === 0 && results.every((item) => item.status !== "failed");
|
|
console.log(JSON.stringify({
|
|
ok,
|
|
mode,
|
|
mutation: mode === "run",
|
|
node: options.node || null,
|
|
lane: options.lane || null,
|
|
repositories,
|
|
repositoryCount: repositories.length,
|
|
diskBefore,
|
|
diskAfter,
|
|
actualDiskReclaimBytes,
|
|
imageCount: images.length,
|
|
containerCount: containers.length,
|
|
activeContainers: containers.filter((container) => container.state === "running").slice(0, 20),
|
|
candidateCount: candidates.length,
|
|
selectedCount: selected.length,
|
|
deferredCount: Math.max(0, candidates.length - selected.length),
|
|
protectedCount: protectedImages.length,
|
|
estimatedReclaimBytes: selected.reduce((sum, image) => sum + Number(image.sizeBytes || 0), 0),
|
|
estimatedReclaimHuman: human(selected.reduce((sum, image) => sum + Number(image.sizeBytes || 0), 0)),
|
|
candidates,
|
|
selected,
|
|
protected: protectedImages,
|
|
results,
|
|
errors,
|
|
policy: {
|
|
minAgeHours,
|
|
keepPerRepository,
|
|
limit,
|
|
dockerPruneUsed: false,
|
|
dockerVolumesTouched: false,
|
|
matchedRepositoriesOnly: true,
|
|
protectedContainerImages: true,
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function nodeRuntimeLegacyDockerRegistryVolumeCleanupNodeScript(): string {
|
|
return String.raw`
|
|
const childProcess = require("child_process");
|
|
|
|
const options = JSON.parse(process.argv[2] || "{}");
|
|
const mode = options.mode === "run" ? "run" : "plan";
|
|
const target = options.target || {};
|
|
const requireK8sRegistryReady = options.requireK8sRegistryReady !== false;
|
|
|
|
function run(args, extraEnv) {
|
|
const result = childProcess.spawnSync(args[0], args.slice(1), {
|
|
encoding: "utf8",
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
env: { ...process.env, ...(extraEnv || {}) },
|
|
});
|
|
return {
|
|
command: args,
|
|
exitCode: typeof result.status === "number" ? result.status : null,
|
|
signal: result.signal || null,
|
|
stdout: result.stdout || "",
|
|
stderr: result.stderr || "",
|
|
ok: result.status === 0,
|
|
};
|
|
}
|
|
|
|
function parseJson(text) {
|
|
try { return JSON.parse(text); } catch { return null; }
|
|
}
|
|
|
|
function diskSnapshot() {
|
|
const result = run(["df", "-B1", "/"]);
|
|
const line = result.stdout.trim().split(/\r?\n/)[1] || "";
|
|
const cols = line.trim().split(/\s+/);
|
|
const usePercent = Number(String(cols[4] || "0").replace(/%$/, ""));
|
|
return {
|
|
filesystem: cols[0] || null,
|
|
sizeBytes: Number(cols[1] || 0),
|
|
usedBytes: Number(cols[2] || 0),
|
|
availableBytes: Number(cols[3] || 0),
|
|
usePercent: Number.isFinite(usePercent) ? usePercent : null,
|
|
mount: cols[5] || "/",
|
|
};
|
|
}
|
|
|
|
function duBytes(path) {
|
|
const bytes = run(["du", "-sb", path]);
|
|
if (bytes.ok) {
|
|
const value = Number((bytes.stdout.trim().split(/\s+/)[0] || "0"));
|
|
if (Number.isFinite(value)) return value;
|
|
}
|
|
const kib = run(["du", "-sk", path]);
|
|
const value = Number((kib.stdout.trim().split(/\s+/)[0] || "0"));
|
|
return Number.isFinite(value) ? value * 1024 : null;
|
|
}
|
|
|
|
function human(bytes) {
|
|
const value = Number(bytes || 0);
|
|
if (!Number.isFinite(value) || value <= 0) return "0B";
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let scaled = value;
|
|
let index = 0;
|
|
while (scaled >= 1024 && index < units.length - 1) {
|
|
scaled /= 1024;
|
|
index += 1;
|
|
}
|
|
return String(Math.round(scaled * 10) / 10) + units[index];
|
|
}
|
|
|
|
function inspectContainer(name) {
|
|
const result = run(["docker", "container", "inspect", name]);
|
|
if (!result.ok) return { ok: false, result, container: null };
|
|
const parsed = parseJson(result.stdout);
|
|
return { ok: Array.isArray(parsed) && parsed.length > 0, result, container: Array.isArray(parsed) ? parsed[0] : null };
|
|
}
|
|
|
|
function listContainers() {
|
|
const list = run(["docker", "ps", "-a", "--no-trunc", "--format", "{{.ID}}"]);
|
|
if (!list.ok) return [];
|
|
const ids = [...new Set(list.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean))];
|
|
if (ids.length === 0) return [];
|
|
const inspect = run(["docker", "container", "inspect", ...ids]);
|
|
const parsed = parseJson(inspect.stdout);
|
|
return Array.isArray(parsed) ? parsed : [];
|
|
}
|
|
|
|
function k8sRegistryStatus(registry) {
|
|
const env = { KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" };
|
|
const namespace = String(registry.namespace || "");
|
|
const deploymentName = String(registry.deploymentName || "");
|
|
const pvcName = String(registry.pvcName || "");
|
|
const deployResult = run(["kubectl", "-n", namespace, "get", "deployment", deploymentName, "-o", "json"], env);
|
|
const pvcResult = run(["kubectl", "-n", namespace, "get", "pvc", pvcName, "-o", "json"], env);
|
|
const endpoint = String(registry.endpoint || "");
|
|
const curlResult = endpoint.length > 0 ? run(["curl", "-fsS", "--max-time", "5", "http://" + endpoint + "/v2/"]) : { ok: false, exitCode: null, stderr: "endpoint missing" };
|
|
const deployment = parseJson(deployResult.stdout) || {};
|
|
const pvc = parseJson(pvcResult.stdout) || {};
|
|
const desired = Math.max(1, Number((deployment.spec && deployment.spec.replicas) || 1));
|
|
const readyReplicas = Number((deployment.status && deployment.status.readyReplicas) || 0);
|
|
const updatedReplicas = Number((deployment.status && deployment.status.updatedReplicas) || 0);
|
|
const pvcPhase = String((pvc.status && pvc.status.phase) || "");
|
|
return {
|
|
ok: deployResult.ok && pvcResult.ok && curlResult.ok && readyReplicas >= desired && updatedReplicas >= desired && pvcPhase === "Bound",
|
|
namespace,
|
|
deploymentName,
|
|
pvcName,
|
|
desiredReplicas: desired,
|
|
readyReplicas,
|
|
updatedReplicas,
|
|
pvcPhase,
|
|
endpoint,
|
|
endpointOk: curlResult.ok,
|
|
deployExitCode: deployResult.exitCode,
|
|
pvcExitCode: pvcResult.exitCode,
|
|
curlExitCode: curlResult.exitCode,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
const diskBefore = diskSnapshot();
|
|
const inspect = inspectContainer(String(target.dockerContainerName || "registry"));
|
|
if (!inspect.ok || inspect.container === null) {
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
mode,
|
|
mutation: false,
|
|
diskBefore,
|
|
diskAfter: null,
|
|
actualDiskReclaimBytes: null,
|
|
candidate: null,
|
|
k8sRegistry: k8sRegistryStatus(target.k8sRegistry || {}),
|
|
blockers: [],
|
|
actions: { containerRemoved: false, volumeRemoved: false },
|
|
policy: { noCandidateIsOk: true, dockerPruneUsed: false, dockerVolumeRmScoped: false, valuesRedacted: true },
|
|
valuesRedacted: true,
|
|
}));
|
|
process.exit(0);
|
|
}
|
|
|
|
const container = inspect.container;
|
|
const state = container.State || {};
|
|
const config = container.Config || {};
|
|
const mounts = Array.isArray(container.Mounts) ? container.Mounts : [];
|
|
const registryMount = mounts.find((mount) => mount && mount.Type === "volume" && mount.Destination === target.mountDestination);
|
|
const allContainers = listContainers();
|
|
const otherUsers = registryMount === undefined ? [] : allContainers
|
|
.filter((item) => item && item.Id !== container.Id)
|
|
.filter((item) => Array.isArray(item.Mounts) && item.Mounts.some((mount) => mount && mount.Name === registryMount.Name))
|
|
.map((item) => ({ id: String(item.Id || "").slice(0, 12), name: String(item.Name || "").replace(/^\//, ""), state: String((item.State && item.State.Status) || "") }));
|
|
const k8sRegistry = k8sRegistryStatus(target.k8sRegistry || {});
|
|
const volumeSource = registryMount && typeof registryMount.Source === "string" ? registryMount.Source : null;
|
|
const volumeName = registryMount && typeof registryMount.Name === "string" ? registryMount.Name : null;
|
|
const sizeBytes = volumeSource === null ? null : duBytes(volumeSource);
|
|
const candidate = registryMount === undefined ? null : {
|
|
containerName: String(container.Name || "").replace(/^\//, ""),
|
|
containerStatus: String(state.Status || ""),
|
|
running: state.Running === true,
|
|
image: String(config.Image || ""),
|
|
volumeName,
|
|
volumeSource,
|
|
mountDestination: String(registryMount.Destination || ""),
|
|
sizeBytes,
|
|
sizeHuman: human(sizeBytes),
|
|
otherUsers,
|
|
valuesRedacted: true,
|
|
};
|
|
const blockers = [];
|
|
if (registryMount === undefined) blockers.push("registry-volume-mount-missing");
|
|
if (state.Running === true || String(state.Status || "") === "running") blockers.push("docker-registry-container-running");
|
|
if (String(config.Image || "") !== String(target.dockerImage || "")) blockers.push("docker-registry-image-mismatch");
|
|
if (otherUsers.length > 0) blockers.push("docker-volume-has-other-container-users");
|
|
if (volumeName === null || volumeName.length === 0) blockers.push("docker-volume-name-missing");
|
|
if (volumeSource === null || !volumeSource.startsWith("/var/lib/docker/volumes/") || !volumeSource.endsWith("/_data")) blockers.push("docker-volume-source-not-under-docker-volumes");
|
|
if (requireK8sRegistryReady && k8sRegistry.ok !== true) blockers.push("replacement-k8s-registry-not-ready");
|
|
|
|
let actions = { containerRemoved: false, volumeRemoved: false, containerRmExitCode: null, volumeRmExitCode: null, stderrTail: "" };
|
|
let diskAfter = null;
|
|
let mutation = false;
|
|
if (mode === "run" && blockers.length === 0 && candidate !== null) {
|
|
const removeContainer = run(["docker", "rm", String(target.dockerContainerName || "registry")]);
|
|
const removeVolume = removeContainer.ok ? run(["docker", "volume", "rm", String(volumeName)]) : { ok: false, exitCode: null, stderr: "container removal failed" };
|
|
actions = {
|
|
containerRemoved: removeContainer.ok,
|
|
volumeRemoved: removeVolume.ok,
|
|
containerRmExitCode: removeContainer.exitCode,
|
|
volumeRmExitCode: removeVolume.exitCode,
|
|
stderrTail: (removeContainer.stderr + "\n" + removeVolume.stderr).trim().slice(-1000),
|
|
};
|
|
mutation = removeContainer.ok && removeVolume.ok;
|
|
if (!removeContainer.ok) blockers.push("docker-container-rm-failed");
|
|
if (removeContainer.ok && !removeVolume.ok) blockers.push("docker-volume-rm-failed");
|
|
diskAfter = diskSnapshot();
|
|
}
|
|
|
|
if (diskAfter === null) diskAfter = mode === "run" ? diskSnapshot() : null;
|
|
const actualDiskReclaimBytes = diskAfter === null ? null : Math.max(0, Number(diskBefore.usedBytes || 0) - Number(diskAfter.usedBytes || 0));
|
|
console.log(JSON.stringify({
|
|
ok: blockers.length === 0,
|
|
mode,
|
|
mutation,
|
|
diskBefore,
|
|
diskAfter,
|
|
actualDiskReclaimBytes,
|
|
candidate,
|
|
k8sRegistry,
|
|
blockers,
|
|
actions,
|
|
policy: {
|
|
requireK8sRegistryReady,
|
|
dockerPruneUsed: false,
|
|
dockerVolumeRmScoped: mode === "run" && mutation,
|
|
dockerComposeTouched: false,
|
|
databaseTouched: false,
|
|
matchedExitedRegistryContainerOnly: true,
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function listNodeRuntimePipelineRunNames(spec: HwlabRuntimeLaneSpec): Set<string> {
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pipelinerun",
|
|
"-o",
|
|
'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}',
|
|
], 60);
|
|
if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`);
|
|
return new Set(result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean));
|
|
}
|
|
|
|
function listNodeRuntimeCiPodClaims(spec: HwlabRuntimeLaneSpec): NodeRuntimeCiPodClaimRow[] {
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pod",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .spec.volumes}}{{if .persistentVolumeClaim}}{{.persistentVolumeClaim.claimName}}{{","}}{{end}}{{end}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} Pod PVC mounts: ${result.stderr.trim().slice(0, 1000)}`);
|
|
return result.stdout.split(/\r?\n/u).map((line) => {
|
|
const [name = "", phase = "", claimsRaw = ""] = line.trim().split("\t");
|
|
return { name, phase, claims: claimsRaw.split(",").map((claim) => claim.trim()).filter(Boolean) };
|
|
}).filter((item) => item.name.length > 0);
|
|
}
|
|
|
|
function listNodeRuntimeCiPvcs(spec: HwlabRuntimeLaneSpec): NodeRuntimeCiPvcRow[] {
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pvc",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PVCs: ${result.stderr.trim().slice(0, 1000)}`);
|
|
return result.stdout.split(/\r?\n/u).map((line) => {
|
|
const [name = "", volumeName = "", phase = "", ownerKind = "", ownerName = "", storage = ""] = line.trim().split("\t");
|
|
return { name, volumeName, phase, ownerKind, ownerName, storage };
|
|
}).filter((item) => item.name.length > 0);
|
|
}
|
|
|
|
export function listNodeRuntimeOrphanedCiPvcs(spec: HwlabRuntimeLaneSpec, limit: number): NodeRuntimeCiPvcRow[] {
|
|
const pipelineRuns = listNodeRuntimePipelineRunNames(spec);
|
|
const activeClaims = new Set(listNodeRuntimeCiPodClaims(spec)
|
|
.filter((pod) => pod.phase !== "Succeeded" && pod.phase !== "Failed")
|
|
.flatMap((pod) => pod.claims));
|
|
return listNodeRuntimeCiPvcs(spec)
|
|
.filter((item) => /^pvc-[a-z0-9]+$/u.test(item.name))
|
|
.filter((item) => item.ownerKind === "PipelineRun" && item.ownerName.length > 0)
|
|
.filter((item) => !pipelineRuns.has(item.ownerName))
|
|
.filter((item) => !activeClaims.has(item.name))
|
|
.sort((a, b) => a.name.localeCompare(b.name))
|
|
.slice(0, limit);
|
|
}
|
|
|
|
export function listNodeRuntimeReleasedCiPvs(spec: HwlabRuntimeLaneSpec, limit: number): NodeRuntimeReleasedPvRow[] {
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"get",
|
|
"pv",
|
|
"-o",
|
|
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\t"}{.spec.local.path}{"\\t"}{.spec.hostPath.path}{"\\n"}{end}',
|
|
], 60);
|
|
if (!isCommandSuccess(result)) throw new Error(`failed to list released ${HWLAB_CI_NAMESPACE} PVs: ${result.stderr.trim().slice(0, 1000)}`);
|
|
return result.stdout.split(/\r?\n/u).map((line) => {
|
|
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = "", localPath = "", hostPath = ""] = line.trim().split("\t");
|
|
return { name, createdAt, phase, storageClass, reclaimPolicy, claimNamespace, claimName, capacity, hostPath: safeNodeRuntimeStoragePath(localPath) ?? safeNodeRuntimeStoragePath(hostPath) };
|
|
})
|
|
.filter((item) => item.name.length > 0)
|
|
.filter((item) => item.phase === "Released")
|
|
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
|
.filter((item) => item.claimNamespace === HWLAB_CI_NAMESPACE && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
|
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
|
.slice(0, limit);
|
|
}
|
|
|
|
function safeNodeRuntimeStoragePath(value: string): string | null {
|
|
if (!value.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
|
if (value.includes("\0") || value.split("/").some((part) => part === "..")) return null;
|
|
return value;
|
|
}
|
|
|
|
function deleteNodeRuntimeCiPvcs(spec: HwlabRuntimeLaneSpec, names: string[], timeoutSeconds: number): CommandResult {
|
|
if (names.length === 0) return { command: [], cwd: repoRoot, exitCode: 0, stdout: "no orphaned PVC candidates", stderr: "", signal: null, timedOut: false };
|
|
return runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "delete", "pvc", ...names, "--ignore-not-found=true", "--wait=true", `--timeout=${timeoutSeconds}s`], timeoutSeconds);
|
|
}
|
|
|
|
function deleteNodeRuntimePersistentVolumes(spec: HwlabRuntimeLaneSpec, names: string[], timeoutSeconds: number): CommandResult {
|
|
if (names.length === 0) return { command: [], cwd: repoRoot, exitCode: 0, stdout: "no released PV candidates", stderr: "", signal: null, timedOut: false };
|
|
return runNodeK3sArgs(spec, ["kubectl", "delete", "pv", ...names, "--ignore-not-found=true", "--wait=true", `--timeout=${timeoutSeconds}s`], timeoutSeconds);
|
|
}
|