diff --git a/scripts/src/ci.ts b/scripts/src/ci.ts index 6b5b8784..3eedddf8 100644 --- a/scripts/src/ci.ts +++ b/scripts/src/ci.ts @@ -162,6 +162,14 @@ interface CiCleanupRunsOptions { confirm: boolean; } +interface CiCleanupFailedPodsOptions { + target: CiTarget; + namespace: string; + minAgeMinutes: number; + limit: number; + confirm: boolean; +} + type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required" | "registry-not-installed" | "registry-unhealthy" | "remote-command-timeout" | "ssh-helper-command-shape-incompatible" | "ci-runner-not-ready"; type PublishPreflightControlChannel = "backend-core" | "database" | "provider" | "registry"; type PublishPreflightDetailedChannel = "backend-core-api" | "provider-dispatch" | "provider-host-ssh" | "database" | "artifact-registry"; @@ -295,6 +303,27 @@ function ciCleanupRunsOptions(args: string[]): CiCleanupRunsOptions { }; } +function ciCleanupFailedPodsOptions(args: string[]): CiCleanupFailedPodsOptions { + const limit = numberOption(args, "--limit", 50); + if (limit <= 0 || limit > 500) throw new Error("ci cleanup-failed-pods --limit must be an integer between 1 and 500"); + const confirm = boolFlag(args, "--confirm"); + if (confirm && boolFlag(args, "--dry-run")) throw new Error("ci cleanup-failed-pods accepts only one of --confirm or --dry-run"); + return { + target: ciTarget(providerIdOption(args)), + namespace: requireK8sName(stringOption(args, "--namespace") ?? "unidesk-ci", "--namespace"), + minAgeMinutes: numberOption(args, "--min-age-minutes", 60), + limit, + confirm, + }; +} + +function requireK8sName(value: string, option: string): string { + if (!/^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$/u.test(value)) { + throw new Error(`${option} must be a Kubernetes DNS label`); + } + return value; +} + function countTextLines(value: string): number { if (value.length === 0) return 0; const breaks = value.match(/\r\n|\r|\n/gu)?.length ?? 0; @@ -721,7 +750,7 @@ function publishPreflightFailedScopes(preflight: PublishPreflight): string[] { function ciRunnerPreflightScript(sourceHostPath: string): string { return [ - "set -euo pipefail", + "set -eu", ...d601K3sGuardShellLines(d601Kubeconfig), "printf 'provider_host_ssh=ok\\n'", "printf 'kubectl='", @@ -1309,6 +1338,94 @@ async function cleanupRuns(config: UniDeskConfig, options: CiCleanupRunsOptions) }; } +async function cleanupFailedPods(config: UniDeskConfig, options: CiCleanupFailedPodsOptions): Promise> { + const script = [ + "set -eu", + ...ciTargetGuardShellLines(options.target, { passOutput: "stderr" }), + `namespace=${shellQuote(options.namespace)}`, + `min_age_minutes=${String(options.minAgeMinutes)}`, + `limit=${String(options.limit)}`, + `confirm=${options.confirm ? "true" : "false"}`, + "tmp=$(mktemp /tmp/unidesk-ci-failed-pods.XXXXXX.tsv)", + "selected=$(mktemp /tmp/unidesk-ci-failed-pods-selected.XXXXXX.txt)", + "trap 'rm -f \"$tmp\" \"$selected\"' EXIT", + "kubectl -n \"$namespace\" get pods --field-selector=status.phase=Failed --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,CREATED:.metadata.creationTimestamp,PHASE:.status.phase,REASON:.status.reason' --no-headers > \"$tmp\" || true", + "now_epoch=$(date -u +%s)", + "candidate_count=0", + "while IFS= read -r line; do", + " set -- $line", + " name=${1:-}", + " created=${2:-}", + " phase=${3:-}", + " reason=${4:-}", + " [ -n \"$name\" ] || continue", + " [ \"$phase\" = \"Failed\" ] || continue", + " created_epoch=$(date -u -d \"$created\" +%s 2>/dev/null || printf '0')", + " [ \"$created_epoch\" -gt 0 ] || continue", + " age_minutes=$(( (now_epoch - created_epoch) / 60 ))", + " [ \"$age_minutes\" -ge \"$min_age_minutes\" ] || continue", + " [ \"$candidate_count\" -lt \"$limit\" ] || continue", + " candidate_count=$((candidate_count + 1))", + " printf 'candidate\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \"$name\" \"$created\" \"$phase\" \"$age_minutes\" \"$reason\"", + " printf '%s\\n' \"$name\" >> \"$selected\"", + "done < \"$tmp\"", + "delete_exit=0", + "delete_stdout=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.out)", + "delete_stderr=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.err)", + "trap 'rm -f \"$tmp\" \"$selected\" \"$delete_stdout\" \"$delete_stderr\"' EXIT", + "if [ \"$confirm\" = \"true\" ] && [ -s \"$selected\" ]; then", + " xargs -r kubectl -n \"$namespace\" delete pod --ignore-not-found=true --wait=false < \"$selected\" > \"$delete_stdout\" 2> \"$delete_stderr\" || delete_exit=$?", + "fi", + "printf 'candidateCount\\t%s\\n' \"$candidate_count\"", + "if [ \"$confirm\" = \"true\" ]; then printf 'deletedPodCount\\t%s\\n' \"$(wc -l < \"$selected\" | tr -d ' ')\"; else printf 'deletedPodCount\\t0\\n'; fi", + "printf 'deleteExit\\t%s\\n' \"$delete_exit\"", + "printf 'deleteStdoutTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stdout\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"", + "printf 'deleteStderrTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stderr\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"", + "exit \"$delete_exit\"", + ].join("\n"); + const result = await runSshCommandCapture(config, `${options.target.providerId}:k3s`, ["sh"], script); + const fields = new Map(); + const candidates: Record[] = []; + for (const line of result.stdout.split(/\r?\n/u)) { + if (!line.trim()) continue; + const [kind = "", ...rest] = line.split("\t"); + if (kind === "candidate") { + const [name = "", createdAt = "", phase = "", ageMinutes = "", reason = ""] = rest; + candidates.push({ name, namespace: options.namespace, phase, createdAt, ageMinutes: Number(ageMinutes), reason }); + continue; + } + fields.set(kind, rest.join("\t")); + } + const deleteExit = Number(fields.get("deleteExit") ?? result.exitCode); + const ok = result.exitCode === 0 && deleteExit === 0; + return { + ok, + command: "ci cleanup-failed-pods", + mode: options.confirm ? "confirmed-cleanup" : "dry-run", + mutation: options.confirm, + providerId: options.target.providerId, + namespace: options.namespace, + minAgeMinutes: options.minAgeMinutes, + limit: options.limit, + candidateCount: Number(fields.get("candidateCount") ?? candidates.length), + candidates, + deletedPodCount: Number(fields.get("deletedPodCount") ?? 0), + deletion: { + exitCode: deleteExit, + stdoutTail: fields.get("deleteStdoutTail") ?? "", + stderrTail: fields.get("deleteStderrTail") ?? "", + }, + probe: { + exitCode: result.exitCode, + stdoutTail: tailTextLines(result.stdout, 40), + stderrTail: tailTextLines(result.stderr, 20), + }, + next: options.confirm + ? { status: `bun scripts/cli.ts ci status --provider-id ${options.target.providerId}` } + : { confirm: `bun scripts/cli.ts ci cleanup-failed-pods --provider-id ${options.target.providerId} --namespace ${options.namespace} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` }, + }; +} + async function install(config: UniDeskConfig, options: CiInstallOptions): Promise> { if (!existsSync(rootPath(options.target.pipelineManifest))) { throw new Error("CI manifests are missing"); @@ -3180,7 +3297,7 @@ function catalogArtifactDescriptor(artifact: CiCatalogArtifact): Record { const catalog = loadCiCatalog(); return { - command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs|cleanup-runs", + command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs|cleanup-runs|cleanup-failed-pods", description: "Manage native k3s Tekton CI on D601 or G14. CI may publish commit-pinned image artifacts, but it intentionally does not deploy CD.", examples: [ "bun scripts/cli.ts ci install", @@ -3202,6 +3319,8 @@ export function ciHelp(): Record { "bun scripts/cli.ts ci logs [--provider-id G14] [--tail-lines 80]", "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --dry-run", "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --confirm", + "bun scripts/cli.ts ci cleanup-failed-pods --provider-id D601 --namespace unidesk-ci --min-age-minutes 60 --dry-run", + "bun scripts/cli.ts ci cleanup-failed-pods --provider-id D601 --namespace unidesk-ci --min-age-minutes 60 --confirm", ], tekton: { pipelineVersion: tektonPipelineVersion, @@ -3271,6 +3390,13 @@ export function ciHelp(): Record { selector: "status.phase in Succeeded,Failed", guard: "--confirm required for deletion; Running/Pending pods are never selected", }, + cleanupFailedPods: { + defaultNamespace: "unidesk-ci", + defaultMinAgeMinutes: 60, + defaultLimit: 50, + scope: "Kubernetes pods with status.phase=Failed only; does not touch Running/Pending pods, workloads, PVCs, Secrets, or Tekton resources", + defaultMode: "dry-run", + }, }; } @@ -3379,7 +3505,8 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi } if (action === "logs") return logs(config, nameArg ?? "", ciTarget(providerIdOption(args)), ciLogsOptions(args)); if (action === "cleanup-runs") return cleanupRuns(config, ciCleanupRunsOptions(args)); - throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs, cleanup-runs"); + if (action === "cleanup-failed-pods") return cleanupFailedPods(config, ciCleanupFailedPodsOptions(args)); + throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs, cleanup-runs, cleanup-failed-pods"); } export function startCiInstallJob(providerId = d601ProviderId): Record {