pikasTech-unidesk/scripts/src/ci/cleanup.ts

// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. cleanup module for scripts/src/ci.ts.
// SPEC: PJ2026-01060308 cicd-yaml-targets draft-2026-06-25-cicd-yaml-targets.

// Moved mechanically from scripts/src/ci.ts:1206-1428 for #903.

import { randomUUID } from "node:crypto";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, posix as posixPath } from "node:path";
import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "../ci-catalog";
import { runCommand } from "../command";
import { type UniDeskConfig, repoRoot, rootPath } from "../config";
import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "../deploy-ssh-identity";
import { jobWithTail, listJobs, readJob, startJob } from "../jobs";
import { coreInternalFetch } from "../microservices";
import {
  artifactRegistryReadonlyResultFromCommand,
  buildArtifactRegistryReadonlyProbe,
  parseArtifactRegistryOptions,
  type ArtifactRegistryReadonlyProbe,
} from "../artifact-registry";
import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard";
import { runSshCommandCapture } from "../ssh";

import type { CiCleanupFailedPodsOptions, CiCleanupRunsOptions, CiTarget } from "./types";
import { ciTargetGuardShellLines, shellQuote, tailTextLines } from "./options";
import { runRemoteKubectl } from "./remote";
import { ciTarget, ciTargetSourceSummary, tektonPipelineVersion, tektonTriggersVersion } from "./types";

export async function status(target = ciTarget(null)): Promise<Record<string, unknown>> {
  const summary = await runRemoteKubectl([
    "set -euo pipefail",
    "printf 'tekton_pipelines='",
    "kubectl get deploy -n tekton-pipelines -o name 2>/dev/null | tr '\\n' ' ' || true",
    "printf '\\ntekton_triggers='",
    "kubectl get deploy -n tekton-pipelines-resolvers -o name 2>/dev/null | tr '\\n' ' ' || true",
    "printf '\\nunidesk_ci='",
    "kubectl get pipeline,task,pipelinerun,eventlistener,svc -n unidesk-ci -o name 2>/dev/null | tr '\\n' ' ' || true",
    "printf '\\n'",
  ].join("\n"), 60_000, 45_000, target);
  return {
    ok: true,
    providerId: target.providerId,
    target: ciTargetSourceSummary(target),
    orchestrator: "native-k3s",
    tekton: {
      pipelineVersion: tektonPipelineVersion,
      triggersVersion: tektonTriggersVersion,
    },
    summary: summary.stdout.trim(),
  };
}

export interface CiCleanupPodCandidate {
  name: string;
  phase: "Succeeded" | "Failed";
  createdAt: string;
  ageMinutes: number;
  selected: boolean;
  selectedReason: string;
}

export function cleanupRunsQueryScript(): string {
  const jsonPath = "{range .items[*]}{.metadata.name}{\"\\t\"}{.status.phase}{\"\\t\"}{.metadata.creationTimestamp}{\"\\n\"}{end}";
  return [
    "set -eu",
    `kubectl get pods -n unidesk-ci -o jsonpath=${shellQuote(jsonPath)} || true`,
  ].join("\n");
}

export function parseCleanupPodCandidates(stdout: string, generatedAtMs: number, minAgeMinutes: number, limit: number): CiCleanupPodCandidate[] {
  const byName = new Map<string, Omit<CiCleanupPodCandidate, "selected" | "selectedReason">>();
  for (const line of stdout.split(/\r?\n/u)) {
    const [name, phaseRaw, createdAt] = line.trim().split("\t");
    if (!name || !createdAt) continue;
    if (phaseRaw !== "Succeeded" && phaseRaw !== "Failed") continue;
    if (!/^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$/u.test(name)) continue;
    const createdAtMs = Date.parse(createdAt);
    if (!Number.isFinite(createdAtMs)) continue;
    const ageMinutes = Math.max(0, Math.floor((generatedAtMs - createdAtMs) / 60_000));
    if (ageMinutes < minAgeMinutes) continue;
    byName.set(name, { name, phase: phaseRaw, createdAt, ageMinutes });
  }
  const sorted = Array.from(byName.values()).sort((left, right) => {
    const byAge = right.ageMinutes - left.ageMinutes;
    return byAge !== 0 ? byAge : left.name.localeCompare(right.name);
  });
  const selectedNames = new Set(sorted.slice(0, limit).map((item) => item.name));
  return sorted.map((item) => ({
    ...item,
    selected: selectedNames.has(item.name),
    selectedReason: selectedNames.has(item.name) ? "terminal-pod-age-within-limit" : "over-limit",
  }));
}

export function cleanupRunsDeleteScript(names: string[]): string {
  const lines = ["set -eu"];
  for (let index = 0; index < names.length; index += 50) {
    const chunk = names.slice(index, index + 50).map(shellQuote).join(" ");
    lines.push(`kubectl -n unidesk-ci delete pod ${chunk} --ignore-not-found=true --wait=false`);
  }
  return lines.join("\n");
}

export async function runCleanupCapture(config: UniDeskConfig, target: CiTarget, script: string) {
  const command = [
    "set -eu",
    ...ciTargetGuardShellLines(target, { passOutput: "stderr" }),
    script,
  ].join("\n");
  return runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], command);
}

export async function cleanupRuns(config: UniDeskConfig, options: CiCleanupRunsOptions): Promise<Record<string, unknown>> {
  const generatedAt = new Date();
  const query = await runCleanupCapture(config, options.target, cleanupRunsQueryScript());
  const queryOk = "ok" in query ? query.ok : query.exitCode === 0;
  if (!queryOk) {
    return {
      ok: false,
      command: "ci cleanup-runs",
      providerId: options.target.providerId,
      namespace: "unidesk-ci",
      mutation: false,
      failureKind: "ci-cleanup-query-failed",
      query: {
        exitCode: query.exitCode,
        stdoutTail: tailTextLines(query.stdout, 80),
        stderrTail: tailTextLines(query.stderr, 80),
      },
    };
  }
  const candidates = parseCleanupPodCandidates(query.stdout, generatedAt.getTime(), options.minAgeMinutes, options.limit);
  const selected = candidates.filter((item) => item.selected);
  const deletion = options.confirm && selected.length > 0
    ? await runCleanupCapture(config, options.target, cleanupRunsDeleteScript(selected.map((item) => item.name)))
    : null;
  const deletionOk = deletion === null ? true : ("ok" in deletion ? deletion.ok : deletion.exitCode === 0);
  const ok = deletionOk;
  return {
    ok,
    command: "ci cleanup-runs",
    providerId: options.target.providerId,
    namespace: "unidesk-ci",
    generatedAt: generatedAt.toISOString(),
    mode: options.confirm ? "confirmed-cleanup" : "dry-run",
    minAgeMinutes: options.minAgeMinutes,
    limit: options.limit,
    mutation: options.confirm,
    candidateCount: candidates.length,
    selectedPodCount: selected.length,
    candidates: candidates.slice(0, Math.min(candidates.length, 120)),
    truncated: candidates.length > 120,
    deletedPodCount: deletionOk && deletion !== null ? selected.length : 0,
    deletion: deletion === null ? null : {
      exitCode: deletion.exitCode,
      stdoutTail: tailTextLines(deletion.stdout, 120),
      stderrTail: tailTextLines(deletion.stderr, 80),
    },
    next: options.confirm
      ? { status: `bun scripts/cli.ts ci status --target ${options.target.targetId}` }
      : { confirm: `bun scripts/cli.ts ci cleanup-runs --target ${options.target.targetId} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
  };
}

export async function cleanupFailedPods(config: UniDeskConfig, options: CiCleanupFailedPodsOptions): Promise<Record<string, unknown>> {
  const script = [
    "set -eu",
    ...ciTargetGuardShellLines(options.target, { passOutput: "stderr" }),
    `namespace=${shellQuote(options.namespace)}`,
    `min_age_minutes=${String(options.minAgeMinutes)}`,
    `limit=${String(options.limit)}`,
    `confirm=${options.confirm ? "true" : "false"}`,
    "tmp=$(mktemp /tmp/unidesk-ci-failed-pods.XXXXXX.tsv)",
    "selected=$(mktemp /tmp/unidesk-ci-failed-pods-selected.XXXXXX.txt)",
    "trap 'rm -f \"$tmp\" \"$selected\"' EXIT",
    "kubectl -n \"$namespace\" get pods --field-selector=status.phase=Failed --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,CREATED:.metadata.creationTimestamp,PHASE:.status.phase,REASON:.status.reason' --no-headers > \"$tmp\" || true",
    "now_epoch=$(date -u +%s)",
    "candidate_count=0",
    "while IFS= read -r line; do",
    "  set -- $line",
    "  name=${1:-}",
    "  created=${2:-}",
    "  phase=${3:-}",
    "  reason=${4:-}",
    "  [ -n \"$name\" ] || continue",
    "  [ \"$phase\" = \"Failed\" ] || continue",
    "  created_epoch=$(date -u -d \"$created\" +%s 2>/dev/null || printf '0')",
    "  [ \"$created_epoch\" -gt 0 ] || continue",
    "  age_minutes=$(( (now_epoch - created_epoch) / 60 ))",
    "  [ \"$age_minutes\" -ge \"$min_age_minutes\" ] || continue",
    "  [ \"$candidate_count\" -lt \"$limit\" ] || continue",
    "  candidate_count=$((candidate_count + 1))",
    "  printf 'candidate\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \"$name\" \"$created\" \"$phase\" \"$age_minutes\" \"$reason\"",
    "  printf '%s\\n' \"$name\" >> \"$selected\"",
    "done < \"$tmp\"",
    "delete_exit=0",
    "delete_stdout=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.out)",
    "delete_stderr=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.err)",
    "trap 'rm -f \"$tmp\" \"$selected\" \"$delete_stdout\" \"$delete_stderr\"' EXIT",
    "if [ \"$confirm\" = \"true\" ] && [ -s \"$selected\" ]; then",
    "  xargs -r kubectl -n \"$namespace\" delete pod --ignore-not-found=true --wait=false < \"$selected\" > \"$delete_stdout\" 2> \"$delete_stderr\" || delete_exit=$?",
    "fi",
    "printf 'candidateCount\\t%s\\n' \"$candidate_count\"",
    "if [ \"$confirm\" = \"true\" ]; then printf 'deletedPodCount\\t%s\\n' \"$(wc -l < \"$selected\" | tr -d ' ')\"; else printf 'deletedPodCount\\t0\\n'; fi",
    "printf 'deleteExit\\t%s\\n' \"$delete_exit\"",
    "printf 'deleteStdoutTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stdout\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"",
    "printf 'deleteStderrTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stderr\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"",
    "exit \"$delete_exit\"",
  ].join("\n");
  const result = await runSshCommandCapture(config, `${options.target.providerId}:k3s`, ["sh"], script);
  const fields = new Map<string, string>();
  const candidates: Record<string, unknown>[] = [];
  for (const line of result.stdout.split(/\r?\n/u)) {
    if (!line.trim()) continue;
    const [kind = "", ...rest] = line.split("\t");
    if (kind === "candidate") {
      const [name = "", createdAt = "", phase = "", ageMinutes = "", reason = ""] = rest;
      candidates.push({ name, namespace: options.namespace, phase, createdAt, ageMinutes: Number(ageMinutes), reason });
      continue;
    }
    fields.set(kind, rest.join("\t"));
  }
  const deleteExit = Number(fields.get("deleteExit") ?? result.exitCode);
  const ok = result.exitCode === 0 && deleteExit === 0;
  return {
    ok,
    command: "ci cleanup-failed-pods",
    mode: options.confirm ? "confirmed-cleanup" : "dry-run",
    mutation: options.confirm,
    providerId: options.target.providerId,
    namespace: options.namespace,
    minAgeMinutes: options.minAgeMinutes,
    limit: options.limit,
    candidateCount: Number(fields.get("candidateCount") ?? candidates.length),
    candidates,
    deletedPodCount: Number(fields.get("deletedPodCount") ?? 0),
    deletion: {
      exitCode: deleteExit,
      stdoutTail: fields.get("deleteStdoutTail") ?? "",
      stderrTail: fields.get("deleteStderrTail") ?? "",
    },
    probe: {
      exitCode: result.exitCode,
      stdoutTail: tailTextLines(result.stdout, 40),
      stderrTail: tailTextLines(result.stderr, 20),
    },
    next: options.confirm
      ? { status: `bun scripts/cli.ts ci status --target ${options.target.targetId}` }
      : { confirm: `bun scripts/cli.ts ci cleanup-failed-pods --target ${options.target.targetId} --namespace ${options.namespace} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
  };
}