fix: stabilize observability runtime apply

2026-06-19 14:19:55 +00:00
parent d58b46f6ab
commit 66f71bec62
3 changed files with 388 additions and 48 deletions
@@ -155,6 +155,13 @@ interface CiLogsOptions {
  capture: "dispatch-task" | "ssh-stream";
 }

+interface CiCleanupRunsOptions {
+  target: CiTarget;
+  minAgeMinutes: number;
+  limit: number;
+  confirm: boolean;
+}
+
 type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required" | "registry-not-installed" | "registry-unhealthy" | "remote-command-timeout" | "ssh-helper-command-shape-incompatible" | "ci-runner-not-ready";
 type PublishPreflightControlChannel = "backend-core" | "database" | "provider" | "registry";
 type PublishPreflightDetailedChannel = "backend-core-api" | "provider-dispatch" | "provider-host-ssh" | "database" | "artifact-registry";
@@ -275,6 +282,19 @@ function ciLogsOptions(args: string[]): CiLogsOptions {
  return { tailLines, capture: args.includes("--dispatch-task") ? "dispatch-task" : "ssh-stream" };
 }

+function ciCleanupRunsOptions(args: string[]): CiCleanupRunsOptions {
+  const limit = numberOption(args, "--limit", 50);
+  if (limit <= 0 || limit > 500) throw new Error("ci cleanup-runs --limit must be an integer between 1 and 500");
+  const confirm = boolFlag(args, "--confirm");
+  if (confirm && boolFlag(args, "--dry-run")) throw new Error("ci cleanup-runs accepts only one of --confirm or --dry-run");
+  return {
+    target: ciTarget(providerIdOption(args)),
+    minAgeMinutes: numberOption(args, "--min-age-minutes", 60),
+    limit,
+    confirm,
+  };
+}
+
 function countTextLines(value: string): number {
  if (value.length === 0) return 0;
  const breaks = value.match(/\r\n|\r|\n/gu)?.length ?? 0;
@@ -956,7 +976,7 @@ async function runRemoteKubectl(script: string, waitMs = 60_000, remoteTimeoutMs

 async function runRemoteKubectlRaw(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise<DispatchResult> {
  const command = [
-    "set -euo pipefail",
+    "set -eu",
    ...ciTargetGuardShellLines(target, { passOutput: "stderr" }),
    script,
  ].join("\n");
@@ -1177,6 +1197,118 @@ async function status(target = ciTarget(null)): Promise<Record<string, unknown>>
  };
 }

+interface CiCleanupPodCandidate {
+  name: string;
+  phase: "Succeeded" | "Failed";
+  createdAt: string;
+  ageMinutes: number;
+  selected: boolean;
+  selectedReason: string;
+}
+
+function cleanupRunsQueryScript(): string {
+  const jsonPath = "{range .items[*]}{.metadata.name}{\"\\t\"}{.status.phase}{\"\\t\"}{.metadata.creationTimestamp}{\"\\n\"}{end}";
+  return [
+    "set -eu",
+    `kubectl get pods -n unidesk-ci -o jsonpath=${shellQuote(jsonPath)} || true`,
+  ].join("\n");
+}
+
+function parseCleanupPodCandidates(stdout: string, generatedAtMs: number, minAgeMinutes: number, limit: number): CiCleanupPodCandidate[] {
+  const byName = new Map<string, Omit<CiCleanupPodCandidate, "selected" | "selectedReason">>();
+  for (const line of stdout.split(/\r?\n/u)) {
+    const [name, phaseRaw, createdAt] = line.trim().split("\t");
+    if (!name || !createdAt) continue;
+    if (phaseRaw !== "Succeeded" && phaseRaw !== "Failed") continue;
+    if (!/^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$/u.test(name)) continue;
+    const createdAtMs = Date.parse(createdAt);
+    if (!Number.isFinite(createdAtMs)) continue;
+    const ageMinutes = Math.max(0, Math.floor((generatedAtMs - createdAtMs) / 60_000));
+    if (ageMinutes < minAgeMinutes) continue;
+    byName.set(name, { name, phase: phaseRaw, createdAt, ageMinutes });
+  }
+  const sorted = Array.from(byName.values()).sort((left, right) => {
+    const byAge = right.ageMinutes - left.ageMinutes;
+    return byAge !== 0 ? byAge : left.name.localeCompare(right.name);
+  });
+  const selectedNames = new Set(sorted.slice(0, limit).map((item) => item.name));
+  return sorted.map((item) => ({
+    ...item,
+    selected: selectedNames.has(item.name),
+    selectedReason: selectedNames.has(item.name) ? "terminal-pod-age-within-limit" : "over-limit",
+  }));
+}
+
+function cleanupRunsDeleteScript(names: string[]): string {
+  const lines = ["set -eu"];
+  for (let index = 0; index < names.length; index += 50) {
+    const chunk = names.slice(index, index + 50).map(shellQuote).join(" ");
+    lines.push(`kubectl -n unidesk-ci delete pod ${chunk} --ignore-not-found=true --wait=false`);
+  }
+  return lines.join("\n");
+}
+
+async function runCleanupCapture(config: UniDeskConfig, target: CiTarget, script: string) {
+  const command = [
+    "set -eu",
+    ...ciTargetGuardShellLines(target, { passOutput: "stderr" }),
+    script,
+  ].join("\n");
+  return runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], command);
+}
+
+async function cleanupRuns(config: UniDeskConfig, options: CiCleanupRunsOptions): Promise<Record<string, unknown>> {
+  const generatedAt = new Date();
+  const query = await runCleanupCapture(config, options.target, cleanupRunsQueryScript());
+  const queryOk = "ok" in query ? query.ok : query.exitCode === 0;
+  if (!queryOk) {
+    return {
+      ok: false,
+      command: "ci cleanup-runs",
+      providerId: options.target.providerId,
+      namespace: "unidesk-ci",
+      mutation: false,
+      failureKind: "ci-cleanup-query-failed",
+      query: {
+        exitCode: query.exitCode,
+        stdoutTail: tailTextLines(query.stdout, 80),
+        stderrTail: tailTextLines(query.stderr, 80),
+      },
+    };
+  }
+  const candidates = parseCleanupPodCandidates(query.stdout, generatedAt.getTime(), options.minAgeMinutes, options.limit);
+  const selected = candidates.filter((item) => item.selected);
+  const deletion = options.confirm && selected.length > 0
+    ? await runCleanupCapture(config, options.target, cleanupRunsDeleteScript(selected.map((item) => item.name)))
+    : null;
+  const deletionOk = deletion === null ? true : ("ok" in deletion ? deletion.ok : deletion.exitCode === 0);
+  const ok = deletionOk;
+  return {
+    ok,
+    command: "ci cleanup-runs",
+    providerId: options.target.providerId,
+    namespace: "unidesk-ci",
+    generatedAt: generatedAt.toISOString(),
+    mode: options.confirm ? "confirmed-cleanup" : "dry-run",
+    minAgeMinutes: options.minAgeMinutes,
+    limit: options.limit,
+    mutation: options.confirm,
+    candidateCount: candidates.length,
+    selectedPodCount: selected.length,
+    candidates: candidates.slice(0, Math.min(candidates.length, 120)),
+    truncated: candidates.length > 120,
+    deletedPodCount: deletionOk && deletion !== null ? selected.length : 0,
+    deletion: deletion === null ? null : {
+      exitCode: deletion.exitCode,
+      stdoutTail: tailTextLines(deletion.stdout, 120),
+      stderrTail: tailTextLines(deletion.stderr, 80),
+    },
+    next: options.confirm
+      ? { status: `bun scripts/cli.ts ci status --provider-id ${options.target.providerId}` }
+      : { confirm: `bun scripts/cli.ts ci cleanup-runs --provider-id ${options.target.providerId} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
+  };
+}
+
 async function install(config: UniDeskConfig, options: CiInstallOptions): Promise<Record<string, unknown>> {
  if (!existsSync(rootPath(options.target.pipelineManifest))) {
    throw new Error("CI manifests are missing");
@@ -3048,7 +3180,7 @@ function catalogArtifactDescriptor(artifact: CiCatalogArtifact): Record<string,
 export function ciHelp(): Record<string, unknown> {
  const catalog = loadCiCatalog();
  return {
-    command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs",
+    command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs|cleanup-runs",
    description: "Manage native k3s Tekton CI on D601 or G14. CI may publish commit-pinned image artifacts, but it intentionally does not deploy CD.",
    examples: [
      "bun scripts/cli.ts ci install",
@@ -3068,6 +3200,8 @@ export function ciHelp(): Record<string, unknown> {
      "bun scripts/cli.ts ci publish-user-service --service frontend --commit <full-sha>",
      "bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000",
      "bun scripts/cli.ts ci logs <runId-or-pipelineRun> [--provider-id G14] [--tail-lines 80]",
+      "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --dry-run",
+      "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --confirm",
    ],
    tekton: {
      pipelineVersion: tektonPipelineVersion,
@@ -3131,6 +3265,12 @@ export function ciHelp(): Record<string, unknown> {
      tailLines: "1..2000",
      reason: "stream capture avoids backend-core task-result compactJson truncating CI log text before the CLI can extract failure hints",
    },
+    cleanupRuns: {
+      defaultMode: "dry-run",
+      namespace: "unidesk-ci",
+      selector: "status.phase in Succeeded,Failed",
+      guard: "--confirm required for deletion; Running/Pending pods are never selected",
+    },
  };
 }

@@ -3238,7 +3378,8 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi
    });
  }
  if (action === "logs") return logs(config, nameArg ?? "", ciTarget(providerIdOption(args)), ciLogsOptions(args));
-  throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs");
+  if (action === "cleanup-runs") return cleanupRuns(config, ciCleanupRunsOptions(args));
+  throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs, cleanup-runs");
 }

 export function startCiInstallJob(providerId = d601ProviderId): Record<string, unknown> {