feat: refine runner and CI cleanup commands
This commit is contained in:
Codex
+130
-3
@@ -162,6 +162,14 @@ interface CiCleanupRunsOptions {
|
||||
confirm: boolean;
|
||||
}
|
||||
|
||||
interface CiCleanupFailedPodsOptions {
|
||||
target: CiTarget;
|
||||
namespace: string;
|
||||
minAgeMinutes: number;
|
||||
limit: number;
|
||||
confirm: boolean;
|
||||
}
|
||||
|
||||
type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required" | "registry-not-installed" | "registry-unhealthy" | "remote-command-timeout" | "ssh-helper-command-shape-incompatible" | "ci-runner-not-ready";
|
||||
type PublishPreflightControlChannel = "backend-core" | "database" | "provider" | "registry";
|
||||
type PublishPreflightDetailedChannel = "backend-core-api" | "provider-dispatch" | "provider-host-ssh" | "database" | "artifact-registry";
|
||||
@@ -295,6 +303,27 @@ function ciCleanupRunsOptions(args: string[]): CiCleanupRunsOptions {
|
||||
};
|
||||
}
|
||||
|
||||
function ciCleanupFailedPodsOptions(args: string[]): CiCleanupFailedPodsOptions {
|
||||
const limit = numberOption(args, "--limit", 50);
|
||||
if (limit <= 0 || limit > 500) throw new Error("ci cleanup-failed-pods --limit must be an integer between 1 and 500");
|
||||
const confirm = boolFlag(args, "--confirm");
|
||||
if (confirm && boolFlag(args, "--dry-run")) throw new Error("ci cleanup-failed-pods accepts only one of --confirm or --dry-run");
|
||||
return {
|
||||
target: ciTarget(providerIdOption(args)),
|
||||
namespace: requireK8sName(stringOption(args, "--namespace") ?? "unidesk-ci", "--namespace"),
|
||||
minAgeMinutes: numberOption(args, "--min-age-minutes", 60),
|
||||
limit,
|
||||
confirm,
|
||||
};
|
||||
}
|
||||
|
||||
function requireK8sName(value: string, option: string): string {
|
||||
if (!/^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$/u.test(value)) {
|
||||
throw new Error(`${option} must be a Kubernetes DNS label`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function countTextLines(value: string): number {
|
||||
if (value.length === 0) return 0;
|
||||
const breaks = value.match(/\r\n|\r|\n/gu)?.length ?? 0;
|
||||
@@ -721,7 +750,7 @@ function publishPreflightFailedScopes(preflight: PublishPreflight): string[] {
|
||||
|
||||
function ciRunnerPreflightScript(sourceHostPath: string): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
"set -eu",
|
||||
...d601K3sGuardShellLines(d601Kubeconfig),
|
||||
"printf 'provider_host_ssh=ok\\n'",
|
||||
"printf 'kubectl='",
|
||||
@@ -1309,6 +1338,94 @@ async function cleanupRuns(config: UniDeskConfig, options: CiCleanupRunsOptions)
|
||||
};
|
||||
}
|
||||
|
||||
async function cleanupFailedPods(config: UniDeskConfig, options: CiCleanupFailedPodsOptions): Promise<Record<string, unknown>> {
|
||||
const script = [
|
||||
"set -eu",
|
||||
...ciTargetGuardShellLines(options.target, { passOutput: "stderr" }),
|
||||
`namespace=${shellQuote(options.namespace)}`,
|
||||
`min_age_minutes=${String(options.minAgeMinutes)}`,
|
||||
`limit=${String(options.limit)}`,
|
||||
`confirm=${options.confirm ? "true" : "false"}`,
|
||||
"tmp=$(mktemp /tmp/unidesk-ci-failed-pods.XXXXXX.tsv)",
|
||||
"selected=$(mktemp /tmp/unidesk-ci-failed-pods-selected.XXXXXX.txt)",
|
||||
"trap 'rm -f \"$tmp\" \"$selected\"' EXIT",
|
||||
"kubectl -n \"$namespace\" get pods --field-selector=status.phase=Failed --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,CREATED:.metadata.creationTimestamp,PHASE:.status.phase,REASON:.status.reason' --no-headers > \"$tmp\" || true",
|
||||
"now_epoch=$(date -u +%s)",
|
||||
"candidate_count=0",
|
||||
"while IFS= read -r line; do",
|
||||
" set -- $line",
|
||||
" name=${1:-}",
|
||||
" created=${2:-}",
|
||||
" phase=${3:-}",
|
||||
" reason=${4:-}",
|
||||
" [ -n \"$name\" ] || continue",
|
||||
" [ \"$phase\" = \"Failed\" ] || continue",
|
||||
" created_epoch=$(date -u -d \"$created\" +%s 2>/dev/null || printf '0')",
|
||||
" [ \"$created_epoch\" -gt 0 ] || continue",
|
||||
" age_minutes=$(( (now_epoch - created_epoch) / 60 ))",
|
||||
" [ \"$age_minutes\" -ge \"$min_age_minutes\" ] || continue",
|
||||
" [ \"$candidate_count\" -lt \"$limit\" ] || continue",
|
||||
" candidate_count=$((candidate_count + 1))",
|
||||
" printf 'candidate\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \"$name\" \"$created\" \"$phase\" \"$age_minutes\" \"$reason\"",
|
||||
" printf '%s\\n' \"$name\" >> \"$selected\"",
|
||||
"done < \"$tmp\"",
|
||||
"delete_exit=0",
|
||||
"delete_stdout=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.out)",
|
||||
"delete_stderr=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.err)",
|
||||
"trap 'rm -f \"$tmp\" \"$selected\" \"$delete_stdout\" \"$delete_stderr\"' EXIT",
|
||||
"if [ \"$confirm\" = \"true\" ] && [ -s \"$selected\" ]; then",
|
||||
" xargs -r kubectl -n \"$namespace\" delete pod --ignore-not-found=true --wait=false < \"$selected\" > \"$delete_stdout\" 2> \"$delete_stderr\" || delete_exit=$?",
|
||||
"fi",
|
||||
"printf 'candidateCount\\t%s\\n' \"$candidate_count\"",
|
||||
"if [ \"$confirm\" = \"true\" ]; then printf 'deletedPodCount\\t%s\\n' \"$(wc -l < \"$selected\" | tr -d ' ')\"; else printf 'deletedPodCount\\t0\\n'; fi",
|
||||
"printf 'deleteExit\\t%s\\n' \"$delete_exit\"",
|
||||
"printf 'deleteStdoutTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stdout\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"",
|
||||
"printf 'deleteStderrTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stderr\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"",
|
||||
"exit \"$delete_exit\"",
|
||||
].join("\n");
|
||||
const result = await runSshCommandCapture(config, `${options.target.providerId}:k3s`, ["sh"], script);
|
||||
const fields = new Map<string, string>();
|
||||
const candidates: Record<string, unknown>[] = [];
|
||||
for (const line of result.stdout.split(/\r?\n/u)) {
|
||||
if (!line.trim()) continue;
|
||||
const [kind = "", ...rest] = line.split("\t");
|
||||
if (kind === "candidate") {
|
||||
const [name = "", createdAt = "", phase = "", ageMinutes = "", reason = ""] = rest;
|
||||
candidates.push({ name, namespace: options.namespace, phase, createdAt, ageMinutes: Number(ageMinutes), reason });
|
||||
continue;
|
||||
}
|
||||
fields.set(kind, rest.join("\t"));
|
||||
}
|
||||
const deleteExit = Number(fields.get("deleteExit") ?? result.exitCode);
|
||||
const ok = result.exitCode === 0 && deleteExit === 0;
|
||||
return {
|
||||
ok,
|
||||
command: "ci cleanup-failed-pods",
|
||||
mode: options.confirm ? "confirmed-cleanup" : "dry-run",
|
||||
mutation: options.confirm,
|
||||
providerId: options.target.providerId,
|
||||
namespace: options.namespace,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
candidateCount: Number(fields.get("candidateCount") ?? candidates.length),
|
||||
candidates,
|
||||
deletedPodCount: Number(fields.get("deletedPodCount") ?? 0),
|
||||
deletion: {
|
||||
exitCode: deleteExit,
|
||||
stdoutTail: fields.get("deleteStdoutTail") ?? "",
|
||||
stderrTail: fields.get("deleteStderrTail") ?? "",
|
||||
},
|
||||
probe: {
|
||||
exitCode: result.exitCode,
|
||||
stdoutTail: tailTextLines(result.stdout, 40),
|
||||
stderrTail: tailTextLines(result.stderr, 20),
|
||||
},
|
||||
next: options.confirm
|
||||
? { status: `bun scripts/cli.ts ci status --provider-id ${options.target.providerId}` }
|
||||
: { confirm: `bun scripts/cli.ts ci cleanup-failed-pods --provider-id ${options.target.providerId} --namespace ${options.namespace} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
|
||||
};
|
||||
}
|
||||
|
||||
async function install(config: UniDeskConfig, options: CiInstallOptions): Promise<Record<string, unknown>> {
|
||||
if (!existsSync(rootPath(options.target.pipelineManifest))) {
|
||||
throw new Error("CI manifests are missing");
|
||||
@@ -3180,7 +3297,7 @@ function catalogArtifactDescriptor(artifact: CiCatalogArtifact): Record<string,
|
||||
export function ciHelp(): Record<string, unknown> {
|
||||
const catalog = loadCiCatalog();
|
||||
return {
|
||||
command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs|cleanup-runs",
|
||||
command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs|cleanup-runs|cleanup-failed-pods",
|
||||
description: "Manage native k3s Tekton CI on D601 or G14. CI may publish commit-pinned image artifacts, but it intentionally does not deploy CD.",
|
||||
examples: [
|
||||
"bun scripts/cli.ts ci install",
|
||||
@@ -3202,6 +3319,8 @@ export function ciHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts ci logs <runId-or-pipelineRun> [--provider-id G14] [--tail-lines 80]",
|
||||
"bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --dry-run",
|
||||
"bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --confirm",
|
||||
"bun scripts/cli.ts ci cleanup-failed-pods --provider-id D601 --namespace unidesk-ci --min-age-minutes 60 --dry-run",
|
||||
"bun scripts/cli.ts ci cleanup-failed-pods --provider-id D601 --namespace unidesk-ci --min-age-minutes 60 --confirm",
|
||||
],
|
||||
tekton: {
|
||||
pipelineVersion: tektonPipelineVersion,
|
||||
@@ -3271,6 +3390,13 @@ export function ciHelp(): Record<string, unknown> {
|
||||
selector: "status.phase in Succeeded,Failed",
|
||||
guard: "--confirm required for deletion; Running/Pending pods are never selected",
|
||||
},
|
||||
cleanupFailedPods: {
|
||||
defaultNamespace: "unidesk-ci",
|
||||
defaultMinAgeMinutes: 60,
|
||||
defaultLimit: 50,
|
||||
scope: "Kubernetes pods with status.phase=Failed only; does not touch Running/Pending pods, workloads, PVCs, Secrets, or Tekton resources",
|
||||
defaultMode: "dry-run",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3379,7 +3505,8 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi
|
||||
}
|
||||
if (action === "logs") return logs(config, nameArg ?? "", ciTarget(providerIdOption(args)), ciLogsOptions(args));
|
||||
if (action === "cleanup-runs") return cleanupRuns(config, ciCleanupRunsOptions(args));
|
||||
throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs, cleanup-runs");
|
||||
if (action === "cleanup-failed-pods") return cleanupFailedPods(config, ciCleanupFailedPodsOptions(args));
|
||||
throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs, cleanup-runs, cleanup-failed-pods");
|
||||
}
|
||||
|
||||
export function startCiInstallJob(providerId = d601ProviderId): Record<string, unknown> {
|
||||
|
||||
Reference in New Issue
Block a user