feat: add hwlab ci cleanup control command
This commit is contained in:
Codex
+1
-1
@@ -57,7 +57,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
|
||||
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
|
||||
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|rerun-current --lane v02 | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane actions, or build/status fixed HWLAB CI tools images through UniDesk G14 routes." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|rerun-current|cleanup-runs --lane v02 | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane and PipelineRun retention actions, or build/status fixed HWLAB CI tools images through UniDesk G14 routes." },
|
||||
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
|
||||
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
|
||||
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
|
||||
|
||||
+156
-8
@@ -49,11 +49,13 @@ interface G14RecordRolloutOptions {
|
||||
}
|
||||
|
||||
interface G14ControlPlaneOptions {
|
||||
action: "status" | "apply" | "rerun-current";
|
||||
lane: "v02";
|
||||
action: "status" | "apply" | "rerun-current" | "cleanup-runs";
|
||||
lane: "v02" | "g14" | "all";
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
timeoutSeconds: number;
|
||||
minAgeMinutes: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
interface G14ToolsImageOptions {
|
||||
@@ -161,20 +163,26 @@ function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||||
|
||||
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "rerun-current") {
|
||||
throw new Error("control-plane usage: status|apply|rerun-current --lane v02 [--dry-run|--confirm]");
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "rerun-current" && actionRaw !== "cleanup-runs") {
|
||||
throw new Error("control-plane usage: status|apply|rerun-current|cleanup-runs --lane v02 [--dry-run|--confirm]");
|
||||
}
|
||||
const lane = optionValue(args, "--lane") ?? "v02";
|
||||
if (actionRaw === "cleanup-runs") {
|
||||
if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
|
||||
} else if (lane !== "v02") {
|
||||
throw new Error("control-plane status/apply/rerun-current currently requires --lane v02");
|
||||
}
|
||||
const lane = optionValue(args, "--lane");
|
||||
if (lane !== "v02") throw new Error("control-plane currently requires --lane v02");
|
||||
const confirm = args.includes("--confirm");
|
||||
const explicitDryRun = args.includes("--dry-run");
|
||||
if (confirm && explicitDryRun) throw new Error("control-plane accepts only one of --confirm or --dry-run");
|
||||
return {
|
||||
action: actionRaw,
|
||||
lane: "v02",
|
||||
lane,
|
||||
confirm,
|
||||
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
||||
limit: positiveIntegerOption(args, "--limit", 20, 200),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -360,6 +368,143 @@ function getPipelineRunCompact(name: string): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] {
|
||||
if (lane === "v02") return ["hwlab-v02-ci-poll-"];
|
||||
if (lane === "g14") return ["hwlab-g14-ci-poll-"];
|
||||
return ["hwlab-v02-ci-poll-", "hwlab-g14-ci-poll-"];
|
||||
}
|
||||
|
||||
function commandErrorSummary(result: CommandJsonResult): string {
|
||||
const remoteStderr = String(nested(result.parsed, ["data", "stderr"]) ?? "").trim();
|
||||
const remoteStdout = String(nested(result.parsed, ["data", "stdout"]) ?? "").trim();
|
||||
const text = [result.stderr.trim(), remoteStderr, remoteStdout, result.stdout.trim()].find((item) => item.length > 0) ?? "";
|
||||
return text.slice(0, 4000);
|
||||
}
|
||||
|
||||
function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||||
const script = [
|
||||
"set -eu",
|
||||
'tmp="$(mktemp)"',
|
||||
'trap \'rm -f "$tmp"\' EXIT',
|
||||
`kubectl get pipelineruns -n ${shellQuote(CI_NAMESPACE)} -o json > "$tmp"`,
|
||||
`export HWLAB_CLEANUP_PREFIXES=${shellQuote(JSON.stringify(pipelinePrefixesForLane(options.lane)))}`,
|
||||
`export HWLAB_CLEANUP_MIN_AGE_MINUTES=${shellQuote(String(options.minAgeMinutes))}`,
|
||||
`export HWLAB_CLEANUP_LIMIT=${shellQuote(String(options.limit))}`,
|
||||
'export HWLAB_CLEANUP_PIPELINERUNS_JSON="$tmp"',
|
||||
"node <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const doc = JSON.parse(fs.readFileSync(process.env.HWLAB_CLEANUP_PIPELINERUNS_JSON, 'utf8'));",
|
||||
"const prefixes = JSON.parse(process.env.HWLAB_CLEANUP_PREFIXES || '[]');",
|
||||
"const minAgeMinutes = Number(process.env.HWLAB_CLEANUP_MIN_AGE_MINUTES || '60');",
|
||||
"const limit = Number(process.env.HWLAB_CLEANUP_LIMIT || '20');",
|
||||
"const now = Date.now();",
|
||||
"const out = (doc.items || [])",
|
||||
" .map((item) => {",
|
||||
" const metadata = item.metadata || {};",
|
||||
" const condition = ((item.status || {}).conditions || [])[0] || {};",
|
||||
" const name = String(metadata.name || '');",
|
||||
" const createdAt = String(metadata.creationTimestamp || '');",
|
||||
" const createdMs = Date.parse(createdAt);",
|
||||
" const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;",
|
||||
" return { name, createdAt, ageMinutes, status: condition.status || null, reason: condition.reason || null };",
|
||||
" })",
|
||||
" .filter((item) => item.name && prefixes.some((prefix) => item.name.startsWith(prefix)))",
|
||||
" .filter((item) => item.status === 'True' || item.status === 'False')",
|
||||
" .filter((item) => typeof item.ageMinutes === 'number' && item.ageMinutes >= minAgeMinutes)",
|
||||
" .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))",
|
||||
" .slice(0, limit);",
|
||||
"console.log(JSON.stringify(out));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = g14K3s(["script", "--", script], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const parsed = JSON.parse(statusText(result) || "[]") as unknown;
|
||||
return Array.isArray(parsed) ? parsed.map((item) => record(item)) : [];
|
||||
}
|
||||
|
||||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): CommandJsonResult {
|
||||
if (pipelineRunNames.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "[]",
|
||||
stderr: "",
|
||||
parsed: [],
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`export WANTED=${shellQuote(JSON.stringify(pipelineRunNames))}`,
|
||||
"kubectl get pvc -n hwlab-ci -o json | node -e 'const fs=require(\"fs\"); const doc=JSON.parse(fs.readFileSync(0,\"utf8\")); const wanted=new Set(JSON.parse(process.env.WANTED)); const out=(doc.items||[]).filter((pvc)=> (pvc.metadata?.ownerReferences||[]).some((ref)=>ref.kind===\"PipelineRun\"&&wanted.has(ref.name))).map((pvc)=>({name:pvc.metadata.name,volume:pvc.spec?.volumeName||null,phase:pvc.status?.phase||null,owner:(pvc.metadata.ownerReferences||[]).find((ref)=>ref.kind===\"PipelineRun\")?.name||null})); console.log(JSON.stringify(out));' ",
|
||||
].join("\n");
|
||||
return g14K3s(["script", "--", script], 60_000);
|
||||
}
|
||||
|
||||
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
if (names.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "no candidates",
|
||||
stderr: "",
|
||||
parsed: null,
|
||||
};
|
||||
}
|
||||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
|
||||
}
|
||||
|
||||
function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listCleanupPipelineRuns(options);
|
||||
const candidateNames = candidates.map((item) => String(item.name));
|
||||
const pvcResult = listOwnedWorkspacePvcs(candidateNames);
|
||||
let ownedPvcs: unknown[] = [];
|
||||
if (isCommandSuccess(pvcResult)) {
|
||||
try {
|
||||
ownedPvcs = JSON.parse(statusText(pvcResult) || "[]") as unknown[];
|
||||
} catch {
|
||||
ownedPvcs = [];
|
||||
}
|
||||
}
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 control-plane cleanup-runs",
|
||||
mode: "dry-run",
|
||||
lane: options.lane,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
ownedPvcs,
|
||||
ownedPvcCount: ownedPvcs.length,
|
||||
mutation: false,
|
||||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane ${options.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` },
|
||||
};
|
||||
}
|
||||
const deletion = deletePipelineRuns(candidateNames, options.timeoutSeconds * 1000);
|
||||
return {
|
||||
ok: isCommandSuccess(deletion),
|
||||
command: "hwlab g14 control-plane cleanup-runs",
|
||||
mode: "confirmed-cleanup",
|
||||
lane: options.lane,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
deletedPipelineRuns: candidateNames,
|
||||
deletedPipelineRunCount: candidateNames.length,
|
||||
ownedPvcsBefore: ownedPvcs,
|
||||
ownedPvcCountBefore: ownedPvcs.length,
|
||||
deletion,
|
||||
followUp: {
|
||||
status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
|
||||
diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function runV02RenderCheck(sourceCommit: string): CommandJsonResult {
|
||||
const renderDir = v02RenderDir(sourceCommit);
|
||||
return v02WorkspaceScript([
|
||||
@@ -496,6 +641,7 @@ function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Reco
|
||||
}
|
||||
|
||||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
||||
const sourceCommit = getV02Head();
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
|
||||
@@ -1430,11 +1576,13 @@ export function hwlabG14Help(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane rerun-current --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||||
],
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap helper, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; control-plane status/apply/rerun-current uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources only.",
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup helper, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; control-plane status/apply/rerun-current/cleanup-runs uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources and completed PipelineRun workspace retention only.",
|
||||
defaults: {
|
||||
repo: HWLAB_REPO,
|
||||
base: G14_SOURCE_BRANCH,
|
||||
|
||||
Reference in New Issue
Block a user