fix: stabilize hwlab ci workspace cleanup
This commit is contained in:
Codex
+1
-1
@@ -57,7 +57,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
|
||||
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
|
||||
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|rerun-current|cleanup-runs --lane v02 | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane and PipelineRun retention actions, or build/status fixed HWLAB CI tools images through UniDesk G14 routes." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|rerun-current|cleanup-runs|cleanup-released-pvs | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane and CI workspace retention actions, or build/status fixed HWLAB CI tools images through UniDesk G14 routes." },
|
||||
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
|
||||
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
|
||||
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
|
||||
|
||||
+151
-66
@@ -49,7 +49,7 @@ interface G14RecordRolloutOptions {
|
||||
}
|
||||
|
||||
interface G14ControlPlaneOptions {
|
||||
action: "status" | "apply" | "rerun-current" | "cleanup-runs";
|
||||
action: "status" | "apply" | "rerun-current" | "cleanup-runs" | "cleanup-released-pvs";
|
||||
lane: "v02" | "g14" | "all";
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
@@ -163,12 +163,14 @@ function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||||
|
||||
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "rerun-current" && actionRaw !== "cleanup-runs") {
|
||||
throw new Error("control-plane usage: status|apply|rerun-current|cleanup-runs --lane v02 [--dry-run|--confirm]");
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "rerun-current" && actionRaw !== "cleanup-runs" && actionRaw !== "cleanup-released-pvs") {
|
||||
throw new Error("control-plane usage: status|apply|rerun-current --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
|
||||
}
|
||||
const lane = optionValue(args, "--lane") ?? "v02";
|
||||
const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
|
||||
if (actionRaw === "cleanup-runs") {
|
||||
if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
|
||||
} else if (actionRaw === "cleanup-released-pvs") {
|
||||
if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
|
||||
} else if (lane !== "v02") {
|
||||
throw new Error("control-plane status/apply/rerun-current currently requires --lane v02");
|
||||
}
|
||||
@@ -382,65 +384,69 @@ function commandErrorSummary(result: CommandJsonResult): string {
|
||||
}
|
||||
|
||||
function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||||
const script = [
|
||||
"set -eu",
|
||||
'tmp="$(mktemp)"',
|
||||
'trap \'rm -f "$tmp"\' EXIT',
|
||||
`kubectl get pipelineruns -n ${shellQuote(CI_NAMESPACE)} -o json > "$tmp"`,
|
||||
`export HWLAB_CLEANUP_PREFIXES=${shellQuote(JSON.stringify(pipelinePrefixesForLane(options.lane)))}`,
|
||||
`export HWLAB_CLEANUP_MIN_AGE_MINUTES=${shellQuote(String(options.minAgeMinutes))}`,
|
||||
`export HWLAB_CLEANUP_LIMIT=${shellQuote(String(options.limit))}`,
|
||||
'export HWLAB_CLEANUP_PIPELINERUNS_JSON="$tmp"',
|
||||
"node <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const doc = JSON.parse(fs.readFileSync(process.env.HWLAB_CLEANUP_PIPELINERUNS_JSON, 'utf8'));",
|
||||
"const prefixes = JSON.parse(process.env.HWLAB_CLEANUP_PREFIXES || '[]');",
|
||||
"const minAgeMinutes = Number(process.env.HWLAB_CLEANUP_MIN_AGE_MINUTES || '60');",
|
||||
"const limit = Number(process.env.HWLAB_CLEANUP_LIMIT || '20');",
|
||||
"const now = Date.now();",
|
||||
"const out = (doc.items || [])",
|
||||
" .map((item) => {",
|
||||
" const metadata = item.metadata || {};",
|
||||
" const condition = ((item.status || {}).conditions || [])[0] || {};",
|
||||
" const name = String(metadata.name || '');",
|
||||
" const createdAt = String(metadata.creationTimestamp || '');",
|
||||
" const createdMs = Date.parse(createdAt);",
|
||||
" const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;",
|
||||
" return { name, createdAt, ageMinutes, status: condition.status || null, reason: condition.reason || null };",
|
||||
" })",
|
||||
" .filter((item) => item.name && prefixes.some((prefix) => item.name.startsWith(prefix)))",
|
||||
" .filter((item) => item.status === 'True' || item.status === 'False')",
|
||||
" .filter((item) => typeof item.ageMinutes === 'number' && item.ageMinutes >= minAgeMinutes)",
|
||||
" .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))",
|
||||
" .slice(0, limit);",
|
||||
"console.log(JSON.stringify(out));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = g14K3s(["script", "--", script], 60_000);
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pipelinerun",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const parsed = JSON.parse(statusText(result) || "[]") as unknown;
|
||||
return Array.isArray(parsed) ? parsed.map((item) => record(item)) : [];
|
||||
const prefixes = pipelinePrefixesForLane(options.lane);
|
||||
const now = Date.now();
|
||||
return statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", createdAt = "", status = "", reason = ""] = line.split("\t");
|
||||
const createdMs = Date.parse(createdAt);
|
||||
const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
||||
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
|
||||
})
|
||||
.filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
|
||||
.filter((item) => item.status === "True" || item.status === "False")
|
||||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, options.limit);
|
||||
}
|
||||
|
||||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): CommandJsonResult {
|
||||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unknown>[] {
|
||||
if (pipelineRunNames.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "[]",
|
||||
stderr: "",
|
||||
parsed: [],
|
||||
};
|
||||
return [];
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`export WANTED=${shellQuote(JSON.stringify(pipelineRunNames))}`,
|
||||
"kubectl get pvc -n hwlab-ci -o json | node -e 'const fs=require(\"fs\"); const doc=JSON.parse(fs.readFileSync(0,\"utf8\")); const wanted=new Set(JSON.parse(process.env.WANTED)); const out=(doc.items||[]).filter((pvc)=> (pvc.metadata?.ownerReferences||[]).some((ref)=>ref.kind===\"PipelineRun\"&&wanted.has(ref.name))).map((pvc)=>({name:pvc.metadata.name,volume:pvc.spec?.volumeName||null,phase:pvc.status?.phase||null,owner:(pvc.metadata.ownerReferences||[]).find((ref)=>ref.kind===\"PipelineRun\")?.name||null})); console.log(JSON.stringify(out));' ",
|
||||
].join("\n");
|
||||
return g14K3s(["script", "--", script], 60_000);
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pvc",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const wanted = new Set(pipelineRunNames);
|
||||
return statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
|
||||
return {
|
||||
name,
|
||||
volume: volume || null,
|
||||
phase: phase || null,
|
||||
ownerKind: ownerKind || null,
|
||||
owner: owner || null,
|
||||
};
|
||||
})
|
||||
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
|
||||
}
|
||||
|
||||
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
@@ -457,18 +463,94 @@ function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResu
|
||||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
|
||||
}
|
||||
|
||||
function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pv",
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
return statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t");
|
||||
return {
|
||||
name,
|
||||
createdAt,
|
||||
phase,
|
||||
storageClass,
|
||||
reclaimPolicy,
|
||||
claimNamespace,
|
||||
claimName,
|
||||
capacity,
|
||||
hostPath: claimNamespace === CI_NAMESPACE && claimName.length > 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null,
|
||||
};
|
||||
})
|
||||
.filter((item) => item.phase === "Released")
|
||||
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
||||
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, options.limit);
|
||||
}
|
||||
|
||||
function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
if (names.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "no candidates",
|
||||
stderr: "",
|
||||
parsed: null,
|
||||
};
|
||||
}
|
||||
return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs);
|
||||
}
|
||||
|
||||
function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listReleasedCiWorkspacePvs(options);
|
||||
const candidateNames = candidates.map((item) => String(item.name));
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||||
mode: "dry-run",
|
||||
lane: options.lane,
|
||||
limit: options.limit,
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
mutation: false,
|
||||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
|
||||
};
|
||||
}
|
||||
const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000);
|
||||
return {
|
||||
ok: isCommandSuccess(deletion),
|
||||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||||
mode: "confirmed-cleanup",
|
||||
lane: options.lane,
|
||||
limit: options.limit,
|
||||
deletedPersistentVolumes: candidateNames,
|
||||
deletedPersistentVolumeCount: candidateNames.length,
|
||||
candidatesBefore: candidates,
|
||||
deletion,
|
||||
followUp: {
|
||||
diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
storage: "bun scripts/cli.ts ssh G14 script -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listCleanupPipelineRuns(options);
|
||||
const candidateNames = candidates.map((item) => String(item.name));
|
||||
const pvcResult = listOwnedWorkspacePvcs(candidateNames);
|
||||
let ownedPvcs: unknown[] = [];
|
||||
if (isCommandSuccess(pvcResult)) {
|
||||
try {
|
||||
ownedPvcs = JSON.parse(statusText(pvcResult) || "[]") as unknown[];
|
||||
} catch {
|
||||
ownedPvcs = [];
|
||||
}
|
||||
}
|
||||
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
@@ -642,6 +724,7 @@ function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Reco
|
||||
|
||||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
||||
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
||||
const sourceCommit = getV02Head();
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
|
||||
@@ -1578,11 +1661,13 @@ export function hwlabG14Help(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab g14 control-plane rerun-current --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||||
],
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup helper, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; control-plane status/apply/rerun-current/cleanup-runs uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources and completed PipelineRun workspace retention only.",
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup helper, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; control-plane status/apply/rerun-current/cleanup-runs/cleanup-released-pvs uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources and completed CI workspace retention only.",
|
||||
defaults: {
|
||||
repo: HWLAB_REPO,
|
||||
base: G14_SOURCE_BRANCH,
|
||||
|
||||
Reference in New Issue
Block a user