fix: add node scoped hwlab ci cleanup
This commit is contained in:
Codex
@@ -78,6 +78,25 @@ interface NodeRuntimeRenderResult {
|
||||
readonly location: NodeRuntimeRenderLocation;
|
||||
}
|
||||
|
||||
interface NodeRuntimeCleanupPipelineRunRow {
|
||||
name: string;
|
||||
createdAt: string | null;
|
||||
ageMinutes: number | null;
|
||||
status: string | null;
|
||||
reason: string | null;
|
||||
selected?: boolean;
|
||||
selectedReason?: string;
|
||||
}
|
||||
|
||||
interface NodeRuntimeCleanupOptions {
|
||||
minAgeMinutes: number;
|
||||
limit: number;
|
||||
sourceCommit?: string;
|
||||
pipelineRun?: string;
|
||||
targetPipelineRun?: string;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface NodeSecretOptions {
|
||||
action: SecretAction;
|
||||
node: string;
|
||||
@@ -195,6 +214,7 @@ const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key";
|
||||
const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key";
|
||||
const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
|
||||
const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
|
||||
const HWLAB_CI_NAMESPACE = "hwlab-ci";
|
||||
|
||||
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
||||
if (args.length === 0) return hwlabNodeHelp();
|
||||
@@ -280,7 +300,7 @@ async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomai
|
||||
}
|
||||
if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) {
|
||||
if (scoped.action === "status") return nodeRuntimeControlPlaneStatus(scoped);
|
||||
if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration") {
|
||||
if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration" || scoped.action === "cleanup-runs") {
|
||||
if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped);
|
||||
return nodeRuntimeControlPlaneRun(scoped);
|
||||
}
|
||||
@@ -1130,6 +1150,15 @@ function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
function compactRuntimeCommandStats(result: CommandResult): Record<string, unknown> {
|
||||
return {
|
||||
exitCode: result.exitCode,
|
||||
stdoutBytes: Buffer.byteLength(result.stdout),
|
||||
stderrBytes: Buffer.byteLength(result.stderr),
|
||||
timedOut: result.timedOut,
|
||||
};
|
||||
}
|
||||
|
||||
function parseLastJsonLineObject(text: string): Record<string, unknown> {
|
||||
for (const line of text.split(/\r?\n/u).reverse()) {
|
||||
const trimmed = line.trim();
|
||||
@@ -1214,7 +1243,7 @@ function nodeRuntimeUnsupportedAction(scoped: ReturnType<typeof parseNodeScopedD
|
||||
lane: scoped.lane,
|
||||
mutation: false,
|
||||
degradedReason: "unsupported-node-scoped-runtime-action",
|
||||
message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/runtime-image/runtime-migration",
|
||||
message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/cleanup-runs/runtime-image/runtime-migration",
|
||||
expected: nodeRuntimeExpected(scoped.spec),
|
||||
};
|
||||
}
|
||||
@@ -1430,9 +1459,336 @@ function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDel
|
||||
if (scoped.action === "apply") return nodeRuntimeApply(scoped);
|
||||
if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped);
|
||||
if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped);
|
||||
if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped);
|
||||
return nodeRuntimeUnsupportedAction(scoped);
|
||||
}
|
||||
|
||||
function parseNodeRuntimeCleanupOptions(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): NodeRuntimeCleanupOptions {
|
||||
const sourceCommitRaw = optionValue(scoped.originalArgs, "--source-commit");
|
||||
const pipelineRunRaw = optionValue(scoped.originalArgs, "--pipeline-run");
|
||||
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) {
|
||||
throw new Error("control-plane cleanup-runs accepts only one of --source-commit or --pipeline-run");
|
||||
}
|
||||
const sourceCommit = sourceCommitRaw?.toLowerCase();
|
||||
if (sourceCommit !== undefined && !/^[0-9a-f]{40}$/u.test(sourceCommit)) {
|
||||
throw new Error("--source-commit must be a full 40-character git sha for cleanup-runs");
|
||||
}
|
||||
const pipelineRun = pipelineRunRaw === undefined ? undefined : validateNodeRuntimePipelineRunName(scoped.spec, pipelineRunRaw);
|
||||
return {
|
||||
minAgeMinutes: positiveIntegerOption(scoped.originalArgs, "--min-age-minutes", 60, 10080),
|
||||
limit: positiveIntegerOption(scoped.originalArgs, "--limit", 20, 200),
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
targetPipelineRun: pipelineRun ?? (sourceCommit === undefined ? undefined : nodeRuntimePipelineRunName(scoped.spec, sourceCommit)),
|
||||
dryRun: scoped.dryRun || !scoped.confirm,
|
||||
};
|
||||
}
|
||||
|
||||
function validateNodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, value: string): string {
|
||||
const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||||
if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) {
|
||||
throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-<sha>[-rerun] PipelineRun name for --lane ${spec.lane}`);
|
||||
}
|
||||
return value.toLowerCase();
|
||||
}
|
||||
|
||||
function nodeRuntimeCleanupRuns(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
const options = parseNodeRuntimeCleanupOptions(scoped);
|
||||
const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
||||
const candidates = listNodeRuntimeCleanupPipelineRuns(scoped.spec, options);
|
||||
const selectedPipelineRuns = candidates
|
||||
.filter((item) => item.selected !== false)
|
||||
.map((item) => item.name);
|
||||
const ownedResources = listNodeRuntimeCleanupOwnedResources(scoped.spec, selectedPipelineRuns);
|
||||
const command = `hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane}`;
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command,
|
||||
mode: "dry-run",
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
namespace: HWLAB_CI_NAMESPACE,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
sourceCommit: options.sourceCommit,
|
||||
pipelineRun: options.pipelineRun,
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
selectedPipelineRuns,
|
||||
selectedPipelineRunCount: selectedPipelineRuns.length,
|
||||
ownedResources,
|
||||
ciObjectCounts: beforeCounts,
|
||||
mutation: false,
|
||||
next: {
|
||||
confirm: [
|
||||
command,
|
||||
`--min-age-minutes ${options.minAgeMinutes}`,
|
||||
`--limit ${options.limit}`,
|
||||
options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`,
|
||||
options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`,
|
||||
"--confirm",
|
||||
"--wait",
|
||||
].filter(Boolean).join(" "),
|
||||
},
|
||||
};
|
||||
}
|
||||
const deletion = deleteNodeRuntimeCleanupRuns(scoped.spec, selectedPipelineRuns, scoped.timeoutSeconds);
|
||||
const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
||||
return {
|
||||
ok: isCommandSuccess(deletion),
|
||||
command,
|
||||
mode: "confirmed-cleanup",
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
namespace: HWLAB_CI_NAMESPACE,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
sourceCommit: options.sourceCommit,
|
||||
pipelineRun: options.pipelineRun,
|
||||
deletedPipelineRuns: selectedPipelineRuns,
|
||||
deletedPipelineRunCount: selectedPipelineRuns.length,
|
||||
ownedResourcesBefore: ownedResources,
|
||||
ciObjectCountsBefore: beforeCounts,
|
||||
ciObjectCountsAfter: afterCounts,
|
||||
deletion: compactRuntimeCommand(deletion),
|
||||
mutation: isCommandSuccess(deletion),
|
||||
degradedReason: isCommandSuccess(deletion) ? undefined : "node-runtime-ci-cleanup-delete-failed",
|
||||
next: {
|
||||
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
rerunStatus: options.targetPipelineRun === undefined
|
||||
? undefined
|
||||
: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${options.targetPipelineRun}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function listNodeRuntimeCleanupPipelineRuns(spec: HwlabRuntimeLaneSpec, options: NodeRuntimeCleanupOptions): NodeRuntimeCleanupPipelineRunRow[] {
|
||||
const result = runNodeK3sArgs(spec, [
|
||||
"kubectl",
|
||||
"-n",
|
||||
HWLAB_CI_NAMESPACE,
|
||||
"get",
|
||||
"pipelinerun",
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
||||
], 60);
|
||||
if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`);
|
||||
const rows = nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout);
|
||||
if (options.targetPipelineRun !== undefined) {
|
||||
const target = rows.find((item) => item.name === options.targetPipelineRun);
|
||||
if (target === undefined) {
|
||||
return [{
|
||||
name: options.targetPipelineRun,
|
||||
createdAt: null,
|
||||
ageMinutes: null,
|
||||
status: null,
|
||||
reason: "target-pipelinerun-not-found",
|
||||
selected: false,
|
||||
}];
|
||||
}
|
||||
if (target.status !== "True" && target.status !== "False") {
|
||||
return [{ ...target, selected: false, selectedReason: "target-pipelinerun-not-terminal" }];
|
||||
}
|
||||
if (target.ageMinutes === null || target.ageMinutes < options.minAgeMinutes) {
|
||||
return [{ ...target, selected: false, selectedReason: target.ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }];
|
||||
}
|
||||
return [target];
|
||||
}
|
||||
const prefix = `${spec.pipelineRunPrefix}-`;
|
||||
const terminalRuns = rows
|
||||
.filter((item) => item.name.startsWith(prefix))
|
||||
.filter((item) => item.status === "True" || item.status === "False")
|
||||
.sort((left, right) => String(left.createdAt ?? "").localeCompare(String(right.createdAt ?? "")));
|
||||
const protectedLatest = terminalRuns
|
||||
.slice()
|
||||
.sort((left, right) => String(right.createdAt ?? "").localeCompare(String(left.createdAt ?? "")))[0]?.name ?? null;
|
||||
return terminalRuns
|
||||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
||||
.map((item) => item.name === protectedLatest ? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" } : item)
|
||||
.slice(0, options.limit);
|
||||
}
|
||||
|
||||
function nodeRuntimeCleanupPipelineRunRowsFromText(text: string): NodeRuntimeCleanupPipelineRunRow[] {
|
||||
const now = Date.now();
|
||||
return text.split(/\r?\n/u).map((line) => {
|
||||
const [name = "", createdAtRaw = "", statusRaw = "", reasonRaw = ""] = line.trim().split("\t");
|
||||
const createdAt = createdAtRaw.length > 0 ? createdAtRaw : null;
|
||||
const createdMs = createdAt === null ? NaN : Date.parse(createdAt);
|
||||
return {
|
||||
name,
|
||||
createdAt,
|
||||
ageMinutes: Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null,
|
||||
status: statusRaw || null,
|
||||
reason: reasonRaw || null,
|
||||
};
|
||||
}).filter((item) => item.name.length > 0);
|
||||
}
|
||||
|
||||
function listNodeRuntimeCleanupOwnedResources(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[]): Record<string, unknown> {
|
||||
const previewLimit = 24;
|
||||
if (pipelineRunNames.length === 0) {
|
||||
return {
|
||||
taskRunPreview: [],
|
||||
podPreview: [],
|
||||
pvcPreview: [],
|
||||
previewLimit,
|
||||
truncated: false,
|
||||
taskRunCount: 0,
|
||||
podCount: 0,
|
||||
pvcCount: 0,
|
||||
};
|
||||
}
|
||||
const wanted = new Set(pipelineRunNames);
|
||||
const taskRunsResult = runNodeK3sArgs(spec, [
|
||||
"kubectl",
|
||||
"-n",
|
||||
HWLAB_CI_NAMESPACE,
|
||||
"get",
|
||||
"taskrun",
|
||||
"-o",
|
||||
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "Succeeded"}}{{.status}}{{"\\t"}}{{.reason}}{{end}}{{end}}{{"\\n"}}{{end}}',
|
||||
], 60);
|
||||
const podsResult = runNodeK3sArgs(spec, [
|
||||
"kubectl",
|
||||
"-n",
|
||||
HWLAB_CI_NAMESPACE,
|
||||
"get",
|
||||
"pod",
|
||||
"-o",
|
||||
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\n"}}{{end}}',
|
||||
], 60);
|
||||
const pvcsResult = runNodeK3sArgs(spec, [
|
||||
"kubectl",
|
||||
"-n",
|
||||
HWLAB_CI_NAMESPACE,
|
||||
"get",
|
||||
"pvc",
|
||||
"-o",
|
||||
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}',
|
||||
], 60);
|
||||
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimeCleanupOwnedTaskRunsFromText(taskRunsResult.stdout, wanted) : [];
|
||||
const pods = isCommandSuccess(podsResult) ? nodeRuntimeCleanupOwnedPodsFromText(podsResult.stdout, wanted) : [];
|
||||
const pvcs = isCommandSuccess(pvcsResult) ? nodeRuntimeCleanupOwnedPvcsFromText(pvcsResult.stdout, wanted) : [];
|
||||
return {
|
||||
taskRunPreview: taskRuns.slice(0, previewLimit),
|
||||
podPreview: pods.slice(0, previewLimit),
|
||||
pvcPreview: pvcs.slice(0, previewLimit),
|
||||
previewLimit,
|
||||
truncated: taskRuns.length > previewLimit || pods.length > previewLimit || pvcs.length > previewLimit,
|
||||
taskRunCount: taskRuns.length,
|
||||
podCount: pods.length,
|
||||
pvcCount: pvcs.length,
|
||||
query: {
|
||||
taskRuns: compactRuntimeCommandStats(taskRunsResult),
|
||||
pods: compactRuntimeCommandStats(podsResult),
|
||||
pvcs: compactRuntimeCommandStats(pvcsResult),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeCleanupOwnedTaskRunsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
||||
return text.split(/\r?\n/u).map((line) => {
|
||||
const [name = "", pipelineRun = "", status = "", reason = ""] = line.trim().split("\t");
|
||||
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
|
||||
return {
|
||||
name,
|
||||
pipelineRun,
|
||||
status: status || null,
|
||||
reason: reason || null,
|
||||
};
|
||||
}).filter((item): item is Record<string, unknown> => item !== null);
|
||||
}
|
||||
|
||||
function nodeRuntimeCleanupOwnedPodsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
||||
return text.split(/\r?\n/u).map((line) => {
|
||||
const [name = "", pipelineRun = "", phase = "", nodeName = ""] = line.trim().split("\t");
|
||||
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
|
||||
return {
|
||||
name,
|
||||
pipelineRun,
|
||||
phase: phase || null,
|
||||
nodeName: nodeName || null,
|
||||
};
|
||||
}).filter((item): item is Record<string, unknown> => item !== null);
|
||||
}
|
||||
|
||||
function nodeRuntimeCleanupOwnedPvcsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
||||
return text.split(/\r?\n/u).map((line) => {
|
||||
const [name = "", volumeName = "", phase = "", ownerKind = "", ownerName = "", storage = ""] = line.trim().split("\t");
|
||||
if (name.length === 0 || ownerKind !== "PipelineRun" || !wanted.has(ownerName)) return null;
|
||||
return {
|
||||
name,
|
||||
pipelineRun: ownerName,
|
||||
phase: phase || null,
|
||||
volumeName: volumeName || null,
|
||||
storage: storage || null,
|
||||
};
|
||||
}).filter((item): item is Record<string, unknown> => item !== null);
|
||||
}
|
||||
|
||||
function deleteNodeRuntimeCleanupRuns(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[], timeoutSeconds: number): CommandResult {
|
||||
if (pipelineRunNames.length === 0) {
|
||||
return {
|
||||
command: [],
|
||||
cwd: repoRoot,
|
||||
exitCode: 0,
|
||||
stdout: "no candidates",
|
||||
stderr: "",
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
|
||||
"names_file=$(mktemp)",
|
||||
"cat > \"$names_file\"",
|
||||
"deleted_pipeline_runs=$(grep -c . \"$names_file\" | tr -d ' ')",
|
||||
"xargs -r -n 50 kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=false < \"$names_file\"",
|
||||
"deleted_pod_groups=0",
|
||||
"deleted_taskrun_groups=0",
|
||||
"explicit_owned_cleanup=skipped-large-batch",
|
||||
"if [ \"$deleted_pipeline_runs\" -le 40 ]; then",
|
||||
" explicit_owned_cleanup=executed",
|
||||
" while IFS= read -r pipeline_run; do",
|
||||
" [ -n \"$pipeline_run\" ] || continue",
|
||||
" kubectl -n \"$namespace\" delete pod -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||||
" deleted_pod_groups=$((deleted_pod_groups + 1))",
|
||||
" kubectl -n \"$namespace\" delete taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||||
" deleted_taskrun_groups=$((deleted_taskrun_groups + 1))",
|
||||
" done < \"$names_file\"",
|
||||
"fi",
|
||||
"printf 'deletedPipelineRunCount\\t%s\\n' \"$deleted_pipeline_runs\"",
|
||||
"printf 'deletedTaskRunLabelGroups\\t%s\\n' \"$deleted_taskrun_groups\"",
|
||||
"printf 'deletedPodLabelGroups\\t%s\\n' \"$deleted_pod_groups\"",
|
||||
"printf 'explicitOwnedCleanup\\t%s\\n' \"$explicit_owned_cleanup\"",
|
||||
"rm -f \"$names_file\"",
|
||||
].join("\n");
|
||||
return runNodeK3sScript(spec, script, timeoutSeconds, pipelineRunNames.join("\n") + "\n");
|
||||
}
|
||||
|
||||
function nodeRuntimeCiObjectCounts(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const script = [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
|
||||
"count_kind() { kubectl -n \"$namespace\" get \"$1\" -o name 2>/dev/null | wc -l | tr -d ' '; }",
|
||||
"printf 'pipelineRuns\\t%s\\n' \"$(count_kind pipelinerun)\"",
|
||||
"printf 'taskRuns\\t%s\\n' \"$(count_kind taskrun)\"",
|
||||
"printf 'pods\\t%s\\n' \"$(count_kind pod)\"",
|
||||
"printf 'pvcs\\t%s\\n' \"$(count_kind pvc)\"",
|
||||
].join("\n");
|
||||
const result = runNodeK3sScript(spec, script, 30);
|
||||
const fields = keyValueLinesFromText(statusText(result));
|
||||
return {
|
||||
pipelineRuns: numericField(fields.pipelineRuns),
|
||||
taskRuns: numericField(fields.taskRuns),
|
||||
pods: numericField(fields.pods),
|
||||
pvcs: numericField(fields.pvcs),
|
||||
result: compactRuntimeCommand(result),
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeBaseImageCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
const action = scoped.runtimeImageAction;
|
||||
if (action === null) {
|
||||
@@ -2510,6 +2866,9 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
||||
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
|
||||
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
|
||||
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
|
||||
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.status === "Unknown"
|
||||
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
|
||||
: null;
|
||||
const workloads = namespaceExists
|
||||
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
|
||||
: null;
|
||||
@@ -2528,6 +2887,9 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
||||
const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && workloadsReady && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready));
|
||||
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
|
||||
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
|
||||
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
|
||||
? pipelineRunDiagnostics.degradedReason
|
||||
: "pipelinerun-not-succeeded";
|
||||
const publicReady = publicProbes.ready === true;
|
||||
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
|
||||
const fullStatus = {
|
||||
@@ -2561,6 +2923,7 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
||||
result: compactRuntimeCommand(argo),
|
||||
},
|
||||
pipelineRun: pipelineRunProbe,
|
||||
pipelineRunDiagnostics,
|
||||
runtime: {
|
||||
ready: runtimeReady,
|
||||
namespace: spec.runtimeNamespace,
|
||||
@@ -2593,7 +2956,7 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
||||
? publicReady
|
||||
? gitMirrorReady ? undefined : "git-mirror-pending-flush"
|
||||
: "public-probe-not-ready"
|
||||
: "pipelinerun-not-succeeded"
|
||||
: pipelineRunDegradedReason
|
||||
: "argo-not-synced-healthy"
|
||||
: namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing"
|
||||
: "control-plane-not-ready",
|
||||
@@ -2661,6 +3024,7 @@ function joinUrlPath(baseUrl: string, suffix: string): string {
|
||||
|
||||
function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
const pipelineRun = record(status.pipelineRun);
|
||||
const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics);
|
||||
const argo = record(status.argo);
|
||||
const runtime = record(status.runtime);
|
||||
const publicProbes = record(status.publicProbes);
|
||||
@@ -2688,6 +3052,14 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
|
||||
message: pipelineRun.message ?? null,
|
||||
createdAt: pipelineRun.createdAt ?? null,
|
||||
ready: pipelineRun.status === "True",
|
||||
diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : {
|
||||
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
|
||||
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
|
||||
podCount: pipelineRunDiagnostics.podCount ?? null,
|
||||
pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [],
|
||||
unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [],
|
||||
schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [],
|
||||
},
|
||||
},
|
||||
argo: {
|
||||
application: argo.application ?? null,
|
||||
@@ -2746,6 +3118,9 @@ function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: Re
|
||||
if (reason === "pipelinerun-not-succeeded") {
|
||||
return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
||||
}
|
||||
if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") {
|
||||
return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`;
|
||||
}
|
||||
if (reason === "public-probe-not-ready") {
|
||||
return `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
|
||||
}
|
||||
@@ -2765,6 +3140,80 @@ function nodeRuntimeStatusCommand(scoped: ReturnType<typeof parseNodeScopedDeleg
|
||||
].filter(Boolean).join(" ");
|
||||
}
|
||||
|
||||
function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
||||
const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
||||
const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
||||
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRuns(parseJsonRecordFromText(taskRunsResult.stdout)) : [];
|
||||
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPods(parseJsonRecordFromText(podsResult.stdout)) : [];
|
||||
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
|
||||
const unscheduledPods = pods.filter((item) => item.scheduled === false);
|
||||
const schedulingMessages = unscheduledPods
|
||||
.map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "")
|
||||
.filter((message) => message.length > 0);
|
||||
const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message));
|
||||
return {
|
||||
ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0,
|
||||
pipelineRun,
|
||||
taskRuns,
|
||||
pods,
|
||||
taskRunCount: taskRuns.length,
|
||||
podCount: pods.length,
|
||||
pendingTaskRuns,
|
||||
unscheduledPods,
|
||||
schedulingMessages,
|
||||
degradedReason: tooManyPods
|
||||
? "node-runtime-ci-pod-capacity-exhausted"
|
||||
: unscheduledPods.length > 0
|
||||
? "node-runtime-ci-pod-unschedulable"
|
||||
: pendingTaskRuns.length > 0
|
||||
? "node-runtime-ci-taskrun-pending"
|
||||
: undefined,
|
||||
query: {
|
||||
taskRuns: compactRuntimeCommand(taskRunsResult),
|
||||
pods: compactRuntimeCommand(podsResult),
|
||||
},
|
||||
next: tooManyPods || unscheduledPods.length > 0
|
||||
? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` }
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
||||
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
||||
return items.map((item) => {
|
||||
const metadata = record(item.metadata);
|
||||
const status = record(item.status);
|
||||
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
||||
const condition = conditions[0] ?? {};
|
||||
return {
|
||||
name: metadata.name ?? null,
|
||||
status: condition.status ?? null,
|
||||
reason: condition.reason ?? null,
|
||||
message: condition.message ?? null,
|
||||
podName: status.podName ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function nodeRuntimePipelineDiagnosticPods(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
||||
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
||||
return items.map((item) => {
|
||||
const metadata = record(item.metadata);
|
||||
const spec = record(item.spec);
|
||||
const status = record(item.status);
|
||||
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
||||
const scheduled = conditions.find((condition) => condition.type === "PodScheduled");
|
||||
return {
|
||||
name: metadata.name ?? null,
|
||||
phase: status.phase ?? null,
|
||||
nodeName: spec.nodeName ?? null,
|
||||
scheduled: scheduled === undefined ? null : scheduled.status === "True",
|
||||
scheduledReason: scheduled?.reason ?? null,
|
||||
scheduledMessage: scheduled?.message ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function nodeRuntimeRenderToken(): string {
|
||||
return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user