|
|
|
@@ -78,6 +78,25 @@ interface NodeRuntimeRenderResult {
|
|
|
|
|
readonly location: NodeRuntimeRenderLocation;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface NodeRuntimeCleanupPipelineRunRow {
|
|
|
|
|
name: string;
|
|
|
|
|
createdAt: string | null;
|
|
|
|
|
ageMinutes: number | null;
|
|
|
|
|
status: string | null;
|
|
|
|
|
reason: string | null;
|
|
|
|
|
selected?: boolean;
|
|
|
|
|
selectedReason?: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface NodeRuntimeCleanupOptions {
|
|
|
|
|
minAgeMinutes: number;
|
|
|
|
|
limit: number;
|
|
|
|
|
sourceCommit?: string;
|
|
|
|
|
pipelineRun?: string;
|
|
|
|
|
targetPipelineRun?: string;
|
|
|
|
|
dryRun: boolean;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface NodeSecretOptions {
|
|
|
|
|
action: SecretAction;
|
|
|
|
|
node: string;
|
|
|
|
@@ -195,6 +214,7 @@ const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key";
|
|
|
|
|
const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key";
|
|
|
|
|
const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
|
|
|
|
|
const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
|
|
|
|
|
const HWLAB_CI_NAMESPACE = "hwlab-ci";
|
|
|
|
|
|
|
|
|
|
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
|
|
|
|
if (args.length === 0) return hwlabNodeHelp();
|
|
|
|
@@ -280,7 +300,7 @@ async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomai
|
|
|
|
|
}
|
|
|
|
|
if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) {
|
|
|
|
|
if (scoped.action === "status") return nodeRuntimeControlPlaneStatus(scoped);
|
|
|
|
|
if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration") {
|
|
|
|
|
if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration" || scoped.action === "cleanup-runs") {
|
|
|
|
|
if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped);
|
|
|
|
|
return nodeRuntimeControlPlaneRun(scoped);
|
|
|
|
|
}
|
|
|
|
@@ -1130,6 +1150,15 @@ function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function compactRuntimeCommandStats(result: CommandResult): Record<string, unknown> {
|
|
|
|
|
return {
|
|
|
|
|
exitCode: result.exitCode,
|
|
|
|
|
stdoutBytes: Buffer.byteLength(result.stdout),
|
|
|
|
|
stderrBytes: Buffer.byteLength(result.stderr),
|
|
|
|
|
timedOut: result.timedOut,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function parseLastJsonLineObject(text: string): Record<string, unknown> {
|
|
|
|
|
for (const line of text.split(/\r?\n/u).reverse()) {
|
|
|
|
|
const trimmed = line.trim();
|
|
|
|
@@ -1214,7 +1243,7 @@ function nodeRuntimeUnsupportedAction(scoped: ReturnType<typeof parseNodeScopedD
|
|
|
|
|
lane: scoped.lane,
|
|
|
|
|
mutation: false,
|
|
|
|
|
degradedReason: "unsupported-node-scoped-runtime-action",
|
|
|
|
|
message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/runtime-image/runtime-migration",
|
|
|
|
|
message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/cleanup-runs/runtime-image/runtime-migration",
|
|
|
|
|
expected: nodeRuntimeExpected(scoped.spec),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
@@ -1430,9 +1459,336 @@ function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDel
|
|
|
|
|
if (scoped.action === "apply") return nodeRuntimeApply(scoped);
|
|
|
|
|
if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped);
|
|
|
|
|
if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped);
|
|
|
|
|
if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped);
|
|
|
|
|
return nodeRuntimeUnsupportedAction(scoped);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function parseNodeRuntimeCleanupOptions(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): NodeRuntimeCleanupOptions {
|
|
|
|
|
const sourceCommitRaw = optionValue(scoped.originalArgs, "--source-commit");
|
|
|
|
|
const pipelineRunRaw = optionValue(scoped.originalArgs, "--pipeline-run");
|
|
|
|
|
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) {
|
|
|
|
|
throw new Error("control-plane cleanup-runs accepts only one of --source-commit or --pipeline-run");
|
|
|
|
|
}
|
|
|
|
|
const sourceCommit = sourceCommitRaw?.toLowerCase();
|
|
|
|
|
if (sourceCommit !== undefined && !/^[0-9a-f]{40}$/u.test(sourceCommit)) {
|
|
|
|
|
throw new Error("--source-commit must be a full 40-character git sha for cleanup-runs");
|
|
|
|
|
}
|
|
|
|
|
const pipelineRun = pipelineRunRaw === undefined ? undefined : validateNodeRuntimePipelineRunName(scoped.spec, pipelineRunRaw);
|
|
|
|
|
return {
|
|
|
|
|
minAgeMinutes: positiveIntegerOption(scoped.originalArgs, "--min-age-minutes", 60, 10080),
|
|
|
|
|
limit: positiveIntegerOption(scoped.originalArgs, "--limit", 20, 200),
|
|
|
|
|
sourceCommit,
|
|
|
|
|
pipelineRun,
|
|
|
|
|
targetPipelineRun: pipelineRun ?? (sourceCommit === undefined ? undefined : nodeRuntimePipelineRunName(scoped.spec, sourceCommit)),
|
|
|
|
|
dryRun: scoped.dryRun || !scoped.confirm,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function validateNodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, value: string): string {
|
|
|
|
|
const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
|
|
|
if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) {
|
|
|
|
|
throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-<sha>[-rerun] PipelineRun name for --lane ${spec.lane}`);
|
|
|
|
|
}
|
|
|
|
|
return value.toLowerCase();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeCleanupRuns(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
|
|
|
const options = parseNodeRuntimeCleanupOptions(scoped);
|
|
|
|
|
const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
|
|
|
const candidates = listNodeRuntimeCleanupPipelineRuns(scoped.spec, options);
|
|
|
|
|
const selectedPipelineRuns = candidates
|
|
|
|
|
.filter((item) => item.selected !== false)
|
|
|
|
|
.map((item) => item.name);
|
|
|
|
|
const ownedResources = listNodeRuntimeCleanupOwnedResources(scoped.spec, selectedPipelineRuns);
|
|
|
|
|
const command = `hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane}`;
|
|
|
|
|
if (options.dryRun) {
|
|
|
|
|
return {
|
|
|
|
|
ok: true,
|
|
|
|
|
command,
|
|
|
|
|
mode: "dry-run",
|
|
|
|
|
node: scoped.node,
|
|
|
|
|
lane: scoped.lane,
|
|
|
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
|
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
|
|
|
limit: options.limit,
|
|
|
|
|
sourceCommit: options.sourceCommit,
|
|
|
|
|
pipelineRun: options.pipelineRun,
|
|
|
|
|
candidates,
|
|
|
|
|
candidateCount: candidates.length,
|
|
|
|
|
selectedPipelineRuns,
|
|
|
|
|
selectedPipelineRunCount: selectedPipelineRuns.length,
|
|
|
|
|
ownedResources,
|
|
|
|
|
ciObjectCounts: beforeCounts,
|
|
|
|
|
mutation: false,
|
|
|
|
|
next: {
|
|
|
|
|
confirm: [
|
|
|
|
|
command,
|
|
|
|
|
`--min-age-minutes ${options.minAgeMinutes}`,
|
|
|
|
|
`--limit ${options.limit}`,
|
|
|
|
|
options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`,
|
|
|
|
|
options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`,
|
|
|
|
|
"--confirm",
|
|
|
|
|
"--wait",
|
|
|
|
|
].filter(Boolean).join(" "),
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const deletion = deleteNodeRuntimeCleanupRuns(scoped.spec, selectedPipelineRuns, scoped.timeoutSeconds);
|
|
|
|
|
const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
|
|
|
return {
|
|
|
|
|
ok: isCommandSuccess(deletion),
|
|
|
|
|
command,
|
|
|
|
|
mode: "confirmed-cleanup",
|
|
|
|
|
node: scoped.node,
|
|
|
|
|
lane: scoped.lane,
|
|
|
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
|
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
|
|
|
limit: options.limit,
|
|
|
|
|
sourceCommit: options.sourceCommit,
|
|
|
|
|
pipelineRun: options.pipelineRun,
|
|
|
|
|
deletedPipelineRuns: selectedPipelineRuns,
|
|
|
|
|
deletedPipelineRunCount: selectedPipelineRuns.length,
|
|
|
|
|
ownedResourcesBefore: ownedResources,
|
|
|
|
|
ciObjectCountsBefore: beforeCounts,
|
|
|
|
|
ciObjectCountsAfter: afterCounts,
|
|
|
|
|
deletion: compactRuntimeCommand(deletion),
|
|
|
|
|
mutation: isCommandSuccess(deletion),
|
|
|
|
|
degradedReason: isCommandSuccess(deletion) ? undefined : "node-runtime-ci-cleanup-delete-failed",
|
|
|
|
|
next: {
|
|
|
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
|
|
|
rerunStatus: options.targetPipelineRun === undefined
|
|
|
|
|
? undefined
|
|
|
|
|
: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${options.targetPipelineRun}`,
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function listNodeRuntimeCleanupPipelineRuns(spec: HwlabRuntimeLaneSpec, options: NodeRuntimeCleanupOptions): NodeRuntimeCleanupPipelineRunRow[] {
|
|
|
|
|
const result = runNodeK3sArgs(spec, [
|
|
|
|
|
"kubectl",
|
|
|
|
|
"-n",
|
|
|
|
|
HWLAB_CI_NAMESPACE,
|
|
|
|
|
"get",
|
|
|
|
|
"pipelinerun",
|
|
|
|
|
"-o",
|
|
|
|
|
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
|
|
|
|
], 60);
|
|
|
|
|
if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`);
|
|
|
|
|
const rows = nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout);
|
|
|
|
|
if (options.targetPipelineRun !== undefined) {
|
|
|
|
|
const target = rows.find((item) => item.name === options.targetPipelineRun);
|
|
|
|
|
if (target === undefined) {
|
|
|
|
|
return [{
|
|
|
|
|
name: options.targetPipelineRun,
|
|
|
|
|
createdAt: null,
|
|
|
|
|
ageMinutes: null,
|
|
|
|
|
status: null,
|
|
|
|
|
reason: "target-pipelinerun-not-found",
|
|
|
|
|
selected: false,
|
|
|
|
|
}];
|
|
|
|
|
}
|
|
|
|
|
if (target.status !== "True" && target.status !== "False") {
|
|
|
|
|
return [{ ...target, selected: false, selectedReason: "target-pipelinerun-not-terminal" }];
|
|
|
|
|
}
|
|
|
|
|
if (target.ageMinutes === null || target.ageMinutes < options.minAgeMinutes) {
|
|
|
|
|
return [{ ...target, selected: false, selectedReason: target.ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }];
|
|
|
|
|
}
|
|
|
|
|
return [target];
|
|
|
|
|
}
|
|
|
|
|
const prefix = `${spec.pipelineRunPrefix}-`;
|
|
|
|
|
const terminalRuns = rows
|
|
|
|
|
.filter((item) => item.name.startsWith(prefix))
|
|
|
|
|
.filter((item) => item.status === "True" || item.status === "False")
|
|
|
|
|
.sort((left, right) => String(left.createdAt ?? "").localeCompare(String(right.createdAt ?? "")));
|
|
|
|
|
const protectedLatest = terminalRuns
|
|
|
|
|
.slice()
|
|
|
|
|
.sort((left, right) => String(right.createdAt ?? "").localeCompare(String(left.createdAt ?? "")))[0]?.name ?? null;
|
|
|
|
|
return terminalRuns
|
|
|
|
|
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
|
|
|
|
.map((item) => item.name === protectedLatest ? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" } : item)
|
|
|
|
|
.slice(0, options.limit);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeCleanupPipelineRunRowsFromText(text: string): NodeRuntimeCleanupPipelineRunRow[] {
|
|
|
|
|
const now = Date.now();
|
|
|
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
|
|
|
const [name = "", createdAtRaw = "", statusRaw = "", reasonRaw = ""] = line.trim().split("\t");
|
|
|
|
|
const createdAt = createdAtRaw.length > 0 ? createdAtRaw : null;
|
|
|
|
|
const createdMs = createdAt === null ? NaN : Date.parse(createdAt);
|
|
|
|
|
return {
|
|
|
|
|
name,
|
|
|
|
|
createdAt,
|
|
|
|
|
ageMinutes: Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null,
|
|
|
|
|
status: statusRaw || null,
|
|
|
|
|
reason: reasonRaw || null,
|
|
|
|
|
};
|
|
|
|
|
}).filter((item) => item.name.length > 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function listNodeRuntimeCleanupOwnedResources(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[]): Record<string, unknown> {
|
|
|
|
|
const previewLimit = 24;
|
|
|
|
|
if (pipelineRunNames.length === 0) {
|
|
|
|
|
return {
|
|
|
|
|
taskRunPreview: [],
|
|
|
|
|
podPreview: [],
|
|
|
|
|
pvcPreview: [],
|
|
|
|
|
previewLimit,
|
|
|
|
|
truncated: false,
|
|
|
|
|
taskRunCount: 0,
|
|
|
|
|
podCount: 0,
|
|
|
|
|
pvcCount: 0,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const wanted = new Set(pipelineRunNames);
|
|
|
|
|
const taskRunsResult = runNodeK3sArgs(spec, [
|
|
|
|
|
"kubectl",
|
|
|
|
|
"-n",
|
|
|
|
|
HWLAB_CI_NAMESPACE,
|
|
|
|
|
"get",
|
|
|
|
|
"taskrun",
|
|
|
|
|
"-o",
|
|
|
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "Succeeded"}}{{.status}}{{"\\t"}}{{.reason}}{{end}}{{end}}{{"\\n"}}{{end}}',
|
|
|
|
|
], 60);
|
|
|
|
|
const podsResult = runNodeK3sArgs(spec, [
|
|
|
|
|
"kubectl",
|
|
|
|
|
"-n",
|
|
|
|
|
HWLAB_CI_NAMESPACE,
|
|
|
|
|
"get",
|
|
|
|
|
"pod",
|
|
|
|
|
"-o",
|
|
|
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\n"}}{{end}}',
|
|
|
|
|
], 60);
|
|
|
|
|
const pvcsResult = runNodeK3sArgs(spec, [
|
|
|
|
|
"kubectl",
|
|
|
|
|
"-n",
|
|
|
|
|
HWLAB_CI_NAMESPACE,
|
|
|
|
|
"get",
|
|
|
|
|
"pvc",
|
|
|
|
|
"-o",
|
|
|
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}',
|
|
|
|
|
], 60);
|
|
|
|
|
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimeCleanupOwnedTaskRunsFromText(taskRunsResult.stdout, wanted) : [];
|
|
|
|
|
const pods = isCommandSuccess(podsResult) ? nodeRuntimeCleanupOwnedPodsFromText(podsResult.stdout, wanted) : [];
|
|
|
|
|
const pvcs = isCommandSuccess(pvcsResult) ? nodeRuntimeCleanupOwnedPvcsFromText(pvcsResult.stdout, wanted) : [];
|
|
|
|
|
return {
|
|
|
|
|
taskRunPreview: taskRuns.slice(0, previewLimit),
|
|
|
|
|
podPreview: pods.slice(0, previewLimit),
|
|
|
|
|
pvcPreview: pvcs.slice(0, previewLimit),
|
|
|
|
|
previewLimit,
|
|
|
|
|
truncated: taskRuns.length > previewLimit || pods.length > previewLimit || pvcs.length > previewLimit,
|
|
|
|
|
taskRunCount: taskRuns.length,
|
|
|
|
|
podCount: pods.length,
|
|
|
|
|
pvcCount: pvcs.length,
|
|
|
|
|
query: {
|
|
|
|
|
taskRuns: compactRuntimeCommandStats(taskRunsResult),
|
|
|
|
|
pods: compactRuntimeCommandStats(podsResult),
|
|
|
|
|
pvcs: compactRuntimeCommandStats(pvcsResult),
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeCleanupOwnedTaskRunsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
|
|
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
|
|
|
const [name = "", pipelineRun = "", status = "", reason = ""] = line.trim().split("\t");
|
|
|
|
|
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
|
|
|
|
|
return {
|
|
|
|
|
name,
|
|
|
|
|
pipelineRun,
|
|
|
|
|
status: status || null,
|
|
|
|
|
reason: reason || null,
|
|
|
|
|
};
|
|
|
|
|
}).filter((item): item is Record<string, unknown> => item !== null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeCleanupOwnedPodsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
|
|
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
|
|
|
const [name = "", pipelineRun = "", phase = "", nodeName = ""] = line.trim().split("\t");
|
|
|
|
|
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
|
|
|
|
|
return {
|
|
|
|
|
name,
|
|
|
|
|
pipelineRun,
|
|
|
|
|
phase: phase || null,
|
|
|
|
|
nodeName: nodeName || null,
|
|
|
|
|
};
|
|
|
|
|
}).filter((item): item is Record<string, unknown> => item !== null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeCleanupOwnedPvcsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
|
|
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
|
|
|
const [name = "", volumeName = "", phase = "", ownerKind = "", ownerName = "", storage = ""] = line.trim().split("\t");
|
|
|
|
|
if (name.length === 0 || ownerKind !== "PipelineRun" || !wanted.has(ownerName)) return null;
|
|
|
|
|
return {
|
|
|
|
|
name,
|
|
|
|
|
pipelineRun: ownerName,
|
|
|
|
|
phase: phase || null,
|
|
|
|
|
volumeName: volumeName || null,
|
|
|
|
|
storage: storage || null,
|
|
|
|
|
};
|
|
|
|
|
}).filter((item): item is Record<string, unknown> => item !== null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function deleteNodeRuntimeCleanupRuns(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[], timeoutSeconds: number): CommandResult {
|
|
|
|
|
if (pipelineRunNames.length === 0) {
|
|
|
|
|
return {
|
|
|
|
|
command: [],
|
|
|
|
|
cwd: repoRoot,
|
|
|
|
|
exitCode: 0,
|
|
|
|
|
stdout: "no candidates",
|
|
|
|
|
stderr: "",
|
|
|
|
|
signal: null,
|
|
|
|
|
timedOut: false,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const script = [
|
|
|
|
|
"set -eu",
|
|
|
|
|
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
|
|
|
|
|
"names_file=$(mktemp)",
|
|
|
|
|
"cat > \"$names_file\"",
|
|
|
|
|
"deleted_pipeline_runs=$(grep -c . \"$names_file\" | tr -d ' ')",
|
|
|
|
|
"xargs -r -n 50 kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=false < \"$names_file\"",
|
|
|
|
|
"deleted_pod_groups=0",
|
|
|
|
|
"deleted_taskrun_groups=0",
|
|
|
|
|
"explicit_owned_cleanup=skipped-large-batch",
|
|
|
|
|
"if [ \"$deleted_pipeline_runs\" -le 40 ]; then",
|
|
|
|
|
" explicit_owned_cleanup=executed",
|
|
|
|
|
" while IFS= read -r pipeline_run; do",
|
|
|
|
|
" [ -n \"$pipeline_run\" ] || continue",
|
|
|
|
|
" kubectl -n \"$namespace\" delete pod -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
|
|
|
" deleted_pod_groups=$((deleted_pod_groups + 1))",
|
|
|
|
|
" kubectl -n \"$namespace\" delete taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
|
|
|
" deleted_taskrun_groups=$((deleted_taskrun_groups + 1))",
|
|
|
|
|
" done < \"$names_file\"",
|
|
|
|
|
"fi",
|
|
|
|
|
"printf 'deletedPipelineRunCount\\t%s\\n' \"$deleted_pipeline_runs\"",
|
|
|
|
|
"printf 'deletedTaskRunLabelGroups\\t%s\\n' \"$deleted_taskrun_groups\"",
|
|
|
|
|
"printf 'deletedPodLabelGroups\\t%s\\n' \"$deleted_pod_groups\"",
|
|
|
|
|
"printf 'explicitOwnedCleanup\\t%s\\n' \"$explicit_owned_cleanup\"",
|
|
|
|
|
"rm -f \"$names_file\"",
|
|
|
|
|
].join("\n");
|
|
|
|
|
return runNodeK3sScript(spec, script, timeoutSeconds, pipelineRunNames.join("\n") + "\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeCiObjectCounts(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
|
|
|
const script = [
|
|
|
|
|
"set +e",
|
|
|
|
|
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
|
|
|
|
|
"count_kind() { kubectl -n \"$namespace\" get \"$1\" -o name 2>/dev/null | wc -l | tr -d ' '; }",
|
|
|
|
|
"printf 'pipelineRuns\\t%s\\n' \"$(count_kind pipelinerun)\"",
|
|
|
|
|
"printf 'taskRuns\\t%s\\n' \"$(count_kind taskrun)\"",
|
|
|
|
|
"printf 'pods\\t%s\\n' \"$(count_kind pod)\"",
|
|
|
|
|
"printf 'pvcs\\t%s\\n' \"$(count_kind pvc)\"",
|
|
|
|
|
].join("\n");
|
|
|
|
|
const result = runNodeK3sScript(spec, script, 30);
|
|
|
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
|
|
|
return {
|
|
|
|
|
pipelineRuns: numericField(fields.pipelineRuns),
|
|
|
|
|
taskRuns: numericField(fields.taskRuns),
|
|
|
|
|
pods: numericField(fields.pods),
|
|
|
|
|
pvcs: numericField(fields.pvcs),
|
|
|
|
|
result: compactRuntimeCommand(result),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeBaseImageCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
|
|
|
const action = scoped.runtimeImageAction;
|
|
|
|
|
if (action === null) {
|
|
|
|
@@ -2510,6 +2866,9 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
|
|
|
|
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
|
|
|
|
|
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
|
|
|
|
|
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
|
|
|
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.status === "Unknown"
|
|
|
|
|
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
|
|
|
|
|
: null;
|
|
|
|
|
const workloads = namespaceExists
|
|
|
|
|
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
|
|
|
|
|
: null;
|
|
|
|
@@ -2528,6 +2887,9 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
|
|
|
|
const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && workloadsReady && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready));
|
|
|
|
|
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
|
|
|
|
|
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
|
|
|
|
|
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
|
|
|
|
|
? pipelineRunDiagnostics.degradedReason
|
|
|
|
|
: "pipelinerun-not-succeeded";
|
|
|
|
|
const publicReady = publicProbes.ready === true;
|
|
|
|
|
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
|
|
|
|
|
const fullStatus = {
|
|
|
|
@@ -2561,6 +2923,7 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
|
|
|
|
result: compactRuntimeCommand(argo),
|
|
|
|
|
},
|
|
|
|
|
pipelineRun: pipelineRunProbe,
|
|
|
|
|
pipelineRunDiagnostics,
|
|
|
|
|
runtime: {
|
|
|
|
|
ready: runtimeReady,
|
|
|
|
|
namespace: spec.runtimeNamespace,
|
|
|
|
@@ -2593,7 +2956,7 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
|
|
|
|
|
? publicReady
|
|
|
|
|
? gitMirrorReady ? undefined : "git-mirror-pending-flush"
|
|
|
|
|
: "public-probe-not-ready"
|
|
|
|
|
: "pipelinerun-not-succeeded"
|
|
|
|
|
: pipelineRunDegradedReason
|
|
|
|
|
: "argo-not-synced-healthy"
|
|
|
|
|
: namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing"
|
|
|
|
|
: "control-plane-not-ready",
|
|
|
|
@@ -2661,6 +3024,7 @@ function joinUrlPath(baseUrl: string, suffix: string): string {
|
|
|
|
|
|
|
|
|
|
function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
|
|
|
const pipelineRun = record(status.pipelineRun);
|
|
|
|
|
const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics);
|
|
|
|
|
const argo = record(status.argo);
|
|
|
|
|
const runtime = record(status.runtime);
|
|
|
|
|
const publicProbes = record(status.publicProbes);
|
|
|
|
@@ -2688,6 +3052,14 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
|
|
|
|
|
message: pipelineRun.message ?? null,
|
|
|
|
|
createdAt: pipelineRun.createdAt ?? null,
|
|
|
|
|
ready: pipelineRun.status === "True",
|
|
|
|
|
diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : {
|
|
|
|
|
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
|
|
|
|
|
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
|
|
|
|
|
podCount: pipelineRunDiagnostics.podCount ?? null,
|
|
|
|
|
pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [],
|
|
|
|
|
unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [],
|
|
|
|
|
schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [],
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
argo: {
|
|
|
|
|
application: argo.application ?? null,
|
|
|
|
@@ -2746,6 +3118,9 @@ function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: Re
|
|
|
|
|
if (reason === "pipelinerun-not-succeeded") {
|
|
|
|
|
return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
|
|
|
|
}
|
|
|
|
|
if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") {
|
|
|
|
|
return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`;
|
|
|
|
|
}
|
|
|
|
|
if (reason === "public-probe-not-ready") {
|
|
|
|
|
return `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
|
|
|
|
|
}
|
|
|
|
@@ -2765,6 +3140,80 @@ function nodeRuntimeStatusCommand(scoped: ReturnType<typeof parseNodeScopedDeleg
|
|
|
|
|
].filter(Boolean).join(" ");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
|
|
|
|
const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
|
|
|
|
const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
|
|
|
|
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRuns(parseJsonRecordFromText(taskRunsResult.stdout)) : [];
|
|
|
|
|
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPods(parseJsonRecordFromText(podsResult.stdout)) : [];
|
|
|
|
|
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
|
|
|
|
|
const unscheduledPods = pods.filter((item) => item.scheduled === false);
|
|
|
|
|
const schedulingMessages = unscheduledPods
|
|
|
|
|
.map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "")
|
|
|
|
|
.filter((message) => message.length > 0);
|
|
|
|
|
const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message));
|
|
|
|
|
return {
|
|
|
|
|
ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0,
|
|
|
|
|
pipelineRun,
|
|
|
|
|
taskRuns,
|
|
|
|
|
pods,
|
|
|
|
|
taskRunCount: taskRuns.length,
|
|
|
|
|
podCount: pods.length,
|
|
|
|
|
pendingTaskRuns,
|
|
|
|
|
unscheduledPods,
|
|
|
|
|
schedulingMessages,
|
|
|
|
|
degradedReason: tooManyPods
|
|
|
|
|
? "node-runtime-ci-pod-capacity-exhausted"
|
|
|
|
|
: unscheduledPods.length > 0
|
|
|
|
|
? "node-runtime-ci-pod-unschedulable"
|
|
|
|
|
: pendingTaskRuns.length > 0
|
|
|
|
|
? "node-runtime-ci-taskrun-pending"
|
|
|
|
|
: undefined,
|
|
|
|
|
query: {
|
|
|
|
|
taskRuns: compactRuntimeCommand(taskRunsResult),
|
|
|
|
|
pods: compactRuntimeCommand(podsResult),
|
|
|
|
|
},
|
|
|
|
|
next: tooManyPods || unscheduledPods.length > 0
|
|
|
|
|
? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` }
|
|
|
|
|
: undefined,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
|
|
|
|
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
|
|
|
|
return items.map((item) => {
|
|
|
|
|
const metadata = record(item.metadata);
|
|
|
|
|
const status = record(item.status);
|
|
|
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
|
|
|
|
const condition = conditions[0] ?? {};
|
|
|
|
|
return {
|
|
|
|
|
name: metadata.name ?? null,
|
|
|
|
|
status: condition.status ?? null,
|
|
|
|
|
reason: condition.reason ?? null,
|
|
|
|
|
message: condition.message ?? null,
|
|
|
|
|
podName: status.podName ?? null,
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimePipelineDiagnosticPods(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
|
|
|
|
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
|
|
|
|
return items.map((item) => {
|
|
|
|
|
const metadata = record(item.metadata);
|
|
|
|
|
const spec = record(item.spec);
|
|
|
|
|
const status = record(item.status);
|
|
|
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
|
|
|
|
const scheduled = conditions.find((condition) => condition.type === "PodScheduled");
|
|
|
|
|
return {
|
|
|
|
|
name: metadata.name ?? null,
|
|
|
|
|
phase: status.phase ?? null,
|
|
|
|
|
nodeName: spec.nodeName ?? null,
|
|
|
|
|
scheduled: scheduled === undefined ? null : scheduled.status === "True",
|
|
|
|
|
scheduledReason: scheduled?.reason ?? null,
|
|
|
|
|
scheduledMessage: scheduled?.message ?? null,
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function nodeRuntimeRenderToken(): string {
|
|
|
|
|
return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
|
|
|
|
}
|
|
|
|
|
Codex