fix(cicd): add NC01 PaC status closeout

This commit is contained in:
Codex
2026-07-08 19:55:43 +02:00
parent 1ae53ffc4d
commit e3951035f3
16 changed files with 644 additions and 37 deletions
+8
View File
@@ -378,6 +378,7 @@ export async function statusPipelinesAsCodeLane(config: UniDeskConfig, options:
...(expectedGitopsRevision === null || argoSyncedToGitops ? [] : ["argo-revision-stale"]),
...(runtimePayload.runtimeNamespaceExists === true ? [] : ["runtime-namespace-missing"]),
...(manager.deploymentExists === true ? [] : ["manager-deployment-missing"]),
...(manager.deploymentExists === true && manager.ready !== true ? ["manager-not-ready"] : []),
...(manager.deploymentExists !== true || sourceCommit === null || managerSourceMatchesExpected ? [] : ["manager-source-stale"]),
...(manager.serviceExists === true ? [] : ["manager-service-missing"]),
...(spec.database.mode === "external-postgres" && database.secretPresent !== true ? ["database-secret-missing"] : []),
@@ -446,9 +447,12 @@ export async function statusPipelinesAsCodeLane(config: UniDeskConfig, options:
namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
manager: {
deploymentExists: manager.deploymentExists ?? false,
ready: manager.ready ?? false,
serviceExists: manager.serviceExists ?? false,
sourceCommit: stringOrNull(manager.sourceCommit),
sourceMatchesExpected: managerSourceMatchesExpected,
replicas: manager.replicas ?? null,
conditions: manager.conditions ?? [],
envIdentity: manager.envIdentity ?? null,
},
databaseSecretPresent: database.secretPresent ?? null,
@@ -590,6 +594,7 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio
...(argo.exists !== true || expectedGitopsRevision === null || argoSyncedToGitops ? [] : ["argo-revision-stale"]),
...(runtimePayload.runtimeNamespaceExists === true ? [] : ["runtime-namespace-missing"]),
...(manager.deploymentExists === true ? [] : ["manager-deployment-missing"]),
...(manager.deploymentExists === true && manager.ready !== true ? ["manager-not-ready"] : []),
...(manager.deploymentExists !== true || sourceCommit === null || managerSourceMatchesExpected ? [] : ["manager-source-stale"]),
...(manager.serviceExists === true ? [] : ["manager-service-missing"]),
...(spec.database.mode === "external-postgres" && database.secretPresent !== true ? ["database-secret-missing"] : []),
@@ -678,9 +683,12 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio
namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
manager: {
deploymentExists: manager.deploymentExists ?? false,
ready: manager.ready ?? false,
serviceExists: manager.serviceExists ?? false,
sourceCommit: stringOrNull(manager.sourceCommit),
sourceMatchesExpected: managerSourceMatchesExpected,
replicas: manager.replicas ?? null,
conditions: manager.conditions ?? [],
},
databaseSecretPresent: database.secretPresent ?? null,
secretsReady: secrets.ready ?? null,
+6 -2
View File
@@ -43,7 +43,7 @@ import { agentRunGetKindHelp, runAgentRunResourceCommand } from "./resource-acti
import { runAgentRunRestCompatCommand, runGitMirrorJob, startAsyncAgentRunJob } from "./rest-bridge";
import { exposeAgentRun, restartYamlLane, secretSync, triggerCurrent } from "./trigger";
import { unsupported } from "./utils";
import { cleanupReleasedPvs, cleanupRunners, cleanupRuns, cleanupSessionPvcs, refresh } from "./yaml-lane";
import { cleanupLocalPostgres, cleanupReleasedPvs, cleanupRunners, cleanupRuns, cleanupSessionPvcs, refresh } from "./yaml-lane";
export function agentRunHelp(): unknown {
return {
@@ -92,6 +92,8 @@ export function agentRunHelp(): unknown {
"bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --confirm",
"bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --dry-run",
"bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --confirm",
"bun scripts/cli.ts agentrun control-plane cleanup-local-postgres --node NC01 --lane nc01-v02 --dry-run",
"bun scripts/cli.ts agentrun control-plane cleanup-local-postgres --node NC01 --lane nc01-v02 --confirm",
"bun scripts/cli.ts agentrun git-mirror status",
"bun scripts/cli.ts agentrun git-mirror status --full",
"bun scripts/cli.ts agentrun git-mirror sync --confirm",
@@ -147,6 +149,7 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str
if (action === "cleanup-session-pvcs") {
return await cleanupSessionPvcs(config, parseCleanupSessionPvcsOptions(actionArgs));
}
if (action === "cleanup-local-postgres") return await cleanupLocalPostgres(config, parseLaneConfirmOptions(actionArgs));
if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(actionArgs));
}
if (group === "git-mirror") {
@@ -275,7 +278,7 @@ export function agentRunHelpText(args: string[]): string {
return [
"Usage: bun scripts/cli.ts agentrun control-plane <action> [options]",
"",
"Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-session-pvcs, cleanup-released-pvs",
"Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-session-pvcs, cleanup-local-postgres, cleanup-released-pvs",
"Examples:",
" bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02",
" bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run",
@@ -288,6 +291,7 @@ export function agentRunHelpText(args: string[]): string {
" bun scripts/cli.ts agentrun control-plane trigger-current --dry-run",
" bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --dry-run",
" bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node JD01 --lane jd01-v02 --dry-run",
" bun scripts/cli.ts agentrun control-plane cleanup-local-postgres --node NC01 --lane nc01-v02 --dry-run",
" bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
].join("\n");
}
+10 -2
View File
@@ -50,6 +50,14 @@ export function agentRunControlPlaneStatusCommand(spec: AgentRunLaneSpec, option
].filter(Boolean).join(" ");
}
function deploymentReplicaSummary(manager: Record<string, unknown>): string {
const replicas = record(manager.replicas);
const desired = displayValue(replicas.desired);
const ready = displayValue(replicas.ready);
const available = displayValue(replicas.available);
return `${ready}/${desired} available=${available}`;
}
export function renderAgentRunControlPlaneStatusSummary(result: Record<string, unknown>): RenderedCliResult {
const summary = record(result.summary);
const target = record(result.target);
@@ -104,7 +112,7 @@ export function renderAgentRunControlPlaneStatusSummary(result: Record<string, u
["pipelinerun", yesNo(latestPipelineRun.status === "True"), `run=${displayValue(latestPipelineRun.name)} status=${displayValue(latestPipelineRun.status)} reason=${displayValue(latestPipelineRun.reason)} duration=${displayValue(latestPipelineRun.durationSeconds)}s`],
["image", yesNo(artifact.imageStatus === "reused" || artifact.imageStatus === "built"), `status=${displayValue(artifact.imageStatus)} env=${displayValue(artifact.envIdentity)} digest=${shortSha(artifact.digest)} gitops=${shortSha(artifact.gitopsCommit)}`],
["argo", yesNo(argo.syncedToGitops), `sync=${displayValue(argo.syncStatus)} health=${displayValue(argo.healthStatus)} revision=${shortSha(argo.revision)}`],
["runtime", yesNo(runtimeManager.sourceMatchesExpected), `manager=${shortSha(runtimeManager.sourceCommit)} secrets=${yesNo(runtime.secretsReady)} dbSecret=${yesNo(runtime.databaseSecretPresent)} localPgAbsent=${yesNo(runtime.localPostgresAbsent)}`],
["runtime", yesNo(runtimeManager.ready === true && runtimeManager.sourceMatchesExpected === true), `manager=${shortSha(runtimeManager.sourceCommit)} ready=${yesNo(runtimeManager.ready)} replicas=${deploymentReplicaSummary(runtimeManager)} secrets=${yesNo(runtime.secretsReady)} dbSecret=${yesNo(runtime.databaseSecretPresent)} localPgAbsent=${yesNo(runtime.localPostgresAbsent)}`],
],
),
"",
@@ -151,7 +159,7 @@ export function renderAgentRunControlPlaneStatusSummary(result: Record<string, u
["git-mirror", yesNo(gitMirror.alreadySynced), `read=${yesNo(gitMirror.readReady)} write=${yesNo(gitMirror.writeReady)} gitops=${shortSha(gitMirror.gitopsCommit)}`],
["ci", yesNo(ciRun.status === "True"), `run=${displayValue(ciRun.name)} status=${displayValue(ciRun.status)} reason=${displayValue(ciRun.reason)} evidenceMissing=${yesNo(ci.evidenceMissing)}`],
["argo", yesNo(argo.syncedToGitops), `sync=${displayValue(argo.syncStatus)} health=${displayValue(argo.healthStatus)} revision=${shortSha(argo.revision)}`],
["runtime", yesNo(runtimeManager.sourceMatchesExpected), `manager=${shortSha(runtimeManager.sourceCommit)} secrets=${yesNo(runtime.secretsReady)} dbSecret=${yesNo(runtime.databaseSecretPresent)} localPgAbsent=${yesNo(runtime.localPostgresAbsent)}`],
["runtime", yesNo(runtimeManager.ready === true && runtimeManager.sourceMatchesExpected === true), `manager=${shortSha(runtimeManager.sourceCommit)} ready=${yesNo(runtimeManager.ready)} replicas=${deploymentReplicaSummary(runtimeManager)} secrets=${yesNo(runtime.secretsReady)} dbSecret=${yesNo(runtime.databaseSecretPresent)} localPgAbsent=${yesNo(runtime.localPostgresAbsent)}`],
],
),
"",
+132 -2
View File
@@ -35,7 +35,7 @@ import {
} from "../agentrun-manifests";
import { sha256Fingerprint } from "../platform-infra-ops-library";
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, RefreshOptions } from "./options";
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, LaneConfirmOptions, RefreshOptions } from "./options";
import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror";
import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets";
import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils";
@@ -234,6 +234,105 @@ export async function cleanupSessionPvcs(config: UniDeskConfig, options: Cleanup
};
}
export async function cleanupLocalPostgres(config: UniDeskConfig, options: LaneConfirmOptions): Promise<Record<string, unknown>> {
const { configPath, spec } = resolveAgentRunLaneTarget(options);
const name = spec.deployment.localPostgres.serviceName;
const plan = {
namespace: spec.runtime.namespace,
statefulSet: name,
service: name,
pvcDisposition: "retained",
requiresDatabaseMode: "external-postgres",
valuesPrinted: false,
};
if (spec.database.mode !== "external-postgres" || spec.deployment.localPostgres.enabled || name === null) {
return {
ok: false,
command: "agentrun control-plane cleanup-local-postgres",
mode: "blocked",
mutation: false,
configPath,
target: agentRunLaneSummary(spec),
degradedReason: "lane-is-not-external-postgres",
plan,
valuesPrinted: false,
};
}
const confirmed = options.confirm && !options.dryRun;
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupLocalPostgresScript(spec.runtime.namespace, name, confirmed)]);
const payload = captureJsonPayload(result);
const base = {
ok: result.exitCode === 0 && payload.ok !== false,
command: "agentrun control-plane cleanup-local-postgres",
mode: confirmed ? "confirmed-cleanup" : "dry-run",
mutation: confirmed,
configPath,
target: agentRunLaneSummary(spec),
plan,
result: payload,
capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
valuesPrinted: false,
};
if (!confirmed) {
return {
...base,
next: {
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-local-postgres --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
},
};
}
return {
...base,
next: {
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
},
};
}
export function cleanupLocalPostgresScript(namespace: string, name: string, confirm: boolean): string {
return [
"set +e",
`namespace=${shQuote(namespace)}`,
`name=${shQuote(name)}`,
`confirm=${confirm ? "true" : "false"}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"kubectl -n \"$namespace\" get statefulset \"$name\" -o name > \"$tmp_dir/sts-before\" 2>/dev/null; sts_before=$?",
"kubectl -n \"$namespace\" get service \"$name\" -o name > \"$tmp_dir/svc-before\" 2>/dev/null; svc_before=$?",
"delete_sts_rc=0",
"delete_svc_rc=0",
"if [ \"$confirm\" = \"true\" ]; then",
" kubectl -n \"$namespace\" delete statefulset \"$name\" --ignore-not-found > \"$tmp_dir/delete-sts.out\" 2> \"$tmp_dir/delete-sts.err\"; delete_sts_rc=$?",
" kubectl -n \"$namespace\" delete service \"$name\" --ignore-not-found > \"$tmp_dir/delete-svc.out\" 2> \"$tmp_dir/delete-svc.err\"; delete_svc_rc=$?",
"else",
" : > \"$tmp_dir/delete-sts.out\"; : > \"$tmp_dir/delete-sts.err\"; : > \"$tmp_dir/delete-svc.out\"; : > \"$tmp_dir/delete-svc.err\"",
"fi",
"kubectl -n \"$namespace\" get statefulset \"$name\" -o name > \"$tmp_dir/sts-after\" 2>/dev/null; sts_after=$?",
"kubectl -n \"$namespace\" get service \"$name\" -o name > \"$tmp_dir/svc-after\" 2>/dev/null; svc_after=$?",
"NAMESPACE=\"$namespace\" NAME=\"$name\" CONFIRM=\"$confirm\" STS_BEFORE=\"$sts_before\" SVC_BEFORE=\"$svc_before\" STS_AFTER=\"$sts_after\" SVC_AFTER=\"$svc_after\" DELETE_STS_RC=\"$delete_sts_rc\" DELETE_SVC_RC=\"$delete_svc_rc\" TMP_DIR=\"$tmp_dir\" python3 - <<'PY'",
"import json, os, pathlib",
"base = pathlib.Path(os.environ['TMP_DIR'])",
"def present(name): return os.environ.get(name) == '0'",
"def text(name):",
" try: return (base / name).read_text().strip()",
" except Exception: return ''",
"payload = {",
" 'ok': os.environ.get('DELETE_STS_RC') == '0' and os.environ.get('DELETE_SVC_RC') == '0',",
" 'namespace': os.environ.get('NAMESPACE'),",
" 'name': os.environ.get('NAME'),",
" 'confirmed': os.environ.get('CONFIRM') == 'true',",
" 'before': {'statefulSetPresent': present('STS_BEFORE'), 'servicePresent': present('SVC_BEFORE')},",
" 'after': {'statefulSetPresent': present('STS_AFTER'), 'servicePresent': present('SVC_AFTER')},",
" 'deleted': {'statefulSet': text('delete-sts.out'), 'service': text('delete-svc.out')},",
" 'pvcDisposition': 'retained',",
" 'valuesPrinted': False,",
"}",
"print(json.dumps(payload, ensure_ascii=False))",
"PY",
].join("\n");
}
export function cleanupSessionPvcsScript(options: CleanupSessionPvcsOptions, spec: AgentRunLaneSpec): string {
const retention = spec.deployment.runner.retention.sessionPvcRetention;
const script = readFileSync(rootPath("scripts/src/agentrun/cleanup-session-pvcs.mjs"), "utf8");
@@ -573,6 +672,34 @@ export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun:
" for entry in envs:",
" if entry.get('name') == name: return entry.get('value')",
" return None",
"def int_value(value, default=0):",
" try: return int(value)",
" except Exception: return default",
"def deployment_replicas(deploy):",
" spec = (deploy or {}).get('spec') or {}",
" status = (deploy or {}).get('status') or {}",
" metadata = (deploy or {}).get('metadata') or {}",
" desired = int_value(spec.get('replicas'), 1)",
" return {",
" 'desired': desired,",
" 'ready': int_value(status.get('readyReplicas'), 0),",
" 'available': int_value(status.get('availableReplicas'), 0),",
" 'updated': int_value(status.get('updatedReplicas'), 0),",
" 'unavailable': int_value(status.get('unavailableReplicas'), 0),",
" 'generation': int_value(metadata.get('generation'), 0),",
" 'observedGeneration': int_value(status.get('observedGeneration'), 0),",
" }",
"def deployment_conditions(deploy):",
" return [{'type': c.get('type'), 'status': c.get('status'), 'reason': c.get('reason')} for c in (((deploy or {}).get('status') or {}).get('conditions') or [])]",
"def deployment_ready(deploy):",
" if deploy is None: return False",
" replicas = deployment_replicas(deploy)",
" desired = replicas['desired']",
" if desired == 0: return True",
" observed = replicas['observedGeneration']",
" generation = replicas['generation']",
" generation_current = observed == 0 or generation == 0 or observed >= generation",
" return generation_current and replicas['ready'] >= desired and replicas['available'] >= desired and replicas['updated'] >= desired",
"def pipeline_run_param(obj, name):",
" for entry in (((obj or {}).get('spec') or {}).get('params') or []):",
" if entry.get('name') == name: return entry.get('value')",
@@ -589,6 +716,9 @@ export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun:
"ports = []",
"for port in ((((manager_svc or {}).get('spec') or {}).get('ports')) or []):",
" ports.append({'name': port.get('name'), 'port': port.get('port'), 'targetPort': port.get('targetPort')})",
"manager_replicas = deployment_replicas(manager_deploy)",
"manager_ready = exists('MANAGER_DEPLOY_EXIT') and deployment_ready(manager_deploy)",
"manager_conditions = deployment_conditions(manager_deploy)",
"matching_postgres = [line for line in re.split(r'\\r?\\n', names) if re.search('postgres', line, re.I)][:20]",
"print(json.dumps({",
" 'ok': exists('RUNTIME_NS_EXIT') and exists('CI_NS_EXIT'),",
@@ -598,7 +728,7 @@ export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun:
" 'pipeline': {'exists': exists('PIPELINE_EXIT'), 'name': os.environ.get('pipeline_name')},",
" 'pipelineRun': {'exists': exists('PIPELINERUN_EXIT'), 'name': os.environ.get('pipeline_run') or None, 'status': c.get('status'), 'reason': c.get('reason'), 'sourceCommit': pipeline_run_param(pipeline_run, 'revision'), 'startTime': ((pipeline_run or {}).get('status') or {}).get('startTime'), 'completionTime': ((pipeline_run or {}).get('status') or {}).get('completionTime')},",
" 'argo': {'exists': exists('ARGO_EXIT'), 'namespace': os.environ.get('argo_namespace'), 'application': os.environ.get('argo_application'), 'revision': (((argo or {}).get('status') or {}).get('sync') or {}).get('revision'), 'syncStatus': (((argo or {}).get('status') or {}).get('sync') or {}).get('status'), 'healthStatus': (((argo or {}).get('status') or {}).get('health') or {}).get('status')},",
" 'manager': {'deploymentExists': exists('MANAGER_DEPLOY_EXIT'), 'serviceExists': exists('MANAGER_SVC_EXIT'), 'deployment': os.environ.get('manager_deployment'), 'service': os.environ.get('manager_service'), 'image': ((((((manager_deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [{}])[0]).get('image'), 'sourceCommit': env_value(manager_deploy, 'AGENTRUN_SOURCE_COMMIT'), 'servicePorts': ports},",
" 'manager': {'deploymentExists': exists('MANAGER_DEPLOY_EXIT'), 'serviceExists': exists('MANAGER_SVC_EXIT'), 'ready': manager_ready, 'deployment': os.environ.get('manager_deployment'), 'service': os.environ.get('manager_service'), 'image': ((((((manager_deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [{}])[0]).get('image'), 'sourceCommit': env_value(manager_deploy, 'AGENTRUN_SOURCE_COMMIT'), 'servicePorts': ports, 'replicas': manager_replicas, 'conditions': manager_conditions},",
" 'database': {'secretPresent': exists('DB_SECRET_EXIT'), 'secretName': os.environ.get('database_secret'), 'key': os.environ.get('database_key'), 'keyPresent': os.environ.get('database_key') in (((db_secret or {}).get('data') or {})), 'valuesPrinted': False},",
" 'secrets': secrets,",
" 'localPostgres': {'absent': len(matching_postgres) == 0, 'matchingObjects': matching_postgres},",