fix: align HWLAB v03 native DB controls

This commit is contained in:
Codex
2026-06-09 03:31:10 +00:00
parent 049426e320
commit 9973b0cb9c
6 changed files with 569 additions and 67 deletions
+474 -25
View File
@@ -25,6 +25,8 @@ interface RuntimeSecretSpec {
node: string;
lane: string;
namespace: string;
platformDb: boolean;
platformPostgresService: string;
postgresSecret: string;
postgresStatefulSet: string;
postgresAdminUser: string;
@@ -80,12 +82,11 @@ export function hwlabNodeHelp(): Record<string, unknown> {
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-cloud-api-v03-db --confirm",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
@@ -96,6 +97,9 @@ export function hwlabNodeHelp(): Record<string, unknown> {
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown>> {
const scoped = parseNodeScopedDelegatedOptions(domain, args);
if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") {
return runNodeEndpointBridge(scoped);
}
if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) {
return startNodeDelegatedJob(scoped);
}
@@ -265,6 +269,9 @@ function parseSecretOptions(args: string[]): NodeSecretOptions {
}
if (name === spec.cloudApiDbSecret) {
if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`);
if (actionRaw === "ensure" && spec.platformDb) {
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
}
return {
action: actionRaw,
node,
@@ -283,6 +290,9 @@ function parseSecretOptions(args: string[]): NodeSecretOptions {
if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
}
if (actionRaw === "ensure" && spec.platformDb) {
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
}
return {
action: actionRaw,
node,
@@ -298,23 +308,29 @@ function parseSecretOptions(args: string[]): NodeSecretOptions {
function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec {
const namespace = `hwlab-${input.lane}`;
const platformDb = /^v0*[3-9]\d*$/.test(input.lane);
const platformPostgresService = "g14-platform-postgres";
const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`;
const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`;
return {
node: input.node,
lane: input.lane,
namespace,
platformDb,
platformPostgresService,
postgresSecret: `${namespace}-postgres`,
postgresStatefulSet: `${namespace}-postgres`,
postgresAdminUser: `hwlab_${input.lane}`,
openFgaSecret: `${namespace}-openfga`,
openFgaDbName: "hwlab_openfga",
openFgaDbUser: "hwlab_openfga",
openFgaDbHost: `${namespace}-postgres.${namespace}.svc.cluster.local`,
openFgaDbName: platformDb ? `openfga_${input.lane}` : "hwlab_openfga",
openFgaDbUser: platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga",
openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`,
cloudApiDbSecret: `hwlab-cloud-api-${input.lane}-db`,
cloudApiDbKey: CLOUD_API_DB_KEY,
cloudApiDbName: `hwlab_${input.lane}`,
cloudApiDbUser: `hwlab_${input.lane}`,
cloudApiDbHost: `${namespace}-postgres.${namespace}.svc.cluster.local`,
cloudApiDbUser: platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`,
cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
cloudApiDeployment: "hwlab-cloud-api",
codeAgentProviderSecret: `${namespace}-code-agent-provider`,
codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
@@ -329,11 +345,11 @@ function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
? readMasterAdminApiKey().key
: "";
const script = options.preset === "openfga"
? openFgaSecretScript(options, spec)
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
: options.preset === "master-server-admin-api-key"
? masterAdminApiKeySecretScript(options, spec)
: options.preset === "cloud-api-db"
? cloudApiDbSecretScript(options, spec)
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
: options.preset === "owned-postgres-cleanup"
? ownedPostgresCleanupScript(options, spec)
: codeAgentProviderSecretScript(options, spec);
@@ -356,26 +372,220 @@ function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
mutation: status.mutation === true,
result: compactCommandResult(result),
valuesRedacted: true,
next: ok && options.action === "status" ? undefined : {
ensure: options.action === "cleanup-owned-postgres"
? `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm`
: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm`,
},
next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec),
};
}
function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, string> {
if (options.action === "cleanup-owned-postgres") {
return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
}
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
return {
status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
};
}
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
}
function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}
function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm");
const dryRun = options.dryRun || !options.confirm;
const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds);
const fields = keyValueLinesFromText(statusText(result));
const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes";
const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes";
const afterExcluded = fields.afterEndpointResourcesExcluded === "yes";
const afterIgnored = fields.afterEndpointsIgnoreUpdates === "yes" || fields.afterEndpointSliceIgnoreUpdates === "yes";
const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored;
return {
ok: dryRun ? result.exitCode === 0 : ok,
command: "hwlab nodes control-plane allow-endpoint-bridge",
node: options.node,
lane: options.lane,
namespace: "argocd",
application: fields.application || `hwlab-node-${options.lane}`,
mode: dryRun ? "dry-run" : "confirmed-control-plane-update",
status: {
action: fields.action || null,
dryRun,
mutation: fields.mutation === "true",
before: {
endpointResourcesExcluded: beforeExcluded,
endpointsIgnoreUpdates: fields.beforeEndpointsIgnoreUpdates === "yes",
endpointSliceIgnoreUpdates: fields.beforeEndpointSliceIgnoreUpdates === "yes",
},
after: {
endpointResourcesExcluded: afterExcluded,
endpointsIgnoreUpdates: fields.afterEndpointsIgnoreUpdates === "yes",
endpointSliceIgnoreUpdates: fields.afterEndpointSliceIgnoreUpdates === "yes",
},
patchExitCode: numericField(fields.patchExitCode),
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
refreshExitCode: numericField(fields.refreshExitCode),
exitCode: result.exitCode,
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
summary: !afterExcluded && !afterIgnored
? "Argo tracks HWLAB external Postgres EndpointSlice resources"
: "Argo still excludes or ignores HWLAB external Postgres EndpointSlice resources",
},
result: compactCommandResult(result),
};
}
function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string {
const application = `hwlab-node-${options.lane}`;
return [
"set +e",
"namespace=argocd",
"configmap=argocd-cm",
`application=${shellQuote(application)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"preset=endpoint-bridge-resource-tracking",
"cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
"cm_has_key() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o jsonpath=\"{.data.$1}\" >/tmp/hwlab-argocd-cm-key.out 2>/dev/null && [ -s /tmp/hwlab-argocd-cm-key.out ] && printf yes || printf no; }",
"endpoint_resources_excluded() { exclusions=$(cm_data resource.exclusions); printf '%s' \"$exclusions\" | grep -Eq '(^|[[:space:]])(Endpoints|EndpointSlice)([[:space:]]|$)' && printf yes || printf no; }",
"before_endpoint_resources_excluded=$(endpoint_resources_excluded)",
"before_endpoints_ignore_updates=$(cm_has_key 'resource\\.customizations\\.ignoreResourceUpdates\\.Endpoints')",
"before_endpoint_slice_ignore_updates=$(cm_has_key 'resource\\.customizations\\.ignoreResourceUpdates\\.discovery\\.k8s\\.io_EndpointSlice')",
"action=observed",
"mutation=false",
"patch_exit=",
"rollout_restart_exit=",
"rollout_status_exit=",
"refresh_exit=",
"needs_update=false",
"if [ \"$before_endpoint_resources_excluded\" = yes ] || [ \"$before_endpoints_ignore_updates\" = yes ] || [ \"$before_endpoint_slice_ignore_updates\" = yes ]; then needs_update=true; fi",
"if [ \"$dry_run\" = true ]; then",
" if [ \"$needs_update\" = true ]; then action=would-remove-old-endpoint-exclusions; else action=kept; fi",
"elif [ \"$needs_update\" = false ]; then",
" action=kept",
"else",
" patch_file=$(mktemp /tmp/hwlab-argocd-endpoint-bridge.XXXXXX.json)",
" python3 - <<'PY' >\"$patch_file\"",
"import json",
"desired = '''### Internal Kubernetes resources excluded to reduce watch volume",
"- apiGroups:",
" - coordination.k8s.io",
" kinds:",
" - Lease",
"### Internal Kubernetes Authz/Authn resources excluded to reduce watched events",
"- apiGroups:",
" - authentication.k8s.io",
" - authorization.k8s.io",
" kinds:",
" - SelfSubjectReview",
" - TokenReview",
" - LocalSubjectAccessReview",
" - SelfSubjectAccessReview",
" - SelfSubjectRulesReview",
" - SubjectAccessReview",
"### Intermediate Certificate Request excluded to reduce watched events",
"- apiGroups:",
" - certificates.k8s.io",
" kinds:",
" - CertificateSigningRequest",
"- apiGroups:",
" - cert-manager.io",
" kinds:",
" - CertificateRequest",
"### Cilium internal resources excluded to reduce UI clutter",
"- apiGroups:",
" - cilium.io",
" kinds:",
" - CiliumIdentity",
" - CiliumEndpoint",
" - CiliumEndpointSlice",
"### Kyverno intermediate and reporting resources excluded to reduce watched events",
"- apiGroups:",
" - kyverno.io",
" - reports.kyverno.io",
" - wgpolicyk8s.io",
" kinds:",
" - PolicyReport",
" - ClusterPolicyReport",
" - EphemeralReport",
" - ClusterEphemeralReport",
" - AdmissionReport",
" - ClusterAdmissionReport",
" - BackgroundScanReport",
" - ClusterBackgroundScanReport",
" - UpdateRequest",
"'''",
"print(json.dumps({",
" 'data': {",
" 'resource.exclusions': desired,",
" 'resource.customizations.ignoreResourceUpdates.Endpoints': None,",
" 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice': None,",
" }",
"}))",
"PY",
" kubectl -n \"$namespace\" patch configmap \"$configmap\" --type merge --patch-file \"$patch_file\" >/tmp/hwlab-argocd-endpoint-bridge-patch.out 2>/tmp/hwlab-argocd-endpoint-bridge-patch.err",
" patch_exit=$?",
" rm -f \"$patch_file\"",
" if [ \"$patch_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout restart statefulset/argocd-application-controller >/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.err",
" rollout_restart_exit=$?",
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout status statefulset/argocd-application-controller --timeout=180s >/tmp/hwlab-argocd-endpoint-bridge-rollout-status.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-status.err",
" rollout_status_exit=$?",
" fi",
" kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/hwlab-argocd-endpoint-bridge-refresh.out 2>/tmp/hwlab-argocd-endpoint-bridge-refresh.err",
" refresh_exit=$?",
" if [ \"$rollout_restart_exit\" -ne 0 ]; then action=rollout-restart-failed",
" elif [ \"$rollout_status_exit\" -ne 0 ]; then action=rollout-status-failed",
" elif [ \"$refresh_exit\" -ne 0 ]; then action=refresh-failed",
" else action=removed-old-endpoint-exclusions; mutation=true; fi",
" else",
" action=patch-failed",
" fi",
"fi",
"after_endpoint_resources_excluded=$(endpoint_resources_excluded)",
"after_endpoints_ignore_updates=$(cm_has_key 'resource\\.customizations\\.ignoreResourceUpdates\\.Endpoints')",
"after_endpoint_slice_ignore_updates=$(cm_has_key 'resource\\.customizations\\.ignoreResourceUpdates\\.discovery\\.k8s\\.io_EndpointSlice')",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'configMap\\t%s\\n' \"$configmap\"",
"printf 'application\\t%s\\n' \"$application\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeEndpointResourcesExcluded\\t%s\\n' \"$before_endpoint_resources_excluded\"",
"printf 'beforeEndpointsIgnoreUpdates\\t%s\\n' \"$before_endpoints_ignore_updates\"",
"printf 'beforeEndpointSliceIgnoreUpdates\\t%s\\n' \"$before_endpoint_slice_ignore_updates\"",
"printf 'afterEndpointResourcesExcluded\\t%s\\n' \"$after_endpoint_resources_excluded\"",
"printf 'afterEndpointsIgnoreUpdates\\t%s\\n' \"$after_endpoints_ignore_updates\"",
"printf 'afterEndpointSliceIgnoreUpdates\\t%s\\n' \"$after_endpoint_slice_ignore_updates\"",
"printf 'patchExitCode\\t%s\\n' \"$patch_exit\"",
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
"printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"",
"if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi",
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
"if [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then exit \"$refresh_exit\"; fi",
].join("\n");
}
function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
const pvc = `data-${spec.postgresSecret}-0`;
const platformService = "g14-platform-postgres";
const postgresService = spec.postgresSecret;
const postgresConfigMap = `${spec.postgresSecret}-init`;
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
`postgres_service=${shellQuote(postgresService)}`,
`postgres_configmap=${shellQuote(postgresConfigMap)}`,
`pvc=${shellQuote(pvc)}`,
`platform_service=${shellQuote(platformService)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
@@ -388,20 +598,44 @@ function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSec
"before_pvc_phase=$(phase_of_pvc)",
"before_pv=$(pv_name)",
"before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
"before_service_exists=$(exists_flag service \"$postgres_service\")",
"before_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
"platform_service_exists=$(exists_flag service \"$platform_service\")",
"action=observed",
"mutation=false",
"delete_statefulset_exit=",
"delete_service_exit=",
"delete_configmap_exit=",
"delete_secret_exit=",
"delete_pvc_exit=",
"before_any_owned=false",
"for flag in \"$before_statefulset_exists\" \"$before_service_exists\" \"$before_configmap_exists\" \"$before_secret_exists\" \"$before_pvc_exists\"; do",
" if [ \"$flag\" = yes ]; then before_any_owned=true; fi",
"done",
"if [ \"$dry_run\" = true ]; then",
" if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
" if [ \"$before_any_owned\" = true ]; then action=would-delete; else action=already-absent; fi",
"else",
" kubectl -n \"$namespace\" delete statefulset \"$postgres_statefulset\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-statefulset-delete.out 2>/tmp/hwlab-owned-postgres-statefulset-delete.err",
" delete_statefulset_exit=$?",
" kubectl -n \"$namespace\" delete service \"$postgres_service\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-service-delete.out 2>/tmp/hwlab-owned-postgres-service-delete.err",
" delete_service_exit=$?",
" kubectl -n \"$namespace\" delete configmap \"$postgres_configmap\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-configmap-delete.out 2>/tmp/hwlab-owned-postgres-configmap-delete.err",
" delete_configmap_exit=$?",
" kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err",
" delete_secret_exit=$?",
" kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err",
" delete_pvc_exit=$?",
" if [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then",
" if [ \"$before_secret_exists\" = yes ] || [ \"$before_pvc_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
" for _ in $(seq 1 30); do",
" current_statefulset=$(exists_flag statefulset \"$postgres_statefulset\")",
" current_service=$(exists_flag service \"$postgres_service\")",
" current_configmap=$(exists_flag configmap \"$postgres_configmap\")",
" current_secret=$(exists_flag secret \"$postgres_secret\")",
" current_pvc=$(exists_flag pvc \"$pvc\")",
" if [ \"$current_statefulset\" != yes ] && [ \"$current_service\" != yes ] && [ \"$current_configmap\" != yes ] && [ \"$current_secret\" != yes ] && [ \"$current_pvc\" != yes ]; then break; fi",
" sleep 2",
" done",
" if [ \"$delete_statefulset_exit\" -eq 0 ] && [ \"$delete_service_exit\" -eq 0 ] && [ \"$delete_configmap_exit\" -eq 0 ] && [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then",
" if [ \"$before_any_owned\" = true ]; then action=deleted; mutation=true; else action=already-absent; fi",
" else",
" action=delete-failed",
" fi",
@@ -411,8 +645,13 @@ function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSec
"after_pvc_phase=$(phase_of_pvc)",
"after_pv=$(pv_name)",
"after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
"after_service_exists=$(exists_flag service \"$postgres_service\")",
"after_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$postgres_secret\"",
"printf 'statefulSet\\t%s\\n' \"$postgres_statefulset\"",
"printf 'service\\t%s\\n' \"$postgres_service\"",
"printf 'configMap\\t%s\\n' \"$postgres_configmap\"",
"printf 'pvc\\t%s\\n' \"$pvc\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
@@ -423,21 +662,119 @@ function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSec
"printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"",
"printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"",
"printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"",
"printf 'beforeServiceExists\\t%s\\n' \"$before_service_exists\"",
"printf 'beforeConfigMapExists\\t%s\\n' \"$before_configmap_exists\"",
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
"printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"",
"printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"",
"printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"",
"printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"",
"printf 'afterServiceExists\\t%s\\n' \"$after_service_exists\"",
"printf 'afterConfigMapExists\\t%s\\n' \"$after_configmap_exists\"",
"printf 'deleteStatefulSetExitCode\\t%s\\n' \"$delete_statefulset_exit\"",
"printf 'deleteServiceExitCode\\t%s\\n' \"$delete_service_exit\"",
"printf 'deleteConfigMapExitCode\\t%s\\n' \"$delete_configmap_exit\"",
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
"printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"",
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
"if [ \"$before_statefulset_exists\" = yes ] || [ \"$after_statefulset_exists\" = yes ]; then exit 45; fi",
"if [ \"$after_statefulset_exists\" = yes ] || [ \"$after_service_exists\" = yes ] || [ \"$after_configmap_exists\" = yes ] || [ \"$after_secret_exists\" = yes ] || [ \"$after_pvc_exists\" = yes ]; then exit 45; fi",
"if [ -n \"$delete_statefulset_exit\" ] && [ \"$delete_statefulset_exit\" != 0 ]; then exit \"$delete_statefulset_exit\"; fi",
"if [ -n \"$delete_service_exit\" ] && [ \"$delete_service_exit\" != 0 ]; then exit \"$delete_service_exit\"; fi",
"if [ -n \"$delete_configmap_exit\" ] && [ \"$delete_configmap_exit\" != 0 ]; then exit \"$delete_configmap_exit\"; fi",
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
"if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi",
].join("\n");
}
function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
const isOpenFga = options.preset === "openfga";
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`,
`database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey)}`,
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
`legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`,
`platform_service=${shellQuote(spec.platformPostgresService)}`,
`platform_host=${shellQuote(spec.platformPostgresService)}`,
`platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`,
`db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`,
`db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`,
`db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost)}`,
`selected_key=${shellQuote(options.key ?? "")}`,
`preset=${shellQuote(options.preset)}`,
"dry_run=true",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
"resource_exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
"endpointslice_exists_flag() { kubectl -n \"$namespace\" get endpointslice -l \"kubernetes.io/service-name=$1\" -o name 2>/dev/null | grep -q . && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"uri_has_platform_host=no",
"uri_has_db_name=no",
"uri_has_db_user=no",
"uri_matches_expected() {",
" uri=$1",
" uri_has_platform_host=no",
" uri_has_db_name=no",
" uri_has_db_user=no",
" case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*) uri_has_platform_host=yes ;; esac",
" case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac",
" case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac",
"}",
"exists=$(secret_exists_flag \"$name\")",
"legacy_postgres_exists=$(secret_exists_flag \"$legacy_postgres_secret\")",
"uri_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
"uri_present=$([ -n \"$uri_b64\" ] && printf yes || printf no)",
"uri_bytes=$(decoded_length \"$uri_b64\")",
"uri_value=$(decoded_value \"$uri_b64\")",
"authn_b64=$(secret_b64_key \"$name\" \"$authn_key\")",
"authn_present=$([ -n \"$authn_b64\" ] && printf yes || printf no)",
"authn_bytes=$(decoded_length \"$authn_b64\")",
"pg_password_b64=$(secret_b64_key \"$name\" \"$postgres_password_key\")",
"pg_password_present=$([ -n \"$pg_password_b64\" ] && printf yes || printf no)",
"pg_password_bytes=$(decoded_length \"$pg_password_b64\")",
"platform_service_exists=$(resource_exists_flag service \"$platform_service\")",
"platform_endpoints_exists=$(resource_exists_flag endpoints \"$platform_service\")",
"platform_endpointslice_exists=$(endpointslice_exists_flag \"$platform_service\")",
"uri_matches_expected \"$uri_value\"",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'key\\t%s\\n' \"$database_url_key\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\tobserved\\n'",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\tfalse\\n'",
"printf 'platformDbMode\\ttrue\\n'",
"printf 'afterExists\\t%s\\n' \"$exists\"",
"printf 'afterDatabaseUrlPresent\\t%s\\n' \"$uri_present\"",
"printf 'afterDatabaseUrlBytes\\t%s\\n' \"$uri_bytes\"",
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$uri_present\"",
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$uri_bytes\"",
"printf 'afterAuthnPresent\\t%s\\n' \"$authn_present\"",
"printf 'afterAuthnBytes\\t%s\\n' \"$authn_bytes\"",
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$pg_password_present\"",
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$pg_password_bytes\"",
"printf 'legacyPostgresSecret\\t%s\\n' \"$legacy_postgres_secret\"",
"printf 'legacyPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
"printf 'afterPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
"printf 'platformService\\t%s\\n' \"$platform_service\"",
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
"printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"",
"printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"",
"printf 'dbName\\t%s\\n' \"$db_name\"",
"printf 'dbUser\\t%s\\n' \"$db_user\"",
"printf 'dbHost\\t%s\\n' \"$db_host\"",
"printf 'dbHostMatchesPlatform\\t%s\\n' \"$uri_has_platform_host\"",
"printf 'dbNameMatchesExpected\\t%s\\n' \"$uri_has_db_name\"",
"printf 'dbUserMatchesExpected\\t%s\\n' \"$uri_has_db_user\"",
"uri_value=",
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
].join("\n");
}
function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
@@ -896,44 +1233,57 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS
function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
const fields = keyValueLinesFromText(text);
if (fields.preset === "owned-postgres-cleanup") {
const absent = fields.afterSecretExists !== "yes" && fields.afterPvcExists !== "yes";
const oldWorkloadAbsent = fields.afterStatefulSetExists !== "yes";
const absent = fields.afterStatefulSetExists !== "yes" &&
fields.afterServiceExists !== "yes" &&
fields.afterConfigMapExists !== "yes" &&
fields.afterSecretExists !== "yes" &&
fields.afterPvcExists !== "yes";
const platformServiceReady = fields.platformServiceExists === "yes";
return {
ok: commandOk && absent && oldWorkloadAbsent && platformServiceReady,
ok: commandOk && absent && platformServiceReady,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.postgresSecret,
statefulSet: fields.statefulSet || spec.postgresStatefulSet,
service: fields.service || spec.postgresSecret,
configMap: fields.configMap || `${spec.postgresSecret}-init`,
pvc: fields.pvc || `data-${spec.postgresSecret}-0`,
preset: "owned-postgres-cleanup",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
before: {
statefulSetExists: fields.beforeStatefulSetExists === "yes",
serviceExists: fields.beforeServiceExists === "yes",
configMapExists: fields.beforeConfigMapExists === "yes",
secretExists: fields.beforeSecretExists === "yes",
pvcExists: fields.beforePvcExists === "yes",
pvcPhase: fields.beforePvcPhase || null,
persistentVolume: fields.beforePersistentVolume || null,
statefulSetExists: fields.beforeStatefulSetExists === "yes",
},
after: {
statefulSetExists: fields.afterStatefulSetExists === "yes",
serviceExists: fields.afterServiceExists === "yes",
configMapExists: fields.afterConfigMapExists === "yes",
secretExists: fields.afterSecretExists === "yes",
pvcExists: fields.afterPvcExists === "yes",
pvcPhase: fields.afterPvcPhase || null,
persistentVolume: fields.afterPersistentVolume || null,
statefulSetExists: fields.afterStatefulSetExists === "yes",
},
platformService: {
name: "g14-platform-postgres",
exists: platformServiceReady,
},
deleteStatefulSetExitCode: numericField(fields.deleteStatefulSetExitCode),
deleteServiceExitCode: numericField(fields.deleteServiceExitCode),
deleteConfigMapExitCode: numericField(fields.deleteConfigMapExitCode),
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
deletePvcExitCode: numericField(fields.deletePvcExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: absent
? `${fields.secret || spec.postgresSecret} and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent`
: `${fields.secret || spec.postgresSecret} or ${fields.pvc || `data-${spec.postgresSecret}-0`} still exists`,
? `${fields.statefulSet || spec.postgresStatefulSet}, ${fields.service || spec.postgresSecret}, ${fields.configMap || `${spec.postgresSecret}-init`}, ${fields.secret || spec.postgresSecret}, and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent`
: `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`,
};
}
if (fields.preset === "master-server-admin-api-key") {
@@ -1002,6 +1352,54 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
if (fields.preset === "cloud-api-db") {
const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes);
const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes);
if (fields.platformDbMode === "true") {
const keysHealthy = fields.afterExists === "yes" &&
fields.afterDatabaseUrlPresent === "yes" &&
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
(fields.platformEndpointsExists === "yes" || fields.platformEndpointSliceExists === "yes");
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
fields.dbNameMatchesExpected === "yes" &&
fields.dbUserMatchesExpected === "yes";
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.cloudApiDbSecret,
key: fields.key || spec.cloudApiDbKey,
preset: "cloud-api-db",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
platformDbMode: true,
after: {
exists: fields.afterExists === "yes",
databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes },
},
legacyPostgresSecret: {
name: fields.legacyPostgresSecret || spec.postgresSecret,
exists: fields.legacyPostgresSecretExists === "yes",
},
platformService: {
name: fields.platformService || spec.platformPostgresService,
exists: fields.platformServiceExists === "yes",
endpointsExist: fields.platformEndpointsExists === "yes",
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
},
dbName: fields.dbName || spec.cloudApiDbName,
dbUser: fields.dbUser || spec.cloudApiDbUser,
dbHost: fields.dbHost || spec.cloudApiDbHost,
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy
? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}`
: `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`,
};
}
const keysHealthy = fields.afterExists === "yes" &&
fields.afterDatabaseUrlPresent === "yes" &&
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
@@ -1054,6 +1452,57 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
const afterAuthnBytes = numericField(fields.afterAuthnBytes);
const afterUriBytes = numericField(fields.afterDatastoreUriBytes);
const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes);
if (fields.platformDbMode === "true") {
const keysHealthy = fields.afterExists === "yes" &&
fields.afterAuthnPresent === "yes" &&
fields.afterDatastoreUriPresent === "yes" &&
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
typeof afterUriBytes === "number" && afterUriBytes > 0;
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
(fields.platformEndpointsExists === "yes" || fields.platformEndpointSliceExists === "yes");
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
fields.dbNameMatchesExpected === "yes" &&
fields.dbUserMatchesExpected === "yes";
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.openFgaSecret,
preset: fields.preset || "openfga",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
platformDbMode: true,
after: {
exists: fields.afterExists === "yes",
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
},
legacyPostgresSecret: {
name: fields.legacyPostgresSecret || spec.postgresSecret,
exists: fields.legacyPostgresSecretExists === "yes",
},
platformService: {
name: fields.platformService || spec.platformPostgresService,
exists: fields.platformServiceExists === "yes",
endpointsExist: fields.platformEndpointsExists === "yes",
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
},
dbName: fields.dbName || spec.openFgaDbName,
dbUser: fields.dbUser || spec.openFgaDbUser,
dbHost: fields.dbHost || spec.openFgaDbHost,
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy
? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.platformService || spec.platformPostgresService}`
: `${fields.secret || spec.openFgaSecret} datastore-uri is not aligned to platform DB`,
};
}
const keysHealthy = fields.afterExists === "yes" &&
fields.afterPostgresSecretExists === "yes" &&
fields.afterAuthnPresent === "yes" &&