import { existsSync, readFileSync } from "node:fs"; import { repoRoot, type Config } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { runHwlabG14Command } from "./hwlab-g14"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes"; type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; type DelegatedNodeDomain = "control-plane" | "git-mirror"; interface NodeSecretOptions { action: SecretAction; node: string; lane: string; name: string; key?: string; preset: SecretPreset; dryRun: boolean; confirm: boolean; timeoutSeconds: number; } interface RuntimeSecretSpec { node: string; lane: string; namespace: string; platformDb: boolean; platformPostgresService: string; postgresSecret: string; postgresStatefulSet: string; postgresAdminUser: string; openFgaSecret: string; openFgaDbName: string; openFgaDbUser: string; openFgaDbHost: string; masterAdminApiKeySecret: string; cloudApiDbSecret: string; cloudApiDbKey: string; cloudApiDbName: string; cloudApiDbUser: string; cloudApiDbHost: string; cloudApiDeployment: string; obsoleteHwpodDbSecret: string; obsoleteHwpodDbName: string; obsoleteHwpodDbUser: string; codeAgentProviderSecret: string; codeAgentProviderSourceNamespace: string; codeAgentProviderSourceSecret: string; fieldManager: string; } const MASTER_ADMIN_API_KEY_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env"; const MASTER_ADMIN_API_KEY_KEY = "api-key"; const OPENFGA_AUTHN_KEY = "authn-preshared-key"; const OPENFGA_DATASTORE_URI_KEY = "datastore-uri"; const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password"; const CLOUD_API_DB_KEY = "database-url"; const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key"; const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key"; const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02"; const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider"; export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise> { if (args.length === 0 || args.includes("--help") || args.includes("-h")) return hwlabNodeHelp(); const [domain] = args; if (domain === "control-plane" || domain === "git-mirror") { return runNodeDelegatedDomain(_config, domain, args.slice(1)); } if (domain !== "secret") { return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret" }; } const options = parseSecretOptions(args.slice(1)); return runNodeSecret(options); } export function hwlabNodeHelp(): Record { return { ok: true, command: "hwlab nodes", description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.", configPath: hwlabRuntimeLaneConfigPath(), examples: [ "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run", "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm", ], }; } async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise> { const scoped = parseNodeScopedDelegatedOptions(domain, args); if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") { return runNodeEndpointBridge(scoped); } if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) { return startNodeDelegatedJob(scoped); } if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) { return startNodeDelegatedJob(scoped); } const delegatedArgs = stripOption(args, "--node"); const result = await runHwlabG14Command(config, [domain, ...delegatedArgs]); return rewriteDelegatedNodeResult(result, scoped); } function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): { domain: DelegatedNodeDomain; action: string; node: string; lane: HwlabRuntimeLane; confirm: boolean; dryRun: boolean; wait: boolean; timeoutSeconds: number; originalArgs: string[]; } { const [actionRaw] = args; if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`); const node = requiredOption(args, "--node"); assertNodeId(node); const laneRaw = requiredOption(args, "--lane"); if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`); const spec = hwlabRuntimeLaneSpec(laneRaw); if (spec.nodeId !== node) throw new Error(`lane ${laneRaw} is configured for node ${spec.nodeId}; got --node ${node}`); const confirm = args.includes("--confirm"); const dryRun = args.includes("--dry-run"); if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`); return { domain, action: actionRaw, node, lane: laneRaw, confirm, dryRun, wait: args.includes("--wait"), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600), originalArgs: [...args], }; } function startNodeDelegatedJob(options: ReturnType): Record { const commandArgs = [ "hwlab", "nodes", options.domain, ...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]), "--node", options.node, "--lane", options.lane, "--confirm", "--timeout-seconds", String(options.timeoutSeconds), "--wait", ]; const command = ["bun", "scripts/cli.ts", ...commandArgs]; const job = startJob( `hwlab_nodes_${options.lane}_${options.domain}_${options.action}`, command, `Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`, ); return { ok: true, command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`, node: options.node, lane: options.lane, mode: "async-job", reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking", job, statusCommand: `bun scripts/cli.ts job status ${job.id}`, waitCommand: command.join(" "), }; } function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType): Record { const rewritten = rewriteDelegatedNodeValue(value, scoped); const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record : { value: rewritten }; return { ...result, node: scoped.node, commandFamily: "hwlab nodes", }; } function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType): unknown { if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped); if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped)); if (typeof value !== "object" || value === null) return value; return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)])); } function rewriteDelegatedNodeString(value: string, scoped: ReturnType): string { const replaceCommand = (text: string, domain: DelegatedNodeDomain) => { const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); return text .replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`) .replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`); }; return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror"); } function parseSecretOptions(args: string[]): NodeSecretOptions { const [actionRaw] = args; if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") { throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]"); } const node = requiredOption(args, "--node"); assertNodeId(node); const lane = requiredOption(args, "--lane"); assertLane(lane); const spec = runtimeSecretSpec({ node, lane }); const name = optionValue(args, "--name") ?? spec.openFgaSecret; const key = optionValue(args, "--key"); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run"); if (actionRaw === "cleanup-owned-postgres") { if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres"); if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key"); if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`); return { action: actionRaw, node, lane, name: spec.postgresSecret, preset: "owned-postgres-cleanup", confirm, dryRun: explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (actionRaw === "cleanup-obsolete") { if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup"); if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key"); if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`); return { action: actionRaw, node, lane, name, preset: "obsolete-secret-cleanup", confirm, dryRun: explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (name === spec.masterAdminApiKeySecret) { if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`); return { action: actionRaw, node, lane, name, key: key ?? MASTER_ADMIN_API_KEY_KEY, preset: "master-server-admin-api-key", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (name === spec.codeAgentProviderSecret) { if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) { throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`); } return { action: actionRaw, node, lane, name, key, preset: "code-agent-provider", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (name === spec.cloudApiDbSecret) { if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`); if (actionRaw === "ensure" && spec.platformDb) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { action: actionRaw, node, lane, name, key: key ?? spec.cloudApiDbKey, preset: "cloud-api-db", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900), }; } if (name !== spec.openFgaSecret) { throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`); } if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) { throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`); } if (actionRaw === "ensure" && spec.platformDb) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { action: actionRaw, node, lane, name, key, preset: "openfga", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec { const namespace = `hwlab-${input.lane}`; const platformDb = /^v0*[3-9]\d*$/.test(input.lane); const platformPostgresService = "g14-platform-postgres"; const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`; const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`; return { node: input.node, lane: input.lane, namespace, platformDb, platformPostgresService, postgresSecret: `${namespace}-postgres`, postgresStatefulSet: `${namespace}-postgres`, postgresAdminUser: `hwlab_${input.lane}`, openFgaSecret: `${namespace}-openfga`, openFgaDbName: platformDb ? `openfga_${input.lane}` : "hwlab_openfga", openFgaDbUser: platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga", openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`, cloudApiDbSecret: `hwlab-cloud-api-${input.lane}-db`, cloudApiDbKey: CLOUD_API_DB_KEY, cloudApiDbName: `hwlab_${input.lane}`, cloudApiDbUser: platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`, cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, cloudApiDeployment: "hwlab-cloud-api", obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`, obsoleteHwpodDbName: `hwpod_${input.lane}`, obsoleteHwpodDbUser: `hwpod_${input.lane}_app`, codeAgentProviderSecret: `${namespace}-code-agent-provider`, codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE, codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET, fieldManager: `unidesk-hwlab-node-${input.lane}-secret`, }; } function runNodeSecret(options: NodeSecretOptions): Record { const spec = runtimeSecretSpec(options); if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec); const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun ? readMasterAdminApiKey().key : ""; const script = options.preset === "openfga" ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec) : options.preset === "master-server-admin-api-key" ? masterAdminApiKeySecretScript(options, spec) : options.preset === "cloud-api-db" ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec) : options.preset === "owned-postgres-cleanup" ? ownedPostgresCleanupScript(options, spec) : codeAgentProviderSecretScript(options, spec); const result = runTransScript(options.node, script, input, options.timeoutSeconds); const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec); const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0; const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0; const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true; const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true; return { ok, command: `hwlab nodes secret ${options.action}`, node: options.node, lane: options.lane, namespace: spec.namespace, secret: options.name, key: options.key ?? null, preset: options.preset, mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure", status, mutation: status.mutation === true, result: compactCommandResult(result), valuesRedacted: true, next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec), }; } function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { if (options.action === "cleanup-owned-postgres") { return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` }; } if (options.action === "cleanup-obsolete") { return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` }; } if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) { return { status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`, controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`, }; } return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; } function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { return runCommand(["/root/.local/bin/trans", node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds); const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec); const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true }; const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds); const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec); const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false); const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true; return { ok, command: `hwlab nodes secret ${options.action}`, node: options.node, lane: options.lane, namespace: spec.namespace, secret: options.name, key: null, preset: options.preset, mode: options.dryRun ? "dry-run" : "confirmed-delete", status: { ok, preset: "obsolete-hwpod-db-cleanup", dryRun: options.dryRun, mutation, kubernetesSecret: kubernetesStatus, platformDatabase: platformDbStatus, hostMutationSkipped: !options.dryRun && hostOptions.dryRun, summary: ok ? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove` : `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`, }, mutation, result: { kubernetesSecret: compactCommandResult(kubernetesResult), platformDatabase: compactCommandResult(platformDbResult), }, valuesRedacted: true, next: ok ? undefined : nextSecretCommand(options, spec), }; } function runNodeEndpointBridge(options: ReturnType): Record { if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm"); const dryRun = options.dryRun || !options.confirm; const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds); const fields = keyValueLinesFromText(statusText(result)); const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes"; const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes"; const afterExcluded = fields.afterEndpointResourcesExcluded === "yes"; const afterIgnored = fields.afterEndpointsIgnoreUpdates === "yes" || fields.afterEndpointSliceIgnoreUpdates === "yes"; const beforeExtraEndpointSlices = splitWhitespaceField(fields.beforeExtraEndpointSliceNames); const afterExtraEndpointSlices = splitWhitespaceField(fields.afterExtraEndpointSliceNames); const beforeLegacyEndpoints = fields.beforeLegacyEndpointsExists === "yes"; const afterLegacyEndpoints = fields.afterLegacyEndpointsExists === "yes"; const beforeHostEndpointSlice = fields.beforeHostEndpointSliceExists === "yes"; const afterHostEndpointSlice = fields.afterHostEndpointSliceExists === "yes"; const bridgeReady = !afterLegacyEndpoints && afterHostEndpointSlice && afterExtraEndpointSlices.length === 0; const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored && bridgeReady; return { ok: dryRun ? result.exitCode === 0 : ok, command: "hwlab nodes control-plane allow-endpoint-bridge", node: options.node, lane: options.lane, namespace: "argocd", application: fields.application || `hwlab-node-${options.lane}`, mode: dryRun ? "dry-run" : "confirmed-control-plane-update", status: { action: fields.action || null, dryRun, mutation: fields.mutation === "true", before: { endpointResourcesExcluded: beforeExcluded, endpointsIgnoreUpdates: fields.beforeEndpointsIgnoreUpdates === "yes", endpointSliceIgnoreUpdates: fields.beforeEndpointSliceIgnoreUpdates === "yes", legacyEndpointsExist: beforeLegacyEndpoints, hostEndpointSliceExists: beforeHostEndpointSlice, extraEndpointSlices: beforeExtraEndpointSlices, }, after: { endpointResourcesExcluded: afterExcluded, endpointsIgnoreUpdates: fields.afterEndpointsIgnoreUpdates === "yes", endpointSliceIgnoreUpdates: fields.afterEndpointSliceIgnoreUpdates === "yes", legacyEndpointsExist: afterLegacyEndpoints, hostEndpointSliceExists: afterHostEndpointSlice, extraEndpointSlices: afterExtraEndpointSlices, }, runtimeNamespace: fields.runtimeNamespace || `hwlab-${options.lane}`, platformService: fields.platformService || "g14-platform-postgres", hostEndpointSlice: fields.hostEndpointSlice || "g14-platform-postgres-host", patchExitCode: numericField(fields.patchExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), deleteLegacyEndpointsExitCode: numericField(fields.deleteLegacyEndpointsExitCode), deleteExtraEndpointSlicesExitCode: numericField(fields.deleteExtraEndpointSlicesExitCode), refreshExitCode: numericField(fields.refreshExitCode), exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), summary: !afterExcluded && !afterIgnored && bridgeReady ? "Argo tracks HWLAB external Postgres EndpointSlice and no legacy Endpoints remain" : "Argo endpoint bridge is not in final Service plus EndpointSlice shape", }, result: compactCommandResult(result), }; } function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string { const application = `hwlab-node-${options.lane}`; const runtimeNamespace = `hwlab-${options.lane}`; return [ "set +e", "namespace=argocd", `runtime_namespace=${shellQuote(runtimeNamespace)}`, "configmap=argocd-cm", `application=${shellQuote(application)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "platform_service=g14-platform-postgres", "host_endpointslice=g14-platform-postgres-host", "preset=endpoint-bridge-resource-tracking", "cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", "cm_has_key() { value=$(cm_data \"$1\"); [ -n \"$value\" ] && [ \"$value\" != \"\" ] && printf yes || printf no; }", "endpoint_resources_excluded() { exclusions=$(cm_data resource.exclusions); printf '%s' \"$exclusions\" | grep -Eq '(^|[[:space:]])(Endpoints|EndpointSlice)([[:space:]]|$)' && printf yes || printf no; }", "resource_exists() { kubectl -n \"$runtime_namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "extra_endpoint_slices() { kubectl -n \"$runtime_namespace\" get endpointslice -l \"kubernetes.io/service-name=$platform_service\" -o name 2>/dev/null | sed \"/\\/$host_endpointslice$/d\" | tr '\\n' ' ' | sed 's/[[:space:]]*$//'; }", "wait_runtime_bridge_clean() {", " for _ in $(seq 1 30); do", " current_legacy=$(resource_exists endpoints \"$platform_service\")", " current_extra=$(extra_endpoint_slices)", " current_host=$(resource_exists endpointslice \"$host_endpointslice\")", " if [ \"$current_legacy\" != yes ] && [ -z \"$current_extra\" ] && [ \"$current_host\" = yes ]; then return 0; fi", " sleep 2", " done", " return 1", "}", "before_endpoint_resources_excluded=$(endpoint_resources_excluded)", "before_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')", "before_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')", "before_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")", "before_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")", "before_extra_endpoint_slice_names=$(extra_endpoint_slices)", "needs_argo_update=false", "if [ \"$before_endpoint_resources_excluded\" = yes ] || [ \"$before_endpoints_ignore_updates\" = yes ] || [ \"$before_endpoint_slice_ignore_updates\" = yes ]; then needs_argo_update=true; fi", "needs_runtime_cleanup=false", "if [ \"$before_legacy_endpoints_exists\" = yes ] || [ -n \"$before_extra_endpoint_slice_names\" ]; then needs_runtime_cleanup=true; fi", "action=observed", "mutation=false", "patch_exit=", "rollout_restart_exit=", "rollout_status_exit=", "delete_legacy_endpoints_exit=", "delete_extra_endpointslices_exit=", "refresh_exit=", "if [ \"$dry_run\" = true ]; then", " if [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-old-endpoint-exclusions-and-legacy-endpoints", " elif [ \"$needs_argo_update\" = true ]; then action=would-remove-old-endpoint-exclusions", " elif [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-legacy-endpoints", " else action=kept; fi", "else", " if [ \"$needs_argo_update\" = true ]; then", " patch_file=$(mktemp /tmp/hwlab-argocd-endpoint-bridge.XXXXXX.json)", " python3 - <<'PY' >\"$patch_file\"", "import json", "desired = '''### Internal Kubernetes resources excluded to reduce watch volume", "- apiGroups:", " - coordination.k8s.io", " kinds:", " - Lease", "### Internal Kubernetes Authz/Authn resources excluded to reduce watched events", "- apiGroups:", " - authentication.k8s.io", " - authorization.k8s.io", " kinds:", " - SelfSubjectReview", " - TokenReview", " - LocalSubjectAccessReview", " - SelfSubjectAccessReview", " - SelfSubjectRulesReview", " - SubjectAccessReview", "### Intermediate Certificate Request excluded to reduce watched events", "- apiGroups:", " - certificates.k8s.io", " kinds:", " - CertificateSigningRequest", "- apiGroups:", " - cert-manager.io", " kinds:", " - CertificateRequest", "### Cilium internal resources excluded to reduce UI clutter", "- apiGroups:", " - cilium.io", " kinds:", " - CiliumIdentity", " - CiliumEndpoint", " - CiliumEndpointSlice", "### Kyverno intermediate and reporting resources excluded to reduce watched events", "- apiGroups:", " - kyverno.io", " - reports.kyverno.io", " - wgpolicyk8s.io", " kinds:", " - PolicyReport", " - ClusterPolicyReport", " - EphemeralReport", " - ClusterEphemeralReport", " - AdmissionReport", " - ClusterAdmissionReport", " - BackgroundScanReport", " - ClusterBackgroundScanReport", " - UpdateRequest", "'''", "print(json.dumps({", " 'data': {", " 'resource.exclusions': desired,", " 'resource.customizations.ignoreResourceUpdates.Endpoints': None,", " 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice': None,", " }", "}))", "PY", " kubectl -n \"$namespace\" patch configmap \"$configmap\" --type merge --patch-file \"$patch_file\" >/tmp/hwlab-argocd-endpoint-bridge-patch.out 2>/tmp/hwlab-argocd-endpoint-bridge-patch.err", " patch_exit=$?", " rm -f \"$patch_file\"", " if [ \"$patch_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout restart statefulset/argocd-application-controller >/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.err", " rollout_restart_exit=$?", " if [ \"$rollout_restart_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status statefulset/argocd-application-controller --timeout=180s >/tmp/hwlab-argocd-endpoint-bridge-rollout-status.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-status.err", " rollout_status_exit=$?", " fi", " fi", " fi", " if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then action=patch-failed", " elif [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", " else", " if [ \"$needs_runtime_cleanup\" = true ]; then", " kubectl -n \"$runtime_namespace\" delete endpoints \"$platform_service\" --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpoints-delete.out 2>/tmp/hwlab-platform-postgres-endpoints-delete.err", " delete_legacy_endpoints_exit=$?", " wait_runtime_bridge_clean", " remaining_extra=$(extra_endpoint_slices)", " if [ -n \"$remaining_extra\" ]; then", " kubectl -n \"$runtime_namespace\" delete $remaining_extra --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpointslices-delete.out 2>/tmp/hwlab-platform-postgres-endpointslices-delete.err", " delete_extra_endpointslices_exit=$?", " wait_runtime_bridge_clean", " fi", " fi", " if [ \"$needs_argo_update\" = true ] || [ \"$needs_runtime_cleanup\" = true ]; then", " kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/hwlab-argocd-endpoint-bridge-refresh.out 2>/tmp/hwlab-argocd-endpoint-bridge-refresh.err", " refresh_exit=$?", " fi", " if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then action=delete-legacy-endpoints-failed", " elif [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then action=delete-extra-endpointslices-failed", " elif [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then action=refresh-failed", " elif [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=removed-old-endpoint-exclusions-and-legacy-endpoints; mutation=true", " elif [ \"$needs_argo_update\" = true ]; then action=removed-old-endpoint-exclusions; mutation=true", " elif [ \"$needs_runtime_cleanup\" = true ]; then action=removed-legacy-endpoints; mutation=true", " else action=kept; fi", " fi", "fi", "after_endpoint_resources_excluded=$(endpoint_resources_excluded)", "after_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')", "after_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')", "after_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")", "after_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")", "after_extra_endpoint_slice_names=$(extra_endpoint_slices)", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"", "printf 'configMap\\t%s\\n' \"$configmap\"", "printf 'application\\t%s\\n' \"$application\"", "printf 'platformService\\t%s\\n' \"$platform_service\"", "printf 'hostEndpointSlice\\t%s\\n' \"$host_endpointslice\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeEndpointResourcesExcluded\\t%s\\n' \"$before_endpoint_resources_excluded\"", "printf 'beforeEndpointsIgnoreUpdates\\t%s\\n' \"$before_endpoints_ignore_updates\"", "printf 'beforeEndpointSliceIgnoreUpdates\\t%s\\n' \"$before_endpoint_slice_ignore_updates\"", "printf 'beforeLegacyEndpointsExists\\t%s\\n' \"$before_legacy_endpoints_exists\"", "printf 'beforeHostEndpointSliceExists\\t%s\\n' \"$before_host_endpointslice_exists\"", "printf 'beforeExtraEndpointSliceNames\\t%s\\n' \"$before_extra_endpoint_slice_names\"", "printf 'afterEndpointResourcesExcluded\\t%s\\n' \"$after_endpoint_resources_excluded\"", "printf 'afterEndpointsIgnoreUpdates\\t%s\\n' \"$after_endpoints_ignore_updates\"", "printf 'afterEndpointSliceIgnoreUpdates\\t%s\\n' \"$after_endpoint_slice_ignore_updates\"", "printf 'afterLegacyEndpointsExists\\t%s\\n' \"$after_legacy_endpoints_exists\"", "printf 'afterHostEndpointSliceExists\\t%s\\n' \"$after_host_endpointslice_exists\"", "printf 'afterExtraEndpointSliceNames\\t%s\\n' \"$after_extra_endpoint_slice_names\"", "printf 'patchExitCode\\t%s\\n' \"$patch_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", "printf 'deleteLegacyEndpointsExitCode\\t%s\\n' \"$delete_legacy_endpoints_exit\"", "printf 'deleteExtraEndpointSlicesExitCode\\t%s\\n' \"$delete_extra_endpointslices_exit\"", "printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"", "if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi", "if [ \"$dry_run\" != true ] && { [ \"$after_legacy_endpoints_exists\" = yes ] || [ -n \"$after_extra_endpoint_slice_names\" ] || [ \"$after_host_endpointslice_exists\" != yes ]; }; then exit 47; fi", "if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi", "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", "if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then exit \"$delete_legacy_endpoints_exit\"; fi", "if [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then exit \"$delete_extra_endpointslices_exit\"; fi", "if [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then exit \"$refresh_exit\"; fi", ].join("\n"); } function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const pvc = `data-${spec.postgresSecret}-0`; const platformService = "g14-platform-postgres"; const postgresService = spec.postgresSecret; const postgresConfigMap = `${spec.postgresSecret}-init`; return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `postgres_secret=${shellQuote(spec.postgresSecret)}`, `postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`, `postgres_service=${shellQuote(postgresService)}`, `postgres_configmap=${shellQuote(postgresConfigMap)}`, `pvc=${shellQuote(pvc)}`, `platform_service=${shellQuote(platformService)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "preset=owned-postgres-cleanup", "exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }", "pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }", "phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }", "before_secret_exists=$(exists_flag secret \"$postgres_secret\")", "before_pvc_exists=$(exists_flag pvc \"$pvc\")", "before_pvc_phase=$(phase_of_pvc)", "before_pv=$(pv_name)", "before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")", "before_service_exists=$(exists_flag service \"$postgres_service\")", "before_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")", "platform_service_exists=$(exists_flag service \"$platform_service\")", "action=observed", "mutation=false", "delete_statefulset_exit=", "delete_service_exit=", "delete_configmap_exit=", "delete_secret_exit=", "delete_pvc_exit=", "before_any_owned=false", "for flag in \"$before_statefulset_exists\" \"$before_service_exists\" \"$before_configmap_exists\" \"$before_secret_exists\" \"$before_pvc_exists\"; do", " if [ \"$flag\" = yes ]; then before_any_owned=true; fi", "done", "if [ \"$dry_run\" = true ]; then", " if [ \"$before_any_owned\" = true ]; then action=would-delete; else action=already-absent; fi", "else", " kubectl -n \"$namespace\" delete statefulset \"$postgres_statefulset\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-statefulset-delete.out 2>/tmp/hwlab-owned-postgres-statefulset-delete.err", " delete_statefulset_exit=$?", " kubectl -n \"$namespace\" delete service \"$postgres_service\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-service-delete.out 2>/tmp/hwlab-owned-postgres-service-delete.err", " delete_service_exit=$?", " kubectl -n \"$namespace\" delete configmap \"$postgres_configmap\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-configmap-delete.out 2>/tmp/hwlab-owned-postgres-configmap-delete.err", " delete_configmap_exit=$?", " kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err", " delete_secret_exit=$?", " kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err", " delete_pvc_exit=$?", " for _ in $(seq 1 30); do", " current_statefulset=$(exists_flag statefulset \"$postgres_statefulset\")", " current_service=$(exists_flag service \"$postgres_service\")", " current_configmap=$(exists_flag configmap \"$postgres_configmap\")", " current_secret=$(exists_flag secret \"$postgres_secret\")", " current_pvc=$(exists_flag pvc \"$pvc\")", " if [ \"$current_statefulset\" != yes ] && [ \"$current_service\" != yes ] && [ \"$current_configmap\" != yes ] && [ \"$current_secret\" != yes ] && [ \"$current_pvc\" != yes ]; then break; fi", " sleep 2", " done", " if [ \"$delete_statefulset_exit\" -eq 0 ] && [ \"$delete_service_exit\" -eq 0 ] && [ \"$delete_configmap_exit\" -eq 0 ] && [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then", " if [ \"$before_any_owned\" = true ]; then action=deleted; mutation=true; else action=already-absent; fi", " else", " action=delete-failed", " fi", "fi", "after_secret_exists=$(exists_flag secret \"$postgres_secret\")", "after_pvc_exists=$(exists_flag pvc \"$pvc\")", "after_pvc_phase=$(phase_of_pvc)", "after_pv=$(pv_name)", "after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")", "after_service_exists=$(exists_flag service \"$postgres_service\")", "after_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$postgres_secret\"", "printf 'statefulSet\\t%s\\n' \"$postgres_statefulset\"", "printf 'service\\t%s\\n' \"$postgres_service\"", "printf 'configMap\\t%s\\n' \"$postgres_configmap\"", "printf 'pvc\\t%s\\n' \"$pvc\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", "printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"", "printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"", "printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"", "printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"", "printf 'beforeServiceExists\\t%s\\n' \"$before_service_exists\"", "printf 'beforeConfigMapExists\\t%s\\n' \"$before_configmap_exists\"", "printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", "printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"", "printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"", "printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"", "printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"", "printf 'afterServiceExists\\t%s\\n' \"$after_service_exists\"", "printf 'afterConfigMapExists\\t%s\\n' \"$after_configmap_exists\"", "printf 'deleteStatefulSetExitCode\\t%s\\n' \"$delete_statefulset_exit\"", "printf 'deleteServiceExitCode\\t%s\\n' \"$delete_service_exit\"", "printf 'deleteConfigMapExitCode\\t%s\\n' \"$delete_configmap_exit\"", "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", "printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"", "if [ \"$platform_service_exists\" != yes ]; then exit 44; fi", "if [ \"$after_statefulset_exists\" = yes ] || [ \"$after_service_exists\" = yes ] || [ \"$after_configmap_exists\" = yes ] || [ \"$after_secret_exists\" = yes ] || [ \"$after_pvc_exists\" = yes ]; then exit 45; fi", "if [ -n \"$delete_statefulset_exit\" ] && [ \"$delete_statefulset_exit\" != 0 ]; then exit \"$delete_statefulset_exit\"; fi", "if [ -n \"$delete_service_exit\" ] && [ \"$delete_service_exit\" != 0 ]; then exit \"$delete_service_exit\"; fi", "if [ -n \"$delete_configmap_exit\" ] && [ \"$delete_configmap_exit\" != 0 ]; then exit \"$delete_configmap_exit\"; fi", "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", "if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi", ].join("\n"); } function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `secret=${shellQuote(options.name)}`, `expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "preset=obsolete-secret-cleanup", "exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }", "workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)", "workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)", "workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)", "before_exists=$(exists_flag)", "action=observed", "mutation=false", "delete_secret_exit=", "if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi", "if [ \"$action\" = observed ]; then", " if [ \"$workload_refs_present\" = yes ]; then", " action=blocked-referenced", " elif [ \"$dry_run\" = true ]; then", " if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi", " else", " kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err", " delete_secret_exit=$?", " for _ in $(seq 1 15); do", " current_exists=$(exists_flag)", " if [ \"$current_exists\" != yes ]; then break; fi", " sleep 1", " done", " if [ \"$delete_secret_exit\" -eq 0 ]; then", " if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi", " else", " action=delete-failed", " fi", " fi", "fi", "after_exists=$(exists_flag)", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_exists\"", "printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"", "printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"", "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", "if [ \"$action\" = unsupported-secret ]; then exit 43; fi", "if [ \"$workload_refs_present\" = yes ]; then exit 46; fi", "if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi", "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", ].join("\n"); } function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `db_name=${shellQuote(spec.obsoleteHwpodDbName)}`, `db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "preset=obsolete-platform-db-cleanup", "database_exists_flag() {", " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)", " code=$?", " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", " [ \"$output\" = t ] && printf yes || printf no", "}", "role_exists_flag() {", " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)", " code=$?", " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", " [ \"$output\" = t ] && printf yes || printf no", "}", "before_database_exists=$(database_exists_flag)", "before_database_probe_exit=$?", "before_role_exists=$(role_exists_flag)", "before_role_probe_exit=$?", "action=observed", "mutation=false", "drop_database_exit=", "drop_role_exit=", "before_any=false", "if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi", "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then", " action=probe-failed", "elif [ \"$dry_run\" = true ]; then", " if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi", "else", " if [ \"$before_database_exists\" = yes ]; then", " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'", "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();", "DROP DATABASE IF EXISTS :\"db_name\";", "SQL", " drop_database_exit=$?", " else", " drop_database_exit=0", " fi", " if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then", " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'", "DROP ROLE IF EXISTS :\"db_user\";", "SQL", " drop_role_exit=$?", " elif [ \"$drop_database_exit\" -eq 0 ]; then", " drop_role_exit=0", " else", " drop_role_exit=", " fi", " if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then", " if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi", " else", " action=drop-failed", " fi", "fi", "after_database_exists=$(database_exists_flag)", "after_database_probe_exit=$?", "after_role_exists=$(role_exists_flag)", "after_role_probe_exit=$?", "printf 'database\\t%s\\n' \"$db_name\"", "printf 'role\\t%s\\n' \"$db_user\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"", "printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"", "printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"", "printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"", "printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"", "printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"", "printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"", "printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"", "printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"", "printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"", "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi", "if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi", "if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi", "if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi", ].join("\n"); } function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const isOpenFga = options.preset === "openfga"; const platformEndpointSlice = `${spec.platformPostgresService}-host`; return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`, `database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey)}`, `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, `legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`, `platform_service=${shellQuote(spec.platformPostgresService)}`, `platform_endpointslice=${shellQuote(platformEndpointSlice)}`, `platform_host=${shellQuote(spec.platformPostgresService)}`, `platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`, `db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`, `db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`, `db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost)}`, `selected_key=${shellQuote(options.key ?? "")}`, `preset=${shellQuote(options.preset)}`, "dry_run=true", "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", "resource_exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "endpointslice_exists_flag() { kubectl -n \"$namespace\" get endpointslice \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "uri_has_platform_host=no", "uri_has_db_name=no", "uri_has_db_user=no", "uri_matches_expected() {", " uri=$1", " uri_has_platform_host=no", " uri_has_db_name=no", " uri_has_db_user=no", " case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*) uri_has_platform_host=yes ;; esac", " case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac", " case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac", "}", "exists=$(secret_exists_flag \"$name\")", "legacy_postgres_exists=$(secret_exists_flag \"$legacy_postgres_secret\")", "uri_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", "uri_present=$([ -n \"$uri_b64\" ] && printf yes || printf no)", "uri_bytes=$(decoded_length \"$uri_b64\")", "uri_value=$(decoded_value \"$uri_b64\")", "authn_b64=$(secret_b64_key \"$name\" \"$authn_key\")", "authn_present=$([ -n \"$authn_b64\" ] && printf yes || printf no)", "authn_bytes=$(decoded_length \"$authn_b64\")", "pg_password_b64=$(secret_b64_key \"$name\" \"$postgres_password_key\")", "pg_password_present=$([ -n \"$pg_password_b64\" ] && printf yes || printf no)", "pg_password_bytes=$(decoded_length \"$pg_password_b64\")", "platform_service_exists=$(resource_exists_flag service \"$platform_service\")", "platform_endpoints_exists=$(resource_exists_flag endpoints \"$platform_service\")", "platform_endpointslice_exists=$(endpointslice_exists_flag \"$platform_endpointslice\")", "uri_matches_expected \"$uri_value\"", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$name\"", "printf 'key\\t%s\\n' \"$database_url_key\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\tobserved\\n'", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\tfalse\\n'", "printf 'platformDbMode\\ttrue\\n'", "printf 'afterExists\\t%s\\n' \"$exists\"", "printf 'afterDatabaseUrlPresent\\t%s\\n' \"$uri_present\"", "printf 'afterDatabaseUrlBytes\\t%s\\n' \"$uri_bytes\"", "printf 'afterDatastoreUriPresent\\t%s\\n' \"$uri_present\"", "printf 'afterDatastoreUriBytes\\t%s\\n' \"$uri_bytes\"", "printf 'afterAuthnPresent\\t%s\\n' \"$authn_present\"", "printf 'afterAuthnBytes\\t%s\\n' \"$authn_bytes\"", "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$pg_password_present\"", "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$pg_password_bytes\"", "printf 'legacyPostgresSecret\\t%s\\n' \"$legacy_postgres_secret\"", "printf 'legacyPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"", "printf 'afterPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"", "printf 'platformService\\t%s\\n' \"$platform_service\"", "printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"", "printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"", "printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"", "printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"", "printf 'dbName\\t%s\\n' \"$db_name\"", "printf 'dbUser\\t%s\\n' \"$db_user\"", "printf 'dbHost\\t%s\\n' \"$db_host\"", "printf 'dbHostMatchesPlatform\\t%s\\n' \"$uri_has_platform_host\"", "printf 'dbNameMatchesExpected\\t%s\\n' \"$uri_has_db_name\"", "printf 'dbUserMatchesExpected\\t%s\\n' \"$uri_has_db_user\"", "uri_value=", "if [ \"$platform_service_exists\" != yes ]; then exit 44; fi", ].join("\n"); } function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `openfga_secret=${shellQuote(spec.openFgaSecret)}`, `postgres_secret=${shellQuote(spec.postgresSecret)}`, `postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`, `postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`, `selected_key=${shellQuote(options.key ?? "")}`, `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, `datastore_uri_key=${shellQuote(OPENFGA_DATASTORE_URI_KEY)}`, `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, `db_name=${shellQuote(spec.openFgaDbName)}`, `db_user=${shellQuote(spec.openFgaDbUser)}`, `db_host=${shellQuote(spec.openFgaDbHost)}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, "preset=openfga", "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", "probe_db() {", " role_result=unknown", " database_result=unknown", " probe_exit=missing-postgres-admin-secret", " if [ -n \"$postgres_admin_password\" ]; then", " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", " role_exit=$?", " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", " database_exit=$?", " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", " fi", "}", "before_exists=$(secret_exists_flag \"$openfga_secret\")", "before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", "before_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", "before_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", "before_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", "before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)", "before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)", "before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)", "before_authn_bytes=$(decoded_length \"$before_authn_b64\")", "before_uri_bytes=$(decoded_length \"$before_uri_b64\")", "before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")", "authn_value=$(decoded_value \"$before_authn_b64\")", "datastore_uri=$(decoded_value \"$before_uri_b64\")", "pg_password=$(decoded_value \"$before_pg_password_b64\")", "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", "probe_db", "db_role_exists_before=$role_result", "db_database_exists_before=$database_result", "db_probe_exit_before=$probe_exit", "action=observed", "mutation=false", "postgres_secret_exit=", "postgres_rollout_exit=", "apply_exit=", "db_ensure_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_secret=false", " [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true", " [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true", " [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true", " missing_db=false", " [ \"$db_role_exists_before\" = t ] || missing_db=true", " [ \"$db_database_exists_before\" = t ] || missing_db=true", " if [ \"$dry_run\" = true ]; then", " if [ \"$before_postgres_exists\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", " elif ! command -v openssl >/dev/null 2>&1; then", " action=openssl-missing; apply_exit=127", " else", " [ -n \"$postgres_admin_password\" ] || postgres_admin_password=$(openssl rand -hex 24)", " kubectl -n \"$namespace\" create secret generic \"$postgres_secret\" --from-literal=\"POSTGRES_PASSWORD=$postgres_admin_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " postgres_secret_exit=$?", " if [ \"$postgres_secret_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status \"statefulset/$postgres_statefulset\" --timeout=120s >/tmp/hwlab-postgres-rollout.out 2>/tmp/hwlab-postgres-rollout.err", " postgres_rollout_exit=$?", " fi", " if [ \"$postgres_secret_exit\" -ne 0 ]; then action=postgres-secret-failed; apply_exit=$postgres_secret_exit", " elif [ \"$postgres_rollout_exit\" -ne 0 ]; then action=postgres-rollout-failed; apply_exit=$postgres_rollout_exit", " else", " [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)", " [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)", " [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"", " kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " apply_exit=$?", " if [ \"$apply_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'", "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", "\\gexec", "ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';", "SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')", "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')", "\\gexec", "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", "SQL", " db_ensure_exit=$?", " if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi", " else action=apply-failed; fi", " fi", " fi", "fi", "after_exists=$(secret_exists_flag \"$openfga_secret\")", "after_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", "after_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", "after_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", "after_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", "after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)", "after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)", "after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)", "after_authn_bytes=$(decoded_length \"$after_authn_b64\")", "after_uri_bytes=$(decoded_length \"$after_uri_b64\")", "after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")", "probe_db", "db_role_exists_after=$role_result", "db_database_exists_after=$database_result", "db_probe_exit_after=$probe_exit", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$openfga_secret\"", "printf 'key\\t%s\\n' \"$selected_key\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeExists\\t%s\\n' \"$before_exists\"", "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterPostgresSecretExists\\t%s\\n' \"$after_postgres_exists\"", "printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"", "printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"", "printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"", "printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"", "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"", "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"", "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", "printf 'postgresSecretExitCode\\t%s\\n' \"$postgres_secret_exit\"", "printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", "authn_value= datastore_uri= pg_password= postgres_admin_password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", ].join("\n"); } function masterAdminApiKeySecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(spec.masterAdminApiKeySecret)}`, `api_key_name=${shellQuote(MASTER_ADMIN_API_KEY_KEY)}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, "preset=master-server-admin-api-key", "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-12; value=; fi; }", "before_exists=$(secret_exists_flag)", "before_api_key_b64=$(secret_b64_key \"$api_key_name\")", "before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)", "before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")", "action=observed", "mutation=false", "apply_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_secret=false", " [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true", " if [ \"$dry_run\" = true ]; then", " if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi", " else", " api_key=$(cat)", " case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac", " if [ -z \"$apply_exit\" ]; then", " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " apply_exit=$?", " if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi", " fi", " api_key=", " fi", "fi", "after_exists=$(secret_exists_flag)", "after_api_key_b64=$(secret_b64_key \"$api_key_name\")", "after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)", "after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")", "after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$name\"", "printf 'key\\t%s\\n' \"$api_key_name\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"", "printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"", "printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", ].join("\n"); } function codeAgentProviderSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(spec.codeAgentProviderSecret)}`, `source_namespace=${shellQuote(spec.codeAgentProviderSourceNamespace)}`, `source_name=${shellQuote(spec.codeAgentProviderSourceSecret)}`, `openai_key=${shellQuote(CODE_AGENT_PROVIDER_OPENAI_KEY)}`, `opencode_key=${shellQuote(CODE_AGENT_PROVIDER_OPENCODE_KEY)}`, `selected_key=${shellQuote(options.key ?? "")}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, "preset=code-agent-provider", "secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "before_exists=$(secret_exists_flag \"$namespace\" \"$name\")", "before_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")", "before_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")", "source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")", "source_openai_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$openai_key\")", "source_opencode_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$opencode_key\")", "before_openai_present=$([ -n \"$before_openai_b64\" ] && printf yes || printf no)", "before_opencode_present=$([ -n \"$before_opencode_b64\" ] && printf yes || printf no)", "source_openai_present=$([ -n \"$source_openai_b64\" ] && printf yes || printf no)", "source_opencode_present=$([ -n \"$source_opencode_b64\" ] && printf yes || printf no)", "before_openai_bytes=$(decoded_length \"$before_openai_b64\")", "before_opencode_bytes=$(decoded_length \"$before_opencode_b64\")", "source_openai_bytes=$(decoded_length \"$source_openai_b64\")", "source_opencode_bytes=$(decoded_length \"$source_opencode_b64\")", "action=observed", "mutation=false", "apply_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_target=false", " [ \"$before_exists\" = yes ] && { [ \"$before_openai_bytes\" -gt 0 ] || [ \"$before_opencode_bytes\" -gt 0 ]; } || missing_target=true", " missing_source=false", " [ \"$source_exists\" = yes ] && { [ \"$source_openai_bytes\" -gt 0 ] || [ \"$source_opencode_bytes\" -gt 0 ]; } || missing_source=true", " if [ \"$missing_source\" = true ]; then", " action=source-missing-provider-key", " apply_exit=44", " elif [ \"$dry_run\" = true ]; then", " if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi", " elif [ \"$missing_target\" = false ]; then", " action=kept", " else", " cat </dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", "probe_db() {", " role_result=unknown", " database_result=unknown", " probe_exit=missing-postgres-admin-secret", " if [ -n \"$postgres_admin_password\" ]; then", " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", " role_exit=$?", " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", " database_exit=$?", " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", " fi", "}", "before_exists=$(secret_exists_flag \"$name\")", "before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", "before_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", "before_url_present=$([ -n \"$before_url_b64\" ] && printf yes || printf no)", "before_url_bytes=$(decoded_length \"$before_url_b64\")", "database_url=$(decoded_value \"$before_url_b64\")", "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", "postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)", "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", "probe_db", "db_role_exists_before=$role_result", "db_database_exists_before=$database_result", "db_probe_exit_before=$probe_exit", "action=observed", "mutation=false", "apply_exit=", "db_ensure_exit=", "rollout_restart_exit=", "rollout_status_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_secret=false", " [ \"$before_url_present\" = yes ] && [ \"$before_url_bytes\" -gt 0 ] || missing_secret=true", " missing_db=false", " [ \"$db_role_exists_before\" = t ] || missing_db=true", " [ \"$db_database_exists_before\" = t ] || missing_db=true", " if [ \"$dry_run\" = true ]; then", " if [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", " elif [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ -z \"$postgres_admin_password\" ]; then", " action=postgres-admin-secret-missing", " apply_exit=44", " elif [ \"$missing_secret\" = false ] && [ \"$missing_db\" = false ]; then", " action=kept", " else", " [ -n \"$database_url\" ] || database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"", " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$database_url_key=$database_url\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " apply_exit=$?", " if [ \"$apply_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$postgres_admin_password\" >/tmp/hwlab-cloud-api-db-psql.out 2>/tmp/hwlab-cloud-api-db-psql.err <<'SQL'", "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", "\\gexec", "ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';", "SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')", "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')", "\\gexec", "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", "SQL", " db_ensure_exit=$?", " if [ \"$db_ensure_exit\" -eq 0 ]; then", " if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then", " kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-cloud-api-rollout-restart.out 2>/tmp/hwlab-cloud-api-rollout-restart.err", " rollout_restart_exit=$?", " if [ \"$rollout_restart_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-cloud-api-rollout-status.out 2>/tmp/hwlab-cloud-api-rollout-status.err", " rollout_status_exit=$?", " fi", " fi", " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", " else action=ensured; mutation=true; fi", " else action=db-ensure-failed; fi", " else action=apply-failed; fi", " fi", "fi", "after_exists=$(secret_exists_flag \"$name\")", "after_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", "after_url_present=$([ -n \"$after_url_b64\" ] && printf yes || printf no)", "after_url_bytes=$(decoded_length \"$after_url_b64\")", "probe_db", "db_role_exists_after=$role_result", "db_database_exists_after=$database_result", "db_probe_exit_after=$probe_exit", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$name\"", "printf 'key\\t%s\\n' \"$database_url_key\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeExists\\t%s\\n' \"$before_exists\"", "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", "printf 'beforeDatabaseUrlPresent\\t%s\\n' \"$before_url_present\"", "printf 'beforeDatabaseUrlBytes\\t%s\\n' \"$before_url_bytes\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterDatabaseUrlPresent\\t%s\\n' \"$after_url_present\"", "printf 'afterDatabaseUrlBytes\\t%s\\n' \"$after_url_bytes\"", "printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"", "printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"", "printf 'dbName\\t%s\\n' \"$db_name\"", "printf 'dbUser\\t%s\\n' \"$db_user\"", "printf 'dbHost\\t%s\\n' \"$db_host\"", "printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"", "printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"", "printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"", "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", "database_url= postgres_admin_password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", ].join("\n"); } function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { const fields = keyValueLinesFromText(text); if (fields.preset === "owned-postgres-cleanup") { const absent = fields.afterStatefulSetExists !== "yes" && fields.afterServiceExists !== "yes" && fields.afterConfigMapExists !== "yes" && fields.afterSecretExists !== "yes" && fields.afterPvcExists !== "yes"; const platformServiceReady = fields.platformServiceExists === "yes"; return { ok: commandOk && absent && platformServiceReady, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.postgresSecret, statefulSet: fields.statefulSet || spec.postgresStatefulSet, service: fields.service || spec.postgresSecret, configMap: fields.configMap || `${spec.postgresSecret}-init`, pvc: fields.pvc || `data-${spec.postgresSecret}-0`, preset: "owned-postgres-cleanup", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", before: { statefulSetExists: fields.beforeStatefulSetExists === "yes", serviceExists: fields.beforeServiceExists === "yes", configMapExists: fields.beforeConfigMapExists === "yes", secretExists: fields.beforeSecretExists === "yes", pvcExists: fields.beforePvcExists === "yes", pvcPhase: fields.beforePvcPhase || null, persistentVolume: fields.beforePersistentVolume || null, }, after: { statefulSetExists: fields.afterStatefulSetExists === "yes", serviceExists: fields.afterServiceExists === "yes", configMapExists: fields.afterConfigMapExists === "yes", secretExists: fields.afterSecretExists === "yes", pvcExists: fields.afterPvcExists === "yes", pvcPhase: fields.afterPvcPhase || null, persistentVolume: fields.afterPersistentVolume || null, }, platformService: { name: "g14-platform-postgres", exists: platformServiceReady, }, deleteStatefulSetExitCode: numericField(fields.deleteStatefulSetExitCode), deleteServiceExitCode: numericField(fields.deleteServiceExitCode), deleteConfigMapExitCode: numericField(fields.deleteConfigMapExitCode), deleteSecretExitCode: numericField(fields.deleteSecretExitCode), deletePvcExitCode: numericField(fields.deletePvcExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: absent ? `${fields.statefulSet || spec.postgresStatefulSet}, ${fields.service || spec.postgresSecret}, ${fields.configMap || `${spec.postgresSecret}-init`}, ${fields.secret || spec.postgresSecret}, and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent` : `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`, }; } if (fields.preset === "obsolete-secret-cleanup") { const absent = fields.afterSecretExists !== "yes"; const refsAbsent = fields.workloadRefsPresent !== "yes"; const dryRun = fields.dryRun === "true"; return { ok: commandOk && refsAbsent && (dryRun || absent), namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.obsoleteHwpodDbSecret, preset: "obsolete-secret-cleanup", action: fields.action || null, dryRun, mutation: fields.mutation === "true", before: { secretExists: fields.beforeSecretExists === "yes", }, after: { secretExists: fields.afterSecretExists === "yes", }, workloadRefs: { present: fields.workloadRefsPresent === "yes", preview: fields.workloadRefsPreview || "", }, deleteSecretExitCode: numericField(fields.deleteSecretExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: refsAbsent && (dryRun || absent) ? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}` : `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`, }; } if (fields.preset === "master-server-admin-api-key") { const afterBytes = numericField(fields.afterApiKeyBytes); const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.masterAdminApiKeySecret, preset: "master-server-admin-api-key", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } }, applyExitCode: numericField(fields.applyExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} exists` : `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} missing`, }; } if (fields.preset === "code-agent-provider") { const beforeOpenaiBytes = numericField(fields.beforeOpenaiBytes); const beforeOpencodeBytes = numericField(fields.beforeOpencodeBytes); const sourceOpenaiBytes = numericField(fields.sourceOpenaiBytes); const sourceOpencodeBytes = numericField(fields.sourceOpencodeBytes); const afterOpenaiBytes = numericField(fields.afterOpenaiBytes); const afterOpencodeBytes = numericField(fields.afterOpencodeBytes); const openaiReady = fields.afterOpenaiPresent === "yes" && typeof afterOpenaiBytes === "number" && afterOpenaiBytes > 0; const opencodeReady = fields.afterOpencodePresent === "yes" && typeof afterOpencodeBytes === "number" && afterOpencodeBytes > 0; const healthy = fields.afterExists === "yes" && (openaiReady || opencodeReady); return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.codeAgentProviderSecret, preset: "code-agent-provider", source: { namespace: fields.sourceNamespace || spec.codeAgentProviderSourceNamespace, secret: fields.sourceSecret || spec.codeAgentProviderSourceSecret, exists: fields.sourceExists === "yes", openaiApiKey: { keyPresent: fields.sourceOpenaiPresent === "yes", valueBytes: sourceOpenaiBytes }, opencodeApiKey: { keyPresent: fields.sourceOpencodePresent === "yes", valueBytes: sourceOpencodeBytes }, }, selectedKey: fields.selectedKey || null, action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", before: { exists: fields.beforeExists === "yes", openaiApiKey: { keyPresent: fields.beforeOpenaiPresent === "yes", valueBytes: beforeOpenaiBytes }, opencodeApiKey: { keyPresent: fields.beforeOpencodePresent === "yes", valueBytes: beforeOpencodeBytes }, }, after: { exists: fields.afterExists === "yes", openaiApiKey: { keyPresent: fields.afterOpenaiPresent === "yes", valueBytes: afterOpenaiBytes }, opencodeApiKey: { keyPresent: fields.afterOpencodePresent === "yes", valueBytes: afterOpencodeBytes }, requiredAnyProviderKeyPresent: openaiReady || opencodeReady, }, applyExitCode: numericField(fields.applyExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`, }; } if (fields.preset === "cloud-api-db") { const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes); const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes); if (fields.platformDbMode === "true") { const keysHealthy = fields.afterExists === "yes" && fields.afterDatabaseUrlPresent === "yes" && typeof afterUrlBytes === "number" && afterUrlBytes > 0; const platformBridgeHealthy = fields.platformServiceExists === "yes" && fields.platformEndpointsExists !== "yes" && fields.platformEndpointSliceExists === "yes"; const uriHealthy = fields.dbHostMatchesPlatform === "yes" && fields.dbNameMatchesExpected === "yes" && fields.dbUserMatchesExpected === "yes"; const healthy = keysHealthy && platformBridgeHealthy && uriHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.cloudApiDbSecret, key: fields.key || spec.cloudApiDbKey, preset: "cloud-api-db", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", platformDbMode: true, after: { exists: fields.afterExists === "yes", databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes }, }, legacyPostgresSecret: { name: fields.legacyPostgresSecret || spec.postgresSecret, exists: fields.legacyPostgresSecretExists === "yes", }, platformService: { name: fields.platformService || spec.platformPostgresService, exists: fields.platformServiceExists === "yes", endpointsExist: fields.platformEndpointsExists === "yes", legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes", endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`, endpointSliceExists: fields.platformEndpointSliceExists === "yes", }, dbName: fields.dbName || spec.cloudApiDbName, dbUser: fields.dbUser || spec.cloudApiDbUser, dbHost: fields.dbHost || spec.cloudApiDbHost, dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes", dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes", dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes", exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`, }; } const keysHealthy = fields.afterExists === "yes" && fields.afterDatabaseUrlPresent === "yes" && typeof afterUrlBytes === "number" && afterUrlBytes > 0; const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; const healthy = keysHealthy && databaseHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.cloudApiDbSecret, key: fields.key || spec.cloudApiDbKey, preset: "cloud-api-db", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", before: { exists: fields.beforeExists === "yes", postgresSecretExists: fields.beforePostgresSecretExists === "yes", databaseUrl: { keyPresent: fields.beforeDatabaseUrlPresent === "yes", valueBytes: beforeUrlBytes }, database: { roleExists: fields.dbRoleExistsBefore || "unknown", databaseExists: fields.dbDatabaseExistsBefore || "unknown", probeExitCode: fields.dbProbeExitCodeBefore || null, }, }, after: { exists: fields.afterExists === "yes", databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes }, database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null, }, }, postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes", postgresSecret: fields.postgresSecret || spec.postgresSecret, dbName: fields.dbName || spec.cloudApiDbName, dbUser: fields.dbUser || spec.cloudApiDbUser, dbHost: fields.dbHost || spec.cloudApiDbHost, cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, applyExitCode: numericField(fields.applyExitCode), dbEnsureExitCode: numericField(fields.dbEnsureExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`, }; } const afterAuthnBytes = numericField(fields.afterAuthnBytes); const afterUriBytes = numericField(fields.afterDatastoreUriBytes); const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes); if (fields.platformDbMode === "true") { const keysHealthy = fields.afterExists === "yes" && fields.afterAuthnPresent === "yes" && fields.afterDatastoreUriPresent === "yes" && typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && typeof afterUriBytes === "number" && afterUriBytes > 0; const platformBridgeHealthy = fields.platformServiceExists === "yes" && fields.platformEndpointsExists !== "yes" && fields.platformEndpointSliceExists === "yes"; const uriHealthy = fields.dbHostMatchesPlatform === "yes" && fields.dbNameMatchesExpected === "yes" && fields.dbUserMatchesExpected === "yes"; const healthy = keysHealthy && platformBridgeHealthy && uriHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.openFgaSecret, preset: fields.preset || "openfga", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", platformDbMode: true, after: { exists: fields.afterExists === "yes", authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, }, legacyPostgresSecret: { name: fields.legacyPostgresSecret || spec.postgresSecret, exists: fields.legacyPostgresSecretExists === "yes", }, platformService: { name: fields.platformService || spec.platformPostgresService, exists: fields.platformServiceExists === "yes", endpointsExist: fields.platformEndpointsExists === "yes", legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes", endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`, endpointSliceExists: fields.platformEndpointSliceExists === "yes", }, dbName: fields.dbName || spec.openFgaDbName, dbUser: fields.dbUser || spec.openFgaDbUser, dbHost: fields.dbHost || spec.openFgaDbHost, dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes", dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes", dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes", exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.platformService || spec.platformPostgresService}` : `${fields.secret || spec.openFgaSecret} datastore-uri is not aligned to platform DB`, }; } const keysHealthy = fields.afterExists === "yes" && fields.afterPostgresSecretExists === "yes" && fields.afterAuthnPresent === "yes" && fields.afterDatastoreUriPresent === "yes" && fields.afterPostgresPasswordPresent === "yes" && typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && typeof afterUriBytes === "number" && afterUriBytes > 0 && typeof afterPasswordBytes === "number" && afterPasswordBytes > 0; const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; const healthy = keysHealthy && databaseHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.openFgaSecret, preset: fields.preset || "openfga", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", after: { exists: fields.afterExists === "yes", postgresSecretExists: fields.afterPostgresSecretExists === "yes", authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null }, }, postgresSecretExitCode: numericField(fields.postgresSecretExitCode), postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode), applyExitCode: numericField(fields.applyExitCode), dbEnsureExitCode: numericField(fields.dbEnsureExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.openFgaSecret} keys and Postgres database exist` : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`, }; } function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { const fields = keyValueLinesFromText(text); const dryRun = fields.dryRun === "true"; const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown"; const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown"; const probesOk = fields.beforeDatabaseExists !== "unknown" && fields.beforeRoleExists !== "unknown" && fields.afterDatabaseExists !== "unknown" && fields.afterRoleExists !== "unknown"; return { ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)), database: fields.database || spec.obsoleteHwpodDbName, role: fields.role || spec.obsoleteHwpodDbUser, preset: "obsolete-platform-db-cleanup", action: fields.action || null, dryRun, mutation: fields.mutation === "true", before: { databaseExists: fields.beforeDatabaseExists === "yes", roleExists: fields.beforeRoleExists === "yes", }, after: { databaseExists: fields.afterDatabaseExists === "yes", roleExists: fields.afterRoleExists === "yes", }, beforeProbeExitCode: { database: numericField(fields.beforeDatabaseProbeExitCode), role: numericField(fields.beforeRoleProbeExitCode), }, afterProbeExitCode: { database: numericField(fields.afterDatabaseProbeExitCode), role: numericField(fields.afterRoleProbeExitCode), }, dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode), dropRoleExitCode: numericField(fields.dropRoleExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: probesOk && (dryRun || (databaseAbsent && roleAbsent)) ? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}` : `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`, }; } export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record { return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane })); } function readMasterAdminApiKey(): { key: string; source: string } { if (!existsSync(MASTER_ADMIN_API_KEY_ENV)) throw new Error(`HWLAB_API_KEY source missing: ${MASTER_ADMIN_API_KEY_ENV}`); const content = readFileSync(MASTER_ADMIN_API_KEY_ENV, "utf8"); const match = content.match(/^HWLAB_API_KEY=(.+)$/m); const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, ""); if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${MASTER_ADMIN_API_KEY_ENV}`); return { key: raw, source: MASTER_ADMIN_API_KEY_ENV }; } function optionValue(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index === -1) return undefined; const value = args[index + 1]; if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); return value; } function requiredOption(args: string[], name: string): string { const value = optionValue(args, name); if (value === undefined) throw new Error(`${name} is required`); return value; } function stripOption(args: string[], name: string): string[] { return stripOptions(args, [name]); } function stripOptions(args: string[], names: readonly string[]): string[] { const remove = new Set(names); const without: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; if (remove.has(arg)) { if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1; continue; } without.push(arg); } return without; } function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const raw = optionValue(args, name); if (raw === undefined) return defaultValue; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return Math.min(value, maxValue); } function assertLane(value: string): void { if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`); } function assertNodeId(value: string): void { if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`); } function shellQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; } function statusText(result: CommandResult): string { return result.stdout || result.stderr; } function keyValueLinesFromText(text: string): Record { const fields: Record = {}; for (const line of text.split(/\r?\n/u)) { const index = line.indexOf("\t"); if (index <= 0) continue; fields[line.slice(0, index)] = line.slice(index + 1); } return fields; } function numericField(value: string | undefined): number | null { if (value === undefined || value === "") return null; const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function splitWhitespaceField(value: string | undefined): string[] { if (!value) return []; return value.split(/\s+/u).filter(Boolean); } function compactCommandResult(result: CommandResult): Record { return { command: compactCommand(result.command), exitCode: result.exitCode, stdoutBytes: result.stdout.length, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), timedOut: result.timedOut, }; } function compactCommand(command: string[]): string[] { const scriptIndex = command.indexOf("--"); if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "