diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml index 5434c07f..031c5788 100644 --- a/config/platform-db/postgres-pk01.yaml +++ b/config/platform-db/postgres-pk01.yaml @@ -230,6 +230,36 @@ postgres: user: agentrun_v02 address: 74.48.78.17/32 method: scram-sha-256 + - type: hostssl + database: hwlab_d601_v03 + user: hwlab_d601_v03_app + address: 10.0.8.0/22 + method: scram-sha-256 + - type: hostssl + database: postgres + user: hwlab_d601_v03_app + address: 10.0.8.0/22 + method: scram-sha-256 + - type: hostssl + database: hwlab_d601_v03 + user: hwlab_d601_v03_app + address: 36.49.29.73/32 + method: scram-sha-256 + - type: hostssl + database: postgres + user: hwlab_d601_v03_app + address: 36.49.29.73/32 + method: scram-sha-256 + - type: hostssl + database: hwlab_d601_v03 + user: hwlab_d601_v03_app + address: 74.48.78.17/32 + method: scram-sha-256 + - type: hostssl + database: postgres + user: hwlab_d601_v03_app + address: 74.48.78.17/32 + method: scram-sha-256 secrets: source: master-local @@ -291,6 +321,20 @@ secrets: AGENTRUN_V02_DB_NAME: agentrun_v02 randomHex: AGENTRUN_V02_DB_PASSWORD: 32 + - name: hwlab-d601-v03-db-credentials + sourceRef: platform-db/hwlab-d601-v03-db.env + type: env + requiredKeys: + - HWLAB_D601_V03_DB_USER + - HWLAB_D601_V03_DB_PASSWORD + - HWLAB_D601_V03_DB_NAME + createIfMissing: + enabled: true + values: + HWLAB_D601_V03_DB_USER: hwlab_d601_v03_app + HWLAB_D601_V03_DB_NAME: hwlab_d601_v03 + randomHex: + HWLAB_D601_V03_DB_PASSWORD: 32 objects: roles: @@ -330,6 +374,15 @@ objects: createdb: false createrole: false superuser: false + - name: hwlab_d601_v03_app + passwordRef: + sourceRef: platform-db/hwlab-d601-v03-db.env + key: HWLAB_D601_V03_DB_PASSWORD + login: true + attributes: + createdb: false + createrole: false + superuser: false databases: - name: sub2api owner: sub2api @@ -351,6 +404,11 @@ objects: encoding: UTF8 locale: C.UTF-8 extensions: [] + - name: hwlab_d601_v03 + owner: hwlab_d601_v03_app + encoding: UTF8 + locale: C.UTF-8 + extensions: [] exports: connectionStrings: @@ -414,6 +472,36 @@ exports: - scope: agentrun-v02 secret: agentrun-v02-mgr-db key: DATABASE_URL + - name: hwlab-d601-v03-cloud-api-database-url + sourceSecretRef: platform-db/hwlab-d601-v03-db.env + render: + envKey: DATABASE_URL + format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require + variables: + PGHOST: 82.156.23.220 + writeToSecretSource: + sourceRef: hwlab/d601-v03-cloud-api-db.env + key: DATABASE_URL + mode: update-or-insert + consumers: + - scope: hwlab-v03 + secret: hwlab-cloud-api-v03-db + key: DATABASE_URL + - name: hwlab-d601-v03-openfga-datastore-uri + sourceSecretRef: platform-db/hwlab-d601-v03-db.env + render: + envKey: DATASTORE_URI + format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require&uselibpqcompat=true + variables: + PGHOST: 82.156.23.220 + writeToSecretSource: + sourceRef: hwlab/d601-v03-openfga-db.env + key: DATASTORE_URI + mode: update-or-insert + consumers: + - scope: hwlab-v03 + secret: hwlab-v03-openfga + key: DATASTORE_URI backup: phase: minimum-restoreable @@ -453,6 +541,9 @@ observability: - kind: psql-app-role database: agentrun_v02 user: agentrun_v02 + - kind: psql-app-role + database: hwlab_d601_v03 + user: hwlab_d601_v03_app - kind: disk-free path: /var/lib/postgresql/16/main minFreeGiB: 10 diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 332a9e55..2e6bf429 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -47,7 +47,11 @@ interface RuntimeSecretSpec { lane: string; namespace: string; platformDb: boolean; + runtimeLaneSpec?: HwlabRuntimeLaneSpec; + externalPostgres?: NonNullable; platformPostgresService: string; + platformPostgresEndpointAddress?: string; + platformPostgresEndpointSlice: string; postgresSecret: string; postgresStatefulSet: string; postgresAdminUser: string; @@ -2959,7 +2963,7 @@ function parseSecretOptions(args: string[]): NodeSecretOptions { } if (name === spec.cloudApiDbSecret) { if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`); - if (actionRaw === "ensure" && spec.platformDb) { + if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { @@ -2980,7 +2984,7 @@ function parseSecretOptions(args: string[]): NodeSecretOptions { if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) { throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`); } - if (actionRaw === "ensure" && spec.platformDb) { + if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { @@ -2998,32 +3002,43 @@ function parseSecretOptions(args: string[]): NodeSecretOptions { function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec { const namespace = `hwlab-${input.lane}`; - const platformDb = /^v0*[3-9]\d*$/.test(input.lane); - const platformPostgresService = "g14-platform-postgres"; + const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined; + const externalPostgres = runtimeLaneSpec?.externalPostgres; + const platformDb = externalPostgres !== undefined || /^v0*[3-9]\d*$/.test(input.lane); + const platformPostgresService = externalPostgres?.serviceName ?? "g14-platform-postgres"; const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`; const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`; + const platformPostgresEndpointSlice = `${platformPostgresService}-host`; + const openFgaDbName = externalPostgres?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga"); + const openFgaDbUser = externalPostgres?.openfga.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga"); + const cloudApiDbName = externalPostgres?.database ?? `hwlab_${input.lane}`; + const cloudApiDbUser = externalPostgres?.cloudApi.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`); return { node: input.node, lane: input.lane, namespace, platformDb, + ...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }), + ...(externalPostgres === undefined ? {} : { externalPostgres }), platformPostgresService, + platformPostgresEndpointAddress: externalPostgres?.endpointAddress, + platformPostgresEndpointSlice, postgresSecret: `${namespace}-postgres`, postgresStatefulSet: `${namespace}-postgres`, postgresAdminUser: `hwlab_${input.lane}`, - openFgaSecret: `${namespace}-openfga`, - openFgaDbName: platformDb ? `openfga_${input.lane}` : "hwlab_openfga", - openFgaDbUser: platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga", + openFgaSecret: externalPostgres?.openfga.secretName ?? `${namespace}-openfga`, + openFgaDbName, + openFgaDbUser, openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`, bootstrapAdminSecret: `${namespace}-bootstrap-admin`, bootstrapAdminPasswordHashKey: BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE, bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET, - cloudApiDbSecret: `hwlab-cloud-api-${input.lane}-db`, - cloudApiDbKey: CLOUD_API_DB_KEY, - cloudApiDbName: `hwlab_${input.lane}`, - cloudApiDbUser: platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`, + cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? `hwlab-cloud-api-${input.lane}-db`, + cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? CLOUD_API_DB_KEY, + cloudApiDbName, + cloudApiDbUser, cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, cloudApiDeployment: "hwlab-cloud-api", obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`, @@ -3039,6 +3054,9 @@ function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecret function runNodeSecret(options: NodeSecretOptions): Record { const spec = runtimeSecretSpec(options); if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec); + if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) { + return runExternalPostgresSecretEnsure(options, spec); + } const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun ? readMasterAdminApiKey().key : ""; @@ -3084,6 +3102,9 @@ function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): if (options.action === "cleanup-obsolete") { return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` }; } + if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) { + return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; + } if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) { return { status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`, @@ -3093,6 +3114,50 @@ function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; } +function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { + const runtimeLaneSpec = spec.runtimeLaneSpec; + if (runtimeLaneSpec === undefined) { + return { + ok: false, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-ensure", + mutation: false, + degradedReason: "external-postgres-runtime-lane-spec-missing", + valuesRedacted: true, + }; + } + const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds); + const shouldReadStatus = sync !== null && sync.ok === true; + const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null; + const status = statusResult === null + ? null + : secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec); + const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true); + return { + ok, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-ensure", + status: status ?? sync, + externalPostgresSecretSync: sync, + mutation: sync?.mutation === true, + result: statusResult === null ? null : compactCommandResult(statusResult), + valuesRedacted: true, + next: ok ? undefined : nextSecretCommand(options, spec), + }; +} + function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record { return { mode: exposure.mode, @@ -3921,7 +3986,7 @@ function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const pvc = `data-${spec.postgresSecret}-0`; - const platformService = "g14-platform-postgres"; + const platformService = spec.platformPostgresService; const postgresService = spec.postgresSecret; const postgresConfigMap = `${spec.postgresSecret}-init`; return [ @@ -4176,12 +4241,14 @@ function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: Runti function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const isOpenFga = options.preset === "openfga"; - const platformEndpointSlice = `${spec.platformPostgresService}-host`; + const platformEndpointSlice = spec.platformPostgresEndpointSlice; + const expectedUriHost = spec.platformPostgresEndpointAddress ?? (isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost); + const databaseUrlKey = isOpenFga ? spec.externalPostgres?.openfga.secretKey ?? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey; return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`, - `database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey)}`, + `database_url_key=${shellQuote(databaseUrlKey)}`, `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, `legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`, @@ -4189,9 +4256,10 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS `platform_endpointslice=${shellQuote(platformEndpointSlice)}`, `platform_host=${shellQuote(spec.platformPostgresService)}`, `platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`, + `platform_endpoint_address=${shellQuote(spec.platformPostgresEndpointAddress ?? "")}`, `db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`, `db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`, - `db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost)}`, + `db_host=${shellQuote(expectedUriHost)}`, `selected_key=${shellQuote(options.key ?? "")}`, `preset=${shellQuote(options.preset)}`, "dry_run=true", @@ -4209,7 +4277,10 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS " uri_has_platform_host=no", " uri_has_db_name=no", " uri_has_db_user=no", - " case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*) uri_has_platform_host=yes ;; esac", + " case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*|*\"@$db_host:\"*|*\"@$db_host/\"*) uri_has_platform_host=yes ;; esac", + " if [ -n \"$platform_endpoint_address\" ]; then", + " case \"$uri\" in *\"@$platform_endpoint_address:\"*|*\"@$platform_endpoint_address/\"*) uri_has_platform_host=yes ;; esac", + " fi", " case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac", " case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac", "}", @@ -4254,6 +4325,7 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS "printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"", "printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"", "printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"", + "printf 'platformEndpointAddress\\t%s\\n' \"$platform_endpoint_address\"", "printf 'dbName\\t%s\\n' \"$db_name\"", "printf 'dbUser\\t%s\\n' \"$db_user\"", "printf 'dbHost\\t%s\\n' \"$db_host\"",