diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml index 8f9df1e2..9af4c4c2 100644 --- a/config/platform-db/postgres-pk01.yaml +++ b/config/platform-db/postgres-pk01.yaml @@ -230,36 +230,6 @@ postgres: user: agentrun_v02 address: 74.48.78.17/32 method: scram-sha-256 - - type: hostssl - database: hwlab_d601_v03 - user: hwlab_d601_v03_app - address: 10.0.8.0/22 - method: scram-sha-256 - - type: hostssl - database: postgres - user: hwlab_d601_v03_app - address: 10.0.8.0/22 - method: scram-sha-256 - - type: hostssl - database: hwlab_d601_v03 - user: hwlab_d601_v03_app - address: 36.49.29.0/24 - method: scram-sha-256 - - type: hostssl - database: postgres - user: hwlab_d601_v03_app - address: 36.49.29.0/24 - method: scram-sha-256 - - type: hostssl - database: hwlab_d601_v03 - user: hwlab_d601_v03_app - address: 74.48.78.17/32 - method: scram-sha-256 - - type: hostssl - database: postgres - user: hwlab_d601_v03_app - address: 74.48.78.17/32 - method: scram-sha-256 secrets: source: master-local @@ -321,21 +291,6 @@ secrets: AGENTRUN_V02_DB_NAME: agentrun_v02 randomHex: AGENTRUN_V02_DB_PASSWORD: 32 - - name: hwlab-d601-v03-db-credentials - sourceRef: platform-db/hwlab-d601-v03-db.env - type: env - requiredKeys: - - HWLAB_D601_V03_DB_USER - - HWLAB_D601_V03_DB_PASSWORD - - HWLAB_D601_V03_DB_NAME - createIfMissing: - enabled: true - values: - HWLAB_D601_V03_DB_USER: hwlab_d601_v03_app - HWLAB_D601_V03_DB_NAME: hwlab_d601_v03 - randomHex: - HWLAB_D601_V03_DB_PASSWORD: 32 - objects: roles: - name: sub2api @@ -374,15 +329,6 @@ objects: createdb: false createrole: false superuser: false - - name: hwlab_d601_v03_app - passwordRef: - sourceRef: platform-db/hwlab-d601-v03-db.env - key: HWLAB_D601_V03_DB_PASSWORD - login: true - attributes: - createdb: false - createrole: false - superuser: false databases: - name: sub2api owner: sub2api @@ -404,11 +350,6 @@ objects: encoding: UTF8 locale: C.UTF-8 extensions: [] - - name: hwlab_d601_v03 - owner: hwlab_d601_v03_app - encoding: UTF8 - locale: C.UTF-8 - extensions: [] exports: connectionStrings: @@ -472,36 +413,6 @@ exports: - scope: agentrun-v02 secret: agentrun-v02-mgr-db key: DATABASE_URL - - name: hwlab-d601-v03-cloud-api-database-url - sourceSecretRef: platform-db/hwlab-d601-v03-db.env - render: - envKey: DATABASE_URL - format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require - variables: - PGHOST: 82.156.23.220 - writeToSecretSource: - sourceRef: hwlab/d601-v03-cloud-api-db.env - key: DATABASE_URL - mode: update-or-insert - consumers: - - scope: hwlab-v03 - secret: hwlab-cloud-api-v03-db - key: DATABASE_URL - - name: hwlab-d601-v03-openfga-datastore-uri - sourceSecretRef: platform-db/hwlab-d601-v03-db.env - render: - envKey: DATASTORE_URI - format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require - variables: - PGHOST: 82.156.23.220 - writeToSecretSource: - sourceRef: hwlab/d601-v03-openfga-db.env - key: DATASTORE_URI - mode: update-or-insert - consumers: - - scope: hwlab-v03 - secret: hwlab-v03-openfga - key: DATASTORE_URI backup: phase: minimum-restoreable @@ -541,9 +452,6 @@ observability: - kind: psql-app-role database: agentrun_v02 user: agentrun_v02 - - kind: psql-app-role - database: hwlab_d601_v03 - user: hwlab_d601_v03_app - kind: disk-free path: /var/lib/postgresql/16/main minFreeGiB: 10 diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 76dc05f0..e4661f78 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -12244,6 +12244,30 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "parse_database_url() {", + " uri=$1", + " uri_host= uri_user= uri_database= uri_sslmode= uri_password_present=no", + " if [ -n \"$uri\" ]; then", + " rest=${uri#*://}", + " rest_no_query=${rest%%\\?*}", + " auth=${rest_no_query%@*}", + " host_path=${rest_no_query#*@}", + " if [ \"$host_path\" != \"$rest_no_query\" ]; then", + " uri_user=${auth%%:*}", + " if [ \"$auth\" != \"$uri_user\" ]; then uri_password_present=yes; fi", + " else", + " host_path=$rest_no_query", + " fi", + " host_port=${host_path%%/*}", + " uri_host=${host_port%%:*}", + " uri_database=${host_path#*/}", + " if [ \"$uri_database\" = \"$host_path\" ]; then uri_database=; fi", + " uri_database=${uri_database%%\\?*}", + " case \"$uri\" in", + " *sslmode=*) uri_sslmode=${uri#*sslmode=}; uri_sslmode=${uri_sslmode%%\\&*} ;;", + " esac", + " fi", + "}", "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", "probe_db() {", " role_result=unknown", @@ -12271,6 +12295,12 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec "before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")", "authn_value=$(decoded_value \"$before_authn_b64\")", "datastore_uri=$(decoded_value \"$before_uri_b64\")", + "parse_database_url \"$datastore_uri\"", + "before_uri_host=$uri_host", + "before_uri_user=$uri_user", + "before_uri_database=$uri_database", + "before_uri_sslmode=$uri_sslmode", + "before_uri_password_present=$uri_password_present", "pg_password=$(decoded_value \"$before_pg_password_b64\")", "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", "probe_db", @@ -12404,6 +12434,17 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec "after_authn_bytes=$(decoded_length \"$after_authn_b64\")", "after_uri_bytes=$(decoded_length \"$after_uri_b64\")", "after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")", + "after_datastore_uri=$(decoded_value \"$after_uri_b64\")", + "parse_database_url \"$after_datastore_uri\"", + "after_uri_host=$uri_host", + "after_uri_user=$uri_user", + "after_uri_database=$uri_database", + "after_uri_sslmode=$uri_sslmode", + "after_uri_password_present=$uri_password_present", + "expected_uri_prefix=\"postgres://$db_user:\"", + "expected_uri_suffix=\"@$db_host:5432/$db_name?sslmode=disable\"", + "case \"$datastore_uri\" in \"$expected_uri_prefix\"*\"$expected_uri_suffix\") before_uri_matches_expected=yes ;; *) before_uri_matches_expected=no ;; esac", + "case \"$after_datastore_uri\" in \"$expected_uri_prefix\"*\"$expected_uri_suffix\") after_uri_matches_expected=yes ;; *) after_uri_matches_expected=no ;; esac", "probe_db", "db_role_exists_after=$role_result", "db_database_exists_after=$database_result", @@ -12423,6 +12464,18 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec "printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"", "printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"", "printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"", + "printf 'beforeDatastoreUriHost\\t%s\\n' \"$before_uri_host\"", + "printf 'beforeDatastoreUriUser\\t%s\\n' \"$before_uri_user\"", + "printf 'beforeDatastoreUriDatabase\\t%s\\n' \"$before_uri_database\"", + "printf 'beforeDatastoreUriSslmode\\t%s\\n' \"$before_uri_sslmode\"", + "printf 'beforeDatastoreUriPasswordPresent\\t%s\\n' \"$before_uri_password_present\"", + "printf 'beforeDatastoreUriMatchesExpected\\t%s\\n' \"$before_uri_matches_expected\"", + "printf 'afterDatastoreUriHost\\t%s\\n' \"$after_uri_host\"", + "printf 'afterDatastoreUriUser\\t%s\\n' \"$after_uri_user\"", + "printf 'afterDatastoreUriDatabase\\t%s\\n' \"$after_uri_database\"", + "printf 'afterDatastoreUriSslmode\\t%s\\n' \"$after_uri_sslmode\"", + "printf 'afterDatastoreUriPasswordPresent\\t%s\\n' \"$after_uri_password_present\"", + "printf 'afterDatastoreUriMatchesExpected\\t%s\\n' \"$after_uri_matches_expected\"", "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"", "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"", "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", @@ -12438,7 +12491,7 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec "printf 'migrateWaitExitCode\\t%s\\n' \"$migrate_wait_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", - "authn_value= datastore_uri= pg_password= postgres_admin_password=", + "authn_value= datastore_uri= after_datastore_uri= pg_password= postgres_admin_password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", "if [ -n \"$migrate_apply_exit\" ] && [ \"$migrate_apply_exit\" != 0 ]; then exit \"$migrate_apply_exit\"; fi", @@ -12881,6 +12934,7 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS `db_user=${shellQuote(spec.cloudApiDbUser)}`, `db_host=${shellQuote(spec.cloudApiDbHost)}`, `cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`, + "db_consumer_deployments=\"hwlab-cloud-api hwlab-user-billing hwlab-workbench-runtime\"", `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, @@ -12908,6 +12962,12 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS "before_url_present=$([ -n \"$before_url_b64\" ] && printf yes || printf no)", "before_url_bytes=$(decoded_length \"$before_url_b64\")", "database_url=$(decoded_value \"$before_url_b64\")", + "parse_database_url \"$database_url\"", + "before_url_host=$uri_host", + "before_url_user=$uri_user", + "before_url_database=$uri_database", + "before_url_sslmode=$uri_sslmode", + "before_url_password_present=$uri_password_present", "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", "postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)", "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", @@ -12957,11 +13017,19 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS " db_ensure_exit=$?", " if [ \"$db_ensure_exit\" -eq 0 ]; then", " if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then", - " kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-cloud-api-rollout-restart.out 2>/tmp/hwlab-cloud-api-rollout-restart.err", - " rollout_restart_exit=$?", + " rollout_restart_exit=0", + " for deployment in $db_consumer_deployments; do", + " kubectl -n \"$namespace\" rollout restart \"deployment/$deployment\" >/tmp/hwlab-db-consumer-rollout-restart-$deployment.out 2>/tmp/hwlab-db-consumer-rollout-restart-$deployment.err", + " rc=$?", + " if [ \"$rc\" -ne 0 ]; then rollout_restart_exit=$rc; break; fi", + " done", " if [ \"$rollout_restart_exit\" -eq 0 ]; then", - " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-cloud-api-rollout-status.out 2>/tmp/hwlab-cloud-api-rollout-status.err", - " rollout_status_exit=$?", + " rollout_status_exit=0", + " for deployment in $db_consumer_deployments; do", + " kubectl -n \"$namespace\" rollout status \"deployment/$deployment\" --timeout=180s >/tmp/hwlab-db-consumer-rollout-status-$deployment.out 2>/tmp/hwlab-db-consumer-rollout-status-$deployment.err", + " rc=$?", + " if [ \"$rc\" -ne 0 ]; then rollout_status_exit=$rc; break; fi", + " done", " fi", " fi", " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", @@ -12975,6 +13043,17 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS "after_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", "after_url_present=$([ -n \"$after_url_b64\" ] && printf yes || printf no)", "after_url_bytes=$(decoded_length \"$after_url_b64\")", + "after_database_url=$(decoded_value \"$after_url_b64\")", + "parse_database_url \"$after_database_url\"", + "after_url_host=$uri_host", + "after_url_user=$uri_user", + "after_url_database=$uri_database", + "after_url_sslmode=$uri_sslmode", + "after_url_password_present=$uri_password_present", + "expected_url_prefix=\"postgres://$db_user:\"", + "expected_url_suffix=\"@$db_host:5432/$db_name?sslmode=disable\"", + "case \"$database_url\" in \"$expected_url_prefix\"*\"$expected_url_suffix\") before_url_matches_expected=yes ;; *) before_url_matches_expected=no ;; esac", + "case \"$after_database_url\" in \"$expected_url_prefix\"*\"$expected_url_suffix\") after_url_matches_expected=yes ;; *) after_url_matches_expected=no ;; esac", "probe_db", "db_role_exists_after=$role_result", "db_database_exists_after=$database_result", @@ -12990,9 +13069,21 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", "printf 'beforeDatabaseUrlPresent\\t%s\\n' \"$before_url_present\"", "printf 'beforeDatabaseUrlBytes\\t%s\\n' \"$before_url_bytes\"", + "printf 'beforeDatabaseUrlHost\\t%s\\n' \"$before_url_host\"", + "printf 'beforeDatabaseUrlUser\\t%s\\n' \"$before_url_user\"", + "printf 'beforeDatabaseUrlDatabase\\t%s\\n' \"$before_url_database\"", + "printf 'beforeDatabaseUrlSslmode\\t%s\\n' \"$before_url_sslmode\"", + "printf 'beforeDatabaseUrlPasswordPresent\\t%s\\n' \"$before_url_password_present\"", + "printf 'beforeDatabaseUrlMatchesExpected\\t%s\\n' \"$before_url_matches_expected\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterDatabaseUrlPresent\\t%s\\n' \"$after_url_present\"", "printf 'afterDatabaseUrlBytes\\t%s\\n' \"$after_url_bytes\"", + "printf 'afterDatabaseUrlHost\\t%s\\n' \"$after_url_host\"", + "printf 'afterDatabaseUrlUser\\t%s\\n' \"$after_url_user\"", + "printf 'afterDatabaseUrlDatabase\\t%s\\n' \"$after_url_database\"", + "printf 'afterDatabaseUrlSslmode\\t%s\\n' \"$after_url_sslmode\"", + "printf 'afterDatabaseUrlPasswordPresent\\t%s\\n' \"$after_url_password_present\"", + "printf 'afterDatabaseUrlMatchesExpected\\t%s\\n' \"$after_url_matches_expected\"", "printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"", "printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"", "printf 'dbName\\t%s\\n' \"$db_name\"", @@ -13005,11 +13096,12 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", + "printf 'dbConsumerDeployments\\t%s\\n' \"$db_consumer_deployments\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", - "database_url= postgres_admin_password=", + "database_url= after_database_url= postgres_admin_password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", @@ -13117,6 +13209,7 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number mutation: fields.mutation === "true", after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } }, cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, + dbConsumerDeployments: fields.dbConsumerDeployments ? fields.dbConsumerDeployments.split(/\s+/u).filter(Boolean) : [spec.cloudApiDeployment], applyExitCode: numericField(fields.applyExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), @@ -13306,7 +13399,11 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number fields.afterDatabaseUrlPresent === "yes" && typeof afterUrlBytes === "number" && afterUrlBytes > 0; const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; - const healthy = keysHealthy && databaseHealthy; + const expectedDbHost = fields.dbHost || spec.cloudApiDbHost; + const expectedDbUser = fields.dbUser || spec.cloudApiDbUser; + const expectedDbName = fields.dbName || spec.cloudApiDbName; + const actualUrlAligned = fields.afterDatabaseUrlMatchesExpected === "yes"; + const healthy = keysHealthy && databaseHealthy && actualUrlAligned; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, @@ -13319,7 +13416,16 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number before: { exists: fields.beforeExists === "yes", postgresSecretExists: fields.beforePostgresSecretExists === "yes", - databaseUrl: { keyPresent: fields.beforeDatabaseUrlPresent === "yes", valueBytes: beforeUrlBytes }, + databaseUrl: { + keyPresent: fields.beforeDatabaseUrlPresent === "yes", + valueBytes: beforeUrlBytes, + host: fields.beforeDatabaseUrlHost || null, + user: fields.beforeDatabaseUrlUser || null, + database: fields.beforeDatabaseUrlDatabase || null, + sslmode: fields.beforeDatabaseUrlSslmode || null, + passwordPresent: fields.beforeDatabaseUrlPasswordPresent === "yes", + alignedToExpected: fields.beforeDatabaseUrlMatchesExpected === "yes", + }, database: { roleExists: fields.dbRoleExistsBefore || "unknown", databaseExists: fields.dbDatabaseExistsBefore || "unknown", @@ -13328,7 +13434,16 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number }, after: { exists: fields.afterExists === "yes", - databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes }, + databaseUrl: { + keyPresent: fields.afterDatabaseUrlPresent === "yes", + valueBytes: afterUrlBytes, + host: fields.afterDatabaseUrlHost || null, + user: fields.afterDatabaseUrlUser || null, + database: fields.afterDatabaseUrlDatabase || null, + sslmode: fields.afterDatabaseUrlSslmode || null, + passwordPresent: fields.afterDatabaseUrlPasswordPresent === "yes", + alignedToExpected: actualUrlAligned, + }, database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", @@ -13337,9 +13452,17 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number }, postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes", postgresSecret: fields.postgresSecret || spec.postgresSecret, - dbName: fields.dbName || spec.cloudApiDbName, - dbUser: fields.dbUser || spec.cloudApiDbUser, - dbHost: fields.dbHost || spec.cloudApiDbHost, + dbName: expectedDbName, + dbUser: expectedDbUser, + dbHost: expectedDbHost, + expectedDatabaseUrl: { + host: expectedDbHost, + user: expectedDbUser, + database: expectedDbName, + sslmode: "disable", + }, + databaseUrlDrift: !actualUrlAligned, + ...(!actualUrlAligned ? { degradedReason: "cloud-api-db-secret-drift" } : {}), cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, applyExitCode: numericField(fields.applyExitCode), dbEnsureExitCode: numericField(fields.dbEnsureExitCode), @@ -13348,7 +13471,11 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, - summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`, + summary: healthy + ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to expected local database` + : !actualUrlAligned + ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.afterDatabaseUrlHost || "-"} ${fields.afterDatabaseUrlDatabase || "-"} as ${fields.afterDatabaseUrlUser || "-"}, expected ${expectedDbHost} ${expectedDbName} as ${expectedDbUser}` + : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`, }; } const afterAuthnBytes = numericField(fields.afterAuthnBytes); @@ -13414,10 +13541,14 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number fields.afterDatastoreUriPresent === "yes" && fields.afterPostgresPasswordPresent === "yes" && typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && - typeof afterUriBytes === "number" && afterUriBytes > 0 && - typeof afterPasswordBytes === "number" && afterPasswordBytes > 0; + typeof afterUriBytes === "number" && afterUriBytes > 0 && + typeof afterPasswordBytes === "number" && afterPasswordBytes > 0; const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; - const healthy = keysHealthy && databaseHealthy; + const expectedOpenFgaHost = fields.dbHost || spec.openFgaDbHost; + const expectedOpenFgaUser = fields.dbUser || spec.openFgaDbUser; + const expectedOpenFgaDbName = fields.dbName || spec.openFgaDbName; + const datastoreUriAligned = fields.afterDatastoreUriMatchesExpected === "yes"; + const healthy = keysHealthy && databaseHealthy && datastoreUriAligned; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, @@ -13430,10 +13561,27 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number exists: fields.afterExists === "yes", postgresSecretExists: fields.afterPostgresSecretExists === "yes", authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, - datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, + datastoreUri: { + keyPresent: fields.afterDatastoreUriPresent === "yes", + valueBytes: afterUriBytes, + host: fields.afterDatastoreUriHost || null, + user: fields.afterDatastoreUriUser || null, + database: fields.afterDatastoreUriDatabase || null, + sslmode: fields.afterDatastoreUriSslmode || null, + passwordPresent: fields.afterDatastoreUriPasswordPresent === "yes", + alignedToExpected: datastoreUriAligned, + }, postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null }, }, + expectedDatastoreUri: { + host: expectedOpenFgaHost, + user: expectedOpenFgaUser, + database: expectedOpenFgaDbName, + sslmode: "disable", + }, + datastoreUriDrift: !datastoreUriAligned, + ...(!datastoreUriAligned ? { degradedReason: "openfga-datastore-uri-drift" } : {}), postgresSecretExitCode: numericField(fields.postgresSecretExitCode), postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode), applyExitCode: numericField(fields.applyExitCode), @@ -13447,7 +13595,11 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, - summary: healthy ? `${fields.secret || spec.openFgaSecret} keys and Postgres database exist` : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`, + summary: healthy + ? `${fields.secret || spec.openFgaSecret} datastore-uri points to expected local database` + : !datastoreUriAligned + ? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.afterDatastoreUriHost || "-"} ${fields.afterDatastoreUriDatabase || "-"} as ${fields.afterDatastoreUriUser || "-"}, expected ${expectedOpenFgaHost} ${expectedOpenFgaDbName} as ${expectedOpenFgaUser}` + : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`, }; } diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index 3b7aa652..fb1b8ad5 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -1419,7 +1419,7 @@ function compactSpanList(value: unknown, limit: number): unknown[] { return { name: span.name ?? null, service: span.service ?? null, - attributes: compactRecord(attrs, ["failureKind", "terminalStatus", "status", "eventType", "idleMs", "waitingFor", "lastEventLabel", "http.route", "http.status_code", "http.response.status_code", "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome", "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind", "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status", "hwlab.live_builds.artifact_catalog_status"]), + attributes: compactRecord(attrs, ["failureKind", "terminalStatus", "status", "eventType", "idleMs", "waitingFor", "lastEventLabel", "http.route", "http.status_code", "http.response.status_code", "workbench.session_id", "workbench.trace_id", "workbench.turn_id", "workbench.read_model.route", "workbench.read_model.count", "workbench.read_model.family", "workbench.read_model.status", "workbench.read_model.reason", "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome", "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind", "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status", "hwlab.live_builds.artifact_catalog_status"]), }; }); } @@ -2097,6 +2097,10 @@ IMPORTANT_ATTRS = [ "workbench.ui.resource_decoded_body_size", "workbench.ui.resource_next_hop_protocol", "workbench.ui.resource_server_timing", + "workbench.session_id", "workbench.trace_id", "workbench.turn_id", + "workbench.read_model.route", "workbench.read_model.count", + "workbench.read_model.family", "workbench.read_model.status", + "workbench.read_model.reason", "db.system", "db.operation.name", "db.sql.table", "db.query.arg_count", "db.index.expected", "db.pool.max_open", "db.pool.open_connections", "db.pool.in_use", "db.pool.idle", "db.pool.wait_count", @@ -2388,6 +2392,10 @@ IMPORTANT_ATTRS = [ "workbench.ui.resource_decoded_body_size", "workbench.ui.resource_next_hop_protocol", "workbench.ui.resource_server_timing", + "workbench.session_id", "workbench.trace_id", "workbench.turn_id", + "workbench.read_model.route", "workbench.read_model.count", + "workbench.read_model.family", "workbench.read_model.status", + "workbench.read_model.reason", "http.target", "http.url", "url.path", "db.system", "db.operation.name", "db.sql.table", "db.query.arg_count", "db.index.expected", "db.pool.max_open", "db.pool.open_connections", @@ -2812,6 +2820,10 @@ IMPORTANT_ATTRS = [ "workbench.ui.resource_decoded_body_size", "workbench.ui.resource_next_hop_protocol", "workbench.ui.resource_server_timing", + "workbench.session_id", "workbench.trace_id", "workbench.turn_id", + "workbench.read_model.route", "workbench.read_model.count", + "workbench.read_model.family", "workbench.read_model.status", + "workbench.read_model.reason", "db.system", "db.operation.name", "db.sql.table", "db.query.arg_count", "db.index.expected", "db.pool.max_open", "db.pool.open_connections", "db.pool.in_use", "db.pool.idle", "db.pool.wait_count",