From 93273a12ef7b549251f66c7796050c36534272d6 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 12 Jun 2026 05:08:37 +0000 Subject: [PATCH] feat: add PK01 host postgres platform-db CLI --- config/platform-db/postgres-pk01.yaml | 214 +++ scripts/cli.ts | 9 + scripts/src/check.ts | 1 + scripts/src/help.ts | 3 + scripts/src/platform-db.ts | 1792 +++++++++++++++++++++++++ 5 files changed, 2019 insertions(+) create mode 100644 config/platform-db/postgres-pk01.yaml create mode 100644 scripts/src/platform-db.ts diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml new file mode 100644 index 00000000..1fa6d534 --- /dev/null +++ b/config/platform-db/postgres-pk01.yaml @@ -0,0 +1,214 @@ +version: 1 +kind: postgres-host-cluster + +metadata: + id: pk01-platform-postgres + description: PK01 host-native PostgreSQL 16 for UniDesk platform external state + owner: unidesk + relatedIssues: + - 280 + - 281 + +cluster: + role: primary + environment: platform + exposure: private-only + haPhase: standalone-with-backup + futureHaModes: + - primary-standby + - managed-rds + +node: + id: PK01 + route: PK01 + mode: host-systemd + osFamily: ubuntu + requiredHostFacts: + minCpu: 2 + minMemoryGiB: 3.5 + minDiskFreeGiB: 20 + requireSwap: true + swap: + ensure: true + size: 4G + path: /swapfile + +postgres: + package: + manager: apt + version: "16" + repo: pgdg-archive + repoUrl: https://apt-archive.postgresql.org/pub/repos/apt + suite: focal-pgdg + component: main + signingKeyUrl: https://www.postgresql.org/media/keys/ACCC4CF8.asc + signedBy: /usr/share/keyrings/postgresql-pgdg.gpg + sourceList: /etc/apt/sources.list.d/pgdg.list + service: + name: postgresql + enabled: true + state: running + paths: + dataDir: /var/lib/postgresql/16/main + configDir: /etc/postgresql/16/main + logDir: /var/log/postgresql + network: + port: 5432 + listenAddresses: + - 127.0.0.1 + - 10.0.8.3 + - 0.0.0.0 + connectionHost: db.pikapython.com + publicDns: db.pikapython.com + transport: postgres-native-tls + sslmode: require + tls: + enabled: true + mode: self-signed-server-cert + commonName: db.pikapython.com + certFile: /etc/postgresql/16/main/server.crt + keyFile: /etc/postgresql/16/main/server.key + futureClientSslmode: verify-full + firewall: + mode: pg-hba-declared-sources + defaultDeny: true + allowSources: + - id: pk01-local + cidr: 127.0.0.1/32 + purpose: local-admin + - id: pk01-vpc + cidr: 10.0.8.0/22 + purpose: private-vpc-clients + - id: master-server-public + cidr: 74.48.78.17/32 + purpose: admin-and-secret-sync + - id: D601-public + cidr: 36.49.29.73/32 + purpose: platform-infra-standby-app + tuning: + maxConnections: 50 + sharedBuffers: 512MB + effectiveCacheSize: 2GB + workMem: 8MB + maintenanceWorkMem: 128MB + walCompression: true + checkpointCompletionTarget: 0.9 + auth: + passwordEncryption: scram-sha-256 + pgHba: + - type: local + database: all + user: postgres + method: peer + - type: host + database: all + user: all + address: 127.0.0.1/32 + method: scram-sha-256 + - type: hostssl + database: sub2api + user: sub2api + address: 10.0.8.0/22 + method: scram-sha-256 + - type: hostssl + database: sub2api + user: sub2api + address: 74.48.78.17/32 + method: scram-sha-256 + - type: hostssl + database: sub2api + user: sub2api + address: 36.49.29.73/32 + method: scram-sha-256 + +secrets: + source: master-local + root: .state/secrets + entries: + - name: sub2api-db-credentials + sourceRef: platform-db/sub2api-db.env + type: env + requiredKeys: + - SUB2API_DB_USER + - SUB2API_DB_PASSWORD + - SUB2API_DB_NAME + createIfMissing: + enabled: true + values: + SUB2API_DB_USER: sub2api + SUB2API_DB_NAME: sub2api + randomHex: + SUB2API_DB_PASSWORD: 32 + +objects: + roles: + - name: sub2api + passwordRef: + sourceRef: platform-db/sub2api-db.env + key: SUB2API_DB_PASSWORD + login: true + attributes: + createdb: false + createrole: false + superuser: false + databases: + - name: sub2api + owner: sub2api + encoding: UTF8 + locale: C.UTF-8 + extensions: [] + +exports: + connectionStrings: + - name: sub2api-database-url + sourceSecretRef: platform-db/sub2api-db.env + render: + envKey: DATABASE_URL + format: postgresql://$(SUB2API_DB_USER):$(SUB2API_DB_PASSWORD)@$(PGHOST):5432/$(SUB2API_DB_NAME)?sslmode=require + variables: + PGHOST: db.pikapython.com + writeToSecretSource: + sourceRef: platform-infra/sub2api.env + key: DATABASE_URL + mode: update-or-insert + consumers: + - scope: platform-infra + secret: sub2api-secrets + key: DATABASE_URL + +backup: + phase: minimum-restoreable + logicalDump: + enabled: true + schedule: "17 3 * * *" + database: sub2api + retentionDays: 14 + destination: + type: pk01-local + path: /var/backups/unidesk/platform-db/pk01/sub2api + encryption: + enabled: false + futureKeyRef: platform-db/backup-age-recipient.txt + physicalWalArchive: + enabled: false + futureTool: pgbackrest + +observability: + statusChecks: + - kind: systemd-active + service: postgresql + - kind: port-listen + host: 127.0.0.1 + port: 5432 + - kind: psql-local + database: postgres + - kind: psql-app-role + database: sub2api + user: sub2api + - kind: disk-free + path: /var/lib/postgresql/16/main + minFreeGiB: 10 + redaction: + neverPrintValues: true + showFingerprint: true + showKeyNames: true diff --git a/scripts/cli.ts b/scripts/cli.ts index 43d58d5e..c6c34ae8 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -24,6 +24,7 @@ import { runCommanderCommand } from "./src/commander"; import { isHelpToken, rootHelp, serverHelp, sshHelp, staticNamespaceHelp } from "./src/help"; import { runServerCleanupCommand } from "./src/server-cleanup"; import { runGcCommand } from "./src/gc"; +import { runPlatformDbCommand } from "./src/platform-db"; const remoteOptions = extractRemoteCliOptions(process.argv.slice(2)); const args = remoteOptions.args; @@ -339,6 +340,14 @@ async function main(): Promise { return; } + if (top === "platform-db") { + const result = await runPlatformDbCommand(readConfig(), args.slice(1)); + const ok = (result as { ok?: unknown }).ok !== false; + emitJson(commandName, result, ok); + if (!ok) process.exitCode = 1; + return; + } + const config = readConfig(); const autoRemoteCiPublishPlan = autoRemoteCiPublishUserServiceDryRunPlan(config, args); if (autoRemoteCiPublishPlan.enabled && autoRemoteCiPublishPlan.host !== null) { diff --git a/scripts/src/check.ts b/scripts/src/check.ts index 44eac86c..eaa21a04 100644 --- a/scripts/src/check.ts +++ b/scripts/src/check.ts @@ -30,6 +30,7 @@ const syntaxFiles = [ "scripts/src/e2e.ts", "scripts/src/help.ts", "scripts/src/commander.ts", + "scripts/src/platform-db.ts", "scripts/src/recovery-guardrails.ts", "scripts/src/server-cleanup.ts", "scripts/src/remote.ts", diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 66019730..d476b273 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -1,5 +1,6 @@ import { ghHelp } from "./gh"; import { authBrokerHelp } from "./auth-broker"; +import { platformDbHelp } from "./platform-db"; export function rootHelp(): unknown { return { @@ -59,6 +60,7 @@ export function rootHelp(): unknown { { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." }, { command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; session follow-up uses send only and the server decides internal steer vs turn." }, { command: "platform-infra sub2api plan|apply|status|validate|codex-pool", description: "Deploy Sub2API in G14 platform-infra, manage the YAML-controlled Codex upstream pool, expose the unified API, and inspect marker sentinel state with low-noise reports without printing API keys." }, + { command: "platform-db postgres plan|status|apply", description: "Manage YAML-declared host-native PostgreSQL 16 on PK01 for Sub2API/platform state, with PostgreSQL native TLS and redacted secret exports." }, { command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." }, { command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." }, { command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run supports --wait-ms N and retry-run reuses the failed run's schedule." }, @@ -693,6 +695,7 @@ export async function staticNamespaceHelp(args: string[]): Promise (await import("./agentrun")).agentRunHelp(), agentRunHelpSummary()); if (top === "platform-infra") return loadHelp(async () => (await import("./platform-infra")).platformInfraHelp(), platformInfraHelpSummary()); + if (top === "platform-db") return platformDbHelp(); if (top === "hwlab" && (sub === "node" || sub === "nodes")) return loadHelp(async () => (await import("./hwlab-node")).hwlabNodeHelp(), hwlabNodeHelpSummary()); if (top === "hwlab" && sub === "g14") return loadHelp(async () => (await import("./hwlab-g14")).hwlabG14Help(), hwlabG14HelpSummary()); if (top === "hwlab") return loadHelp(async () => (await import("./hwlab-cd")).hwlabHelp(), hwlabHelpSummary()); diff --git a/scripts/src/platform-db.ts b/scripts/src/platform-db.ts new file mode 100644 index 00000000..c68957ab --- /dev/null +++ b/scripts/src/platform-db.ts @@ -0,0 +1,1792 @@ +import { createHash, randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; +import type { UniDeskConfig } from "./config"; +import { repoRoot, rootPath } from "./config"; +import { startJob } from "./jobs"; +import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; + +const defaultConfigPath = "config/platform-db/postgres-pk01.yaml"; +const managedHbaStart = "# BEGIN unidesk managed pk01-platform-postgres"; +const managedHbaEnd = "# END unidesk managed pk01-platform-postgres"; + +interface PlatformDbOptions { + configPath: string; + confirm: boolean; + wait: boolean; + dryRun: boolean; + full: boolean; + raw: boolean; +} + +interface PostgresHostConfig { + configPath: string; + version: number; + kind: "postgres-host-cluster"; + metadata: { + id: string; + relatedIssues: number[]; + }; + cluster: { + role: string; + haPhase: string; + }; + node: { + id: string; + route: string; + mode: "host-systemd"; + requiredHostFacts: { + minCpu: number; + minMemoryGiB: number; + minDiskFreeGiB: number; + requireSwap: boolean; + }; + swap: { + ensure: boolean; + size: string; + path: string; + }; + }; + postgres: { + package: { + manager: "apt"; + version: string; + repo: string; + repoUrl: string; + suite: string; + component: string; + signingKeyUrl: string; + signedBy: string; + sourceList: string; + }; + service: { + name: string; + enabled: boolean; + state: string; + }; + paths: { + dataDir: string; + configDir: string; + logDir: string; + }; + network: { + port: number; + listenAddresses: string[]; + connectionHost: string; + publicDns: string; + transport: "postgres-native-tls"; + sslmode: "require"; + tls: { + enabled: boolean; + mode: string; + commonName: string; + certFile: string; + keyFile: string; + futureClientSslmode: string; + }; + firewall: { + mode: string; + defaultDeny: boolean; + allowSources: Array<{ id: string; cidr: string; purpose: string }>; + }; + }; + tuning: { + maxConnections: number; + sharedBuffers: string; + effectiveCacheSize: string; + workMem: string; + maintenanceWorkMem: string; + walCompression: boolean; + checkpointCompletionTarget: number; + }; + auth: { + passwordEncryption: "scram-sha-256"; + pgHba: PgHbaRule[]; + }; + }; + secrets: { + source: "master-local"; + root: string; + entries: SecretEntryConfig[]; + }; + objects: { + roles: Array<{ + name: string; + passwordRef: { sourceRef: string; key: string }; + login: boolean; + attributes: { createdb: boolean; createrole: boolean; superuser: boolean }; + }>; + databases: Array<{ + name: string; + owner: string; + encoding: string; + locale: string; + extensions: string[]; + }>; + }; + exports: { + connectionStrings: ConnectionStringExportConfig[]; + }; + backup: { + logicalDump: { + enabled: boolean; + schedule: string; + database: string; + retentionDays: number; + destination: { type: string; path: string }; + encryption: { enabled: boolean }; + }; + }; +} + +interface PgHbaRule { + type: "local" | "host" | "hostssl"; + database: string; + user: string; + address?: string; + method: string; +} + +interface SecretEntryConfig { + name: string; + sourceRef: string; + type: "env"; + requiredKeys: string[]; + createIfMissing?: { + enabled: boolean; + values?: Record; + randomHex?: Record; + }; +} + +interface ConnectionStringExportConfig { + name: string; + sourceSecretRef: string; + render: { + envKey: string; + format: string; + variables?: Record; + }; + writeToSecretSource: { + sourceRef: string; + key: string; + mode: "update-or-insert"; + }; + consumers: Array<{ scope: string; secret: string; key: string }>; +} + +interface SecretMaterial { + values: Record; + sourcePath: string; + existedBefore: boolean; + missingBefore: string[]; + action: "none" | "create" | "update" | "blocked"; + fingerprint: string | null; +} + +interface SecretInspection { + ok: boolean; + root: string; + entries: Array<{ + name: string; + sourceRef: string; + sourcePath: string; + exists: boolean; + requiredKeys: string[]; + presentKeys: string[]; + missingKeys: string[]; + action: string; + fingerprint: string | null; + }>; + materials: Map; +} + +interface RemoteFacts { + ok: boolean; + observedAt: string | null; + host: { + hostname: string | null; + osPrettyName: string | null; + osCodename: string | null; + arch: string | null; + cpuCount: number | null; + memoryTotalMiB: number | null; + memoryAvailableMiB: number | null; + swapTotalMiB: number | null; + rootAvailGiB: number | null; + targetDataAvailGiB: number | null; + }; + network: { + port5432Listening: boolean; + port5432Line: string | null; + dns: { + hostname: string; + resolves: boolean; + addresses: string[]; + }; + }; + repo: { + releaseUrl: string; + reachable: boolean; + firstLine: string | null; + }; + postgres: { + psqlVersion: string | null; + postgresVersion: string | null; + packageInstalled: boolean; + serviceActive: boolean; + serviceEnabled: boolean; + dataDirExists: boolean; + configDirExists: boolean; + logDirExists: boolean; + roleExists: boolean; + databaseExists: boolean; + serverVersion: string | null; + sslOn: boolean; + sslCertFile: string | null; + sslKeyFile: string | null; + appConnectionOk: boolean | null; + appConnectionSsl: boolean | null; + appConnectionHost: string | null; + }; + raw?: Record; +} + +interface RemoteJobStart { + ok: boolean; + remoteJobId: string | null; + statePath: string | null; + logPath: string | null; + parsed: Record | null; + capture: Record; +} + +export function platformDbHelp(): unknown { + return { + command: "platform-db postgres plan|status|apply", + output: "json", + usage: [ + `bun scripts/cli.ts platform-db postgres plan --config ${defaultConfigPath}`, + `bun scripts/cli.ts platform-db postgres status --config ${defaultConfigPath} [--full|--raw]`, + `bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --dry-run`, + `bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --confirm`, + `bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --confirm --wait`, + ], + description: "Manage YAML-declared host-native PostgreSQL for UniDesk platform state. The PK01 declaration uses PostgreSQL 16 for Sub2API and does not deploy k3s.", + safety: { + configTruth: defaultConfigPath, + defaultMutation: false, + apply: "apply --confirm returns an async UniDesk job; the job uses short trans calls and a remote PK01 job instead of keeping one long SSH session open.", + redaction: "Passwords and full DATABASE_URL values are never printed; output is limited to key names, presence, fingerprints and state.", + noK3s: "PK01 PostgreSQL is host-systemd, not Docker or k3s.", + }, + }; +} + +export async function runPlatformDbCommand(config: UniDeskConfig, args: string[]): Promise> { + const [target, action = "plan"] = args; + if (target !== "postgres") return unsupported(args); + const options = parseOptions(args.slice(2)); + if (action === "plan") return await plan(config, options); + if (action === "status") return await status(config, options); + if (action === "apply") return await apply(config, options); + return unsupported(args); +} + +function unsupported(args: string[]): Record { + return { + ok: false, + error: "unsupported-platform-db-command", + args, + help: platformDbHelp(), + }; +} + +function parseOptions(args: string[]): PlatformDbOptions { + const options: PlatformDbOptions = { + configPath: defaultConfigPath, + confirm: false, + wait: false, + dryRun: false, + full: false, + raw: false, + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--config") { + const raw = args[index + 1]; + if (raw === undefined || raw.trim().length === 0) throw new Error("--config requires a non-empty path"); + options.configPath = raw; + index += 1; + } else if (arg === "--confirm") { + options.confirm = true; + } else if (arg === "--wait") { + options.wait = true; + } else if (arg === "--dry-run") { + options.dryRun = true; + } else if (arg === "--full") { + options.full = true; + } else if (arg === "--raw") { + options.raw = true; + options.full = true; + } else { + throw new Error(`unsupported platform-db option: ${arg}`); + } + } + if (options.confirm && options.dryRun) throw new Error("apply accepts only one of --confirm or --dry-run"); + return options; +} + +async function plan(config: UniDeskConfig, options: PlatformDbOptions): Promise> { + const pg = readPostgresHostConfig(options.configPath); + const secrets = inspectSecrets(pg, false); + const remote = await remoteFacts(config, pg, null); + const facts = remote.parsed; + const checks = facts === null ? [] : desiredChecks(pg, facts, secrets); + return { + ok: remote.capture.exitCode === 0 && facts !== null && secrets.ok, + action: "platform-db-postgres-plan", + mutation: false, + config: configSummary(pg), + secrets: secretSummary(secrets), + remoteFacts: facts, + desired: desiredSummary(pg, secrets, facts), + checks, + next: { + apply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configPath} --confirm`, + status: `bun scripts/cli.ts platform-db postgres status --config ${pg.configPath}`, + issue: "https://github.com/pikasTech/unidesk/issues/281", + }, + remote: compactCapture(remote.capture, { full: options.full || remote.capture.exitCode !== 0 }), + ...(options.raw ? { raw: remote.capture } : {}), + }; +} + +async function status(config: UniDeskConfig, options: PlatformDbOptions): Promise> { + const pg = readPostgresHostConfig(options.configPath); + const secrets = inspectSecrets(pg, false); + const remote = await remoteFacts(config, pg, secrets); + const facts = remote.parsed; + const deploymentHealthy = facts !== null + && facts.postgres.packageInstalled + && facts.postgres.serviceActive + && facts.postgres.sslOn + && facts.postgres.roleExists + && facts.postgres.databaseExists + && facts.network.port5432Listening + && facts.postgres.appConnectionOk === true + && facts.postgres.appConnectionSsl === true; + const cutoverReady = deploymentHealthy && facts.network.dns.resolves; + return { + ok: remote.capture.exitCode === 0 && facts !== null && deploymentHealthy && secrets.ok, + action: "platform-db-postgres-status", + mutation: false, + config: configSummary(pg), + summary: facts === null ? null : { + healthy: deploymentHealthy, + deploymentHealthy, + cutoverReady, + cutoverBlockers: cutoverReady ? [] : ["dns-db-pikapython-unresolved"], + pgVersion: pg.postgres.package.version, + packageInstalled: facts.postgres.packageInstalled, + serviceActive: facts.postgres.serviceActive, + serviceEnabled: facts.postgres.serviceEnabled, + sslOn: facts.postgres.sslOn, + dnsResolves: facts.network.dns.resolves, + dnsAddresses: facts.network.dns.addresses, + roleExists: facts.postgres.roleExists, + databaseExists: facts.postgres.databaseExists, + appConnectionOk: facts.postgres.appConnectionOk, + appConnectionSsl: facts.postgres.appConnectionSsl, + appConnectionHost: facts.postgres.appConnectionHost, + port5432Listening: facts.network.port5432Listening, + serverVersion: facts.postgres.serverVersion, + secretsOk: secrets.ok, + dnsRequiredBeforeSub2ApiCutover: true, + }, + remoteFacts: facts, + secrets: secretSummary(secrets), + remote: compactCapture(remote.capture, { full: options.full || remote.capture.exitCode !== 0 }), + ...(options.raw ? { raw: remote.capture } : {}), + }; +} + +async function apply(config: UniDeskConfig, options: PlatformDbOptions): Promise> { + const pg = readPostgresHostConfig(options.configPath); + if (!options.confirm || options.dryRun) { + const planned = await plan(config, { ...options, dryRun: true, confirm: false }); + return { + ok: planned.ok, + action: "platform-db-postgres-apply", + mode: "dry-run", + mutation: false, + plan: planned, + next: { + confirm: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configPath} --confirm`, + }, + }; + } + if (!options.wait) { + const job = startJob( + "platform_db_postgres_apply", + ["bun", "scripts/cli.ts", "platform-db", "postgres", "apply", "--config", pg.configPath, "--confirm", "--wait"], + "Apply YAML-declared PK01 host-native PostgreSQL 16 through the controlled UniDesk platform-db CLI", + ); + return { + ok: true, + action: "platform-db-postgres-apply", + mode: "async-job", + mutation: true, + job, + statusCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, + next: { + status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, + postgresStatus: `bun scripts/cli.ts platform-db postgres status --config ${pg.configPath}`, + }, + }; + } + + const localState = ensureLocalSecretState(pg); + const start = await startRemoteApplyJob(config, pg, localState); + if (!start.ok || start.remoteJobId === null) { + return { + ok: false, + action: "platform-db-postgres-apply", + mode: "remote-job-start-failed", + mutation: true, + config: configSummary(pg), + localSecrets: localState.summary, + remoteJob: start, + }; + } + const waited = await waitRemoteApplyJob(config, pg, start.remoteJobId); + const finalStatus = await status(config, { ...options, raw: false, full: false }); + return { + ok: waited.ok && finalStatus.ok === true, + action: "platform-db-postgres-apply", + mode: "confirmed-wait", + mutation: true, + config: configSummary(pg), + localSecrets: localState.summary, + remoteJob: { + start, + waited, + }, + finalStatus, + next: { + status: `bun scripts/cli.ts platform-db postgres status --config ${pg.configPath}`, + syncSecrets: "bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm", + }, + }; +} + +function readPostgresHostConfig(pathArg: string): PostgresHostConfig { + const configPath = resolveConfigPath(pathArg); + const parsed = asRecord(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown, configPath); + const version = numberField(parsed, "version", configPath); + const kind = stringField(parsed, "kind", configPath); + if (kind !== "postgres-host-cluster") throw new Error(`${configPath}.kind must be postgres-host-cluster`); + const metadata = objectField(parsed, "metadata", configPath); + const cluster = objectField(parsed, "cluster", configPath); + const node = objectField(parsed, "node", configPath); + const requiredHostFacts = objectField(node, "requiredHostFacts", `${configPath}.node`); + const swap = objectField(node, "swap", `${configPath}.node`); + const postgres = objectField(parsed, "postgres", configPath); + const postgresPackage = objectField(postgres, "package", `${configPath}.postgres`); + const service = objectField(postgres, "service", `${configPath}.postgres`); + const paths = objectField(postgres, "paths", `${configPath}.postgres`); + const network = objectField(postgres, "network", `${configPath}.postgres`); + const tls = objectField(network, "tls", `${configPath}.postgres.network`); + const firewall = objectField(network, "firewall", `${configPath}.postgres.network`); + const tuning = objectField(postgres, "tuning", `${configPath}.postgres`); + const auth = objectField(postgres, "auth", `${configPath}.postgres`); + const secrets = objectField(parsed, "secrets", configPath); + const objects = objectField(parsed, "objects", configPath); + const exportsConfig = objectField(parsed, "exports", configPath); + const backup = objectField(parsed, "backup", configPath); + const logicalDump = objectField(backup, "logicalDump", `${configPath}.backup`); + const destination = objectField(logicalDump, "destination", `${configPath}.backup.logicalDump`); + const encryption = objectField(logicalDump, "encryption", `${configPath}.backup.logicalDump`); + const manager = stringField(postgresPackage, "manager", `${configPath}.postgres.package`); + if (manager !== "apt") throw new Error(`${configPath}.postgres.package.manager must be apt`); + const packageVersion = stringField(postgresPackage, "version", `${configPath}.postgres.package`); + if (packageVersion !== "16") throw new Error(`${configPath}.postgres.package.version must be \"16\" for Sub2API PG16 deployment`); + const passwordEncryption = stringField(auth, "passwordEncryption", `${configPath}.postgres.auth`); + if (passwordEncryption !== "scram-sha-256") throw new Error(`${configPath}.postgres.auth.passwordEncryption must be scram-sha-256`); + const nodeMode = stringField(node, "mode", `${configPath}.node`); + if (nodeMode !== "host-systemd") throw new Error(`${configPath}.node.mode must be host-systemd`); + const secretSource = stringField(secrets, "source", `${configPath}.secrets`); + if (secretSource !== "master-local") throw new Error(`${configPath}.secrets.source must be master-local`); + const entries = arrayOfRecords(secrets.entries, `${configPath}.secrets.entries`).map((entry, index) => secretEntry(entry, `${configPath}.secrets.entries[${index}]`)); + const roles = arrayOfRecords(objects.roles, `${configPath}.objects.roles`).map((role, index) => roleConfig(role, `${configPath}.objects.roles[${index}]`)); + const databases = arrayOfRecords(objects.databases, `${configPath}.objects.databases`).map((database, index) => databaseConfig(database, `${configPath}.objects.databases[${index}]`)); + if (roles.length !== 1 || databases.length !== 1) throw new Error(`${configPath}.objects must declare exactly one role and one database for the first PK01 Sub2API rollout`); + const connectionStrings = arrayOfRecords(exportsConfig.connectionStrings, `${configPath}.exports.connectionStrings`).map((item, index) => connectionStringExport(item, `${configPath}.exports.connectionStrings[${index}]`)); + return { + configPath: relativeConfigPath(configPath), + version, + kind, + metadata: { + id: stringField(metadata, "id", `${configPath}.metadata`), + relatedIssues: numberArrayField(metadata, "relatedIssues", `${configPath}.metadata`), + }, + cluster: { + role: stringField(cluster, "role", `${configPath}.cluster`), + haPhase: stringField(cluster, "haPhase", `${configPath}.cluster`), + }, + node: { + id: stringField(node, "id", `${configPath}.node`), + route: stringField(node, "route", `${configPath}.node`), + mode: nodeMode, + requiredHostFacts: { + minCpu: numberField(requiredHostFacts, "minCpu", `${configPath}.node.requiredHostFacts`), + minMemoryGiB: numberField(requiredHostFacts, "minMemoryGiB", `${configPath}.node.requiredHostFacts`), + minDiskFreeGiB: numberField(requiredHostFacts, "minDiskFreeGiB", `${configPath}.node.requiredHostFacts`), + requireSwap: booleanField(requiredHostFacts, "requireSwap", `${configPath}.node.requiredHostFacts`), + }, + swap: { + ensure: booleanField(swap, "ensure", `${configPath}.node.swap`), + size: stringField(swap, "size", `${configPath}.node.swap`), + path: absolutePathField(swap, "path", `${configPath}.node.swap`), + }, + }, + postgres: { + package: { + manager, + version: packageVersion, + repo: stringField(postgresPackage, "repo", `${configPath}.postgres.package`), + repoUrl: stringField(postgresPackage, "repoUrl", `${configPath}.postgres.package`), + suite: stringField(postgresPackage, "suite", `${configPath}.postgres.package`), + component: stringField(postgresPackage, "component", `${configPath}.postgres.package`), + signingKeyUrl: stringField(postgresPackage, "signingKeyUrl", `${configPath}.postgres.package`), + signedBy: absolutePathField(postgresPackage, "signedBy", `${configPath}.postgres.package`), + sourceList: absolutePathField(postgresPackage, "sourceList", `${configPath}.postgres.package`), + }, + service: { + name: stringField(service, "name", `${configPath}.postgres.service`), + enabled: booleanField(service, "enabled", `${configPath}.postgres.service`), + state: stringField(service, "state", `${configPath}.postgres.service`), + }, + paths: { + dataDir: absolutePathField(paths, "dataDir", `${configPath}.postgres.paths`), + configDir: absolutePathField(paths, "configDir", `${configPath}.postgres.paths`), + logDir: absolutePathField(paths, "logDir", `${configPath}.postgres.paths`), + }, + network: { + port: integerField(network, "port", `${configPath}.postgres.network`), + listenAddresses: stringArrayField(network, "listenAddresses", `${configPath}.postgres.network`), + connectionHost: stringField(network, "connectionHost", `${configPath}.postgres.network`), + publicDns: stringField(network, "publicDns", `${configPath}.postgres.network`), + transport: postgresTransportField(network, "transport", `${configPath}.postgres.network`), + sslmode: postgresSslModeField(network, "sslmode", `${configPath}.postgres.network`), + tls: { + enabled: booleanField(tls, "enabled", `${configPath}.postgres.network.tls`), + mode: stringField(tls, "mode", `${configPath}.postgres.network.tls`), + commonName: stringField(tls, "commonName", `${configPath}.postgres.network.tls`), + certFile: absolutePathField(tls, "certFile", `${configPath}.postgres.network.tls`), + keyFile: absolutePathField(tls, "keyFile", `${configPath}.postgres.network.tls`), + futureClientSslmode: stringField(tls, "futureClientSslmode", `${configPath}.postgres.network.tls`), + }, + firewall: { + mode: stringField(firewall, "mode", `${configPath}.postgres.network.firewall`), + defaultDeny: booleanField(firewall, "defaultDeny", `${configPath}.postgres.network.firewall`), + allowSources: arrayOfRecords(firewall.allowSources, `${configPath}.postgres.network.firewall.allowSources`).map((item, index) => ({ + id: stringField(item, "id", `${configPath}.postgres.network.firewall.allowSources[${index}]`), + cidr: stringField(item, "cidr", `${configPath}.postgres.network.firewall.allowSources[${index}]`), + purpose: stringField(item, "purpose", `${configPath}.postgres.network.firewall.allowSources[${index}]`), + })), + }, + }, + tuning: { + maxConnections: integerField(tuning, "maxConnections", `${configPath}.postgres.tuning`), + sharedBuffers: stringField(tuning, "sharedBuffers", `${configPath}.postgres.tuning`), + effectiveCacheSize: stringField(tuning, "effectiveCacheSize", `${configPath}.postgres.tuning`), + workMem: stringField(tuning, "workMem", `${configPath}.postgres.tuning`), + maintenanceWorkMem: stringField(tuning, "maintenanceWorkMem", `${configPath}.postgres.tuning`), + walCompression: booleanField(tuning, "walCompression", `${configPath}.postgres.tuning`), + checkpointCompletionTarget: numberField(tuning, "checkpointCompletionTarget", `${configPath}.postgres.tuning`), + }, + auth: { + passwordEncryption, + pgHba: arrayOfRecords(auth.pgHba, `${configPath}.postgres.auth.pgHba`).map((item, index) => pgHbaRule(item, `${configPath}.postgres.auth.pgHba[${index}]`)), + }, + }, + secrets: { + source: secretSource, + root: stringField(secrets, "root", `${configPath}.secrets`), + entries, + }, + objects: { roles, databases }, + exports: { connectionStrings }, + backup: { + logicalDump: { + enabled: booleanField(logicalDump, "enabled", `${configPath}.backup.logicalDump`), + schedule: stringField(logicalDump, "schedule", `${configPath}.backup.logicalDump`), + database: stringField(logicalDump, "database", `${configPath}.backup.logicalDump`), + retentionDays: integerField(logicalDump, "retentionDays", `${configPath}.backup.logicalDump`), + destination: { + type: stringField(destination, "type", `${configPath}.backup.logicalDump.destination`), + path: absolutePathField(destination, "path", `${configPath}.backup.logicalDump.destination`), + }, + encryption: { + enabled: booleanField(encryption, "enabled", `${configPath}.backup.logicalDump.encryption`), + }, + }, + }, + }; +} + +function resolveConfigPath(pathArg: string): string { + return isAbsolute(pathArg) ? pathArg : rootPath(pathArg); +} + +function relativeConfigPath(path: string): string { + return path.startsWith(`${repoRoot}/`) ? path.slice(repoRoot.length + 1) : path; +} + +function asRecord(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be a YAML object`); + return value as Record; +} + +function objectField(obj: Record, key: string, path: string): Record { + return asRecord(obj[key], `${path}.${key}`); +} + +function stringField(obj: Record, key: string, path: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value.trim(); +} + +function absolutePathField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!value.startsWith("/")) throw new Error(`${path}.${key} must be an absolute path`); + return value; +} + +function numberField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) throw new Error(`${path}.${key} must be a positive number`); + return value; +} + +function integerField(obj: Record, key: string, path: string): number { + const value = numberField(obj, key, path); + if (!Number.isInteger(value)) throw new Error(`${path}.${key} must be an integer`); + return value; +} + +function booleanField(obj: Record, key: string, path: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + +function postgresTransportField(obj: Record, key: string, path: string): "postgres-native-tls" { + const value = stringField(obj, key, path); + if (value !== "postgres-native-tls") throw new Error(`${path}.${key} must be postgres-native-tls`); + return value; +} + +function postgresSslModeField(obj: Record, key: string, path: string): "require" { + const value = stringField(obj, key, path); + if (value !== "require") throw new Error(`${path}.${key} must be require`); + return value; +} + +function stringArrayField(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== "string" || item.trim().length === 0)) { + throw new Error(`${path}.${key} must be a non-empty string array`); + } + return value.map((item) => (item as string).trim()); +} + +function numberArrayField(obj: Record, key: string, path: string): number[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "number" || !Number.isInteger(item))) { + throw new Error(`${path}.${key} must be an integer array`); + } + return value as number[]; +} + +function arrayOfRecords(value: unknown, path: string): Record[] { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`); + return value.map((item, index) => asRecord(item, `${path}[${index}]`)); +} + +function secretEntry(entry: Record, path: string): SecretEntryConfig { + const type = stringField(entry, "type", path); + if (type !== "env") throw new Error(`${path}.type must be env`); + const createRaw = entry.createIfMissing === undefined ? undefined : objectField(entry, "createIfMissing", path); + const valuesRaw = createRaw?.values === undefined ? undefined : asStringMap(createRaw.values, `${path}.createIfMissing.values`); + const randomHexRaw = createRaw?.randomHex === undefined ? undefined : asNumberMap(createRaw.randomHex, `${path}.createIfMissing.randomHex`); + return { + name: stringField(entry, "name", path), + sourceRef: safeSourceRef(stringField(entry, "sourceRef", path), `${path}.sourceRef`), + type, + requiredKeys: stringArrayField(entry, "requiredKeys", path), + ...(createRaw === undefined ? {} : { + createIfMissing: { + enabled: booleanField(createRaw, "enabled", `${path}.createIfMissing`), + ...(valuesRaw === undefined ? {} : { values: valuesRaw }), + ...(randomHexRaw === undefined ? {} : { randomHex: randomHexRaw }), + }, + }), + }; +} + +function asStringMap(value: unknown, path: string): Record { + const record = asRecord(value, path); + const result: Record = {}; + for (const [key, item] of Object.entries(record)) { + if (typeof item !== "string") throw new Error(`${path}.${key} must be a string`); + result[key] = item; + } + return result; +} + +function asNumberMap(value: unknown, path: string): Record { + const record = asRecord(value, path); + const result: Record = {}; + for (const [key, item] of Object.entries(record)) { + if (typeof item !== "number" || !Number.isInteger(item) || item <= 0 || item > 128) throw new Error(`${path}.${key} must be an integer in 1..128`); + result[key] = item; + } + return result; +} + +function safeSourceRef(value: string, path: string): string { + if (value.startsWith("/") || value.includes("..") || value.includes("\0")) throw new Error(`${path} must be a relative secret source ref without ..`); + return value; +} + +function roleConfig(role: Record, path: string): PostgresHostConfig["objects"]["roles"][number] { + const passwordRef = objectField(role, "passwordRef", path); + const attributes = objectField(role, "attributes", path); + const name = pgIdentifier(stringField(role, "name", path), `${path}.name`); + return { + name, + passwordRef: { + sourceRef: safeSourceRef(stringField(passwordRef, "sourceRef", `${path}.passwordRef`), `${path}.passwordRef.sourceRef`), + key: envKey(stringField(passwordRef, "key", `${path}.passwordRef`), `${path}.passwordRef.key`), + }, + login: booleanField(role, "login", path), + attributes: { + createdb: booleanField(attributes, "createdb", `${path}.attributes`), + createrole: booleanField(attributes, "createrole", `${path}.attributes`), + superuser: booleanField(attributes, "superuser", `${path}.attributes`), + }, + }; +} + +function databaseConfig(database: Record, path: string): PostgresHostConfig["objects"]["databases"][number] { + return { + name: pgIdentifier(stringField(database, "name", path), `${path}.name`), + owner: pgIdentifier(stringField(database, "owner", path), `${path}.owner`), + encoding: stringField(database, "encoding", path), + locale: stringField(database, "locale", path), + extensions: stringArrayFieldAllowEmpty(database, "extensions", path).map((item, index) => pgIdentifier(item, `${path}.extensions[${index}]`)), + }; +} + +function stringArrayFieldAllowEmpty(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.trim().length === 0)) { + throw new Error(`${path}.${key} must be a string array`); + } + return value.map((item) => (item as string).trim()); +} + +function connectionStringExport(item: Record, path: string): ConnectionStringExportConfig { + const render = objectField(item, "render", path); + const writeToSecretSource = objectField(item, "writeToSecretSource", path); + const variables = render.variables === undefined ? undefined : asStringMap(render.variables, `${path}.render.variables`); + const mode = stringField(writeToSecretSource, "mode", `${path}.writeToSecretSource`); + if (mode !== "update-or-insert") throw new Error(`${path}.writeToSecretSource.mode must be update-or-insert`); + return { + name: stringField(item, "name", path), + sourceSecretRef: safeSourceRef(stringField(item, "sourceSecretRef", path), `${path}.sourceSecretRef`), + render: { + envKey: envKey(stringField(render, "envKey", `${path}.render`), `${path}.render.envKey`), + format: stringField(render, "format", `${path}.render`), + ...(variables === undefined ? {} : { variables }), + }, + writeToSecretSource: { + sourceRef: safeSourceRef(stringField(writeToSecretSource, "sourceRef", `${path}.writeToSecretSource`), `${path}.writeToSecretSource.sourceRef`), + key: envKey(stringField(writeToSecretSource, "key", `${path}.writeToSecretSource`), `${path}.writeToSecretSource.key`), + mode, + }, + consumers: arrayOfRecords(item.consumers, `${path}.consumers`).map((consumer, index) => ({ + scope: stringField(consumer, "scope", `${path}.consumers[${index}]`), + secret: stringField(consumer, "secret", `${path}.consumers[${index}]`), + key: envKey(stringField(consumer, "key", `${path}.consumers[${index}]`), `${path}.consumers[${index}].key`), + })), + }; +} + +function pgHbaRule(item: Record, path: string): PgHbaRule { + const type = stringField(item, "type", path); + if (type !== "local" && type !== "host" && type !== "hostssl") throw new Error(`${path}.type must be local, host, or hostssl`); + const method = stringField(item, "method", path); + if (method !== "peer" && method !== "scram-sha-256") throw new Error(`${path}.method must be peer or scram-sha-256`); + const address = item.address === undefined ? undefined : stringField(item, "address", path); + if ((type === "host" || type === "hostssl") && address === undefined) throw new Error(`${path}.address is required for host rules`); + return { + type, + database: stringField(item, "database", path), + user: stringField(item, "user", path), + ...(address === undefined ? {} : { address }), + method, + }; +} + +function pgIdentifier(value: string, path: string): string { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(value)) throw new Error(`${path} must be a safe PostgreSQL identifier`); + return value; +} + +function envKey(value: string, path: string): string { + if (!/^[A-Z_][A-Z0-9_]*$/u.test(value)) throw new Error(`${path} must be an uppercase env key`); + return value; +} + +function configSummary(pg: PostgresHostConfig): Record { + return { + path: pg.configPath, + id: pg.metadata.id, + relatedIssues: pg.metadata.relatedIssues, + node: pg.node.id, + route: pg.node.route, + mode: pg.node.mode, + pgVersion: pg.postgres.package.version, + packageRepo: { + repo: pg.postgres.package.repo, + repoUrl: pg.postgres.package.repoUrl, + suite: pg.postgres.package.suite, + component: pg.postgres.package.component, + sourceList: pg.postgres.package.sourceList, + }, + service: pg.postgres.service.name, + port: pg.postgres.network.port, + listenAddresses: pg.postgres.network.listenAddresses, + connectionHost: pg.postgres.network.connectionHost, + publicDns: pg.postgres.network.publicDns, + transport: pg.postgres.network.transport, + sslmode: pg.postgres.network.sslmode, + tls: { + enabled: pg.postgres.network.tls.enabled, + mode: pg.postgres.network.tls.mode, + commonName: pg.postgres.network.tls.commonName, + certFile: pg.postgres.network.tls.certFile, + keyFile: pg.postgres.network.tls.keyFile, + futureClientSslmode: pg.postgres.network.tls.futureClientSslmode, + }, + haPhase: pg.cluster.haPhase, + noK3s: true, + }; +} + +function inspectSecrets(pg: PostgresHostConfig, materialize: boolean): SecretInspection { + const root = secretRoot(pg); + const materials = new Map(); + const entries = pg.secrets.entries.map((entry) => { + const sourcePath = join(root, entry.sourceRef); + const existedBefore = existsSync(sourcePath); + const existingValues = existedBefore ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; + const nextValues = { ...existingValues }; + for (const [key, value] of Object.entries(entry.createIfMissing?.values ?? {})) { + if (nextValues[key] === undefined) nextValues[key] = value; + } + for (const [key, bytes] of Object.entries(entry.createIfMissing?.randomHex ?? {})) { + if (nextValues[key] === undefined) nextValues[key] = randomBytes(bytes).toString("hex"); + } + const missingBefore = entry.requiredKeys.filter((key) => existingValues[key] === undefined || existingValues[key].length === 0); + const missingAfter = entry.requiredKeys.filter((key) => nextValues[key] === undefined || nextValues[key].length === 0); + const action = missingAfter.length > 0 + ? "blocked" + : !existedBefore + ? "create" + : missingBefore.length > 0 + ? "update" + : "none"; + if (materialize && action !== "blocked" && (action === "create" || action === "update")) { + writeEnvFile(sourcePath, nextValues); + } + const material: SecretMaterial = { + values: nextValues, + sourcePath, + existedBefore, + missingBefore, + action, + fingerprint: missingAfter.length === 0 ? fingerprintValues(nextValues, entry.requiredKeys) : null, + }; + materials.set(entry.sourceRef, material); + return { + name: entry.name, + sourceRef: entry.sourceRef, + sourcePath: redactRepoPath(sourcePath), + exists: existedBefore, + requiredKeys: entry.requiredKeys, + presentKeys: entry.requiredKeys.filter((key) => nextValues[key] !== undefined && nextValues[key].length > 0), + missingKeys: missingAfter, + action, + fingerprint: material.fingerprint, + }; + }); + return { + ok: entries.every((entry) => entry.missingKeys.length === 0), + root: redactRepoPath(root), + entries, + materials, + }; +} + +function ensureLocalSecretState(pg: PostgresHostConfig): { inspection: SecretInspection; summary: Record; exports: Array> } { + const inspection = inspectSecrets(pg, true); + if (!inspection.ok) throw new Error("required secret source keys are missing and cannot be generated from YAML"); + validateSub2ApiSecretConsistency(pg, inspection); + const exports = pg.exports.connectionStrings.map((item) => writeConnectionStringExport(pg, inspection, item)); + return { + inspection, + exports, + summary: { + ok: inspection.ok, + root: inspection.root, + entries: inspection.entries, + exports, + valuesPrinted: false, + }, + }; +} + +function validateSub2ApiSecretConsistency(pg: PostgresHostConfig, inspection: SecretInspection): void { + const role = pg.objects.roles[0]; + const database = pg.objects.databases[0]; + const material = inspection.materials.get(role.passwordRef.sourceRef); + if (material === undefined) throw new Error(`secret source not found for role passwordRef: ${role.passwordRef.sourceRef}`); + if (material.values.SUB2API_DB_USER !== role.name) throw new Error("SUB2API_DB_USER must match YAML objects.roles[0].name"); + if (material.values.SUB2API_DB_NAME !== database.name) throw new Error("SUB2API_DB_NAME must match YAML objects.databases[0].name"); +} + +function writeConnectionStringExport(pg: PostgresHostConfig, inspection: SecretInspection, item: ConnectionStringExportConfig): Record { + const source = inspection.materials.get(item.sourceSecretRef); + if (source === undefined) throw new Error(`export source secret not found: ${item.sourceSecretRef}`); + const rendered = renderConnectionString(item, source.values); + const root = secretRoot(pg); + const targetPath = join(root, item.writeToSecretSource.sourceRef); + const existing = existsSync(targetPath) ? parseEnvFile(readFileSync(targetPath, "utf8")) : {}; + const before = existing[item.writeToSecretSource.key]; + const next = { ...existing, [item.writeToSecretSource.key]: rendered }; + writeEnvFile(targetPath, next); + return { + name: item.name, + sourceRef: item.sourceSecretRef, + targetRef: item.writeToSecretSource.sourceRef, + key: item.writeToSecretSource.key, + action: before === rendered ? "none" : before === undefined ? "create" : "update", + fingerprint: fingerprintValues({ [item.writeToSecretSource.key]: rendered }, [item.writeToSecretSource.key]), + consumers: item.consumers, + valuesPrinted: false, + }; +} + +function renderConnectionString(item: ConnectionStringExportConfig, values: Record): string { + let output = item.render.format; + const merged = { ...values, ...(item.render.variables ?? {}) }; + for (const [key, value] of Object.entries(merged)) { + const rendered = key === "PGHOST" ? value : encodeURIComponent(value); + output = output.replaceAll(`$(${key})`, rendered); + } + if (/\$\([A-Z0-9_]+\)/u.test(output)) throw new Error(`unresolved variable in connection string export ${item.name}`); + return output; +} + +function secretRoot(pg: PostgresHostConfig): string { + return isAbsolute(pg.secrets.root) ? pg.secrets.root : rootPath(pg.secrets.root); +} + +function parseEnvFile(text: string): Record { + const result: Record = {}; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) continue; + result[key] = unquoteEnvValue(line.slice(eq + 1).trim()); + } + return result; +} + +function unquoteEnvValue(value: string): string { + if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith("\"") && value.endsWith("\""))) { + return value.slice(1, -1); + } + return value; +} + +function writeEnvFile(path: string, values: Record): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const lines = Object.keys(values) + .sort() + .map((key) => `${key}=${quoteEnv(values[key])}`); + writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); +} + +function quoteEnv(value: string): string { + if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; + return `'${value.replaceAll("'", "'\"'\"'")}'`; +} + +function fingerprintValues(values: Record, keys: string[]): string { + const material = keys + .slice() + .sort() + .map((key) => `${key}=${values[key] ?? ""}`) + .join("\n"); + return `sha256:${createHash("sha256").update(material).digest("hex")}`; +} + +function secretSummary(secrets: SecretInspection): Record { + return { + ok: secrets.ok, + root: secrets.root, + entries: secrets.entries, + valuesPrinted: false, + }; +} + +function desiredSummary(pg: PostgresHostConfig, secrets: SecretInspection, facts: RemoteFacts | null): Record { + const role = pg.objects.roles[0]; + const database = pg.objects.databases[0]; + return { + package: { + name: `postgresql-${pg.postgres.package.version}`, + repo: pg.postgres.package.repo, + releaseUrl: releaseUrl(pg), + action: facts?.postgres.packageInstalled ? "none" : "install", + }, + host: { + swap: { + ensure: pg.node.swap.ensure, + size: pg.node.swap.size, + path: pg.node.swap.path, + action: facts === null ? "unknown" : (facts.host.swapTotalMiB ?? 0) > 0 ? "none" : "ensure", + }, + minCpu: pg.node.requiredHostFacts.minCpu, + minMemoryGiB: pg.node.requiredHostFacts.minMemoryGiB, + minDiskFreeGiB: pg.node.requiredHostFacts.minDiskFreeGiB, + }, + postgres: { + service: pg.postgres.service.name, + port: pg.postgres.network.port, + listenAddresses: pg.postgres.network.listenAddresses, + connectionHost: pg.postgres.network.connectionHost, + publicDns: pg.postgres.network.publicDns, + transport: pg.postgres.network.transport, + sslmode: pg.postgres.network.sslmode, + tls: { + enabled: pg.postgres.network.tls.enabled, + mode: pg.postgres.network.tls.mode, + commonName: pg.postgres.network.tls.commonName, + certFile: pg.postgres.network.tls.certFile, + keyFile: pg.postgres.network.tls.keyFile, + futureClientSslmode: pg.postgres.network.tls.futureClientSslmode, + }, + pgHbaManagedBlock: { start: managedHbaStart, end: managedHbaEnd, rules: pg.postgres.auth.pgHba.length }, + pgHbaRemoteTransport: "hostssl", + role: { name: role.name, action: facts?.postgres.roleExists ? "none" : "create-or-update" }, + database: { name: database.name, owner: database.owner, action: facts?.postgres.databaseExists ? "none" : "create" }, + }, + secrets: secretSummary(secrets), + exports: pg.exports.connectionStrings.map((item) => ({ + name: item.name, + sourceRef: item.sourceSecretRef, + targetRef: item.writeToSecretSource.sourceRef, + key: item.writeToSecretSource.key, + consumers: item.consumers, + valuesPrinted: false, + })), + backup: pg.backup.logicalDump, + }; +} + +function desiredChecks(pg: PostgresHostConfig, facts: RemoteFacts, secrets: SecretInspection): Array> { + const memoryGiB = (facts.host.memoryTotalMiB ?? 0) / 1024; + const checks = [ + { name: "provider-online", ok: facts.ok, detail: "PK01 remote facts returned JSON" }, + { name: "cpu", ok: (facts.host.cpuCount ?? 0) >= pg.node.requiredHostFacts.minCpu, observed: facts.host.cpuCount, required: pg.node.requiredHostFacts.minCpu }, + { name: "memory", ok: memoryGiB >= pg.node.requiredHostFacts.minMemoryGiB, observedGiB: Number(memoryGiB.toFixed(2)), requiredGiB: pg.node.requiredHostFacts.minMemoryGiB }, + { name: "disk-free", ok: (facts.host.rootAvailGiB ?? 0) >= pg.node.requiredHostFacts.minDiskFreeGiB, observedGiB: facts.host.rootAvailGiB, requiredGiB: pg.node.requiredHostFacts.minDiskFreeGiB }, + { name: "swap", ok: !pg.node.requiredHostFacts.requireSwap || (facts.host.swapTotalMiB ?? 0) > 0 || pg.node.swap.ensure, observedMiB: facts.host.swapTotalMiB, disposition: (facts.host.swapTotalMiB ?? 0) > 0 ? "already-present" : "will-ensure" }, + { name: "port-5432-free-or-postgres", ok: !facts.network.port5432Listening || facts.postgres.packageInstalled, detail: facts.network.port5432Line }, + { name: "pgdg-repo-release", ok: facts.repo.reachable, releaseUrl: facts.repo.releaseUrl, firstLine: facts.repo.firstLine }, + { name: "secrets", ok: secrets.ok, entries: secrets.entries.map((entry) => ({ sourceRef: entry.sourceRef, action: entry.action, missingKeys: entry.missingKeys })) }, + { name: "pg16-required", ok: pg.postgres.package.version === "16", detail: "Sub2API deployment requires PostgreSQL 16 per issue #281 update." }, + { name: "dns-db-pikapython", ok: facts.network.dns.resolves, host: facts.network.dns.hostname, addresses: facts.network.dns.addresses, requiredBeforeSub2ApiCutover: true }, + { name: "postgres-tls-required", ok: pg.postgres.network.tls.enabled && pg.postgres.network.sslmode === "require", transport: pg.postgres.network.transport, sslmode: pg.postgres.network.sslmode }, + { name: "remote-pg-hba-hostssl", ok: pg.postgres.auth.pgHba.filter((rule) => rule.type !== "local" && rule.address !== "127.0.0.1/32").every((rule) => rule.type === "hostssl"), detail: "remote PostgreSQL access must use hostssl so plaintext remote connections are refused" }, + { name: "postgres-ssl-runtime", ok: !facts.postgres.packageInstalled || facts.postgres.sslOn, observed: facts.postgres.sslOn, disposition: facts.postgres.packageInstalled ? "must-be-on" : "will-enable-on-apply" }, + ]; + return checks; +} + +async function remoteFacts(config: UniDeskConfig, pg: PostgresHostConfig, secrets: SecretInspection | null): Promise<{ capture: SshCaptureResult; parsed: RemoteFacts | null }> { + const capture = await runSshCommandCapture(config, pg.node.route, ["script"], factsScript(pg, appProbe(pg, secrets))); + const parsed = parseJsonOutput(capture.stdout) as RemoteFacts | null; + return { capture, parsed }; +} + +function appProbe(pg: PostgresHostConfig, secrets: SecretInspection | null): { user: string; password: string; database: string } | null { + if (secrets === null || !secrets.ok) return null; + const role = pg.objects.roles[0]; + const database = pg.objects.databases[0]; + const material = secrets.materials.get(role.passwordRef.sourceRef); + const password = material?.values[role.passwordRef.key]; + if (password === undefined || password.length === 0) return null; + return { user: role.name, password, database: database.name }; +} + +function factsScript(pg: PostgresHostConfig, probe: { user: string; password: string; database: string } | null): string { + const release = releaseUrl(pg); + const dataDir = pg.postgres.paths.dataDir; + const configDir = pg.postgres.paths.configDir; + const logDir = pg.postgres.paths.logDir; + const role = pg.objects.roles[0].name; + const database = pg.objects.databases[0].name; + const dnsHost = pg.postgres.network.publicDns; + const appHost = pg.postgres.network.listenAddresses.find((item) => item !== "127.0.0.1" && item !== "0.0.0.0") ?? "127.0.0.1"; + const probeEnv = probe === null + ? "" + : `APP_PROBE_USER=${shellQuote(probe.user)}\nAPP_PROBE_DATABASE=${shellQuote(probe.database)}\nAPP_PROBE_PASSWORD=${shellQuote(probe.password)}\nAPP_PROBE_HOST=${shellQuote(appHost)}\nAPP_PROBE_PORT=${shellQuote(String(pg.postgres.network.port))}\n`; + return ` +set +e +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +${probeEnv} +date -Is >"$tmp/observedAt" 2>/dev/null +hostname >"$tmp/hostname" 2>/dev/null +if [ -r /etc/os-release ]; then . /etc/os-release; printf '%s' "$PRETTY_NAME" >"$tmp/osPrettyName"; printf '%s' "$VERSION_CODENAME" >"$tmp/osCodename"; fi +dpkg --print-architecture >"$tmp/arch" 2>/dev/null +nproc --all >"$tmp/cpu" 2>/dev/null +awk '/MemTotal:/ {print int($2/1024)}' /proc/meminfo >"$tmp/memTotal" 2>/dev/null +awk '/MemAvailable:/ {print int($2/1024)}' /proc/meminfo >"$tmp/memAvailable" 2>/dev/null +awk 'NR>1 {sum += $3} END {print int(sum/1024)}' /proc/swaps >"$tmp/swapTotal" 2>/dev/null +df -BG / | awk 'NR==2 {gsub("G","",$4); print $4}' >"$tmp/rootAvail" 2>/dev/null +df -BG "${dataDir}" 2>/dev/null | awk 'NR==2 {gsub("G","",$4); print $4}' >"$tmp/dataAvail" 2>/dev/null +ss -lntup 2>/dev/null | awk '/:5432 / {print; found=1} END {exit found?0:1}' >"$tmp/port5432" 2>/dev/null +printf '%s' "$?" >"$tmp/port5432rc" +getent ahostsv4 "${dnsHost}" >"$tmp/dns" 2>/dev/null +printf '%s' "$?" >"$tmp/dnsrc" +curl -fsSI --max-time 10 "${release}" >"$tmp/release" 2>"$tmp/releaseErr" +printf '%s' "$?" >"$tmp/releaserc" +command -v psql >"$tmp/psqlPath" 2>/dev/null +if [ "$?" -eq 0 ]; then psql --version >"$tmp/psqlVersion" 2>/dev/null; fi +command -v postgres >"$tmp/postgresPath" 2>/dev/null +if [ "$?" -eq 0 ]; then postgres --version >"$tmp/postgresVersion" 2>/dev/null; fi +dpkg-query -W postgresql-${pg.postgres.package.version} >/dev/null 2>&1 && printf 'install ok installed' >"$tmp/packageStatus" || : +systemctl is-active ${shellQuote(pg.postgres.service.name)} >"$tmp/serviceActive" 2>/dev/null +systemctl is-enabled ${shellQuote(pg.postgres.service.name)} >"$tmp/serviceEnabled" 2>/dev/null +[ -d "${dataDir}" ]; printf '%s' "$?" >"$tmp/dataDirRc" +[ -d "${configDir}" ]; printf '%s' "$?" >"$tmp/configDirRc" +[ -d "${logDir}" ]; printf '%s' "$?" >"$tmp/logDirRc" +if command -v runuser >/dev/null 2>&1 && command -v psql >/dev/null 2>&1 && id postgres >/dev/null 2>&1; then + if [ "$(id -u)" -eq 0 ]; then + root_prefix="" + elif command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then + root_prefix="sudo -n" + else + root_prefix="" + fi + if [ -n "$root_prefix" ] || [ "$(id -u)" -eq 0 ]; then + $root_prefix runuser -u postgres -- psql -Atqc "SHOW server_version;" >"$tmp/serverVersion" 2>"$tmp/serverVersionErr" + $root_prefix runuser -u postgres -- psql -Atqc "SHOW ssl;" >"$tmp/sslOn" 2>/dev/null + $root_prefix runuser -u postgres -- psql -Atqc "SHOW ssl_cert_file;" >"$tmp/sslCertFile" 2>/dev/null + $root_prefix runuser -u postgres -- psql -Atqc "SHOW ssl_key_file;" >"$tmp/sslKeyFile" 2>/dev/null + $root_prefix runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_roles WHERE rolname='${role}'" >"$tmp/roleExists" 2>/dev/null + $root_prefix runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='${database}'" >"$tmp/databaseExists" 2>/dev/null + fi +fi +if [ -n "\${APP_PROBE_PASSWORD:-}" ] && command -v psql >/dev/null 2>&1; then + PGPASSWORD="$APP_PROBE_PASSWORD" psql "host=$APP_PROBE_HOST port=$APP_PROBE_PORT user=$APP_PROBE_USER dbname=$APP_PROBE_DATABASE sslmode=require" -Atqc "SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid();" >"$tmp/appSsl" 2>"$tmp/appConnErr" + printf '%s' "$?" >"$tmp/appConnRc" + printf '%s' "$APP_PROBE_HOST" >"$tmp/appConnHost" +fi +python3 - "$tmp" "${release}" <<'PY' +import json, os, sys +tmp, release = sys.argv[1], sys.argv[2] +def text(name): + try: + return open(os.path.join(tmp, name), encoding="utf-8", errors="replace").read().strip() + except FileNotFoundError: + return "" +def int_or_none(name): + raw = text(name) + try: + return int(raw) + except ValueError: + return None +def yes_file(name, expected="0"): + return text(name) == expected +release_rc = text("releaserc") +payload = { + "ok": True, + "observedAt": text("observedAt") or None, + "host": { + "hostname": text("hostname") or None, + "osPrettyName": text("osPrettyName") or None, + "osCodename": text("osCodename") or None, + "arch": text("arch") or None, + "cpuCount": int_or_none("cpu"), + "memoryTotalMiB": int_or_none("memTotal"), + "memoryAvailableMiB": int_or_none("memAvailable"), + "swapTotalMiB": int_or_none("swapTotal") or 0, + "rootAvailGiB": int_or_none("rootAvail"), + "targetDataAvailGiB": int_or_none("dataAvail"), + }, + "network": { + "port5432Listening": text("port5432rc") == "0", + "port5432Line": text("port5432") or None, + "dns": { + "hostname": "${dnsHost}", + "resolves": text("dnsrc") == "0", + "addresses": sorted({line.split()[0] for line in text("dns").splitlines() if line.strip()}), + }, + }, + "repo": { + "releaseUrl": release, + "reachable": release_rc == "0", + "firstLine": (text("release").splitlines() or [None])[0], + }, + "postgres": { + "psqlVersion": text("psqlVersion") or None, + "postgresVersion": text("postgresVersion") or None, + "packageInstalled": "install ok installed" in text("packageStatus"), + "serviceActive": text("serviceActive") == "active", + "serviceEnabled": text("serviceEnabled") == "enabled", + "dataDirExists": yes_file("dataDirRc"), + "configDirExists": yes_file("configDirRc"), + "logDirExists": yes_file("logDirRc"), + "roleExists": text("roleExists") == "1", + "databaseExists": text("databaseExists") == "1", + "serverVersion": text("serverVersion") or None, + "sslOn": text("sslOn").lower() in {"on", "true", "1"}, + "sslCertFile": text("sslCertFile") or None, + "sslKeyFile": text("sslKeyFile") or None, + "appConnectionOk": None if text("appConnRc") == "" else text("appConnRc") == "0", + "appConnectionSsl": None if text("appSsl") == "" else text("appSsl").lower() in {"t", "true", "1", "on"}, + "appConnectionHost": text("appConnHost") or None, + }, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +`; +} + +function releaseUrl(pg: PostgresHostConfig): string { + return `${pg.postgres.package.repoUrl.replace(/\/+$/u, "")}/dists/${pg.postgres.package.suite}/Release`; +} + +async function startRemoteApplyJob(config: UniDeskConfig, pg: PostgresHostConfig, localState: { inspection: SecretInspection }): Promise { + const payload = remoteApplyPayload(pg, localState.inspection); + const script = startRemoteApplyJobScript(pg, payload); + const capture = await runSshCommandCapture(config, pg.node.route, ["script"], script); + const parsed = parseJsonOutput(capture.stdout); + return { + ok: capture.exitCode === 0 && parsed !== null && parsed.ok === true, + remoteJobId: typeof parsed?.remoteJobId === "string" ? parsed.remoteJobId : null, + statePath: typeof parsed?.statePath === "string" ? parsed.statePath : null, + logPath: typeof parsed?.logPath === "string" ? parsed.logPath : null, + parsed, + capture: compactCapture(capture, { full: capture.exitCode !== 0 }), + }; +} + +function remoteApplyPayload(pg: PostgresHostConfig, secrets: SecretInspection): Record { + const role = pg.objects.roles[0]; + const database = pg.objects.databases[0]; + const material = secrets.materials.get(role.passwordRef.sourceRef); + if (material === undefined) throw new Error(`missing material for ${role.passwordRef.sourceRef}`); + const dbPassword = material.values[role.passwordRef.key]; + if (dbPassword === undefined || dbPassword.length === 0) throw new Error(`missing ${role.passwordRef.key}`); + return { + clusterId: pg.metadata.id, + pgVersion: pg.postgres.package.version, + node: pg.node, + package: pg.postgres.package, + service: pg.postgres.service, + paths: pg.postgres.paths, + network: pg.postgres.network, + tls: pg.postgres.network.tls, + tuning: pg.postgres.tuning, + pgHba: pg.postgres.auth.pgHba, + passwordEncryption: pg.postgres.auth.passwordEncryption, + role: { + name: role.name, + password: dbPassword, + login: role.login, + attributes: role.attributes, + }, + database, + backup: pg.backup.logicalDump, + hbaMarkers: { start: managedHbaStart, end: managedHbaEnd }, + }; +} + +function startRemoteApplyJobScript(pg: PostgresHostConfig, payload: Record): string { + const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); + const runner = remoteRunnerScript(); + return ` +set -eu +if ! command -v sudo >/dev/null 2>&1 || ! sudo -n true >/dev/null 2>&1; then + printf '{"ok":false,"error":"sudo-noninteractive-unavailable","message":"PK01 host PostgreSQL apply requires passwordless sudo for root-owned systemd, apt, PostgreSQL config and /root/.unidesk job state.","valuesPrinted":false}\\n' + exit 1 +fi +sudo -n sh <<'ROOT' +set -eu +umask 077 +base="/root/.unidesk/platform-db/jobs" +mkdir -p "$base" +job_id="pk01-pg16-$(date +%Y%m%d%H%M%S)-$$" +job_dir="$base/$job_id" +mkdir -p "$job_dir" +payload="$job_dir/payload.json" +state="$job_dir/state.json" +log="$job_dir/apply.log" +runner="$job_dir/run.sh" +printf '%s' '${encoded}' | base64 -d > "$payload" +cat > "$runner" <<'RUNNER' +${runner} +RUNNER +chmod 700 "$runner" +python3 - "$state" <<'PY' +import json, sys, datetime +path = sys.argv[1] +json.dump({"status":"queued","stage":"created","startedAt":datetime.datetime.utcnow().isoformat()+"Z","finishedAt":None,"exitCode":None}, open(path, "w"), ensure_ascii=False, indent=2) +PY +nohup "$runner" "$payload" "$state" >"$log" 2>&1 & +printf '{"ok":true,"remoteJobId":"%s","statePath":"%s","logPath":"%s","valuesPrinted":false}\\n' "$job_id" "$state" "$log" +ROOT +`; +} + +function remoteRunnerScript(): string { + return String.raw`#!/bin/sh +set -eu +payload="$1" +state="$2" +job_dir="$(dirname "$payload")" +log="$job_dir/apply.log" + +write_state() { + status="$1" + stage="$2" + exit_code="" + if [ "$#" -ge 3 ]; then + exit_code="$3" + fi + python3 - "$state" "$status" "$stage" "$exit_code" <<'PY' +import datetime, json, os, sys +path, status, stage, exit_code = sys.argv[1:5] +try: + current = json.load(open(path, encoding="utf-8")) +except Exception: + current = {} +current["status"] = status +current["stage"] = stage +current["updatedAt"] = datetime.datetime.utcnow().isoformat() + "Z" +if status == "running" and not current.get("startedAt"): + current["startedAt"] = current["updatedAt"] +if status in {"succeeded", "failed"}: + current["finishedAt"] = current["updatedAt"] + current["exitCode"] = int(exit_code or ("0" if status == "succeeded" else "1")) +json.dump(current, open(path, "w"), ensure_ascii=False, indent=2) +PY +} + +fail() { + rc="$1" + stage="$2" + write_state failed "$stage" "$rc" + exit "$rc" +} + +run_stage() { + stage="$1" + shift + write_state running "$stage" + printf '\n### %s\n' "$stage" + "$@" || fail "$?" "$stage" +} + +prepare_apt_sourceparts() { + apt_sourceparts="$job_dir/apt-sourceparts" + rm -rf "$apt_sourceparts" + mkdir -p "$apt_sourceparts" + if [ -f "$SOURCE_LIST" ]; then + cp "$SOURCE_LIST" "$apt_sourceparts/$(basename "$SOURCE_LIST")" + fi + printf '%s' "$apt_sourceparts" +} + +apt_get_update_safe() { + apt_sourceparts="$(prepare_apt_sourceparts)" + env DEBIAN_FRONTEND=noninteractive apt-get \ + -o Dir::Etc::sourceparts="$apt_sourceparts" \ + -o APT::Get::List-Cleanup=0 \ + update +} + +apt_get_install_safe() { + apt_sourceparts="$(prepare_apt_sourceparts)" + env DEBIAN_FRONTEND=noninteractive apt-get \ + -o Dir::Etc::sourceparts="$apt_sourceparts" \ + -o APT::Get::List-Cleanup=0 \ + install -y "$@" +} + +json_get() { + python3 - "$payload" "$1" <<'PY' +import json, sys +data = json.load(open(sys.argv[1], encoding="utf-8")) +value = data +for part in sys.argv[2].split("."): + value = value[part] +if isinstance(value, bool): + print("true" if value else "false") +else: + print(value) +PY +} + +render_file() { + name="$1" + python3 - "$payload" "$name" <<'PY' +import json, sys +data = json.load(open(sys.argv[1], encoding="utf-8")) +name = sys.argv[2] +if name == "pg_hba": + markers = data["hbaMarkers"] + print(markers["start"]) + for rule in data["pgHba"]: + if rule["type"] == "local": + print(f'local {rule["database"]} {rule["user"]} {rule["method"]}') + else: + print(f'{rule["type"]} {rule["database"]} {rule["user"]} {rule["address"]} {rule["method"]}') + print(markers["end"]) +elif name == "sql": + role = data["role"] + name = role["name"] + password = role["password"].replace("'", "''") + attrs = [] + attrs.append("LOGIN" if role.get("login") else "NOLOGIN") + attrs.append("CREATEDB" if role["attributes"].get("createdb") else "NOCREATEDB") + attrs.append("CREATEROLE" if role["attributes"].get("createrole") else "NOCREATEROLE") + attrs.append("SUPERUSER" if role["attributes"].get("superuser") else "NOSUPERUSER") + attr_sql = " ".join(attrs) + print("DO $unidesk$") + print("BEGIN") + print(f" IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{name}') THEN") + print(f" CREATE ROLE {name} {attr_sql} PASSWORD '{password}';") + print(" ELSE") + print(f" ALTER ROLE {name} WITH {attr_sql} PASSWORD '{password}';") + print(" END IF;") + print("END") + print("$unidesk$;") +elif name == "backup_script": + backup = data["backup"] + role = data["role"]["name"] + db = data["database"]["name"] + port = data["network"]["port"] + dest = backup["destination"]["path"] + retention = backup["retentionDays"] + password = data["role"]["password"].replace("'", "'\"'\"'") + print("#!/bin/sh") + print("set -eu") + print(f"dest='{dest}'") + print("mkdir -p \"$dest\"") + print("ts=$(date +%Y%m%dT%H%M%S)") + print(f"PGPASSWORD='{password}' /usr/lib/postgresql/{data['pgVersion']}/bin/pg_dump -h 127.0.0.1 -p {port} -U {role} -d {db} --format=custom --file \"$dest/{db}-$ts.dump\"") + print(f"find \"$dest\" -type f -name '{db}-*.dump' -mtime +{retention} -delete") +PY +} + +write_state running started +trap 'rc=$?; if [ "$rc" -ne 0 ]; then write_state failed trap "$rc"; fi' EXIT + +PG_VERSION="$(json_get pgVersion)" +SERVICE="$(json_get service.name)" +SIGNED_BY="$(json_get package.signedBy)" +SOURCE_LIST="$(json_get package.sourceList)" +REPO_URL="$(json_get package.repoUrl)" +SUITE="$(json_get package.suite)" +COMPONENT="$(json_get package.component)" +KEY_URL="$(json_get package.signingKeyUrl)" +SWAP_ENSURE="$(json_get node.swap.ensure)" +SWAP_PATH="$(json_get node.swap.path)" +SWAP_SIZE="$(json_get node.swap.size)" +CONFIG_DIR="$(json_get paths.configDir)" +DATA_DIR="$(json_get paths.dataDir)" +BACKUP_ENABLED="$(json_get backup.enabled)" +BACKUP_PATH="$(json_get backup.destination.path)" +DB_NAME="$(json_get database.name)" +DB_OWNER="$(json_get database.owner)" +DB_ENCODING="$(json_get database.encoding)" +DB_LOCALE="$(json_get database.locale)" +TLS_ENABLED="$(json_get tls.enabled)" +TLS_COMMON_NAME="$(json_get tls.commonName)" +TLS_CERT_FILE="$(json_get tls.certFile)" +TLS_KEY_FILE="$(json_get tls.keyFile)" +LISTEN_ADDRESSES="$(python3 - "$payload" <<'PY' +import json, sys +data=json.load(open(sys.argv[1], encoding="utf-8")) +print(",".join(data["network"]["listenAddresses"])) +PY +)" + +if [ "$SWAP_ENSURE" = "true" ] && ! awk 'NR>1 {found=1} END {exit found?0:1}' /proc/swaps; then + run_stage ensure-swap fallocate -l "$SWAP_SIZE" "$SWAP_PATH" + chmod 600 "$SWAP_PATH" + run_stage mkswap mkswap "$SWAP_PATH" + run_stage swapon swapon "$SWAP_PATH" + grep -F "$SWAP_PATH none swap sw 0 0" /etc/fstab >/dev/null 2>&1 || printf '%s none swap sw 0 0\n' "$SWAP_PATH" >> /etc/fstab +fi + +run_stage apt-prerequisites apt_get_update_safe +run_stage apt-install-prerequisites apt_get_install_safe ca-certificates curl gnupg openssl +write_state running configure-pgdg-repo +install -d -m 0755 "$(dirname "$SIGNED_BY")" +curl -fsSL "$KEY_URL" | gpg --dearmor > "$SIGNED_BY.tmp" +mv "$SIGNED_BY.tmp" "$SIGNED_BY" +chmod 0644 "$SIGNED_BY" +printf 'deb [signed-by=%s] %s %s %s\n' "$SIGNED_BY" "$REPO_URL" "$SUITE" "$COMPONENT" > "$SOURCE_LIST" +run_stage apt-update-pgdg apt_get_update_safe +run_stage install-postgresql apt_get_install_safe "postgresql-$PG_VERSION" "postgresql-client-$PG_VERSION" +run_stage enable-service systemctl enable "$SERVICE" +run_stage start-service systemctl start "$SERVICE" + +if [ "$TLS_ENABLED" = "true" ]; then + write_state running configure-tls + if [ ! -s "$TLS_CERT_FILE" ] || [ ! -s "$TLS_KEY_FILE" ]; then + tmp_key="$job_dir/server.key" + tmp_cert="$job_dir/server.crt" + openssl req -new -x509 -days 825 -nodes -text -subj "/CN=$TLS_COMMON_NAME" -keyout "$tmp_key" -out "$tmp_cert" + install -o postgres -g postgres -m 0600 "$tmp_key" "$TLS_KEY_FILE" + install -o postgres -g postgres -m 0644 "$tmp_cert" "$TLS_CERT_FILE" + else + chown postgres:postgres "$TLS_KEY_FILE" "$TLS_CERT_FILE" + chmod 0600 "$TLS_KEY_FILE" + chmod 0644 "$TLS_CERT_FILE" + fi +fi + +write_state running configure-postgresql +runuser -u postgres -- psql -v ON_ERROR_STOP=1 < "$tmp_hba" +else + : > "$tmp_hba" +fi +render_file pg_hba >> "$tmp_hba" +install -o postgres -g postgres -m 0640 "$tmp_hba" "$hba" + +run_stage restart-postgresql systemctl restart "$SERVICE" + +write_state running create-role +sql="$job_dir/role.sql" +render_file sql > "$sql" +runuser -u postgres -- psql -v ON_ERROR_STOP=1 < "$sql" + +write_state running create-database +if ! runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1; then + runuser -u postgres -- createdb -O "$DB_OWNER" -E "$DB_ENCODING" --locale="$DB_LOCALE" --template=template0 "$DB_NAME" +fi + +if [ "$BACKUP_ENABLED" = "true" ]; then + write_state running configure-backup + mkdir -p "$BACKUP_PATH" + backup_script="/usr/local/sbin/unidesk-pk01-sub2api-pgdump.sh" + render_file backup_script > "$backup_script" + chmod 700 "$backup_script" + cat > /etc/systemd/system/unidesk-pk01-sub2api-pgdump.service < /etc/systemd/system/unidesk-pk01-sub2api-pgdump.timer </dev/null +runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_roles WHERE rolname='$DB_OWNER'" | grep -q 1 +runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1 +write_state succeeded complete 0 +trap - EXIT +exit 0 +`; +} + +async function waitRemoteApplyJob(config: UniDeskConfig, pg: PostgresHostConfig, remoteJobId: string): Promise> { + const observations: Array> = []; + const maxPolls = 240; + let lastProgressKey: string | null = null; + for (let attempt = 0; attempt < maxPolls; attempt += 1) { + const observation = await readRemoteJob(config, pg, remoteJobId); + observations.push(observation.summary); + const statusValue = observation.status; + const stage = typeof observation.summary.stage === "string" ? observation.summary.stage : null; + const progressKey = `${statusValue ?? "unknown"}:${stage ?? "unknown"}`; + if (attempt === 0 || progressKey !== lastProgressKey || attempt % 6 === 0 || statusValue === "succeeded" || statusValue === "failed") { + lastProgressKey = progressKey; + console.error(JSON.stringify({ + event: "platform-db.postgres.apply.progress", + at: new Date().toISOString(), + remoteJobId, + attempt: attempt + 1, + status: statusValue, + stage, + updatedAt: observation.summary.updatedAt ?? null, + exitCode: observation.summary.exitCode ?? null, + })); + } + if (statusValue === "succeeded" || statusValue === "failed") { + return { + ok: statusValue === "succeeded", + attempts: attempt + 1, + terminal: observation.summary, + recent: observations.slice(-10), + }; + } + await Bun.sleep(5000); + } + return { + ok: false, + attempts: maxPolls, + terminal: null, + recent: observations.slice(-10), + error: "remote apply job did not finish before local polling budget", + }; +} + +async function readRemoteJob(config: UniDeskConfig, pg: PostgresHostConfig, remoteJobId: string): Promise<{ status: string | null; summary: Record }> { + const safeId = remoteJobId.replace(/[^A-Za-z0-9_.-]/gu, ""); + const script = ` +set +e +if ! command -v sudo >/dev/null 2>&1 || ! sudo -n true >/dev/null 2>&1; then + printf '{"status":"unknown","stage":"sudo-unavailable","error":"sudo-noninteractive-unavailable"}' + printf '\\n---LOG---\\n' + exit 1 +fi +sudo -n sh <<'ROOT' +job_dir="/root/.unidesk/platform-db/jobs/${safeId}" +state="$job_dir/state.json" +log="$job_dir/apply.log" +if [ -f "$state" ]; then cat "$state"; else printf '{"status":"missing","stage":"missing"}'; fi +printf '\\n---LOG---\\n' +if [ -f "$log" ]; then tail -80 "$log"; fi +ROOT +`; + const capture = await runSshCommandCapture(config, pg.node.route, ["script"], script); + const [stateText, logTail = ""] = capture.stdout.split("\n---LOG---\n"); + const parsed = parseJsonOutput(stateText) ?? {}; + const statusValue = typeof parsed.status === "string" ? parsed.status : null; + return { + status: statusValue, + summary: { + ok: capture.exitCode === 0, + remoteJobId, + status: statusValue, + stage: parsed.stage ?? null, + updatedAt: parsed.updatedAt ?? null, + finishedAt: parsed.finishedAt ?? null, + exitCode: parsed.exitCode ?? null, + logTail: redactLogTail(logTail), + capture: compactCapture(capture, { full: capture.exitCode !== 0 }), + }, + }; +} + +function redactLogTail(text: string): string { + return text + .split(/\r?\n/u) + .filter((line) => !line.includes("PASSWORD") && !line.includes("postgresql://")) + .slice(-80) + .join("\n"); +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\"'\"'")}'`; +} + +function parseJsonOutput(stdout: string): Record | null { + const trimmed = stdout.trim(); + if (trimmed.length === 0) return null; + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start === -1 || end === -1 || end <= start) return null; + try { + const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; + } catch { + return null; + } +} + +function compactCapture(result: SshCaptureResult, options: { full?: boolean } = {}): Record { + const full = options.full ?? false; + return { + exitCode: result.exitCode, + stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), + stderrBytes: Buffer.byteLength(result.stderr, "utf8"), + stdoutTail: full || result.exitCode !== 0 ? result.stdout.slice(-8000) : "", + stderrTail: full || result.exitCode !== 0 ? result.stderr.slice(-4000) : "", + }; +} + +function redactRepoPath(path: string): string { + return path.startsWith(`${repoRoot}/`) ? path.slice(repoRoot.length + 1) : path; +}