|
|
|
@@ -5,6 +5,7 @@ import { dirname, isAbsolute, join } from "node:path";
|
|
|
|
|
import type { UniDeskConfig } from "./config";
|
|
|
|
|
import { repoRoot, rootPath } from "./config";
|
|
|
|
|
import { startJob } from "./jobs";
|
|
|
|
|
import { MONITOR_CENTRAL_MIGRATIONS, MONITOR_CENTRAL_SCHEMA_VERSION } from "./monitor-central-persistence";
|
|
|
|
|
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
|
|
|
|
import { compactText, fingerprintEnvValues as fingerprintValues, parseEnvFile, parseJsonOutput } from "./platform-infra-ops-library";
|
|
|
|
|
|
|
|
|
@@ -21,6 +22,13 @@ interface PlatformDbOptions {
|
|
|
|
|
raw: boolean;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface MonitorResetConfig {
|
|
|
|
|
database: string;
|
|
|
|
|
owner: string;
|
|
|
|
|
schema: string;
|
|
|
|
|
secretRef: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface PostgresHostConfig {
|
|
|
|
|
configPath: string;
|
|
|
|
|
version: number;
|
|
|
|
@@ -139,6 +147,7 @@ interface PostgresHostConfig {
|
|
|
|
|
encryption: { enabled: boolean };
|
|
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
managedOperations?: { monitorReset: MonitorResetConfig };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface PgHbaRule {
|
|
|
|
@@ -300,6 +309,9 @@ export function platformDbHelp(): unknown {
|
|
|
|
|
`bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --dry-run`,
|
|
|
|
|
`bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --confirm`,
|
|
|
|
|
`bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --confirm --wait`,
|
|
|
|
|
"bun scripts/cli.ts platform-db postgres monitor plan --config config/platform-db/postgres-nc01.yaml",
|
|
|
|
|
"bun scripts/cli.ts platform-db postgres monitor status --config config/platform-db/postgres-nc01.yaml",
|
|
|
|
|
"bun scripts/cli.ts platform-db postgres monitor reset --config config/platform-db/postgres-nc01.yaml --confirm",
|
|
|
|
|
],
|
|
|
|
|
description: "Manage YAML-declared host-native PostgreSQL for UniDesk platform state. The PK01 declaration uses PostgreSQL 16 for Sub2API and does not deploy k3s.",
|
|
|
|
|
safety: {
|
|
|
|
@@ -307,6 +319,7 @@ export function platformDbHelp(): unknown {
|
|
|
|
|
defaultMutation: false,
|
|
|
|
|
exportSecrets: "export-secrets only materializes YAML-declared local Secret source/export files; it does not touch the remote PostgreSQL service.",
|
|
|
|
|
apply: "apply --confirm returns an async UniDesk job; the job uses short trans calls and a remote PK01 job instead of keeping one long SSH session open.",
|
|
|
|
|
monitorReset: "monitor reset is restricted by owning YAML to the dedicated web_probe_monitor database and requires --confirm; it cannot accept a database name or arbitrary SQL.",
|
|
|
|
|
redaction: "Passwords and full DATABASE_URL values are never printed; output is limited to key names, presence, fingerprints and state.",
|
|
|
|
|
noK3s: "PK01 PostgreSQL is host-systemd, not Docker or k3s.",
|
|
|
|
|
},
|
|
|
|
@@ -316,6 +329,11 @@ export function platformDbHelp(): unknown {
|
|
|
|
|
export async function runPlatformDbCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
|
|
|
|
const [target, action = "plan"] = args;
|
|
|
|
|
if (target !== "postgres") return unsupported(args);
|
|
|
|
|
if (action === "monitor") {
|
|
|
|
|
const monitorAction = args[2] ?? "plan";
|
|
|
|
|
const options = parseOptions(args.slice(3));
|
|
|
|
|
return await monitorCommand(config, monitorAction, options);
|
|
|
|
|
}
|
|
|
|
|
const options = parseOptions(args.slice(2));
|
|
|
|
|
if (action === "plan") return await plan(config, options);
|
|
|
|
|
if (action === "status") return await status(config, options);
|
|
|
|
@@ -324,6 +342,53 @@ export async function runPlatformDbCommand(config: UniDeskConfig, args: string[]
|
|
|
|
|
return unsupported(args);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function monitorCommand(config: UniDeskConfig, action: string, options: PlatformDbOptions): Promise<Record<string, unknown>> {
|
|
|
|
|
if (!(["plan", "status", "reset"] as const).includes(action as "plan" | "status" | "reset")) return unsupported(["postgres", "monitor", action]);
|
|
|
|
|
const pg = readPostgresHostConfig(options.configPath);
|
|
|
|
|
const target = requireMonitorResetConfig(pg);
|
|
|
|
|
const base = {
|
|
|
|
|
action: `platform-db-postgres-monitor-${action}`,
|
|
|
|
|
target: "web-probe-monitor",
|
|
|
|
|
config: pg.configPath,
|
|
|
|
|
database: target.database,
|
|
|
|
|
owner: target.owner,
|
|
|
|
|
schema: target.schema,
|
|
|
|
|
schemaVersion: MONITOR_CENTRAL_SCHEMA_VERSION,
|
|
|
|
|
secretRef: target.secretRef,
|
|
|
|
|
valuesPrinted: false,
|
|
|
|
|
};
|
|
|
|
|
if (action === "plan" || (action === "reset" && !options.confirm)) {
|
|
|
|
|
return {
|
|
|
|
|
ok: true,
|
|
|
|
|
...base,
|
|
|
|
|
mutation: false,
|
|
|
|
|
plan: monitorResetPlan(target),
|
|
|
|
|
next: {
|
|
|
|
|
status: `bun scripts/cli.ts platform-db postgres monitor status --config ${pg.configPath}`,
|
|
|
|
|
confirm: `bun scripts/cli.ts platform-db postgres monitor reset --config ${pg.configPath} --confirm`,
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const script = action === "status" ? monitorStatusScript(pg, target) : monitorResetScript(pg, target);
|
|
|
|
|
const capture = await runSshCommandCapture(config, pg.node.route, ["sh"], script);
|
|
|
|
|
const parsed = parseJsonOutput(capture.stdout);
|
|
|
|
|
const commandOk = action === "status"
|
|
|
|
|
? monitorStatusSucceeded(capture.exitCode, parsed)
|
|
|
|
|
: capture.exitCode === 0 && parsed?.ok === true;
|
|
|
|
|
return {
|
|
|
|
|
ok: commandOk,
|
|
|
|
|
...base,
|
|
|
|
|
mutation: action === "reset",
|
|
|
|
|
...(action === "reset" ? { confirmed: true } : {}),
|
|
|
|
|
state: parsed,
|
|
|
|
|
remote: compactCapture(capture, { full: options.full || capture.exitCode !== 0 }),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function monitorStatusSucceeded(exitCode: number, parsed: Record<string, unknown> | null): boolean {
|
|
|
|
|
return exitCode === 0 && parsed?.ok === true && parsed.ready === true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function unsupported(args: string[]): Record<string, unknown> {
|
|
|
|
|
return {
|
|
|
|
|
ok: false,
|
|
|
|
@@ -589,6 +654,7 @@ function readPostgresHostConfig(pathArg: string): PostgresHostConfig {
|
|
|
|
|
const objects = objectField(parsed, "objects", configPath);
|
|
|
|
|
const exportsConfig = objectField(parsed, "exports", configPath);
|
|
|
|
|
const backup = objectField(parsed, "backup", configPath);
|
|
|
|
|
const managedOperations = parsed.managedOperations === undefined ? null : objectField(parsed, "managedOperations", configPath);
|
|
|
|
|
const logicalDump = objectField(backup, "logicalDump", `${configPath}.backup`);
|
|
|
|
|
const destination = objectField(logicalDump, "destination", `${configPath}.backup.logicalDump`);
|
|
|
|
|
const encryption = objectField(logicalDump, "encryption", `${configPath}.backup.logicalDump`);
|
|
|
|
@@ -618,6 +684,13 @@ function readPostgresHostConfig(pathArg: string): PostgresHostConfig {
|
|
|
|
|
databaseNames.add(database.name);
|
|
|
|
|
}
|
|
|
|
|
const connectionStrings = arrayOfRecords(exportsConfig.connectionStrings, `${configPath}.exports.connectionStrings`).map((item, index) => connectionStringExport(item, `${configPath}.exports.connectionStrings[${index}]`));
|
|
|
|
|
const monitorReset = managedOperations === null ? null : monitorResetConfig(objectField(managedOperations, "monitorReset", `${configPath}.managedOperations`), `${configPath}.managedOperations.monitorReset`);
|
|
|
|
|
if (monitorReset !== null) {
|
|
|
|
|
const database = databases.find((item) => item.name === monitorReset.database);
|
|
|
|
|
if (monitorReset.database !== "web_probe_monitor" || monitorReset.owner !== "web_probe_monitor" || monitorReset.schema !== "monitor" || database?.owner !== monitorReset.owner) {
|
|
|
|
|
throw new Error(`${configPath}.managedOperations.monitorReset is restricted to the dedicated web_probe_monitor database/schema`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
configPath: relativeConfigPath(configPath),
|
|
|
|
|
version,
|
|
|
|
@@ -729,6 +802,16 @@ function readPostgresHostConfig(pathArg: string): PostgresHostConfig {
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
...(monitorReset === null ? {} : { managedOperations: { monitorReset } }),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function monitorResetConfig(item: Record<string, unknown>, path: string): MonitorResetConfig {
|
|
|
|
|
return {
|
|
|
|
|
database: pgIdentifier(stringField(item, "database", path), `${path}.database`),
|
|
|
|
|
owner: pgIdentifier(stringField(item, "owner", path), `${path}.owner`),
|
|
|
|
|
schema: pgIdentifier(stringField(item, "schema", path), `${path}.schema`),
|
|
|
|
|
secretRef: stringField(item, "secretRef", path),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -988,6 +1071,91 @@ function configSummary(pg: PostgresHostConfig): Record<string, unknown> {
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function requireMonitorResetConfig(pg: PostgresHostConfig): MonitorResetConfig {
|
|
|
|
|
const target = pg.managedOperations?.monitorReset;
|
|
|
|
|
if (target === undefined) throw new Error(`${pg.configPath}.managedOperations.monitorReset is required`);
|
|
|
|
|
return target;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function monitorResetPlan(target: MonitorResetConfig): string[] {
|
|
|
|
|
return [
|
|
|
|
|
`terminate connections only to ${target.database}`,
|
|
|
|
|
`drop and recreate only ${target.database} owned by ${target.owner}`,
|
|
|
|
|
`apply central Monitor schema ${MONITOR_CENTRAL_SCHEMA_VERSION} as ${target.owner}`,
|
|
|
|
|
"verify schema ready and Monitor business row count is 0",
|
|
|
|
|
"verify sibling YAML-declared database OIDs are unchanged",
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function monitorStatusScript(pg: PostgresHostConfig, target: MonitorResetConfig): string {
|
|
|
|
|
const siblings = pg.objects.databases.filter((database) => database.name !== target.database).map((database) => database.name).join(",");
|
|
|
|
|
const siblingArray = pg.objects.databases.filter((database) => database.name !== target.database).map((database) => sqlLiteral(database.name)).join(", ");
|
|
|
|
|
return `${monitorScriptPrelude()}
|
|
|
|
|
database=${shellQuote(target.database)}
|
|
|
|
|
schema_version=${shellQuote(MONITOR_CENTRAL_SCHEMA_VERSION)}
|
|
|
|
|
siblings=${shellQuote(siblings)}
|
|
|
|
|
database_oid=$(sudo -n runuser -u postgres -- psql -At -d postgres -c "select oid from pg_database where datname = ${sqlLiteral(target.database)};")
|
|
|
|
|
sibling_oids=$(sudo -n runuser -u postgres -- psql -At -d postgres -c "select coalesce(string_agg(datname || ':' || oid, ',' order by datname), '') from pg_database where datname in (${siblingArray});")
|
|
|
|
|
schema_ready=false
|
|
|
|
|
schema_version_observed=""
|
|
|
|
|
business_rows=""
|
|
|
|
|
if [ -n "$database_oid" ]; then
|
|
|
|
|
migration_table=$(sudo -n runuser -u postgres -- psql -At -d "$database" -c "select coalesce(to_regclass('monitor.schema_migrations')::text, '');")
|
|
|
|
|
if [ -n "$migration_table" ]; then
|
|
|
|
|
schema_version_observed=$(sudo -n runuser -u postgres -- psql -At -d "$database" -c "select coalesce((select version from monitor.schema_migrations order by applied_at desc limit 1), '');")
|
|
|
|
|
fi
|
|
|
|
|
if [ "$schema_version_observed" = "$schema_version" ]; then
|
|
|
|
|
schema_ready=true
|
|
|
|
|
business_rows=$(sudo -n runuser -u postgres -- psql -At -d "$database" -c "select (select count(*) from monitor.runners) + (select count(*) from monitor.runs) + (select count(*) from monitor.run_payloads) + (select count(*) from monitor.findings) + (select count(*) from monitor.artifact_locators) + (select count(*) from monitor.timeline_events) + (select count(*) from monitor.runtime_metadata) + (select count(*) from monitor.import_manifests);")
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
empty=false
|
|
|
|
|
ready=false
|
|
|
|
|
if [ "$schema_ready" = true ] && [ "$business_rows" = "0" ]; then empty=true; ready=true; fi
|
|
|
|
|
printf '{"ok":%s,"databaseExists":%s,"schemaReady":%s,"empty":%s,"ready":%s,"databaseOid":"%s","schemaVersion":"%s","businessRows":%s,"siblingDatabaseOids":"%s"}\n' "$ready" "$([ -n "$database_oid" ] && printf true || printf false)" "$schema_ready" "$empty" "$ready" "$database_oid" "$schema_version_observed" "$([ -n "$business_rows" ] && printf '%s' "$business_rows" || printf null)" "$sibling_oids"
|
|
|
|
|
if [ "$ready" != true ]; then exit 1; fi
|
|
|
|
|
`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function monitorResetScript(pg: PostgresHostConfig, target: MonitorResetConfig): string {
|
|
|
|
|
const siblings = pg.objects.databases.filter((database) => database.name !== target.database).map((database) => database.name).join(",");
|
|
|
|
|
const siblingArray = pg.objects.databases.filter((database) => database.name !== target.database).map((database) => sqlLiteral(database.name)).join(", ");
|
|
|
|
|
const migrations = MONITOR_CENTRAL_MIGRATIONS.map((statement) => `${statement};`).join("\n");
|
|
|
|
|
return `${monitorScriptPrelude()}
|
|
|
|
|
database=${shellQuote(target.database)}
|
|
|
|
|
owner=${shellQuote(target.owner)}
|
|
|
|
|
siblings=${shellQuote(siblings)}
|
|
|
|
|
before=$(sudo -n runuser -u postgres -- psql -At -d postgres -c "select coalesce(string_agg(datname || ':' || oid, ',' order by datname), '') from pg_database where datname in (${siblingArray});")
|
|
|
|
|
sudo -n runuser -u postgres -- psql -v ON_ERROR_STOP=1 -d postgres <<'SQL'
|
|
|
|
|
select pg_terminate_backend(pid) from pg_stat_activity where datname = ${sqlLiteral(target.database)} and pid <> pg_backend_pid();
|
|
|
|
|
drop database if exists "${target.database}";
|
|
|
|
|
create database "${target.database}" owner "${target.owner}" encoding 'UTF8' locale 'C.UTF-8' template template0;
|
|
|
|
|
SQL
|
|
|
|
|
sudo -n runuser -u postgres -- psql -v ON_ERROR_STOP=1 -d "$database" <<'SQL'
|
|
|
|
|
set role "${target.owner}";
|
|
|
|
|
${migrations}
|
|
|
|
|
insert into monitor.schema_migrations (version, applied_at) values (${sqlLiteral(MONITOR_CENTRAL_SCHEMA_VERSION)}, now()) on conflict (version) do nothing;
|
|
|
|
|
SQL
|
|
|
|
|
after=$(sudo -n runuser -u postgres -- psql -At -d postgres -c "select coalesce(string_agg(datname || ':' || oid, ',' order by datname), '') from pg_database where datname in (${siblingArray});")
|
|
|
|
|
rows=$(sudo -n runuser -u postgres -- psql -At -d "$database" -c "select (select count(*) from monitor.runners) + (select count(*) from monitor.runs) + (select count(*) from monitor.run_payloads) + (select count(*) from monitor.findings) + (select count(*) from monitor.artifact_locators) + (select count(*) from monitor.timeline_events) + (select count(*) from monitor.runtime_metadata) + (select count(*) from monitor.import_manifests);")
|
|
|
|
|
version=$(sudo -n runuser -u postgres -- psql -At -d "$database" -c "select version from monitor.schema_migrations order by applied_at desc limit 1;")
|
|
|
|
|
if [ "$before" != "$after" ] || [ "$rows" != "0" ] || [ "$version" != ${shellQuote(MONITOR_CENTRAL_SCHEMA_VERSION)} ]; then exit 1; fi
|
|
|
|
|
printf '{"ok":true,"database":"%s","schema":"monitor","schemaVersion":"%s","businessRows":0,"siblingDatabasesUnchanged":true}\n' "$database" "$version"
|
|
|
|
|
`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function monitorScriptPrelude(): string {
|
|
|
|
|
return `set -eu
|
|
|
|
|
if ! command -v sudo >/dev/null 2>&1 || ! sudo -n true >/dev/null 2>&1; then
|
|
|
|
|
printf '{"ok":false,"error":"sudo-noninteractive-unavailable"}\n'
|
|
|
|
|
exit 1
|
|
|
|
|
fi`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sqlLiteral(value: string): string {
|
|
|
|
|
return `'${value.replaceAll("'", "''")}'`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function inspectSecrets(pg: PostgresHostConfig, materialize: boolean): SecretInspection {
|
|
|
|
|
const root = secretRoot(pg);
|
|
|
|
|
const materials = new Map<string, SecretMaterial>();
|
|
|
|
|