Codex
2026 lines
112 KiB
TypeScript
2026 lines
112 KiB
TypeScript
import { existsSync, readFileSync } from "node:fs";
|
|
import { repoRoot, type Config } from "./config";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
import { startJob } from "./jobs";
|
|
import { runHwlabG14Command } from "./hwlab-g14";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes";
|
|
|
|
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete";
|
|
type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
|
|
type DelegatedNodeDomain = "control-plane" | "git-mirror";
|
|
|
|
interface NodeSecretOptions {
|
|
action: SecretAction;
|
|
node: string;
|
|
lane: string;
|
|
name: string;
|
|
key?: string;
|
|
preset: SecretPreset;
|
|
dryRun: boolean;
|
|
confirm: boolean;
|
|
timeoutSeconds: number;
|
|
}
|
|
|
|
interface RuntimeSecretSpec {
|
|
node: string;
|
|
lane: string;
|
|
namespace: string;
|
|
platformDb: boolean;
|
|
platformPostgresService: string;
|
|
postgresSecret: string;
|
|
postgresStatefulSet: string;
|
|
postgresAdminUser: string;
|
|
openFgaSecret: string;
|
|
openFgaDbName: string;
|
|
openFgaDbUser: string;
|
|
openFgaDbHost: string;
|
|
masterAdminApiKeySecret: string;
|
|
cloudApiDbSecret: string;
|
|
cloudApiDbKey: string;
|
|
cloudApiDbName: string;
|
|
cloudApiDbUser: string;
|
|
cloudApiDbHost: string;
|
|
cloudApiDeployment: string;
|
|
obsoleteHwpodDbSecret: string;
|
|
obsoleteHwpodDbName: string;
|
|
obsoleteHwpodDbUser: string;
|
|
codeAgentProviderSecret: string;
|
|
codeAgentProviderSourceNamespace: string;
|
|
codeAgentProviderSourceSecret: string;
|
|
fieldManager: string;
|
|
}
|
|
|
|
const MASTER_ADMIN_API_KEY_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env";
|
|
const MASTER_ADMIN_API_KEY_KEY = "api-key";
|
|
const OPENFGA_AUTHN_KEY = "authn-preshared-key";
|
|
const OPENFGA_DATASTORE_URI_KEY = "datastore-uri";
|
|
const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password";
|
|
const CLOUD_API_DB_KEY = "database-url";
|
|
const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key";
|
|
const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key";
|
|
const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
|
|
const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
|
|
|
|
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
|
if (args.length === 0 || args.includes("--help") || args.includes("-h")) return hwlabNodeHelp();
|
|
const [domain] = args;
|
|
if (domain === "control-plane" || domain === "git-mirror") {
|
|
return runNodeDelegatedDomain(_config, domain, args.slice(1));
|
|
}
|
|
if (domain !== "secret") {
|
|
return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret" };
|
|
}
|
|
const options = parseSecretOptions(args.slice(1));
|
|
return runNodeSecret(options);
|
|
}
|
|
|
|
export function hwlabNodeHelp(): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
command: "hwlab nodes",
|
|
description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.",
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
examples: [
|
|
"bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
|
|
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03",
|
|
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
|
|
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run",
|
|
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm",
|
|
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
|
|
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
|
|
],
|
|
};
|
|
}
|
|
|
|
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown>> {
|
|
const scoped = parseNodeScopedDelegatedOptions(domain, args);
|
|
if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") {
|
|
return runNodeEndpointBridge(scoped);
|
|
}
|
|
if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) {
|
|
return startNodeDelegatedJob(scoped);
|
|
}
|
|
if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) {
|
|
return startNodeDelegatedJob(scoped);
|
|
}
|
|
const delegatedArgs = stripOption(args, "--node");
|
|
const result = await runHwlabG14Command(config, [domain, ...delegatedArgs]);
|
|
return rewriteDelegatedNodeResult(result, scoped);
|
|
}
|
|
|
|
function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): {
|
|
domain: DelegatedNodeDomain;
|
|
action: string;
|
|
node: string;
|
|
lane: HwlabRuntimeLane;
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
wait: boolean;
|
|
timeoutSeconds: number;
|
|
originalArgs: string[];
|
|
} {
|
|
const [actionRaw] = args;
|
|
if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`);
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const laneRaw = requiredOption(args, "--lane");
|
|
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`);
|
|
const spec = hwlabRuntimeLaneSpec(laneRaw);
|
|
if (spec.nodeId !== node) throw new Error(`lane ${laneRaw} is configured for node ${spec.nodeId}; got --node ${node}`);
|
|
const confirm = args.includes("--confirm");
|
|
const dryRun = args.includes("--dry-run");
|
|
if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`);
|
|
return {
|
|
domain,
|
|
action: actionRaw,
|
|
node,
|
|
lane: laneRaw,
|
|
confirm,
|
|
dryRun,
|
|
wait: args.includes("--wait"),
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600),
|
|
originalArgs: [...args],
|
|
};
|
|
}
|
|
|
|
function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const commandArgs = [
|
|
"hwlab",
|
|
"nodes",
|
|
options.domain,
|
|
...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
|
|
"--node",
|
|
options.node,
|
|
"--lane",
|
|
options.lane,
|
|
"--confirm",
|
|
"--timeout-seconds",
|
|
String(options.timeoutSeconds),
|
|
"--wait",
|
|
];
|
|
const command = ["bun", "scripts/cli.ts", ...commandArgs];
|
|
const job = startJob(
|
|
`hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
|
|
command,
|
|
`Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
|
|
);
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: "async-job",
|
|
reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
|
|
job,
|
|
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
|
|
waitCommand: command.join(" "),
|
|
};
|
|
}
|
|
|
|
function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const rewritten = rewriteDelegatedNodeValue(value, scoped);
|
|
const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
|
|
return {
|
|
...result,
|
|
node: scoped.node,
|
|
commandFamily: "hwlab nodes",
|
|
};
|
|
}
|
|
|
|
function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
|
|
if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
|
|
if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
|
|
if (typeof value !== "object" || value === null) return value;
|
|
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
|
|
}
|
|
|
|
function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
|
|
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
return text
|
|
.replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
|
|
.replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
|
|
};
|
|
return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
|
|
}
|
|
|
|
function parseSecretOptions(args: string[]): NodeSecretOptions {
|
|
const [actionRaw] = args;
|
|
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
|
|
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]");
|
|
}
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const lane = requiredOption(args, "--lane");
|
|
assertLane(lane);
|
|
const spec = runtimeSecretSpec({ node, lane });
|
|
const name = optionValue(args, "--name") ?? spec.openFgaSecret;
|
|
const key = optionValue(args, "--key");
|
|
const confirm = args.includes("--confirm");
|
|
const explicitDryRun = args.includes("--dry-run");
|
|
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
|
if (actionRaw === "cleanup-owned-postgres") {
|
|
if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
|
|
if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
|
|
if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name: spec.postgresSecret,
|
|
preset: "owned-postgres-cleanup",
|
|
confirm,
|
|
dryRun: explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (actionRaw === "cleanup-obsolete") {
|
|
if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup");
|
|
if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key");
|
|
if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
preset: "obsolete-secret-cleanup",
|
|
confirm,
|
|
dryRun: explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.masterAdminApiKeySecret) {
|
|
if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? MASTER_ADMIN_API_KEY_KEY,
|
|
preset: "master-server-admin-api-key",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.codeAgentProviderSecret) {
|
|
if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) {
|
|
throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key,
|
|
preset: "code-agent-provider",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.cloudApiDbSecret) {
|
|
if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`);
|
|
if (actionRaw === "ensure" && spec.platformDb) {
|
|
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? spec.cloudApiDbKey,
|
|
preset: "cloud-api-db",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
|
|
};
|
|
}
|
|
if (name !== spec.openFgaSecret) {
|
|
throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`);
|
|
}
|
|
if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
|
|
throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
|
|
}
|
|
if (actionRaw === "ensure" && spec.platformDb) {
|
|
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key,
|
|
preset: "openfga",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
|
|
function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec {
|
|
const namespace = `hwlab-${input.lane}`;
|
|
const platformDb = /^v0*[3-9]\d*$/.test(input.lane);
|
|
const platformPostgresService = "g14-platform-postgres";
|
|
const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`;
|
|
const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`;
|
|
return {
|
|
node: input.node,
|
|
lane: input.lane,
|
|
namespace,
|
|
platformDb,
|
|
platformPostgresService,
|
|
postgresSecret: `${namespace}-postgres`,
|
|
postgresStatefulSet: `${namespace}-postgres`,
|
|
postgresAdminUser: `hwlab_${input.lane}`,
|
|
openFgaSecret: `${namespace}-openfga`,
|
|
openFgaDbName: platformDb ? `openfga_${input.lane}` : "hwlab_openfga",
|
|
openFgaDbUser: platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga",
|
|
openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
|
masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`,
|
|
cloudApiDbSecret: `hwlab-cloud-api-${input.lane}-db`,
|
|
cloudApiDbKey: CLOUD_API_DB_KEY,
|
|
cloudApiDbName: `hwlab_${input.lane}`,
|
|
cloudApiDbUser: platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`,
|
|
cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
|
cloudApiDeployment: "hwlab-cloud-api",
|
|
obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`,
|
|
obsoleteHwpodDbName: `hwpod_${input.lane}`,
|
|
obsoleteHwpodDbUser: `hwpod_${input.lane}_app`,
|
|
codeAgentProviderSecret: `${namespace}-code-agent-provider`,
|
|
codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
|
|
codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET,
|
|
fieldManager: `unidesk-hwlab-node-${input.lane}-secret`,
|
|
};
|
|
}
|
|
|
|
function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
|
|
const spec = runtimeSecretSpec(options);
|
|
if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec);
|
|
const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun
|
|
? readMasterAdminApiKey().key
|
|
: "";
|
|
const script = options.preset === "openfga"
|
|
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
|
|
: options.preset === "master-server-admin-api-key"
|
|
? masterAdminApiKeySecretScript(options, spec)
|
|
: options.preset === "cloud-api-db"
|
|
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
|
|
: options.preset === "owned-postgres-cleanup"
|
|
? ownedPostgresCleanupScript(options, spec)
|
|
: codeAgentProviderSecretScript(options, spec);
|
|
const result = runTransScript(options.node, script, input, options.timeoutSeconds);
|
|
const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
|
|
const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
|
|
const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
|
|
const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true;
|
|
const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: options.key ?? null,
|
|
preset: options.preset,
|
|
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure",
|
|
status,
|
|
mutation: status.mutation === true,
|
|
result: compactCommandResult(result),
|
|
valuesRedacted: true,
|
|
next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|
|
|
|
function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, string> {
|
|
if (options.action === "cleanup-owned-postgres") {
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
|
|
}
|
|
if (options.action === "cleanup-obsolete") {
|
|
return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` };
|
|
}
|
|
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
|
return {
|
|
status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
|
|
controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
|
|
};
|
|
}
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
|
|
}
|
|
|
|
function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand(["/root/.local/bin/trans", node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds);
|
|
const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec);
|
|
const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true };
|
|
const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds);
|
|
const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec);
|
|
const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false);
|
|
const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: null,
|
|
preset: options.preset,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-delete",
|
|
status: {
|
|
ok,
|
|
preset: "obsolete-hwpod-db-cleanup",
|
|
dryRun: options.dryRun,
|
|
mutation,
|
|
kubernetesSecret: kubernetesStatus,
|
|
platformDatabase: platformDbStatus,
|
|
hostMutationSkipped: !options.dryRun && hostOptions.dryRun,
|
|
summary: ok
|
|
? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove`
|
|
: `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`,
|
|
},
|
|
mutation,
|
|
result: {
|
|
kubernetesSecret: compactCommandResult(kubernetesResult),
|
|
platformDatabase: compactCommandResult(platformDbResult),
|
|
},
|
|
valuesRedacted: true,
|
|
next: ok ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|
|
|
|
function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm");
|
|
const dryRun = options.dryRun || !options.confirm;
|
|
const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes";
|
|
const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes";
|
|
const afterExcluded = fields.afterEndpointResourcesExcluded === "yes";
|
|
const afterIgnored = fields.afterEndpointsIgnoreUpdates === "yes" || fields.afterEndpointSliceIgnoreUpdates === "yes";
|
|
const beforeExtraEndpointSlices = splitWhitespaceField(fields.beforeExtraEndpointSliceNames);
|
|
const afterExtraEndpointSlices = splitWhitespaceField(fields.afterExtraEndpointSliceNames);
|
|
const beforeLegacyEndpoints = fields.beforeLegacyEndpointsExists === "yes";
|
|
const afterLegacyEndpoints = fields.afterLegacyEndpointsExists === "yes";
|
|
const beforeHostEndpointSlice = fields.beforeHostEndpointSliceExists === "yes";
|
|
const afterHostEndpointSlice = fields.afterHostEndpointSliceExists === "yes";
|
|
const bridgeReady = !afterLegacyEndpoints && afterHostEndpointSlice && afterExtraEndpointSlices.length === 0;
|
|
const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored && bridgeReady;
|
|
return {
|
|
ok: dryRun ? result.exitCode === 0 : ok,
|
|
command: "hwlab nodes control-plane allow-endpoint-bridge",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: "argocd",
|
|
application: fields.application || `hwlab-node-${options.lane}`,
|
|
mode: dryRun ? "dry-run" : "confirmed-control-plane-update",
|
|
status: {
|
|
action: fields.action || null,
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
endpointResourcesExcluded: beforeExcluded,
|
|
endpointsIgnoreUpdates: fields.beforeEndpointsIgnoreUpdates === "yes",
|
|
endpointSliceIgnoreUpdates: fields.beforeEndpointSliceIgnoreUpdates === "yes",
|
|
legacyEndpointsExist: beforeLegacyEndpoints,
|
|
hostEndpointSliceExists: beforeHostEndpointSlice,
|
|
extraEndpointSlices: beforeExtraEndpointSlices,
|
|
},
|
|
after: {
|
|
endpointResourcesExcluded: afterExcluded,
|
|
endpointsIgnoreUpdates: fields.afterEndpointsIgnoreUpdates === "yes",
|
|
endpointSliceIgnoreUpdates: fields.afterEndpointSliceIgnoreUpdates === "yes",
|
|
legacyEndpointsExist: afterLegacyEndpoints,
|
|
hostEndpointSliceExists: afterHostEndpointSlice,
|
|
extraEndpointSlices: afterExtraEndpointSlices,
|
|
},
|
|
runtimeNamespace: fields.runtimeNamespace || `hwlab-${options.lane}`,
|
|
platformService: fields.platformService || "g14-platform-postgres",
|
|
hostEndpointSlice: fields.hostEndpointSlice || "g14-platform-postgres-host",
|
|
patchExitCode: numericField(fields.patchExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
deleteLegacyEndpointsExitCode: numericField(fields.deleteLegacyEndpointsExitCode),
|
|
deleteExtraEndpointSlicesExitCode: numericField(fields.deleteExtraEndpointSlicesExitCode),
|
|
refreshExitCode: numericField(fields.refreshExitCode),
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
summary: !afterExcluded && !afterIgnored && bridgeReady
|
|
? "Argo tracks HWLAB external Postgres EndpointSlice and no legacy Endpoints remain"
|
|
: "Argo endpoint bridge is not in final Service plus EndpointSlice shape",
|
|
},
|
|
result: compactCommandResult(result),
|
|
};
|
|
}
|
|
|
|
function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string {
|
|
const application = `hwlab-node-${options.lane}`;
|
|
const runtimeNamespace = `hwlab-${options.lane}`;
|
|
return [
|
|
"set +e",
|
|
"namespace=argocd",
|
|
`runtime_namespace=${shellQuote(runtimeNamespace)}`,
|
|
"configmap=argocd-cm",
|
|
`application=${shellQuote(application)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"platform_service=g14-platform-postgres",
|
|
"host_endpointslice=g14-platform-postgres-host",
|
|
"preset=endpoint-bridge-resource-tracking",
|
|
"cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
|
"cm_has_key() { value=$(cm_data \"$1\"); [ -n \"$value\" ] && [ \"$value\" != \"<no value>\" ] && printf yes || printf no; }",
|
|
"endpoint_resources_excluded() { exclusions=$(cm_data resource.exclusions); printf '%s' \"$exclusions\" | grep -Eq '(^|[[:space:]])(Endpoints|EndpointSlice)([[:space:]]|$)' && printf yes || printf no; }",
|
|
"resource_exists() { kubectl -n \"$runtime_namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"extra_endpoint_slices() { kubectl -n \"$runtime_namespace\" get endpointslice -l \"kubernetes.io/service-name=$platform_service\" -o name 2>/dev/null | sed \"/\\/$host_endpointslice$/d\" | tr '\\n' ' ' | sed 's/[[:space:]]*$//'; }",
|
|
"wait_runtime_bridge_clean() {",
|
|
" for _ in $(seq 1 30); do",
|
|
" current_legacy=$(resource_exists endpoints \"$platform_service\")",
|
|
" current_extra=$(extra_endpoint_slices)",
|
|
" current_host=$(resource_exists endpointslice \"$host_endpointslice\")",
|
|
" if [ \"$current_legacy\" != yes ] && [ -z \"$current_extra\" ] && [ \"$current_host\" = yes ]; then return 0; fi",
|
|
" sleep 2",
|
|
" done",
|
|
" return 1",
|
|
"}",
|
|
"before_endpoint_resources_excluded=$(endpoint_resources_excluded)",
|
|
"before_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')",
|
|
"before_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')",
|
|
"before_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")",
|
|
"before_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")",
|
|
"before_extra_endpoint_slice_names=$(extra_endpoint_slices)",
|
|
"needs_argo_update=false",
|
|
"if [ \"$before_endpoint_resources_excluded\" = yes ] || [ \"$before_endpoints_ignore_updates\" = yes ] || [ \"$before_endpoint_slice_ignore_updates\" = yes ]; then needs_argo_update=true; fi",
|
|
"needs_runtime_cleanup=false",
|
|
"if [ \"$before_legacy_endpoints_exists\" = yes ] || [ -n \"$before_extra_endpoint_slice_names\" ]; then needs_runtime_cleanup=true; fi",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"patch_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"delete_legacy_endpoints_exit=",
|
|
"delete_extra_endpointslices_exit=",
|
|
"refresh_exit=",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-old-endpoint-exclusions-and-legacy-endpoints",
|
|
" elif [ \"$needs_argo_update\" = true ]; then action=would-remove-old-endpoint-exclusions",
|
|
" elif [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-legacy-endpoints",
|
|
" else action=kept; fi",
|
|
"else",
|
|
" if [ \"$needs_argo_update\" = true ]; then",
|
|
" patch_file=$(mktemp /tmp/hwlab-argocd-endpoint-bridge.XXXXXX.json)",
|
|
" python3 - <<'PY' >\"$patch_file\"",
|
|
"import json",
|
|
"desired = '''### Internal Kubernetes resources excluded to reduce watch volume",
|
|
"- apiGroups:",
|
|
" - coordination.k8s.io",
|
|
" kinds:",
|
|
" - Lease",
|
|
"### Internal Kubernetes Authz/Authn resources excluded to reduce watched events",
|
|
"- apiGroups:",
|
|
" - authentication.k8s.io",
|
|
" - authorization.k8s.io",
|
|
" kinds:",
|
|
" - SelfSubjectReview",
|
|
" - TokenReview",
|
|
" - LocalSubjectAccessReview",
|
|
" - SelfSubjectAccessReview",
|
|
" - SelfSubjectRulesReview",
|
|
" - SubjectAccessReview",
|
|
"### Intermediate Certificate Request excluded to reduce watched events",
|
|
"- apiGroups:",
|
|
" - certificates.k8s.io",
|
|
" kinds:",
|
|
" - CertificateSigningRequest",
|
|
"- apiGroups:",
|
|
" - cert-manager.io",
|
|
" kinds:",
|
|
" - CertificateRequest",
|
|
"### Cilium internal resources excluded to reduce UI clutter",
|
|
"- apiGroups:",
|
|
" - cilium.io",
|
|
" kinds:",
|
|
" - CiliumIdentity",
|
|
" - CiliumEndpoint",
|
|
" - CiliumEndpointSlice",
|
|
"### Kyverno intermediate and reporting resources excluded to reduce watched events",
|
|
"- apiGroups:",
|
|
" - kyverno.io",
|
|
" - reports.kyverno.io",
|
|
" - wgpolicyk8s.io",
|
|
" kinds:",
|
|
" - PolicyReport",
|
|
" - ClusterPolicyReport",
|
|
" - EphemeralReport",
|
|
" - ClusterEphemeralReport",
|
|
" - AdmissionReport",
|
|
" - ClusterAdmissionReport",
|
|
" - BackgroundScanReport",
|
|
" - ClusterBackgroundScanReport",
|
|
" - UpdateRequest",
|
|
"'''",
|
|
"print(json.dumps({",
|
|
" 'data': {",
|
|
" 'resource.exclusions': desired,",
|
|
" 'resource.customizations.ignoreResourceUpdates.Endpoints': None,",
|
|
" 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice': None,",
|
|
" }",
|
|
"}))",
|
|
"PY",
|
|
" kubectl -n \"$namespace\" patch configmap \"$configmap\" --type merge --patch-file \"$patch_file\" >/tmp/hwlab-argocd-endpoint-bridge-patch.out 2>/tmp/hwlab-argocd-endpoint-bridge-patch.err",
|
|
" patch_exit=$?",
|
|
" rm -f \"$patch_file\"",
|
|
" if [ \"$patch_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout restart statefulset/argocd-application-controller >/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.err",
|
|
" rollout_restart_exit=$?",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status statefulset/argocd-application-controller --timeout=180s >/tmp/hwlab-argocd-endpoint-bridge-rollout-status.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-status.err",
|
|
" rollout_status_exit=$?",
|
|
" fi",
|
|
" fi",
|
|
" fi",
|
|
" if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then action=patch-failed",
|
|
" elif [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else",
|
|
" if [ \"$needs_runtime_cleanup\" = true ]; then",
|
|
" kubectl -n \"$runtime_namespace\" delete endpoints \"$platform_service\" --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpoints-delete.out 2>/tmp/hwlab-platform-postgres-endpoints-delete.err",
|
|
" delete_legacy_endpoints_exit=$?",
|
|
" wait_runtime_bridge_clean",
|
|
" remaining_extra=$(extra_endpoint_slices)",
|
|
" if [ -n \"$remaining_extra\" ]; then",
|
|
" kubectl -n \"$runtime_namespace\" delete $remaining_extra --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpointslices-delete.out 2>/tmp/hwlab-platform-postgres-endpointslices-delete.err",
|
|
" delete_extra_endpointslices_exit=$?",
|
|
" wait_runtime_bridge_clean",
|
|
" fi",
|
|
" fi",
|
|
" if [ \"$needs_argo_update\" = true ] || [ \"$needs_runtime_cleanup\" = true ]; then",
|
|
" kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/hwlab-argocd-endpoint-bridge-refresh.out 2>/tmp/hwlab-argocd-endpoint-bridge-refresh.err",
|
|
" refresh_exit=$?",
|
|
" fi",
|
|
" if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then action=delete-legacy-endpoints-failed",
|
|
" elif [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then action=delete-extra-endpointslices-failed",
|
|
" elif [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then action=refresh-failed",
|
|
" elif [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=removed-old-endpoint-exclusions-and-legacy-endpoints; mutation=true",
|
|
" elif [ \"$needs_argo_update\" = true ]; then action=removed-old-endpoint-exclusions; mutation=true",
|
|
" elif [ \"$needs_runtime_cleanup\" = true ]; then action=removed-legacy-endpoints; mutation=true",
|
|
" else action=kept; fi",
|
|
" fi",
|
|
"fi",
|
|
"after_endpoint_resources_excluded=$(endpoint_resources_excluded)",
|
|
"after_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')",
|
|
"after_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')",
|
|
"after_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")",
|
|
"after_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")",
|
|
"after_extra_endpoint_slice_names=$(extra_endpoint_slices)",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"",
|
|
"printf 'configMap\\t%s\\n' \"$configmap\"",
|
|
"printf 'application\\t%s\\n' \"$application\"",
|
|
"printf 'platformService\\t%s\\n' \"$platform_service\"",
|
|
"printf 'hostEndpointSlice\\t%s\\n' \"$host_endpointslice\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeEndpointResourcesExcluded\\t%s\\n' \"$before_endpoint_resources_excluded\"",
|
|
"printf 'beforeEndpointsIgnoreUpdates\\t%s\\n' \"$before_endpoints_ignore_updates\"",
|
|
"printf 'beforeEndpointSliceIgnoreUpdates\\t%s\\n' \"$before_endpoint_slice_ignore_updates\"",
|
|
"printf 'beforeLegacyEndpointsExists\\t%s\\n' \"$before_legacy_endpoints_exists\"",
|
|
"printf 'beforeHostEndpointSliceExists\\t%s\\n' \"$before_host_endpointslice_exists\"",
|
|
"printf 'beforeExtraEndpointSliceNames\\t%s\\n' \"$before_extra_endpoint_slice_names\"",
|
|
"printf 'afterEndpointResourcesExcluded\\t%s\\n' \"$after_endpoint_resources_excluded\"",
|
|
"printf 'afterEndpointsIgnoreUpdates\\t%s\\n' \"$after_endpoints_ignore_updates\"",
|
|
"printf 'afterEndpointSliceIgnoreUpdates\\t%s\\n' \"$after_endpoint_slice_ignore_updates\"",
|
|
"printf 'afterLegacyEndpointsExists\\t%s\\n' \"$after_legacy_endpoints_exists\"",
|
|
"printf 'afterHostEndpointSliceExists\\t%s\\n' \"$after_host_endpointslice_exists\"",
|
|
"printf 'afterExtraEndpointSliceNames\\t%s\\n' \"$after_extra_endpoint_slice_names\"",
|
|
"printf 'patchExitCode\\t%s\\n' \"$patch_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"printf 'deleteLegacyEndpointsExitCode\\t%s\\n' \"$delete_legacy_endpoints_exit\"",
|
|
"printf 'deleteExtraEndpointSlicesExitCode\\t%s\\n' \"$delete_extra_endpointslices_exit\"",
|
|
"printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"",
|
|
"if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi",
|
|
"if [ \"$dry_run\" != true ] && { [ \"$after_legacy_endpoints_exists\" = yes ] || [ -n \"$after_extra_endpoint_slice_names\" ] || [ \"$after_host_endpointslice_exists\" != yes ]; }; then exit 47; fi",
|
|
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
"if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then exit \"$delete_legacy_endpoints_exit\"; fi",
|
|
"if [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then exit \"$delete_extra_endpointslices_exit\"; fi",
|
|
"if [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then exit \"$refresh_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
const pvc = `data-${spec.postgresSecret}-0`;
|
|
const platformService = "g14-platform-postgres";
|
|
const postgresService = spec.postgresSecret;
|
|
const postgresConfigMap = `${spec.postgresSecret}-init`;
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
|
|
`postgres_service=${shellQuote(postgresService)}`,
|
|
`postgres_configmap=${shellQuote(postgresConfigMap)}`,
|
|
`pvc=${shellQuote(pvc)}`,
|
|
`platform_service=${shellQuote(platformService)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"preset=owned-postgres-cleanup",
|
|
"exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }",
|
|
"phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }",
|
|
"before_secret_exists=$(exists_flag secret \"$postgres_secret\")",
|
|
"before_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
|
"before_pvc_phase=$(phase_of_pvc)",
|
|
"before_pv=$(pv_name)",
|
|
"before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
|
|
"before_service_exists=$(exists_flag service \"$postgres_service\")",
|
|
"before_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
|
|
"platform_service_exists=$(exists_flag service \"$platform_service\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"delete_statefulset_exit=",
|
|
"delete_service_exit=",
|
|
"delete_configmap_exit=",
|
|
"delete_secret_exit=",
|
|
"delete_pvc_exit=",
|
|
"before_any_owned=false",
|
|
"for flag in \"$before_statefulset_exists\" \"$before_service_exists\" \"$before_configmap_exists\" \"$before_secret_exists\" \"$before_pvc_exists\"; do",
|
|
" if [ \"$flag\" = yes ]; then before_any_owned=true; fi",
|
|
"done",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_any_owned\" = true ]; then action=would-delete; else action=already-absent; fi",
|
|
"else",
|
|
" kubectl -n \"$namespace\" delete statefulset \"$postgres_statefulset\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-statefulset-delete.out 2>/tmp/hwlab-owned-postgres-statefulset-delete.err",
|
|
" delete_statefulset_exit=$?",
|
|
" kubectl -n \"$namespace\" delete service \"$postgres_service\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-service-delete.out 2>/tmp/hwlab-owned-postgres-service-delete.err",
|
|
" delete_service_exit=$?",
|
|
" kubectl -n \"$namespace\" delete configmap \"$postgres_configmap\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-configmap-delete.out 2>/tmp/hwlab-owned-postgres-configmap-delete.err",
|
|
" delete_configmap_exit=$?",
|
|
" kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err",
|
|
" delete_secret_exit=$?",
|
|
" kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err",
|
|
" delete_pvc_exit=$?",
|
|
" for _ in $(seq 1 30); do",
|
|
" current_statefulset=$(exists_flag statefulset \"$postgres_statefulset\")",
|
|
" current_service=$(exists_flag service \"$postgres_service\")",
|
|
" current_configmap=$(exists_flag configmap \"$postgres_configmap\")",
|
|
" current_secret=$(exists_flag secret \"$postgres_secret\")",
|
|
" current_pvc=$(exists_flag pvc \"$pvc\")",
|
|
" if [ \"$current_statefulset\" != yes ] && [ \"$current_service\" != yes ] && [ \"$current_configmap\" != yes ] && [ \"$current_secret\" != yes ] && [ \"$current_pvc\" != yes ]; then break; fi",
|
|
" sleep 2",
|
|
" done",
|
|
" if [ \"$delete_statefulset_exit\" -eq 0 ] && [ \"$delete_service_exit\" -eq 0 ] && [ \"$delete_configmap_exit\" -eq 0 ] && [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then",
|
|
" if [ \"$before_any_owned\" = true ]; then action=deleted; mutation=true; else action=already-absent; fi",
|
|
" else",
|
|
" action=delete-failed",
|
|
" fi",
|
|
"fi",
|
|
"after_secret_exists=$(exists_flag secret \"$postgres_secret\")",
|
|
"after_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
|
"after_pvc_phase=$(phase_of_pvc)",
|
|
"after_pv=$(pv_name)",
|
|
"after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
|
|
"after_service_exists=$(exists_flag service \"$postgres_service\")",
|
|
"after_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$postgres_secret\"",
|
|
"printf 'statefulSet\\t%s\\n' \"$postgres_statefulset\"",
|
|
"printf 'service\\t%s\\n' \"$postgres_service\"",
|
|
"printf 'configMap\\t%s\\n' \"$postgres_configmap\"",
|
|
"printf 'pvc\\t%s\\n' \"$pvc\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
|
"printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"",
|
|
"printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"",
|
|
"printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"",
|
|
"printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"",
|
|
"printf 'beforeServiceExists\\t%s\\n' \"$before_service_exists\"",
|
|
"printf 'beforeConfigMapExists\\t%s\\n' \"$before_configmap_exists\"",
|
|
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
|
"printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"",
|
|
"printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"",
|
|
"printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"",
|
|
"printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"",
|
|
"printf 'afterServiceExists\\t%s\\n' \"$after_service_exists\"",
|
|
"printf 'afterConfigMapExists\\t%s\\n' \"$after_configmap_exists\"",
|
|
"printf 'deleteStatefulSetExitCode\\t%s\\n' \"$delete_statefulset_exit\"",
|
|
"printf 'deleteServiceExitCode\\t%s\\n' \"$delete_service_exit\"",
|
|
"printf 'deleteConfigMapExitCode\\t%s\\n' \"$delete_configmap_exit\"",
|
|
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
|
|
"printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"",
|
|
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
|
|
"if [ \"$after_statefulset_exists\" = yes ] || [ \"$after_service_exists\" = yes ] || [ \"$after_configmap_exists\" = yes ] || [ \"$after_secret_exists\" = yes ] || [ \"$after_pvc_exists\" = yes ]; then exit 45; fi",
|
|
"if [ -n \"$delete_statefulset_exit\" ] && [ \"$delete_statefulset_exit\" != 0 ]; then exit \"$delete_statefulset_exit\"; fi",
|
|
"if [ -n \"$delete_service_exit\" ] && [ \"$delete_service_exit\" != 0 ]; then exit \"$delete_service_exit\"; fi",
|
|
"if [ -n \"$delete_configmap_exit\" ] && [ \"$delete_configmap_exit\" != 0 ]; then exit \"$delete_configmap_exit\"; fi",
|
|
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
|
|
"if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`secret=${shellQuote(options.name)}`,
|
|
`expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"preset=obsolete-secret-cleanup",
|
|
"exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)",
|
|
"workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)",
|
|
"workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)",
|
|
"before_exists=$(exists_flag)",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"delete_secret_exit=",
|
|
"if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi",
|
|
"if [ \"$action\" = observed ]; then",
|
|
" if [ \"$workload_refs_present\" = yes ]; then",
|
|
" action=blocked-referenced",
|
|
" elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
|
|
" else",
|
|
" kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err",
|
|
" delete_secret_exit=$?",
|
|
" for _ in $(seq 1 15); do",
|
|
" current_exists=$(exists_flag)",
|
|
" if [ \"$current_exists\" != yes ]; then break; fi",
|
|
" sleep 1",
|
|
" done",
|
|
" if [ \"$delete_secret_exit\" -eq 0 ]; then",
|
|
" if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
|
|
" else",
|
|
" action=delete-failed",
|
|
" fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(exists_flag)",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"",
|
|
"printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"",
|
|
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
|
|
"if [ \"$action\" = unsupported-secret ]; then exit 43; fi",
|
|
"if [ \"$workload_refs_present\" = yes ]; then exit 46; fi",
|
|
"if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi",
|
|
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`db_name=${shellQuote(spec.obsoleteHwpodDbName)}`,
|
|
`db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"preset=obsolete-platform-db-cleanup",
|
|
"database_exists_flag() {",
|
|
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)",
|
|
" code=$?",
|
|
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
|
|
" [ \"$output\" = t ] && printf yes || printf no",
|
|
"}",
|
|
"role_exists_flag() {",
|
|
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)",
|
|
" code=$?",
|
|
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
|
|
" [ \"$output\" = t ] && printf yes || printf no",
|
|
"}",
|
|
"before_database_exists=$(database_exists_flag)",
|
|
"before_database_probe_exit=$?",
|
|
"before_role_exists=$(role_exists_flag)",
|
|
"before_role_probe_exit=$?",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"drop_database_exit=",
|
|
"drop_role_exit=",
|
|
"before_any=false",
|
|
"if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi",
|
|
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then",
|
|
" action=probe-failed",
|
|
"elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi",
|
|
"else",
|
|
" if [ \"$before_database_exists\" = yes ]; then",
|
|
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'",
|
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();",
|
|
"DROP DATABASE IF EXISTS :\"db_name\";",
|
|
"SQL",
|
|
" drop_database_exit=$?",
|
|
" else",
|
|
" drop_database_exit=0",
|
|
" fi",
|
|
" if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then",
|
|
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'",
|
|
"DROP ROLE IF EXISTS :\"db_user\";",
|
|
"SQL",
|
|
" drop_role_exit=$?",
|
|
" elif [ \"$drop_database_exit\" -eq 0 ]; then",
|
|
" drop_role_exit=0",
|
|
" else",
|
|
" drop_role_exit=",
|
|
" fi",
|
|
" if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then",
|
|
" if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi",
|
|
" else",
|
|
" action=drop-failed",
|
|
" fi",
|
|
"fi",
|
|
"after_database_exists=$(database_exists_flag)",
|
|
"after_database_probe_exit=$?",
|
|
"after_role_exists=$(role_exists_flag)",
|
|
"after_role_probe_exit=$?",
|
|
"printf 'database\\t%s\\n' \"$db_name\"",
|
|
"printf 'role\\t%s\\n' \"$db_user\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"",
|
|
"printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"",
|
|
"printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"",
|
|
"printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"",
|
|
"printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"",
|
|
"printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"",
|
|
"printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"",
|
|
"printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"",
|
|
"printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"",
|
|
"printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"",
|
|
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi",
|
|
"if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi",
|
|
"if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi",
|
|
"if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
const isOpenFga = options.preset === "openfga";
|
|
const platformEndpointSlice = `${spec.platformPostgresService}-host`;
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`,
|
|
`database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey)}`,
|
|
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
|
|
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
|
|
`legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`platform_service=${shellQuote(spec.platformPostgresService)}`,
|
|
`platform_endpointslice=${shellQuote(platformEndpointSlice)}`,
|
|
`platform_host=${shellQuote(spec.platformPostgresService)}`,
|
|
`platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`,
|
|
`db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`,
|
|
`db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`,
|
|
`db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost)}`,
|
|
`selected_key=${shellQuote(options.key ?? "")}`,
|
|
`preset=${shellQuote(options.preset)}`,
|
|
"dry_run=true",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"resource_exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"endpointslice_exists_flag() { kubectl -n \"$namespace\" get endpointslice \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"uri_has_platform_host=no",
|
|
"uri_has_db_name=no",
|
|
"uri_has_db_user=no",
|
|
"uri_matches_expected() {",
|
|
" uri=$1",
|
|
" uri_has_platform_host=no",
|
|
" uri_has_db_name=no",
|
|
" uri_has_db_user=no",
|
|
" case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*) uri_has_platform_host=yes ;; esac",
|
|
" case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac",
|
|
" case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac",
|
|
"}",
|
|
"exists=$(secret_exists_flag \"$name\")",
|
|
"legacy_postgres_exists=$(secret_exists_flag \"$legacy_postgres_secret\")",
|
|
"uri_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
|
|
"uri_present=$([ -n \"$uri_b64\" ] && printf yes || printf no)",
|
|
"uri_bytes=$(decoded_length \"$uri_b64\")",
|
|
"uri_value=$(decoded_value \"$uri_b64\")",
|
|
"authn_b64=$(secret_b64_key \"$name\" \"$authn_key\")",
|
|
"authn_present=$([ -n \"$authn_b64\" ] && printf yes || printf no)",
|
|
"authn_bytes=$(decoded_length \"$authn_b64\")",
|
|
"pg_password_b64=$(secret_b64_key \"$name\" \"$postgres_password_key\")",
|
|
"pg_password_present=$([ -n \"$pg_password_b64\" ] && printf yes || printf no)",
|
|
"pg_password_bytes=$(decoded_length \"$pg_password_b64\")",
|
|
"platform_service_exists=$(resource_exists_flag service \"$platform_service\")",
|
|
"platform_endpoints_exists=$(resource_exists_flag endpoints \"$platform_service\")",
|
|
"platform_endpointslice_exists=$(endpointslice_exists_flag \"$platform_endpointslice\")",
|
|
"uri_matches_expected \"$uri_value\"",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$database_url_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\tobserved\\n'",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\tfalse\\n'",
|
|
"printf 'platformDbMode\\ttrue\\n'",
|
|
"printf 'afterExists\\t%s\\n' \"$exists\"",
|
|
"printf 'afterDatabaseUrlPresent\\t%s\\n' \"$uri_present\"",
|
|
"printf 'afterDatabaseUrlBytes\\t%s\\n' \"$uri_bytes\"",
|
|
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$uri_present\"",
|
|
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$uri_bytes\"",
|
|
"printf 'afterAuthnPresent\\t%s\\n' \"$authn_present\"",
|
|
"printf 'afterAuthnBytes\\t%s\\n' \"$authn_bytes\"",
|
|
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$pg_password_present\"",
|
|
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$pg_password_bytes\"",
|
|
"printf 'legacyPostgresSecret\\t%s\\n' \"$legacy_postgres_secret\"",
|
|
"printf 'legacyPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
|
|
"printf 'afterPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
|
|
"printf 'platformService\\t%s\\n' \"$platform_service\"",
|
|
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
|
|
"printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"",
|
|
"printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"",
|
|
"printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"",
|
|
"printf 'dbName\\t%s\\n' \"$db_name\"",
|
|
"printf 'dbUser\\t%s\\n' \"$db_user\"",
|
|
"printf 'dbHost\\t%s\\n' \"$db_host\"",
|
|
"printf 'dbHostMatchesPlatform\\t%s\\n' \"$uri_has_platform_host\"",
|
|
"printf 'dbNameMatchesExpected\\t%s\\n' \"$uri_has_db_name\"",
|
|
"printf 'dbUserMatchesExpected\\t%s\\n' \"$uri_has_db_user\"",
|
|
"uri_value=",
|
|
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`openfga_secret=${shellQuote(spec.openFgaSecret)}`,
|
|
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
|
|
`postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`,
|
|
`selected_key=${shellQuote(options.key ?? "")}`,
|
|
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
|
|
`datastore_uri_key=${shellQuote(OPENFGA_DATASTORE_URI_KEY)}`,
|
|
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
|
|
`db_name=${shellQuote(spec.openFgaDbName)}`,
|
|
`db_user=${shellQuote(spec.openFgaDbUser)}`,
|
|
`db_host=${shellQuote(spec.openFgaDbHost)}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=openfga",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
|
|
"probe_db() {",
|
|
" role_result=unknown",
|
|
" database_result=unknown",
|
|
" probe_exit=missing-postgres-admin-secret",
|
|
" if [ -n \"$postgres_admin_password\" ]; then",
|
|
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
|
|
" role_exit=$?",
|
|
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
|
|
" database_exit=$?",
|
|
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
|
|
" fi",
|
|
"}",
|
|
"before_exists=$(secret_exists_flag \"$openfga_secret\")",
|
|
"before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
|
|
"before_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")",
|
|
"before_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")",
|
|
"before_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")",
|
|
"postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)",
|
|
"before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)",
|
|
"before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)",
|
|
"before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)",
|
|
"before_authn_bytes=$(decoded_length \"$before_authn_b64\")",
|
|
"before_uri_bytes=$(decoded_length \"$before_uri_b64\")",
|
|
"before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")",
|
|
"authn_value=$(decoded_value \"$before_authn_b64\")",
|
|
"datastore_uri=$(decoded_value \"$before_uri_b64\")",
|
|
"pg_password=$(decoded_value \"$before_pg_password_b64\")",
|
|
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
|
|
"probe_db",
|
|
"db_role_exists_before=$role_result",
|
|
"db_database_exists_before=$database_result",
|
|
"db_probe_exit_before=$probe_exit",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"postgres_secret_exit=",
|
|
"postgres_rollout_exit=",
|
|
"apply_exit=",
|
|
"db_ensure_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_secret=false",
|
|
" [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true",
|
|
" [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true",
|
|
" [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true",
|
|
" missing_db=false",
|
|
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
|
|
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_postgres_exists\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
|
|
" elif ! command -v openssl >/dev/null 2>&1; then",
|
|
" action=openssl-missing; apply_exit=127",
|
|
" else",
|
|
" [ -n \"$postgres_admin_password\" ] || postgres_admin_password=$(openssl rand -hex 24)",
|
|
" kubectl -n \"$namespace\" create secret generic \"$postgres_secret\" --from-literal=\"POSTGRES_PASSWORD=$postgres_admin_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" postgres_secret_exit=$?",
|
|
" if [ \"$postgres_secret_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status \"statefulset/$postgres_statefulset\" --timeout=120s >/tmp/hwlab-postgres-rollout.out 2>/tmp/hwlab-postgres-rollout.err",
|
|
" postgres_rollout_exit=$?",
|
|
" fi",
|
|
" if [ \"$postgres_secret_exit\" -ne 0 ]; then action=postgres-secret-failed; apply_exit=$postgres_secret_exit",
|
|
" elif [ \"$postgres_rollout_exit\" -ne 0 ]; then action=postgres-rollout-failed; apply_exit=$postgres_rollout_exit",
|
|
" else",
|
|
" [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)",
|
|
" [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
|
|
" [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
|
|
" kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'",
|
|
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
|
|
"\\gexec",
|
|
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
|
|
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
|
|
"\\gexec",
|
|
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
|
|
"SQL",
|
|
" db_ensure_exit=$?",
|
|
" if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi",
|
|
" else action=apply-failed; fi",
|
|
" fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag \"$openfga_secret\")",
|
|
"after_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
|
|
"after_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")",
|
|
"after_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")",
|
|
"after_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")",
|
|
"after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)",
|
|
"after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)",
|
|
"after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)",
|
|
"after_authn_bytes=$(decoded_length \"$after_authn_b64\")",
|
|
"after_uri_bytes=$(decoded_length \"$after_uri_b64\")",
|
|
"after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")",
|
|
"probe_db",
|
|
"db_role_exists_after=$role_result",
|
|
"db_database_exists_after=$database_result",
|
|
"db_probe_exit_after=$probe_exit",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$openfga_secret\"",
|
|
"printf 'key\\t%s\\n' \"$selected_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterPostgresSecretExists\\t%s\\n' \"$after_postgres_exists\"",
|
|
"printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"",
|
|
"printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"",
|
|
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"",
|
|
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"",
|
|
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"",
|
|
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"",
|
|
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
|
|
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
|
|
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
|
|
"printf 'postgresSecretExitCode\\t%s\\n' \"$postgres_secret_exit\"",
|
|
"printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
|
|
"authn_value= datastore_uri= pg_password= postgres_admin_password=",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function masterAdminApiKeySecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.masterAdminApiKeySecret)}`,
|
|
`api_key_name=${shellQuote(MASTER_ADMIN_API_KEY_KEY)}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=master-server-admin-api-key",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-12; value=; fi; }",
|
|
"before_exists=$(secret_exists_flag)",
|
|
"before_api_key_b64=$(secret_b64_key \"$api_key_name\")",
|
|
"before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)",
|
|
"before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_secret=false",
|
|
" [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi",
|
|
" else",
|
|
" api_key=$(cat)",
|
|
" case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac",
|
|
" if [ -z \"$apply_exit\" ]; then",
|
|
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
|
|
" fi",
|
|
" api_key=",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag)",
|
|
"after_api_key_b64=$(secret_b64_key \"$api_key_name\")",
|
|
"after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)",
|
|
"after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")",
|
|
"after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$api_key_name\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"",
|
|
"printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"",
|
|
"printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function codeAgentProviderSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.codeAgentProviderSecret)}`,
|
|
`source_namespace=${shellQuote(spec.codeAgentProviderSourceNamespace)}`,
|
|
`source_name=${shellQuote(spec.codeAgentProviderSourceSecret)}`,
|
|
`openai_key=${shellQuote(CODE_AGENT_PROVIDER_OPENAI_KEY)}`,
|
|
`opencode_key=${shellQuote(CODE_AGENT_PROVIDER_OPENCODE_KEY)}`,
|
|
`selected_key=${shellQuote(options.key ?? "")}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=code-agent-provider",
|
|
"secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"before_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
|
|
"before_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")",
|
|
"before_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")",
|
|
"source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")",
|
|
"source_openai_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$openai_key\")",
|
|
"source_opencode_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$opencode_key\")",
|
|
"before_openai_present=$([ -n \"$before_openai_b64\" ] && printf yes || printf no)",
|
|
"before_opencode_present=$([ -n \"$before_opencode_b64\" ] && printf yes || printf no)",
|
|
"source_openai_present=$([ -n \"$source_openai_b64\" ] && printf yes || printf no)",
|
|
"source_opencode_present=$([ -n \"$source_opencode_b64\" ] && printf yes || printf no)",
|
|
"before_openai_bytes=$(decoded_length \"$before_openai_b64\")",
|
|
"before_opencode_bytes=$(decoded_length \"$before_opencode_b64\")",
|
|
"source_openai_bytes=$(decoded_length \"$source_openai_b64\")",
|
|
"source_opencode_bytes=$(decoded_length \"$source_opencode_b64\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_target=false",
|
|
" [ \"$before_exists\" = yes ] && { [ \"$before_openai_bytes\" -gt 0 ] || [ \"$before_opencode_bytes\" -gt 0 ]; } || missing_target=true",
|
|
" missing_source=false",
|
|
" [ \"$source_exists\" = yes ] && { [ \"$source_openai_bytes\" -gt 0 ] || [ \"$source_opencode_bytes\" -gt 0 ]; } || missing_source=true",
|
|
" if [ \"$missing_source\" = true ]; then",
|
|
" action=source-missing-provider-key",
|
|
" apply_exit=44",
|
|
" elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi",
|
|
" elif [ \"$missing_target\" = false ]; then",
|
|
" action=kept",
|
|
" else",
|
|
" cat <<EOF_SECRET | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $name",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" hwlab.pikastech.local/secret-preset: code-agent-provider",
|
|
"type: Opaque",
|
|
"data:",
|
|
" openai-api-key: $source_openai_b64",
|
|
" opencode-api-key: $source_opencode_b64",
|
|
"EOF_SECRET",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then action=copied-from-source; mutation=true; else action=apply-failed; fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
|
|
"after_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")",
|
|
"after_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")",
|
|
"after_openai_present=$([ -n \"$after_openai_b64\" ] && printf yes || printf no)",
|
|
"after_opencode_present=$([ -n \"$after_opencode_b64\" ] && printf yes || printf no)",
|
|
"after_openai_bytes=$(decoded_length \"$after_openai_b64\")",
|
|
"after_opencode_bytes=$(decoded_length \"$after_opencode_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'sourceNamespace\\t%s\\n' \"$source_namespace\"",
|
|
"printf 'sourceSecret\\t%s\\n' \"$source_name\"",
|
|
"printf 'selectedKey\\t%s\\n' \"$selected_key\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforeOpenaiPresent\\t%s\\n' \"$before_openai_present\"",
|
|
"printf 'beforeOpenaiBytes\\t%s\\n' \"$before_openai_bytes\"",
|
|
"printf 'beforeOpencodePresent\\t%s\\n' \"$before_opencode_present\"",
|
|
"printf 'beforeOpencodeBytes\\t%s\\n' \"$before_opencode_bytes\"",
|
|
"printf 'sourceExists\\t%s\\n' \"$source_exists\"",
|
|
"printf 'sourceOpenaiPresent\\t%s\\n' \"$source_openai_present\"",
|
|
"printf 'sourceOpenaiBytes\\t%s\\n' \"$source_openai_bytes\"",
|
|
"printf 'sourceOpencodePresent\\t%s\\n' \"$source_opencode_present\"",
|
|
"printf 'sourceOpencodeBytes\\t%s\\n' \"$source_opencode_bytes\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterOpenaiPresent\\t%s\\n' \"$after_openai_present\"",
|
|
"printf 'afterOpenaiBytes\\t%s\\n' \"$after_openai_bytes\"",
|
|
"printf 'afterOpencodePresent\\t%s\\n' \"$after_opencode_present\"",
|
|
"printf 'afterOpencodeBytes\\t%s\\n' \"$after_opencode_bytes\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.cloudApiDbSecret)}`,
|
|
`database_url_key=${shellQuote(spec.cloudApiDbKey)}`,
|
|
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
|
|
`postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`,
|
|
`db_name=${shellQuote(spec.cloudApiDbName)}`,
|
|
`db_user=${shellQuote(spec.cloudApiDbUser)}`,
|
|
`db_host=${shellQuote(spec.cloudApiDbHost)}`,
|
|
`cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=cloud-api-db",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
|
|
"probe_db() {",
|
|
" role_result=unknown",
|
|
" database_result=unknown",
|
|
" probe_exit=missing-postgres-admin-secret",
|
|
" if [ -n \"$postgres_admin_password\" ]; then",
|
|
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
|
|
" role_exit=$?",
|
|
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
|
|
" database_exit=$?",
|
|
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
|
|
" fi",
|
|
"}",
|
|
"before_exists=$(secret_exists_flag \"$name\")",
|
|
"before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
|
|
"before_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
|
|
"before_url_present=$([ -n \"$before_url_b64\" ] && printf yes || printf no)",
|
|
"before_url_bytes=$(decoded_length \"$before_url_b64\")",
|
|
"database_url=$(decoded_value \"$before_url_b64\")",
|
|
"postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)",
|
|
"postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)",
|
|
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
|
|
"probe_db",
|
|
"db_role_exists_before=$role_result",
|
|
"db_database_exists_before=$database_result",
|
|
"db_probe_exit_before=$probe_exit",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"db_ensure_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_secret=false",
|
|
" [ \"$before_url_present\" = yes ] && [ \"$before_url_bytes\" -gt 0 ] || missing_secret=true",
|
|
" missing_db=false",
|
|
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
|
|
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
|
|
" elif [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ -z \"$postgres_admin_password\" ]; then",
|
|
" action=postgres-admin-secret-missing",
|
|
" apply_exit=44",
|
|
" elif [ \"$missing_secret\" = false ] && [ \"$missing_db\" = false ]; then",
|
|
" action=kept",
|
|
" else",
|
|
" [ -n \"$database_url\" ] || database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"",
|
|
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$database_url_key=$database_url\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$postgres_admin_password\" >/tmp/hwlab-cloud-api-db-psql.out 2>/tmp/hwlab-cloud-api-db-psql.err <<'SQL'",
|
|
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
|
|
"\\gexec",
|
|
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
|
|
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
|
|
"\\gexec",
|
|
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
|
|
"SQL",
|
|
" db_ensure_exit=$?",
|
|
" if [ \"$db_ensure_exit\" -eq 0 ]; then",
|
|
" if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then",
|
|
" kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-cloud-api-rollout-restart.out 2>/tmp/hwlab-cloud-api-rollout-restart.err",
|
|
" rollout_restart_exit=$?",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-cloud-api-rollout-status.out 2>/tmp/hwlab-cloud-api-rollout-status.err",
|
|
" rollout_status_exit=$?",
|
|
" fi",
|
|
" fi",
|
|
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else action=ensured; mutation=true; fi",
|
|
" else action=db-ensure-failed; fi",
|
|
" else action=apply-failed; fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag \"$name\")",
|
|
"after_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
|
|
"after_url_present=$([ -n \"$after_url_b64\" ] && printf yes || printf no)",
|
|
"after_url_bytes=$(decoded_length \"$after_url_b64\")",
|
|
"probe_db",
|
|
"db_role_exists_after=$role_result",
|
|
"db_database_exists_after=$database_result",
|
|
"db_probe_exit_after=$probe_exit",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$database_url_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"",
|
|
"printf 'beforeDatabaseUrlPresent\\t%s\\n' \"$before_url_present\"",
|
|
"printf 'beforeDatabaseUrlBytes\\t%s\\n' \"$before_url_bytes\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterDatabaseUrlPresent\\t%s\\n' \"$after_url_present\"",
|
|
"printf 'afterDatabaseUrlBytes\\t%s\\n' \"$after_url_bytes\"",
|
|
"printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"",
|
|
"printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"",
|
|
"printf 'dbName\\t%s\\n' \"$db_name\"",
|
|
"printf 'dbUser\\t%s\\n' \"$db_user\"",
|
|
"printf 'dbHost\\t%s\\n' \"$db_host\"",
|
|
"printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"",
|
|
"printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"",
|
|
"printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"",
|
|
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
|
|
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
|
|
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
|
|
"printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"database_url= postgres_admin_password=",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const fields = keyValueLinesFromText(text);
|
|
if (fields.preset === "owned-postgres-cleanup") {
|
|
const absent = fields.afterStatefulSetExists !== "yes" &&
|
|
fields.afterServiceExists !== "yes" &&
|
|
fields.afterConfigMapExists !== "yes" &&
|
|
fields.afterSecretExists !== "yes" &&
|
|
fields.afterPvcExists !== "yes";
|
|
const platformServiceReady = fields.platformServiceExists === "yes";
|
|
return {
|
|
ok: commandOk && absent && platformServiceReady,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.postgresSecret,
|
|
statefulSet: fields.statefulSet || spec.postgresStatefulSet,
|
|
service: fields.service || spec.postgresSecret,
|
|
configMap: fields.configMap || `${spec.postgresSecret}-init`,
|
|
pvc: fields.pvc || `data-${spec.postgresSecret}-0`,
|
|
preset: "owned-postgres-cleanup",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
statefulSetExists: fields.beforeStatefulSetExists === "yes",
|
|
serviceExists: fields.beforeServiceExists === "yes",
|
|
configMapExists: fields.beforeConfigMapExists === "yes",
|
|
secretExists: fields.beforeSecretExists === "yes",
|
|
pvcExists: fields.beforePvcExists === "yes",
|
|
pvcPhase: fields.beforePvcPhase || null,
|
|
persistentVolume: fields.beforePersistentVolume || null,
|
|
},
|
|
after: {
|
|
statefulSetExists: fields.afterStatefulSetExists === "yes",
|
|
serviceExists: fields.afterServiceExists === "yes",
|
|
configMapExists: fields.afterConfigMapExists === "yes",
|
|
secretExists: fields.afterSecretExists === "yes",
|
|
pvcExists: fields.afterPvcExists === "yes",
|
|
pvcPhase: fields.afterPvcPhase || null,
|
|
persistentVolume: fields.afterPersistentVolume || null,
|
|
},
|
|
platformService: {
|
|
name: "g14-platform-postgres",
|
|
exists: platformServiceReady,
|
|
},
|
|
deleteStatefulSetExitCode: numericField(fields.deleteStatefulSetExitCode),
|
|
deleteServiceExitCode: numericField(fields.deleteServiceExitCode),
|
|
deleteConfigMapExitCode: numericField(fields.deleteConfigMapExitCode),
|
|
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
|
|
deletePvcExitCode: numericField(fields.deletePvcExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: absent
|
|
? `${fields.statefulSet || spec.postgresStatefulSet}, ${fields.service || spec.postgresSecret}, ${fields.configMap || `${spec.postgresSecret}-init`}, ${fields.secret || spec.postgresSecret}, and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent`
|
|
: `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`,
|
|
};
|
|
}
|
|
if (fields.preset === "obsolete-secret-cleanup") {
|
|
const absent = fields.afterSecretExists !== "yes";
|
|
const refsAbsent = fields.workloadRefsPresent !== "yes";
|
|
const dryRun = fields.dryRun === "true";
|
|
return {
|
|
ok: commandOk && refsAbsent && (dryRun || absent),
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.obsoleteHwpodDbSecret,
|
|
preset: "obsolete-secret-cleanup",
|
|
action: fields.action || null,
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
secretExists: fields.beforeSecretExists === "yes",
|
|
},
|
|
after: {
|
|
secretExists: fields.afterSecretExists === "yes",
|
|
},
|
|
workloadRefs: {
|
|
present: fields.workloadRefsPresent === "yes",
|
|
preview: fields.workloadRefsPreview || "",
|
|
},
|
|
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: refsAbsent && (dryRun || absent)
|
|
? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}`
|
|
: `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`,
|
|
};
|
|
}
|
|
if (fields.preset === "master-server-admin-api-key") {
|
|
const afterBytes = numericField(fields.afterApiKeyBytes);
|
|
const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.masterAdminApiKeySecret,
|
|
preset: "master-server-admin-api-key",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } },
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy ? `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} exists` : `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} missing`,
|
|
};
|
|
}
|
|
if (fields.preset === "code-agent-provider") {
|
|
const beforeOpenaiBytes = numericField(fields.beforeOpenaiBytes);
|
|
const beforeOpencodeBytes = numericField(fields.beforeOpencodeBytes);
|
|
const sourceOpenaiBytes = numericField(fields.sourceOpenaiBytes);
|
|
const sourceOpencodeBytes = numericField(fields.sourceOpencodeBytes);
|
|
const afterOpenaiBytes = numericField(fields.afterOpenaiBytes);
|
|
const afterOpencodeBytes = numericField(fields.afterOpencodeBytes);
|
|
const openaiReady = fields.afterOpenaiPresent === "yes" && typeof afterOpenaiBytes === "number" && afterOpenaiBytes > 0;
|
|
const opencodeReady = fields.afterOpencodePresent === "yes" && typeof afterOpencodeBytes === "number" && afterOpencodeBytes > 0;
|
|
const healthy = fields.afterExists === "yes" && (openaiReady || opencodeReady);
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.codeAgentProviderSecret,
|
|
preset: "code-agent-provider",
|
|
source: {
|
|
namespace: fields.sourceNamespace || spec.codeAgentProviderSourceNamespace,
|
|
secret: fields.sourceSecret || spec.codeAgentProviderSourceSecret,
|
|
exists: fields.sourceExists === "yes",
|
|
openaiApiKey: { keyPresent: fields.sourceOpenaiPresent === "yes", valueBytes: sourceOpenaiBytes },
|
|
opencodeApiKey: { keyPresent: fields.sourceOpencodePresent === "yes", valueBytes: sourceOpencodeBytes },
|
|
},
|
|
selectedKey: fields.selectedKey || null,
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
exists: fields.beforeExists === "yes",
|
|
openaiApiKey: { keyPresent: fields.beforeOpenaiPresent === "yes", valueBytes: beforeOpenaiBytes },
|
|
opencodeApiKey: { keyPresent: fields.beforeOpencodePresent === "yes", valueBytes: beforeOpencodeBytes },
|
|
},
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
openaiApiKey: { keyPresent: fields.afterOpenaiPresent === "yes", valueBytes: afterOpenaiBytes },
|
|
opencodeApiKey: { keyPresent: fields.afterOpencodePresent === "yes", valueBytes: afterOpencodeBytes },
|
|
requiredAnyProviderKeyPresent: openaiReady || opencodeReady,
|
|
},
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`,
|
|
};
|
|
}
|
|
if (fields.preset === "cloud-api-db") {
|
|
const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes);
|
|
const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes);
|
|
if (fields.platformDbMode === "true") {
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterDatabaseUrlPresent === "yes" &&
|
|
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
|
|
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
|
|
fields.platformEndpointsExists !== "yes" &&
|
|
fields.platformEndpointSliceExists === "yes";
|
|
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
|
|
fields.dbNameMatchesExpected === "yes" &&
|
|
fields.dbUserMatchesExpected === "yes";
|
|
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.cloudApiDbSecret,
|
|
key: fields.key || spec.cloudApiDbKey,
|
|
preset: "cloud-api-db",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
platformDbMode: true,
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes },
|
|
},
|
|
legacyPostgresSecret: {
|
|
name: fields.legacyPostgresSecret || spec.postgresSecret,
|
|
exists: fields.legacyPostgresSecretExists === "yes",
|
|
},
|
|
platformService: {
|
|
name: fields.platformService || spec.platformPostgresService,
|
|
exists: fields.platformServiceExists === "yes",
|
|
endpointsExist: fields.platformEndpointsExists === "yes",
|
|
legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes",
|
|
endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`,
|
|
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
|
|
},
|
|
dbName: fields.dbName || spec.cloudApiDbName,
|
|
dbUser: fields.dbUser || spec.cloudApiDbUser,
|
|
dbHost: fields.dbHost || spec.cloudApiDbHost,
|
|
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
|
|
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
|
|
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy
|
|
? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}`
|
|
: `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`,
|
|
};
|
|
}
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterDatabaseUrlPresent === "yes" &&
|
|
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
|
|
const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t";
|
|
const healthy = keysHealthy && databaseHealthy;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.cloudApiDbSecret,
|
|
key: fields.key || spec.cloudApiDbKey,
|
|
preset: "cloud-api-db",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
exists: fields.beforeExists === "yes",
|
|
postgresSecretExists: fields.beforePostgresSecretExists === "yes",
|
|
databaseUrl: { keyPresent: fields.beforeDatabaseUrlPresent === "yes", valueBytes: beforeUrlBytes },
|
|
database: {
|
|
roleExists: fields.dbRoleExistsBefore || "unknown",
|
|
databaseExists: fields.dbDatabaseExistsBefore || "unknown",
|
|
probeExitCode: fields.dbProbeExitCodeBefore || null,
|
|
},
|
|
},
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes },
|
|
database: {
|
|
roleExists: fields.dbRoleExistsAfter || "unknown",
|
|
databaseExists: fields.dbDatabaseExistsAfter || "unknown",
|
|
probeExitCode: fields.dbProbeExitCodeAfter || null,
|
|
},
|
|
},
|
|
postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes",
|
|
postgresSecret: fields.postgresSecret || spec.postgresSecret,
|
|
dbName: fields.dbName || spec.cloudApiDbName,
|
|
dbUser: fields.dbUser || spec.cloudApiDbUser,
|
|
dbHost: fields.dbHost || spec.cloudApiDbHost,
|
|
cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment,
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`,
|
|
};
|
|
}
|
|
const afterAuthnBytes = numericField(fields.afterAuthnBytes);
|
|
const afterUriBytes = numericField(fields.afterDatastoreUriBytes);
|
|
const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes);
|
|
if (fields.platformDbMode === "true") {
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterAuthnPresent === "yes" &&
|
|
fields.afterDatastoreUriPresent === "yes" &&
|
|
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
|
|
typeof afterUriBytes === "number" && afterUriBytes > 0;
|
|
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
|
|
fields.platformEndpointsExists !== "yes" &&
|
|
fields.platformEndpointSliceExists === "yes";
|
|
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
|
|
fields.dbNameMatchesExpected === "yes" &&
|
|
fields.dbUserMatchesExpected === "yes";
|
|
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.openFgaSecret,
|
|
preset: fields.preset || "openfga",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
platformDbMode: true,
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
|
|
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
|
|
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
|
|
},
|
|
legacyPostgresSecret: {
|
|
name: fields.legacyPostgresSecret || spec.postgresSecret,
|
|
exists: fields.legacyPostgresSecretExists === "yes",
|
|
},
|
|
platformService: {
|
|
name: fields.platformService || spec.platformPostgresService,
|
|
exists: fields.platformServiceExists === "yes",
|
|
endpointsExist: fields.platformEndpointsExists === "yes",
|
|
legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes",
|
|
endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`,
|
|
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
|
|
},
|
|
dbName: fields.dbName || spec.openFgaDbName,
|
|
dbUser: fields.dbUser || spec.openFgaDbUser,
|
|
dbHost: fields.dbHost || spec.openFgaDbHost,
|
|
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
|
|
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
|
|
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy
|
|
? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.platformService || spec.platformPostgresService}`
|
|
: `${fields.secret || spec.openFgaSecret} datastore-uri is not aligned to platform DB`,
|
|
};
|
|
}
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterPostgresSecretExists === "yes" &&
|
|
fields.afterAuthnPresent === "yes" &&
|
|
fields.afterDatastoreUriPresent === "yes" &&
|
|
fields.afterPostgresPasswordPresent === "yes" &&
|
|
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
|
|
typeof afterUriBytes === "number" && afterUriBytes > 0 &&
|
|
typeof afterPasswordBytes === "number" && afterPasswordBytes > 0;
|
|
const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t";
|
|
const healthy = keysHealthy && databaseHealthy;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.openFgaSecret,
|
|
preset: fields.preset || "openfga",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
postgresSecretExists: fields.afterPostgresSecretExists === "yes",
|
|
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
|
|
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
|
|
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
|
|
database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null },
|
|
},
|
|
postgresSecretExitCode: numericField(fields.postgresSecretExitCode),
|
|
postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode),
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy ? `${fields.secret || spec.openFgaSecret} keys and Postgres database exist` : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`,
|
|
};
|
|
}
|
|
|
|
function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const fields = keyValueLinesFromText(text);
|
|
const dryRun = fields.dryRun === "true";
|
|
const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown";
|
|
const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown";
|
|
const probesOk = fields.beforeDatabaseExists !== "unknown" &&
|
|
fields.beforeRoleExists !== "unknown" &&
|
|
fields.afterDatabaseExists !== "unknown" &&
|
|
fields.afterRoleExists !== "unknown";
|
|
return {
|
|
ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)),
|
|
database: fields.database || spec.obsoleteHwpodDbName,
|
|
role: fields.role || spec.obsoleteHwpodDbUser,
|
|
preset: "obsolete-platform-db-cleanup",
|
|
action: fields.action || null,
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
databaseExists: fields.beforeDatabaseExists === "yes",
|
|
roleExists: fields.beforeRoleExists === "yes",
|
|
},
|
|
after: {
|
|
databaseExists: fields.afterDatabaseExists === "yes",
|
|
roleExists: fields.afterRoleExists === "yes",
|
|
},
|
|
beforeProbeExitCode: {
|
|
database: numericField(fields.beforeDatabaseProbeExitCode),
|
|
role: numericField(fields.beforeRoleProbeExitCode),
|
|
},
|
|
afterProbeExitCode: {
|
|
database: numericField(fields.afterDatabaseProbeExitCode),
|
|
role: numericField(fields.afterRoleProbeExitCode),
|
|
},
|
|
dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode),
|
|
dropRoleExitCode: numericField(fields.dropRoleExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: probesOk && (dryRun || (databaseAbsent && roleAbsent))
|
|
? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}`
|
|
: `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`,
|
|
};
|
|
}
|
|
|
|
export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record<string, unknown> {
|
|
return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane }));
|
|
}
|
|
|
|
function readMasterAdminApiKey(): { key: string; source: string } {
|
|
if (!existsSync(MASTER_ADMIN_API_KEY_ENV)) throw new Error(`HWLAB_API_KEY source missing: ${MASTER_ADMIN_API_KEY_ENV}`);
|
|
const content = readFileSync(MASTER_ADMIN_API_KEY_ENV, "utf8");
|
|
const match = content.match(/^HWLAB_API_KEY=(.+)$/m);
|
|
const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, "");
|
|
if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${MASTER_ADMIN_API_KEY_ENV}`);
|
|
return { key: raw, source: MASTER_ADMIN_API_KEY_ENV };
|
|
}
|
|
|
|
function optionValue(args: string[], name: string): string | undefined {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return undefined;
|
|
const value = args[index + 1];
|
|
if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
function requiredOption(args: string[], name: string): string {
|
|
const value = optionValue(args, name);
|
|
if (value === undefined) throw new Error(`${name} is required`);
|
|
return value;
|
|
}
|
|
|
|
function stripOption(args: string[], name: string): string[] {
|
|
return stripOptions(args, [name]);
|
|
}
|
|
|
|
function stripOptions(args: string[], names: readonly string[]): string[] {
|
|
const remove = new Set(names);
|
|
const without: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (remove.has(arg)) {
|
|
if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1;
|
|
continue;
|
|
}
|
|
without.push(arg);
|
|
}
|
|
return without;
|
|
}
|
|
|
|
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const raw = optionValue(args, name);
|
|
if (raw === undefined) return defaultValue;
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
function assertLane(value: string): void {
|
|
if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`);
|
|
}
|
|
|
|
function assertNodeId(value: string): void {
|
|
if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`);
|
|
}
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
|
}
|
|
|
|
function statusText(result: CommandResult): string {
|
|
return result.stdout || result.stderr;
|
|
}
|
|
|
|
function keyValueLinesFromText(text: string): Record<string, string> {
|
|
const fields: Record<string, string> = {};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const index = line.indexOf("\t");
|
|
if (index <= 0) continue;
|
|
fields[line.slice(0, index)] = line.slice(index + 1);
|
|
}
|
|
return fields;
|
|
}
|
|
|
|
function numericField(value: string | undefined): number | null {
|
|
if (value === undefined || value === "") return null;
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function splitWhitespaceField(value: string | undefined): string[] {
|
|
if (!value) return [];
|
|
return value.split(/\s+/u).filter(Boolean);
|
|
}
|
|
|
|
function compactCommandResult(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
command: compactCommand(result.command),
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: result.stdout.length,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
function compactCommand(command: string[]): string[] {
|
|
const scriptIndex = command.indexOf("--");
|
|
if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "<script omitted>"];
|
|
return command;
|
|
}
|