From 6b48a9e682d4b5bdee564212d944afc1b27b0d32 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 8 Jun 2026 09:47:43 +0000 Subject: [PATCH] feat: add HWLAB node lane secret CLI --- scripts/cli.ts | 8 + scripts/src/help.ts | 4 +- scripts/src/hwlab-node.ts | 505 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 516 insertions(+), 1 deletion(-) create mode 100644 scripts/src/hwlab-node.ts diff --git a/scripts/cli.ts b/scripts/cli.ts index 3f2e5f17..fa10d227 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -26,6 +26,7 @@ import { isHelpToken, rootHelp, serverHelp, sshHelp, staticNamespaceHelp } from import { runServerCleanupCommand } from "./src/server-cleanup"; import { runHwlabCdCommand } from "./src/hwlab-cd"; import { runHwlabG14Command } from "./src/hwlab-g14"; +import { runHwlabNodeCommand } from "./src/hwlab-node"; import { runGcCommand } from "./src/gc"; import { runAgentRunCommand } from "./src/agentrun"; @@ -290,6 +291,13 @@ async function main(): Promise { } if (top === "hwlab") { + if (sub === "node" || sub === "nodes") { + const result = await runHwlabNodeCommand(readConfig(), args.slice(2)); + const ok = (result as { ok?: unknown }).ok !== false; + emitJson(commandName, result, ok); + if (!ok) process.exitCode = 1; + return; + } if (sub === "g14") { const result = await runHwlabG14Command(readConfig(), args.slice(2)); const ok = (result as { ok?: unknown }).ok !== false; diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 67c40181..6842a8ef 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -2,6 +2,7 @@ import { ghHelp } from "./gh"; import { authBrokerHelp } from "./auth-broker"; import { hwlabHelp } from "./hwlab-cd"; import { hwlabG14Help } from "./hwlab-g14"; +import { hwlabNodeHelp } from "./hwlab-node"; import { agentRunHelp } from "./agentrun"; export function rootHelp(): unknown { @@ -58,7 +59,7 @@ export function rootHelp(): unknown { { command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." }, { command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." }, { command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." }, - { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions through UniDesk G14 routes; long confirmed trigger/sync/flush actions return async jobs by default." }, + { command: "hwlab nodes secret status|ensure --node G14 --lane v03 | hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Manage HWLAB node/lane runtime prerequisites, start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." }, { command: "agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs", description: "Run bounded AgentRun v0.1 Tekton/Argo status, manual PipelineRun trigger, Argo refresh, and completed CI workspace retention through UniDesk G14 routes." }, { command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." }, { command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." }, @@ -615,6 +616,7 @@ export function staticNamespaceHelp(args: string[]): unknown | null { if (top === "auth-broker") return authBrokerHelp(); if (top === "gh") return ghHelp(); if (top === "agentrun") return agentRunHelp(); + if (top === "hwlab" && (sub === "node" || sub === "nodes")) return hwlabNodeHelp(); if (top === "hwlab" && sub === "g14") return hwlabG14Help(); if (top === "hwlab") return hwlabHelp(); return null; diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts new file mode 100644 index 00000000..afa3513a --- /dev/null +++ b/scripts/src/hwlab-node.ts @@ -0,0 +1,505 @@ +import { existsSync, readFileSync } from "node:fs"; +import { repoRoot, type Config } from "./config"; +import { runCommand, type CommandResult } from "./command"; + +type SecretAction = "status" | "ensure"; +type SecretPreset = "openfga" | "master-server-admin-api-key"; + +interface NodeSecretOptions { + action: SecretAction; + node: string; + lane: string; + name: string; + key?: string; + preset: SecretPreset; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; +} + +interface RuntimeSecretSpec { + node: string; + lane: string; + namespace: string; + postgresSecret: string; + postgresStatefulSet: string; + postgresAdminUser: string; + openFgaSecret: string; + openFgaDbName: string; + openFgaDbUser: string; + openFgaDbHost: string; + masterAdminApiKeySecret: string; + fieldManager: string; +} + +const MASTER_ADMIN_API_KEY_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env"; +const MASTER_ADMIN_API_KEY_KEY = "api-key"; +const OPENFGA_AUTHN_KEY = "authn-preshared-key"; +const OPENFGA_DATASTORE_URI_KEY = "datastore-uri"; +const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password"; + +export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise> { + if (args.length === 0 || args.includes("--help") || args.includes("-h")) return hwlabNodeHelp(); + const [domain] = args; + if (domain !== "secret") { + return { ok: false, command: `hwlab node ${domain ?? ""}`.trim(), message: "supported commands: hwlab node secret status|ensure" }; + } + const options = parseSecretOptions(args.slice(1)); + return runNodeSecret(options); +} + +export function hwlabNodeHelp(): Record { + return { + ok: true, + command: "hwlab nodes", + description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.", + examples: [ + "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", + ], + }; +} + +function parseSecretOptions(args: string[]): NodeSecretOptions { + const [actionRaw] = args; + if (actionRaw !== "status" && actionRaw !== "ensure") { + throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key [--dry-run|--confirm]"); + } + const node = optionValue(args, "--node") ?? "G14"; + assertNodeId(node); + const lane = optionValue(args, "--lane") ?? "v03"; + assertLane(lane); + const spec = runtimeSecretSpec({ node, lane }); + const name = optionValue(args, "--name") ?? spec.openFgaSecret; + const key = optionValue(args, "--key"); + const confirm = args.includes("--confirm"); + const explicitDryRun = args.includes("--dry-run"); + if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run"); + if (name === spec.masterAdminApiKeySecret) { + if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`); + return { + action: actionRaw, + node, + lane, + name, + key: key ?? MASTER_ADMIN_API_KEY_KEY, + preset: "master-server-admin-api-key", + confirm, + dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; + } + if (name !== spec.openFgaSecret) { + throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret} or ${spec.masterAdminApiKeySecret} for --lane ${lane}`); + } + if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) { + throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`); + } + return { + action: actionRaw, + node, + lane, + name, + key, + preset: "openfga", + confirm, + dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; +} + +function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec { + const namespace = `hwlab-${input.lane}`; + return { + node: input.node, + lane: input.lane, + namespace, + postgresSecret: `${namespace}-postgres`, + postgresStatefulSet: `${namespace}-postgres`, + postgresAdminUser: `hwlab_${input.lane}`, + openFgaSecret: `${namespace}-openfga`, + openFgaDbName: "hwlab_openfga", + openFgaDbUser: "hwlab_openfga", + openFgaDbHost: `${namespace}-postgres.${namespace}.svc.cluster.local`, + masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`, + fieldManager: `unidesk-hwlab-node-${input.lane}-secret`, + }; +} + +function runNodeSecret(options: NodeSecretOptions): Record { + const spec = runtimeSecretSpec(options); + const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun + ? readMasterAdminApiKey().key + : ""; + const result = runTransScript(options.node, options.preset === "openfga" ? openFgaSecretScript(options, spec) : masterAdminApiKeySecretScript(options, spec), input, options.timeoutSeconds); + const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec); + const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0; + const ok = dryRunOk ? true : status.ok === true; + return { + ok, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : "confirmed-ensure", + status, + mutation: status.mutation === true, + result: compactCommandResult(result), + valuesRedacted: true, + next: ok && options.action === "status" ? undefined : { + ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm`, + }, + }; +} + +function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { + return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `openfga_secret=${shellQuote(spec.openFgaSecret)}`, + `postgres_secret=${shellQuote(spec.postgresSecret)}`, + `postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`, + `postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`, + `selected_key=${shellQuote(options.key ?? "")}`, + `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, + `datastore_uri_key=${shellQuote(OPENFGA_DATASTORE_URI_KEY)}`, + `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, + `db_name=${shellQuote(spec.openFgaDbName)}`, + `db_user=${shellQuote(spec.openFgaDbUser)}`, + `db_host=${shellQuote(spec.openFgaDbHost)}`, + `action_request=${shellQuote(options.action)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `field_manager=${shellQuote(spec.fieldManager)}`, + "preset=openfga", + "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", + "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", + "probe_db() {", + " role_result=unknown", + " database_result=unknown", + " probe_exit=missing-postgres-admin-secret", + " if [ -n \"$postgres_admin_password\" ]; then", + " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", + " role_exit=$?", + " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", + " database_exit=$?", + " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", + " fi", + "}", + "before_exists=$(secret_exists_flag \"$openfga_secret\")", + "before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", + "before_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", + "before_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", + "before_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", + "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", + "before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)", + "before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)", + "before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)", + "before_authn_bytes=$(decoded_length \"$before_authn_b64\")", + "before_uri_bytes=$(decoded_length \"$before_uri_b64\")", + "before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")", + "authn_value=$(decoded_value \"$before_authn_b64\")", + "datastore_uri=$(decoded_value \"$before_uri_b64\")", + "pg_password=$(decoded_value \"$before_pg_password_b64\")", + "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", + "probe_db", + "db_role_exists_before=$role_result", + "db_database_exists_before=$database_result", + "db_probe_exit_before=$probe_exit", + "action=observed", + "mutation=false", + "postgres_secret_exit=", + "postgres_rollout_exit=", + "apply_exit=", + "db_ensure_exit=", + "if [ \"$action_request\" = ensure ]; then", + " missing_secret=false", + " [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true", + " [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true", + " [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true", + " missing_db=false", + " [ \"$db_role_exists_before\" = t ] || missing_db=true", + " [ \"$db_database_exists_before\" = t ] || missing_db=true", + " if [ \"$dry_run\" = true ]; then", + " if [ \"$before_postgres_exists\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", + " elif ! command -v openssl >/dev/null 2>&1; then", + " action=openssl-missing; apply_exit=127", + " else", + " [ -n \"$postgres_admin_password\" ] || postgres_admin_password=$(openssl rand -hex 24)", + " kubectl -n \"$namespace\" create secret generic \"$postgres_secret\" --from-literal=\"POSTGRES_PASSWORD=$postgres_admin_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", + " postgres_secret_exit=$?", + " if [ \"$postgres_secret_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout status \"statefulset/$postgres_statefulset\" --timeout=120s >/tmp/hwlab-postgres-rollout.out 2>/tmp/hwlab-postgres-rollout.err", + " postgres_rollout_exit=$?", + " fi", + " if [ \"$postgres_secret_exit\" -ne 0 ]; then action=postgres-secret-failed; apply_exit=$postgres_secret_exit", + " elif [ \"$postgres_rollout_exit\" -ne 0 ]; then action=postgres-rollout-failed; apply_exit=$postgres_rollout_exit", + " else", + " [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)", + " [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)", + " [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"", + " kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", + " apply_exit=$?", + " if [ \"$apply_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'", + "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", + "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", + "\\gexec", + "ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';", + "SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')", + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')", + "\\gexec", + "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", + "SQL", + " db_ensure_exit=$?", + " if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi", + " else action=apply-failed; fi", + " fi", + " fi", + "fi", + "after_exists=$(secret_exists_flag \"$openfga_secret\")", + "after_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", + "after_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", + "after_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", + "after_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", + "after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)", + "after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)", + "after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)", + "after_authn_bytes=$(decoded_length \"$after_authn_b64\")", + "after_uri_bytes=$(decoded_length \"$after_uri_b64\")", + "after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")", + "probe_db", + "db_role_exists_after=$role_result", + "db_database_exists_after=$database_result", + "db_probe_exit_after=$probe_exit", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$openfga_secret\"", + "printf 'key\\t%s\\n' \"$selected_key\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeExists\\t%s\\n' \"$before_exists\"", + "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", + "printf 'afterExists\\t%s\\n' \"$after_exists\"", + "printf 'afterPostgresSecretExists\\t%s\\n' \"$after_postgres_exists\"", + "printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"", + "printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"", + "printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"", + "printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"", + "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"", + "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"", + "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", + "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", + "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", + "printf 'postgresSecretExitCode\\t%s\\n' \"$postgres_secret_exit\"", + "printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", + "authn_value= datastore_uri= pg_password= postgres_admin_password=", + "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", + "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", + ].join("\n"); +} + +function masterAdminApiKeySecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `name=${shellQuote(spec.masterAdminApiKeySecret)}`, + `api_key_name=${shellQuote(MASTER_ADMIN_API_KEY_KEY)}`, + `action_request=${shellQuote(options.action)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `field_manager=${shellQuote(spec.fieldManager)}`, + "preset=master-server-admin-api-key", + "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-12; value=; fi; }", + "before_exists=$(secret_exists_flag)", + "before_api_key_b64=$(secret_b64_key \"$api_key_name\")", + "before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)", + "before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")", + "action=observed", + "mutation=false", + "apply_exit=", + "if [ \"$action_request\" = ensure ]; then", + " missing_secret=false", + " [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true", + " if [ \"$dry_run\" = true ]; then", + " if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi", + " else", + " api_key=$(cat)", + " case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac", + " if [ -z \"$apply_exit\" ]; then", + " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", + " apply_exit=$?", + " if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi", + " fi", + " api_key=", + " fi", + "fi", + "after_exists=$(secret_exists_flag)", + "after_api_key_b64=$(secret_b64_key \"$api_key_name\")", + "after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)", + "after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")", + "after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$name\"", + "printf 'key\\t%s\\n' \"$api_key_name\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'afterExists\\t%s\\n' \"$after_exists\"", + "printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"", + "printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"", + "printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", + ].join("\n"); +} + +function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { + const fields = keyValueLinesFromText(text); + if (fields.preset === "master-server-admin-api-key") { + const afterBytes = numericField(fields.afterApiKeyBytes); + const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.masterAdminApiKeySecret, + preset: "master-server-admin-api-key", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } }, + applyExitCode: numericField(fields.applyExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy ? `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} exists` : `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} missing`, + }; + } + const afterAuthnBytes = numericField(fields.afterAuthnBytes); + const afterUriBytes = numericField(fields.afterDatastoreUriBytes); + const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes); + const keysHealthy = fields.afterExists === "yes" && + fields.afterPostgresSecretExists === "yes" && + fields.afterAuthnPresent === "yes" && + fields.afterDatastoreUriPresent === "yes" && + fields.afterPostgresPasswordPresent === "yes" && + typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && + typeof afterUriBytes === "number" && afterUriBytes > 0 && + typeof afterPasswordBytes === "number" && afterPasswordBytes > 0; + const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; + const healthy = keysHealthy && databaseHealthy; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.openFgaSecret, + preset: fields.preset || "openfga", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + after: { + exists: fields.afterExists === "yes", + postgresSecretExists: fields.afterPostgresSecretExists === "yes", + authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, + datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, + postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, + database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null }, + }, + postgresSecretExitCode: numericField(fields.postgresSecretExitCode), + postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode), + applyExitCode: numericField(fields.applyExitCode), + dbEnsureExitCode: numericField(fields.dbEnsureExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy ? `${fields.secret || spec.openFgaSecret} keys and Postgres database exist` : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`, + }; +} + +function readMasterAdminApiKey(): { key: string; source: string } { + if (!existsSync(MASTER_ADMIN_API_KEY_ENV)) throw new Error(`HWLAB_API_KEY source missing: ${MASTER_ADMIN_API_KEY_ENV}`); + const content = readFileSync(MASTER_ADMIN_API_KEY_ENV, "utf8"); + const match = content.match(/^HWLAB_API_KEY=(.+)$/m); + const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, ""); + if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${MASTER_ADMIN_API_KEY_ENV}`); + return { key: raw, source: MASTER_ADMIN_API_KEY_ENV }; +} + +function optionValue(args: string[], name: string): string | undefined { + const index = args.indexOf(name); + if (index === -1) return undefined; + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); + return value; +} + +function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { + const raw = optionValue(args, name); + if (raw === undefined) return defaultValue; + const value = Number(raw); + if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); + return Math.min(value, maxValue); +} + +function assertLane(value: string): void { + if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`); +} + +function assertNodeId(value: string): void { + if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`); +} + +function shellQuote(value: string): string { + return `'${value.replace(/'/gu, `'"'"'`)}'`; +} + +function statusText(result: CommandResult): string { + return result.stdout || result.stderr; +} + +function keyValueLinesFromText(text: string): Record { + const fields: Record = {}; + for (const line of text.split(/\r?\n/u)) { + const index = line.indexOf("\t"); + if (index <= 0) continue; + fields[line.slice(0, index)] = line.slice(index + 1); + } + return fields; +} + +function numericField(value: string | undefined): number | null { + if (value === undefined || value === "") return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function compactCommandResult(result: CommandResult): Record { + return { + command: compactCommand(result.command), + exitCode: result.exitCode, + stdoutBytes: result.stdout.length, + stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), + timedOut: result.timedOut, + }; +} + +function compactCommand(command: string[]): string[] { + const scriptIndex = command.indexOf("--"); + if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "