fix(hwlab): bootstrap local postgres runtime secrets (#728)

Co-authored-by: Codex <codex@noreply.local>

.agents/skills/unidesk-cicd/SKILL.md

@@ -100,7 +100,7 @@ bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confi
 
 `hwlab nodes control-plane status` 默认返回 compact commander summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。
 
-`hwlab nodes control-plane sync --confirm` 是 Argo runtime 收敛修复入口：会终止卡住的 running Argo operation、删除失败 hook Job，并在 StatefulSet template 已更新但旧 controller-revision pod 因 `ImagePullBackOff` / `ErrImagePull` / `CrashLoopBackOff` 卡住时受控删除该旧 pod，让 StatefulSet 按最新 revision 重建。不要手工裸删 pod；需要解除这类死锁时走该入口。
+`hwlab nodes control-plane sync --confirm` 是 Argo runtime 收敛修复入口：会先按 YAML `runtimeStore.postgres.mode=local-k3s` 同步本地 postgres bootstrap Secret，再终止卡住的 running Argo operation、删除失败 hook Job，并在 StatefulSet template 已更新但旧 controller-revision pod 因 `ImagePullBackOff` / `ErrImagePull` / `CrashLoopBackOff` 卡住时受控删除该旧 pod，让 StatefulSet 按最新 revision 重建。不要手工裸删 pod；需要解除这类死锁时走该入口。
 
 `hwlab nodes control-plane trigger-current --node <node> --lane <lane> --confirm --wait` 是 node/lane CI/CD 一键入口：按 YAML 解析 source head，执行 git-mirror pre-sync/pre-flush，刷新 control-plane，创建或复用 commit-pinned PipelineRun，等待 PipelineRun 终态，并在终态成功后执行 post-flush。默认输出必须是低噪声 CICD 表格摘要；完整 JSON 只能通过 `--full` 或 `--raw` 展开。`--wait` 默认最多等待 120 秒；超过 120 秒时 CLI 返回 `pending` warning，并直接给出 env-reuse 检查命令 `control-plane status --full` 和 git-mirror 检查命令 `git-mirror status`，不继续长时间阻塞，也不把仍在运行误报为构建失败。
 

config/hwlab-node-lanes.yaml

@@ -147,6 +147,8 @@ lanes:
             statefulSet: hwlab-v03-postgres
             serviceName: hwlab-v03-postgres
             adminUser: hwlab_v03
+            adminPasswordSourceRef: hwlab/d601-v03-postgres.env
+            adminPasswordSourceKey: HWLAB_V03_POSTGRES_PASSWORD
             cloudApi:
               secretName: hwlab-cloud-api-v03-db
               secretKey: database-url

scripts/src/hwlab-node-help.ts

@@ -60,7 +60,7 @@ export function hwlabNodeHelp(): Record<string, unknown> {
     notes: [
       "`trigger-current` skips an already succeeded PipelineRun for the same HWLAB source commit by default.",
       "`trigger-current --confirm --wait` is the one-command CICD path: git-mirror pre-sync/pre-flush, control-plane refresh, PipelineRun create/reuse, bounded wait, and post-flush when terminal.",
-      "`control-plane sync --confirm` terminates a stale running Argo operation, deletes failed Argo hook Jobs, and recreates stale non-ready StatefulSet pods that are still pinned to an old controller revision with pull/backoff errors.",
+      "`control-plane sync --confirm` syncs the YAML-declared local-k3s postgres bootstrap Secret, terminates a stale running Argo operation, deletes failed Argo hook Jobs, and recreates stale non-ready StatefulSet pods that are still pinned to an old controller revision with pull/backoff errors.",
       "`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.",
       "Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not."
     ],

scripts/src/hwlab-node-impl.ts

@@ -2422,6 +2422,20 @@ function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOpti
       headProbe: compactRuntimeCommand(head.result),
     };
   }
+  const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
+  if (localPostgres !== null && localPostgres.ok !== true) {
+    return {
+      ok: false,
+      command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
+      phase: "local-postgres-secret-sync",
+      sourceCommit,
+      localPostgres,
+      degradedReason: "local-postgres-secret-sync-failed",
+    };
+  }
   const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds);
   if (secrets !== null && secrets.ok !== true) {
     return {
@@ -2432,6 +2446,7 @@ function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOpti
       mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
       phase: "external-postgres-secret-sync",
       sourceCommit,
+      localPostgres,
       secrets,
       degradedReason: "external-postgres-secret-sync-failed",
     };
@@ -2446,6 +2461,7 @@ function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOpti
       mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
       phase: "base-image-seed",
       sourceCommit,
+      localPostgres,
       secrets,
       baseImage,
       degradedReason: "node-runtime-base-image-seed-failed",
@@ -2461,6 +2477,7 @@ function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOpti
       mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
       phase: "source-render",
       sourceCommit,
+      localPostgres,
       secrets,
       baseImage,
       renderDir: render.renderDir,
@@ -2485,6 +2502,7 @@ function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOpti
     mutation: !scoped.dryRun && isCommandSuccess(apply),
     sourceCommit,
     expected: nodeRuntimeExpected(spec),
+    localPostgres,
     secrets,
     baseImage,
     renderDir: render.renderDir,
@@ -2546,6 +2564,20 @@ function nodeRuntimeRefresh(scoped: ReturnType<typeof parseNodeScopedDelegatedOp
 
 function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
   const spec = scoped.spec;
+  const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
+  if (localPostgres !== null && localPostgres.ok !== true) {
+    return {
+      ok: false,
+      command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
+      phase: "local-postgres-secret-sync",
+      localPostgres,
+      degradedReason: "local-postgres-secret-sync-failed",
+      next: { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` },
+    };
+  }
   const operation = {
     operation: {
       initiatedBy: { username: "unidesk-hwlab-node-cli" },
@@ -2685,7 +2717,8 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
     node: scoped.node,
     lane: scoped.lane,
     mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
-    mutation: !scoped.dryRun && isCommandSuccess(result),
+    mutation: !scoped.dryRun && (isCommandSuccess(result) || localPostgres?.mutation === true),
+    localPostgres,
     argoApplication: fields.app || spec.app,
     syncSource: {
       repoURL: spec.argoRepoUrl,
@@ -6329,6 +6362,239 @@ function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boo
   };
 }
 
+function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
+  const pg = spec.runtimeStore?.postgres;
+  if (pg?.mode !== "local-k3s") return null;
+  const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`;
+  const sourceRef = pg.adminPasswordSourceRef ?? "";
+  const sourceKey = pg.adminPasswordSourceKey ?? "";
+  if (sourceRef.length === 0 || sourceKey.length === 0) {
+    return {
+      ok: false,
+      mode: dryRun ? "dry-run" : "confirmed-secret-sync",
+      mutation: false,
+      namespace: spec.runtimeNamespace,
+      secretName,
+      secretKey: "POSTGRES_PASSWORD",
+      degradedReason: "local-postgres-secret-source-not-configured",
+      valuesPrinted: false,
+      next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` },
+    };
+  }
+  const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun });
+  if (material.ok !== true) {
+    return {
+      ok: false,
+      mode: dryRun ? "dry-run" : "confirmed-secret-sync",
+      mutation: false,
+      namespace: spec.runtimeNamespace,
+      secretName,
+      secretKey: "POSTGRES_PASSWORD",
+      source: material,
+      degradedReason: material.error ?? "local-postgres-secret-source-missing",
+      valuesPrinted: false,
+      next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=<redacted>` },
+    };
+  }
+  const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? "");
+  const fields = keyValueLinesFromText(statusText(result));
+  const afterBytes = numericField(fields.afterPasswordBytes) ?? 0;
+  const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0;
+  return {
+    ok,
+    mode: dryRun ? "dry-run" : "confirmed-secret-sync",
+    mutation: !dryRun && fields.mutation === "true",
+    namespace: spec.runtimeNamespace,
+    secretName,
+    secretKey: "POSTGRES_PASSWORD",
+    source: {
+      ok: material.ok,
+      sourceRef,
+      sourceKey,
+      sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath),
+      generated: material.generated,
+      fingerprint: material.fingerprint,
+      valueBytes: material.value?.length ?? 0,
+      valuesPrinted: false,
+    },
+    before: {
+      namespaceExists: fields.beforeNamespaceExists === "yes",
+      secretExists: fields.beforeSecretExists === "yes",
+      passwordBytes: numericField(fields.beforePasswordBytes),
+      fingerprint: fields.beforePasswordFingerprint || null,
+    },
+    after: {
+      namespaceExists: fields.afterNamespaceExists === "yes",
+      secretExists: fields.afterSecretExists === "yes",
+      passwordBytes: numericField(fields.afterPasswordBytes),
+      fingerprint: fields.afterPasswordFingerprint || null,
+    },
+    action: fields.action || null,
+    applyExitCode: numericField(fields.applyExitCode),
+    namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode),
+    degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed",
+    result: compactRuntimeCommand(result),
+    valuesPrinted: false,
+  };
+}
+
+function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): {
+  ok: boolean;
+  sourceRef: string;
+  sourceKey: string;
+  sourcePath: string | null;
+  checkedPaths: string[];
+  value: string | null;
+  fingerprint: string | null;
+  generated: boolean;
+  error?: string;
+} {
+  const checkedPaths = localSecretSourcePaths(input.sourceRef);
+  const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null;
+  const sourcePath = existingPath ?? checkedPaths[0] ?? null;
+  if (sourcePath === null) {
+    return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" };
+  }
+  if (existingPath === null && input.dryRun) {
+    return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" };
+  }
+  if (existingPath === null) {
+    const value = randomBytes(32).toString("base64url");
+    mkdirSync(dirname(sourcePath), { recursive: true });
+    writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 });
+    return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
+  }
+  const text = readFileSync(existingPath, "utf8");
+  const values = parseEnvFile(text);
+  const existingValue = values[input.sourceKey];
+  if (existingValue !== undefined && existingValue.length > 0) {
+    return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false };
+  }
+  if (input.dryRun) {
+    return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" };
+  }
+  const value = randomBytes(32).toString("base64url");
+  const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n";
+  writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 });
+  return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
+}
+
+function localSecretSourcePaths(sourceRef: string): string[] {
+  const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
+  const marker = "/.worktree/";
+  const index = repoRoot.indexOf(marker);
+  if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef));
+  return [...new Set(paths)];
+}
+
+function shortSecretFingerprint(value: string): string {
+  return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`;
+}
+
+function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string {
+  return [
+    "set +e",
+    `namespace=${shellQuote(spec.runtimeNamespace)}`,
+    `secret=${shellQuote(secretName)}`,
+    "secret_key=POSTGRES_PASSWORD",
+    `source_ref=${shellQuote(sourceRef)}`,
+    `source_key=${shellQuote(sourceKey)}`,
+    `source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`,
+    `dry_run=${shellQuote(dryRun ? "true" : "false")}`,
+    "field_manager=unidesk-hwlab-node-local-postgres-secret",
+    "password=$(cat)",
+    "password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')",
+    "exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
+    "namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }",
+    "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }",
+    "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
+    "decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }",
+    "before_namespace_exists=$(namespace_exists_flag)",
+    "before_secret_exists=no",
+    "before_b64=",
+    "if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi",
+    "before_password_bytes=$(decoded_length \"$before_b64\")",
+    "before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")",
+    "after_namespace_exists=$before_namespace_exists",
+    "after_secret_exists=$before_secret_exists",
+    "after_password_bytes=$before_password_bytes",
+    "after_password_fingerprint=$before_password_fingerprint",
+    "action=observed",
+    "mutation=false",
+    "namespace_create_exit=",
+    "apply_exit=",
+    "if [ \"$dry_run\" = true ]; then",
+    "  if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi",
+    "elif [ \"$password_bytes\" -le 0 ]; then",
+    "  action=source-empty",
+    "  apply_exit=42",
+    "elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then",
+    "  action=source-mismatch",
+    "  apply_exit=47",
+    "else",
+    "  if [ \"$before_namespace_exists\" != yes ]; then",
+    "    kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err",
+    "    namespace_create_exit=$?",
+    "  else",
+    "    namespace_create_exit=0",
+    "  fi",
+    "  if [ \"$namespace_create_exit\" = 0 ]; then",
+    "    tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)",
+    "    password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')",
+    "    cat >\"$tmp\" <<EOF_SECRET",
+    "apiVersion: v1",
+    "kind: Secret",
+    "metadata:",
+    "  name: $secret",
+    "  namespace: $namespace",
+    "  labels:",
+    "    app.kubernetes.io/part-of: hwlab",
+    "    app.kubernetes.io/managed-by: unidesk",
+    "  annotations:",
+    "    hwlab.pikastech.local/secret-source-ref: $source_ref",
+    "    hwlab.pikastech.local/secret-source-key: $source_key",
+    "    hwlab.pikastech.local/secret-source-fingerprint: $source_fingerprint",
+    "type: Opaque",
+    "data:",
+    "  $secret_key: $password_b64",
+    "EOF_SECRET",
+    "    kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err",
+    "    apply_exit=$?",
+    "    rm -f \"$tmp\"",
+    "    if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
+    "  else",
+    "    action=namespace-create-failed",
+    "    apply_exit=$namespace_create_exit",
+    "  fi",
+    "fi",
+    "after_namespace_exists=$(namespace_exists_flag)",
+    "if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi",
+    "after_password_bytes=$(decoded_length \"$after_b64\")",
+    "after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")",
+    "printf 'namespace\\t%s\\n' \"$namespace\"",
+    "printf 'secret\\t%s\\n' \"$secret\"",
+    "printf 'key\\t%s\\n' \"$secret_key\"",
+    "printf 'sourceRef\\t%s\\n' \"$source_ref\"",
+    "printf 'sourceKey\\t%s\\n' \"$source_key\"",
+    "printf 'dryRun\\t%s\\n' \"$dry_run\"",
+    "printf 'action\\t%s\\n' \"$action\"",
+    "printf 'mutation\\t%s\\n' \"$mutation\"",
+    "printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"",
+    "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
+    "printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"",
+    "printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"",
+    "printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"",
+    "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
+    "printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"",
+    "printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"",
+    "printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"",
+    "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
+    "password=",
+    "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
+    "if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi",
+  ].join("\n");
+}
+
 function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
   const source = spec.baseImageSource ?? "";
   const script = [
@@ -11631,11 +11897,23 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec
     "postgres_rollout_exit=",
     "apply_exit=",
     "db_ensure_exit=",
+    "rollout_restart_exit=",
+    "rollout_status_exit=",
+    "openfga_deployment=hwlab-openfga",
+    "openfga_image=",
+    "migrate_job=",
+    "migrate_apply_exit=",
+    "migrate_wait_exit=",
     "if [ \"$action_request\" = ensure ]; then",
     "  missing_secret=false",
     "  [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true",
     "  [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true",
     "  [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true",
+    "  expected_datastore_uri=",
+    "  if [ -n \"$pg_password\" ]; then",
+    "    expected_datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
+    "    [ \"$datastore_uri\" = \"$expected_datastore_uri\" ] || missing_secret=true",
+    "  fi",
     "  missing_db=false",
     "  [ \"$db_role_exists_before\" = t ] || missing_db=true",
     "  [ \"$db_database_exists_before\" = t ] || missing_db=true",
@@ -11656,7 +11934,7 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec
     "    else",
     "      [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)",
     "      [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
-    "      [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
+    "      datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
     "      kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
     "      apply_exit=$?",
     "      if [ \"$apply_exit\" -eq 0 ]; then",
@@ -11671,7 +11949,60 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec
     "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
     "SQL",
     "        db_ensure_exit=$?",
-    "        if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi",
+    "        if [ \"$db_ensure_exit\" -eq 0 ]; then",
+    "          openfga_image=$(kubectl -n \"$namespace\" get deployment \"$openfga_deployment\" -o 'jsonpath={.spec.template.spec.containers[0].image}' 2>/tmp/hwlab-openfga-image.err)",
+    "          migrate_job=\"$openfga_deployment-migrate-unidesk-$(date +%s)\"",
+    "          tmp=$(mktemp /tmp/hwlab-openfga-migrate.XXXXXX.yaml)",
+    "          cat >\"$tmp\" <<EOF_JOB",
+    "apiVersion: batch/v1",
+    "kind: Job",
+    "metadata:",
+    "  name: $migrate_job",
+    "  namespace: $namespace",
+    "  labels:",
+    "    app.kubernetes.io/part-of: hwlab",
+    "    app.kubernetes.io/managed-by: unidesk",
+    "spec:",
+    "  backoffLimit: 0",
+    "  ttlSecondsAfterFinished: 300",
+    "  template:",
+    "    spec:",
+    "      restartPolicy: Never",
+    "      containers:",
+    "        - name: openfga-migrate",
+    "          image: $openfga_image",
+    "          imagePullPolicy: IfNotPresent",
+    "          args: [\"migrate\"]",
+    "          env:",
+    "            - name: OPENFGA_DATASTORE_ENGINE",
+    "              value: postgres",
+    "            - name: OPENFGA_DATASTORE_URI",
+    "              valueFrom:",
+    "                secretKeyRef:",
+    "                  name: $openfga_secret",
+    "                  key: $datastore_uri_key",
+    "EOF_JOB",
+    "          kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-openfga-migrate-apply.out 2>/tmp/hwlab-openfga-migrate-apply.err",
+    "          migrate_apply_exit=$?",
+    "          rm -f \"$tmp\"",
+    "          if [ \"$migrate_apply_exit\" -eq 0 ]; then",
+    "            kubectl -n \"$namespace\" wait --for=condition=complete \"job/$migrate_job\" --timeout=25s >/tmp/hwlab-openfga-migrate-wait.out 2>/tmp/hwlab-openfga-migrate-wait.err",
+    "            migrate_wait_exit=$?",
+    "          fi",
+    "          if [ \"$migrate_apply_exit\" != 0 ]; then action=migrate-apply-failed",
+    "          elif [ \"$migrate_wait_exit\" != 0 ]; then action=migrate-wait-failed",
+    "          else",
+    "          kubectl -n \"$namespace\" rollout restart \"deployment/$openfga_deployment\" >/tmp/hwlab-openfga-rollout-restart.out 2>/tmp/hwlab-openfga-rollout-restart.err",
+    "          rollout_restart_exit=$?",
+    "          if [ \"$rollout_restart_exit\" -eq 0 ]; then",
+    "            kubectl -n \"$namespace\" rollout status \"deployment/$openfga_deployment\" --timeout=20s >/tmp/hwlab-openfga-rollout-status.out 2>/tmp/hwlab-openfga-rollout-status.err",
+    "            rollout_status_exit=$?",
+    "          fi",
+    "          if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
+    "          elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
+    "          else action=ensured; mutation=true; fi",
+    "          fi",
+    "        else action=db-ensure-failed; fi",
     "      else action=apply-failed; fi",
     "    fi",
     "  fi",
@@ -11715,9 +12046,19 @@ function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec
     "printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"",
     "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
     "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
+    "printf 'openfgaImage\\t%s\\n' \"$openfga_image\"",
+    "printf 'migrateJob\\t%s\\n' \"$migrate_job\"",
+    "printf 'migrateApplyExitCode\\t%s\\n' \"$migrate_apply_exit\"",
+    "printf 'migrateWaitExitCode\\t%s\\n' \"$migrate_wait_exit\"",
+    "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
+    "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
     "authn_value= datastore_uri= pg_password= postgres_admin_password=",
     "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
     "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
+    "if [ -n \"$migrate_apply_exit\" ] && [ \"$migrate_apply_exit\" != 0 ]; then exit \"$migrate_apply_exit\"; fi",
+    "if [ -n \"$migrate_wait_exit\" ] && [ \"$migrate_wait_exit\" != 0 ]; then exit \"$migrate_wait_exit\"; fi",
+    "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
+    "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
   ].join("\n");
 }
 
@@ -12197,6 +12538,11 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS
     "if [ \"$action_request\" = ensure ]; then",
     "  missing_secret=false",
     "  [ \"$before_url_present\" = yes ] && [ \"$before_url_bytes\" -gt 0 ] || missing_secret=true",
+    "  expected_database_url=",
+    "  if [ -n \"$postgres_admin_password\" ]; then",
+    "    expected_database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"",
+    "    [ \"$database_url\" = \"$expected_database_url\" ] || missing_secret=true",
+    "  fi",
     "  missing_db=false",
     "  [ \"$db_role_exists_before\" = t ] || missing_db=true",
     "  [ \"$db_database_exists_before\" = t ] || missing_db=true",
@@ -12208,7 +12554,7 @@ function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretS
     "  elif [ \"$missing_secret\" = false ] && [ \"$missing_db\" = false ]; then",
     "    action=kept",
     "  else",
-    "    [ -n \"$database_url\" ] || database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"",
+    "    database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"",
     "    kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$database_url_key=$database_url\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
     "    apply_exit=$?",
     "    if [ \"$apply_exit\" -eq 0 ]; then",
@@ -12706,6 +13052,12 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
     postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode),
     applyExitCode: numericField(fields.applyExitCode),
     dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
+    openfgaImage: fields.openfgaImage || null,
+    migrateJob: fields.migrateJob || null,
+    migrateApplyExitCode: numericField(fields.migrateApplyExitCode),
+    migrateWaitExitCode: numericField(fields.migrateWaitExitCode),
+    rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
+    rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
     exitCode,
     stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
     valuesRedacted: true,

scripts/src/hwlab-node-lanes.ts

@@ -85,6 +85,8 @@ export interface HwlabRuntimePostgresStoreSpec {
   readonly statefulSet?: string;
   readonly serviceName?: string;
   readonly adminUser?: string;
+  readonly adminPasswordSourceRef?: string;
+  readonly adminPasswordSourceKey?: string;
   readonly cloudApi?: HwlabRuntimePostgresStoreComponentSpec;
   readonly openfga?: HwlabRuntimePostgresStoreComponentSpec;
   readonly poolMax: number;
@@ -650,6 +652,8 @@ function runtimeStoreConfig(value: unknown, path: string): HwlabRuntimeStoreSpec
             ...(postgres.statefulSet === undefined ? {} : { statefulSet: stringField(postgres, "statefulSet", `${path}.postgres`) }),
             ...(postgres.serviceName === undefined ? {} : { serviceName: stringField(postgres, "serviceName", `${path}.postgres`) }),
             ...(postgres.adminUser === undefined ? {} : { adminUser: stringField(postgres, "adminUser", `${path}.postgres`) }),
+            ...(postgres.adminPasswordSourceRef === undefined ? {} : { adminPasswordSourceRef: stringField(postgres, "adminPasswordSourceRef", `${path}.postgres`) }),
+            ...(postgres.adminPasswordSourceKey === undefined ? {} : { adminPasswordSourceKey: secretKeyField(postgres, "adminPasswordSourceKey", `${path}.postgres`) }),
             ...(postgres.cloudApi === undefined ? {} : { cloudApi: runtimeStorePostgresComponentConfig(postgres.cloudApi, `${path}.postgres.cloudApi`) }),
             ...(postgres.openfga === undefined ? {} : { openfga: runtimeStorePostgresComponentConfig(postgres.openfga, `${path}.postgres.openfga`) }),
             poolMax: numberField(postgres, "poolMax", `${path}.postgres`),