import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { repoRoot, rootPath, type Config } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { runHwlabG14Command } from "./hwlab-g14"; import { hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes"; type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; type DelegatedNodeDomain = "control-plane" | "git-mirror"; type NodeRuntimeRenderLocation = "node-host" | "local"; interface NodeRuntimeRenderResult { readonly result: CommandResult; readonly renderDir: string; readonly worktreeDir: string; readonly location: NodeRuntimeRenderLocation; } interface NodeSecretOptions { action: SecretAction; node: string; lane: string; name: string; key?: string; preset: SecretPreset; dryRun: boolean; confirm: boolean; timeoutSeconds: number; } interface NodePublicExposureOptions { action: "public-exposure"; node: string; lane: HwlabRuntimeLane; dryRun: boolean; confirm: boolean; timeoutSeconds: number; spec: HwlabRuntimeLaneSpec; } interface RuntimeSecretSpec { node: string; lane: string; namespace: string; platformDb: boolean; runtimeLaneSpec?: HwlabRuntimeLaneSpec; externalPostgres?: NonNullable; platformPostgresService: string; platformPostgresEndpointAddress?: string; platformPostgresEndpointSlice: string; postgresSecret: string; postgresStatefulSet: string; postgresAdminUser: string; openFgaSecret: string; openFgaDbName: string; openFgaDbUser: string; openFgaDbHost: string; masterAdminApiKeySecret: string; bootstrapAdminSecret: string; bootstrapAdminPasswordHashKey: string; bootstrapAdminSourceNamespace: string; bootstrapAdminSourceSecret: string; cloudApiDbSecret: string; cloudApiDbKey: string; cloudApiDbName: string; cloudApiDbUser: string; cloudApiDbHost: string; cloudApiDeployment: string; obsoleteHwpodDbSecret: string; obsoleteHwpodDbName: string; obsoleteHwpodDbUser: string; codeAgentProviderSecret: string; codeAgentProviderSourceNamespace: string; codeAgentProviderSourceSecret: string; fieldManager: string; } const MASTER_ADMIN_API_KEY_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env"; const MASTER_ADMIN_API_KEY_KEY = "api-key"; const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash"; const BOOTSTRAP_ADMIN_SOURCE_NAMESPACE = "hwlab-v02"; const BOOTSTRAP_ADMIN_SOURCE_SECRET = "hwlab-v02-bootstrap-admin"; const OPENFGA_AUTHN_KEY = "authn-preshared-key"; const OPENFGA_DATASTORE_URI_KEY = "datastore-uri"; const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password"; const CLOUD_API_DB_KEY = "database-url"; const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key"; const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key"; const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02"; const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider"; export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise> { if (args.length === 0) return hwlabNodeHelp(); const [domain] = args; if (domain === "control-plane" && args[1] === "infra") { if (args.length === 2 || args.includes("--help") || args.includes("-h") || args[2] === "help") return hwlabNodeControlPlaneInfraHelp(); return runHwlabNodeControlPlaneInfra(args.slice(2)); } if (args.includes("--help") || args.includes("-h")) return hwlabNodeHelp(); if (domain === "control-plane" || domain === "git-mirror") { return runNodeDelegatedDomain(_config, domain, args.slice(1)); } if (domain !== "secret") { return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret" }; } const options = parseSecretOptions(args.slice(1)); return runNodeSecret(options); } export function hwlabNodeHelp(): Record { return { ok: true, command: "hwlab nodes", description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.", configPath: hwlabRuntimeLaneConfigPath(), examples: [ "bun scripts/cli.ts hwlab nodes control-plane infra plan --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane plan --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-bootstrap-admin", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-bootstrap-admin --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run", "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm", ], }; } async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise> { const scoped = parseNodeScopedDelegatedOptions(domain, args); const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane); if (domain === "control-plane" && scoped.action === "public-exposure") { return runNodePublicExposure({ action: "public-exposure", node: scoped.node, lane: scoped.lane, dryRun: scoped.dryRun || !scoped.confirm, confirm: scoped.confirm, timeoutSeconds: scoped.timeoutSeconds, spec: scoped.spec, }); } if (domain === "control-plane" && scoped.action === "plan") { return nodeRuntimeControlPlanePlan(scoped); } if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) { if (scoped.action === "status") return nodeRuntimeControlPlaneStatus(scoped); if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration") { if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped); return nodeRuntimeControlPlaneRun(scoped); } return nodeRuntimeUnsupportedAction(scoped); } if (domain === "git-mirror" && scoped.node !== defaultSpec.nodeId) { return { ok: false, command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mutation: false, degradedReason: "node-scoped-git-mirror-action-not-enabled", message: "D601 runtime GitOps is declared in UniDesk YAML and triggered by node-scoped control-plane commands; this command intentionally does not delegate D601 git-mirror actions to G14.", expected: nodeRuntimeExpected(scoped.spec), }; } if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") { return runNodeEndpointBridge(scoped); } if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) { return startNodeDelegatedJob(scoped); } if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) { return startNodeDelegatedJob(scoped); } const delegatedArgs = stripOption(args, "--node"); const result = await runHwlabG14Command(config, [domain, ...delegatedArgs]); return rewriteDelegatedNodeResult(result, scoped); } function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): { domain: DelegatedNodeDomain; action: string; node: string; lane: HwlabRuntimeLane; confirm: boolean; dryRun: boolean; wait: boolean; rerun: boolean; allowLiveDbRead: boolean; timeoutSeconds: number; originalArgs: string[]; spec: HwlabRuntimeLaneSpec; } { const [actionRaw] = args; if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`); const node = requiredOption(args, "--node"); assertNodeId(node); const laneRaw = requiredOption(args, "--lane"); if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`); const spec = hwlabRuntimeLaneSpecForNode(laneRaw, node); const confirm = args.includes("--confirm"); const dryRun = args.includes("--dry-run"); if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`); return { domain, action: actionRaw, node, lane: laneRaw, confirm, dryRun, wait: args.includes("--wait"), rerun: args.includes("--rerun"), allowLiveDbRead: args.includes("--allow-live-db-read"), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600), originalArgs: [...args], spec, }; } function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record { return { configPath: hwlabRuntimeLaneConfigPath(), node: spec.nodeId, nodeRoute: spec.nodeRoute, nodeKubeRoute: spec.nodeKubeRoute, lane: spec.lane, sourceBranch: spec.sourceBranch, workspace: spec.workspace, cicdRepo: spec.cicdRepo, git: { sourceUrl: spec.gitUrl, readUrl: spec.gitReadUrl, writeUrl: spec.gitWriteUrl, }, argo: { repoURL: spec.argoRepoUrl, }, gitopsBranch: spec.gitopsBranch, catalogPath: spec.catalogPath, runtimePath: spec.runtimePath, runtimeNamespace: spec.runtimeNamespace, renderDir: spec.runtimeRenderDir, pipeline: spec.pipeline, pipelineRunPrefix: spec.pipelineRunPrefix, serviceAccount: spec.serviceAccountName, argoApplication: spec.app, registryPrefix: spec.registryPrefix, baseImage: { image: spec.baseImage, sourceImage: spec.baseImageSource ?? null, }, buildkit: spec.buildkit === undefined ? null : { sidecarImage: spec.buildkit.sidecarImage, }, dockerBuildProxy: { http: spec.networkProfile.dockerBuildProxy.http, https: spec.networkProfile.dockerBuildProxy.https, all: spec.networkProfile.dockerBuildProxy.all, noProxy: spec.networkProfile.dockerBuildProxy.noProxy, }, stepEnv: spec.stepEnv, public: { webUrl: spec.publicWebUrl, apiUrl: spec.publicApiUrl, }, publicExposure: spec.publicExposure === null ? null : publicExposureSummary(spec.publicExposure), downloadProfile: { id: spec.downloadProfileId, git: spec.downloadProfile.git, npm: spec.downloadProfile.npm, }, observability: spec.observability, runtimeImageRewrites: spec.runtimeImageRewrites, externalPostgres: spec.externalPostgres === undefined ? null : { provider: spec.externalPostgres.provider, configRef: spec.externalPostgres.configRef, serviceName: spec.externalPostgres.serviceName, endpointAddress: spec.externalPostgres.endpointAddress, port: spec.externalPostgres.port, sslmode: spec.externalPostgres.sslmode, database: spec.externalPostgres.database, cloudApi: { secretName: spec.externalPostgres.cloudApi.secretName, secretKey: spec.externalPostgres.cloudApi.secretKey, sourceRef: spec.externalPostgres.cloudApi.sourceRef, envKey: spec.externalPostgres.cloudApi.envKey, role: spec.externalPostgres.cloudApi.role, }, openfga: { secretName: spec.externalPostgres.openfga.secretName, secretKey: spec.externalPostgres.openfga.secretKey, sourceRef: spec.externalPostgres.openfga.sourceRef, envKey: spec.externalPostgres.openfga.envKey, authnKey: spec.externalPostgres.openfga.authnKey ?? null, role: spec.externalPostgres.openfga.role, schema: spec.externalPostgres.openfga.schema ?? null, }, valuesPrinted: false, }, localPostgres: { shouldRender: spec.externalPostgres === undefined, expectedAbsent: spec.externalPostgres !== undefined, }, }; } function nodeRuntimeControlPlanePlan(scoped: ReturnType): Record { return { ok: true, command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, mode: "plan", mutation: false, node: scoped.node, lane: scoped.lane, expected: nodeRuntimeExpected(scoped.spec), checks: { nodeScopedTargetConfigured: true, externalPostgresDeclared: scoped.spec.externalPostgres !== undefined, secretValuesPrinted: false, runtimeNamespace: scoped.spec.runtimeNamespace, localPostgresExpectedAbsent: scoped.spec.externalPostgres !== undefined, publicExposureDeclared: scoped.spec.publicExposure !== null, }, next: { infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`, status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, platformDbStatus: scoped.spec.externalPostgres === undefined ? null : `bun scripts/cli.ts platform-db postgres status --config ${scoped.spec.externalPostgres.configRef}`, publicExposure: scoped.spec.publicExposure === null ? null : `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${scoped.node} --lane ${scoped.lane} --confirm`, }, }; } function compactRuntimeCommand(result: CommandResult): Record { return { exitCode: result.exitCode, stdoutBytes: Buffer.byteLength(result.stdout), stderrBytes: Buffer.byteLength(result.stderr), stdoutTail: result.stdout.trim().slice(-2000), stderrTail: result.stderr.trim().slice(-2000), timedOut: result.timedOut, }; } function nodeRuntimeUnsupportedAction(scoped: ReturnType): Record { return { ok: false, command: `hwlab nodes control-plane ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mutation: false, degradedReason: "unsupported-node-scoped-runtime-action", message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/runtime-migration", expected: nodeRuntimeExpected(scoped.spec), }; } function transPath(): string { return join(repoRoot, "scripts", "trans"); } function runNodeHostScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { return runCommand([transPath(), spec.nodeRoute, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function runNodeHostScriptAsync(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, label: string): CommandResult { const token = nodeRuntimeRenderToken(); const stateDir = `/tmp/unidesk-hwlab-node-runtime/${label}-${token}`; const statusPath = `${stateDir}/status.json`; const stdoutPath = `${stateDir}/stdout.log`; const stderrPath = `${stateDir}/stderr.log`; const runnerPath = `${stateDir}/run.sh`; const startScript = [ "set -eu", `state_dir=${shellQuote(stateDir)}`, `status_path=${shellQuote(statusPath)}`, `stdout_path=${shellQuote(stdoutPath)}`, `stderr_path=${shellQuote(stderrPath)}`, `runner_path=${shellQuote(runnerPath)}`, "rm -rf \"$state_dir\"", "mkdir -p \"$state_dir\"", "cat > \"$runner_path\" <<'UNIDESK_REMOTE_RUNNER'", script, "UNIDESK_REMOTE_RUNNER", "chmod 700 \"$runner_path\"", "python3 - \"$status_path\" <<'PY'", "import datetime, json, sys", "json.dump({'state':'running','ok':None,'pid':None,'startedAt':datetime.datetime.utcnow().isoformat()+'Z','finishedAt':None,'exitCode':None}, open(sys.argv[1], 'w'), indent=2)", "PY", "(", " set +e", " sh \"$runner_path\" >\"$stdout_path\" 2>\"$stderr_path\"", " code=$?", " python3 - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'PY'", "import datetime, json, pathlib, sys", "status_path, stdout_path, stderr_path, code = sys.argv[1:5]", "def tail(path):", " try:", " text = pathlib.Path(path).read_text(errors='replace')", " except FileNotFoundError:", " return ''", " return text[-12000:]", "payload = {'state':'finished','ok':code == '0','pid':None,'startedAt':None,'finishedAt':datetime.datetime.utcnow().isoformat()+'Z','exitCode':int(code),'stdout':tail(stdout_path),'stderr':tail(stderr_path)}", "try:", " current = json.load(open(status_path))", " payload['startedAt'] = current.get('startedAt')", "except Exception:", " pass", "json.dump(payload, open(status_path, 'w'), indent=2)", "PY", ") >/dev/null 2>&1 &", "pid=$!", "python3 - \"$status_path\" \"$pid\" <<'PY'", "import json, sys", "path, pid = sys.argv[1:3]", "payload = json.load(open(path))", "payload['pid'] = int(pid)", "json.dump(payload, open(path, 'w'), indent=2)", "PY", "printf '{\"ok\":true,\"state\":\"started\",\"pid\":%s,\"statusPath\":\"%s\"}\\n' \"$pid\" \"$status_path\"", ].join("\n"); const start = runNodeHostScript(spec, startScript, 55); if (!isCommandSuccess(start)) return start; const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000; let lastStatus: CommandResult | null = null; let payload: Record = {}; while (Date.now() <= deadline) { runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); const status = runNodeHostScript(spec, `cat ${shellQuote(statusPath)} 2>/dev/null || printf '{"state":"missing","ok":false,"exitCode":127,"stdout":"","stderr":"missing async status"}\\n'`, 55); lastStatus = status; payload = parseJsonObject(status.stdout.trim()); if (payload.state === "finished" || payload.ok === true || payload.ok === false) { return commandResultFromAsync(spec, payload, statusPath, false); } } const kill = runNodeHostScript(spec, [ "set +e", `status_path=${shellQuote(statusPath)}`, "pid=$(python3 - \"$status_path\" <<'PY' 2>/dev/null", "import json, sys", "try: print(json.load(open(sys.argv[1])).get('pid') or '')", "except Exception: print('')", "PY", ")", "if [ -n \"$pid\" ]; then kill \"$pid\" 2>/dev/null || true; fi", "cat \"$status_path\" 2>/dev/null || true", ].join("\n"), 55); return { command: [transPath(), spec.nodeRoute, "script", "--", ""], cwd: repoRoot, exitCode: 124, stdout: typeof payload.stdout === "string" ? payload.stdout : "", stderr: [ `remote async command timed out after ${timeoutSeconds}s; statusPath=${statusPath}`, typeof payload.stderr === "string" ? payload.stderr : "", lastStatus?.stderr.trim() ?? "", kill.stderr.trim(), ].filter(Boolean).join("\n"), signal: null, timedOut: true, }; } function parseJsonObject(text: string): Record { try { const parsed = JSON.parse(text) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; } catch { return {}; } } function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record, statusPath: string, timedOut: boolean): CommandResult { return { command: [transPath(), spec.nodeRoute, "script", "--", ""], cwd: repoRoot, exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1, stdout: typeof payload.stdout === "string" ? payload.stdout : "", stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`, signal: null, timedOut, }; } function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult { return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 }); } function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { return runCommand([transPath(), spec.nodeKubeRoute, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function isCommandSuccess(result: CommandResult): boolean { return result.exitCode === 0 && !result.timedOut; } function shortSha(value: string): string { return value.slice(0, 12).toLowerCase(); } function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`; } function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string { const suffix = `/${spec.runtimeRenderDir}`; if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length); return spec.gitopsRoot; } function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 }); if (!isCommandSuccess(result)) return { sourceCommit: null, result }; const match = /[0-9a-f]{40}/iu.exec(statusText(result)); return { sourceCommit: match?.[0].toLowerCase() ?? null, result }; } function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { const gitRetries = Math.max(1, spec.downloadProfile.git.retries); const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); return [ `cicd_repo=${shellQuote(spec.cicdRepo)}`, `cicd_url=${shellQuote(spec.gitUrl)}`, `cicd_branch=${shellQuote(spec.sourceBranch)}`, `cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`, `git_retries=${shellQuote(String(gitRetries))}`, `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", "retry_git() {", " op=$1", " shift", " attempt=1", " while [ \"$attempt\" -le \"$git_retries\" ]; do", " echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2", " run_git \"$@\"", " code=$?", " if [ \"$code\" -eq 0 ]; then return 0; fi", " echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2", " attempt=$((attempt + 1))", " done", " return 1", "}", "mkdir -p \"$(dirname \"$cicd_repo\")\"", "if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", " :", "elif [ -e \"$cicd_repo\" ]; then", " echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", " exit 41", "else", " retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", "fi", "git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"", "git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", "if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then", " rm -rf \"$cicd_repo\"", " retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", " git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", " retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune", "fi", ].join("\n"); } function nodeRuntimeControlPlaneRun(scoped: ReturnType): Record { if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped); if (scoped.action === "sync") return nodeRuntimeSync(scoped); if (scoped.action === "apply") return nodeRuntimeApply(scoped); if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped); if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped); return nodeRuntimeUnsupportedAction(scoped); } function nodeRuntimeMigration(scoped: ReturnType): Record { const spec = scoped.spec; if (scoped.allowLiveDbRead && scoped.confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm"); if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-migration requires --dry-run or --confirm"); const head = resolveNodeRuntimeLaneHead(spec); const sourceCommit = head.sourceCommit; if (sourceCommit === null) { return { ok: false, command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, phase: "source-head", mutation: false, degradedReason: "node-runtime-source-head-unresolved", headProbe: compactRuntimeCommand(head.result), }; } const reportPath = `/tmp/hwlab-${scoped.node.toLowerCase()}-${scoped.lane}-runtime-migration-${shortSha(sourceCommit)}.json`; const migrationArgs = scoped.dryRun ? [ ...(scoped.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]), "--report", reportPath, ] : [ "--apply", "--confirm-dev", "--confirmed-non-production", "--report", reportPath, ]; const result = runNodeK3sArgs(spec, [ "kubectl", "exec", "-n", spec.runtimeNamespace, "deployment/hwlab-cloud-api", "-c", "hwlab-cloud-api", "--", "sh", "-lc", `cd /workspace/hwlab-boot/repo && exec bun cmd/hwlab-cloud-api/migrate.ts ${migrationArgs.map(shellQuote).join(" ")}`, ], scoped.timeoutSeconds); const ok = isCommandSuccess(result); return { ok, command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? scoped.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply", sourceCommit, runtimeNamespace: spec.runtimeNamespace, target: "deployment/hwlab-cloud-api -c hwlab-cloud-api", workingDirectory: "/workspace/hwlab-boot/repo", migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs], reportPath, mutation: !scoped.dryRun && ok, result: compactRuntimeCommand(result), degradedReason: ok ? undefined : "node-runtime-migration-failed", next: scoped.dryRun ? { liveReadDryRun: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --allow-live-db-read --dry-run`, apply: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --confirm`, } : { health: `curl -fsS --max-time 20 ${spec.publicApiUrl}/health/live`, status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, }, }; } function nodeRuntimeApply(scoped: ReturnType): Record { const spec = scoped.spec; const head = resolveNodeRuntimeLaneHead(spec); const sourceCommit = head.sourceCommit; if (sourceCommit === null) { return { ok: false, command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, phase: "source-head", degradedReason: "node-runtime-source-head-unresolved", headProbe: compactRuntimeCommand(head.result), }; } const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds); if (secrets !== null && secrets.ok !== true) { return { ok: false, command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? "dry-run" : "confirmed-apply", phase: "external-postgres-secret-sync", sourceCommit, secrets, degradedReason: "external-postgres-secret-sync-failed", }; } const baseImage = ensureNodeBaseImage(spec, scoped.dryRun, scoped.timeoutSeconds); if (baseImage !== null && baseImage.ok !== true) { return { ok: false, command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? "dry-run" : "confirmed-apply", phase: "base-image-seed", sourceCommit, secrets, baseImage, degradedReason: "node-runtime-base-image-seed-failed", }; } const render = renderNodeRuntimeControlPlane(spec, sourceCommit, scoped.timeoutSeconds); if (!isCommandSuccess(render.result)) { return { ok: false, command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? "dry-run" : "confirmed-apply", phase: "source-render", sourceCommit, secrets, baseImage, renderDir: render.renderDir, renderLocation: render.location, render: compactRuntimeCommand(render.result), degradedReason: "node-runtime-control-plane-render-failed", }; } const apply = render.location === "local" ? applyLocalNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds) : applyNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds); const cleanup = render.location === "local" ? cleanupLocalNodeRuntimeRenderDir(spec, render) : cleanupNodeRuntimeRenderDir(spec, render.renderDir); return { ok: isCommandSuccess(apply), command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? "dry-run" : "confirmed-apply", mutation: !scoped.dryRun && isCommandSuccess(apply), sourceCommit, expected: nodeRuntimeExpected(spec), secrets, baseImage, renderDir: render.renderDir, renderLocation: render.location, render: compactRuntimeCommand(render.result), apply: compactRuntimeCommand(apply), cleanupRenderDir: compactRuntimeCommand(cleanup), degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed", next: scoped.dryRun ? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` } : { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` }, }; } function nodeRuntimeRefresh(scoped: ReturnType): Record { const spec = scoped.spec; const script = [ "set +e", `app=${shellQuote(spec.app)}`, "argo_namespace=argocd", `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`, "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)", "if [ \"$dry_run\" = true ]; then", " output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)", "else", " output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)", "fi", "code=$?", "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)", "printf 'app\\t%s\\n' \"$app\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'before\\t%s\\n' \"$before\"", "printf 'after\\t%s\\n' \"$after\"", "printf 'annotateExitCode\\t%s\\n' \"$code\"", "printf 'annotateOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", "exit \"$code\"", ].join("\n"); const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); const fields = keyValueLinesFromText(statusText(result)); return { ok: isCommandSuccess(result), command: `hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? "dry-run" : "confirmed-refresh", mutation: !scoped.dryRun && isCommandSuccess(result), argoApplication: fields.app || spec.app, before: fields.before || null, after: fields.after || null, annotateExitCode: numericField(fields.annotateExitCode), result: compactRuntimeCommand(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, }; } function nodeRuntimeSync(scoped: ReturnType): Record { const spec = scoped.spec; const operation = { operation: { initiatedBy: { username: "unidesk-hwlab-node-cli" }, retry: { limit: 1 }, sync: { prune: true, revision: spec.gitopsBranch, source: { repoURL: spec.argoRepoUrl, targetRevision: spec.gitopsBranch, path: spec.runtimePath, }, syncOptions: ["CreateNamespace=true"], syncStrategy: { hook: {} }, }, }, }; const patchB64 = Buffer.from(JSON.stringify(operation), "utf8").toString("base64"); const script = [ "set +e", `app=${shellQuote(spec.app)}`, "argo_namespace=argocd", `runtime_namespace=${shellQuote(spec.runtimeNamespace)}`, `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`, `patch_b64=${shellQuote(patchB64)}`, "terminate_patch_file=$(mktemp /tmp/hwlab-node-argocd-terminate.XXXXXX.json)", "patch_file=$(mktemp /tmp/hwlab-node-argocd-sync.XXXXXX.json)", "printf '{\"operation\":null}' > \"$terminate_patch_file\"", "printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"", "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)", "operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)", "terminate_code=0", "terminate_output=", "terminated_operation=false", "if [ \"$operation_phase\" = Running ]; then", " if [ \"$dry_run\" = true ]; then", " terminated_operation=would-terminate", " else", " terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)", " terminate_code=$?", " if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi", " sleep 2", " fi", "fi", "failed_hook_count=0", "deleted_hook_count=0", "failed_hooks=", "deleted_hooks=", "hook_delete_errors=", "for job in $(kubectl -n \"$runtime_namespace\" get job -o name 2>/dev/null || true); do", " tracking=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ index .metadata.annotations \"argocd.argoproj.io/tracking-id\" }}' 2>/dev/null || true)", " failed=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ range .status.conditions }}{{ if or (eq .type \"Failed\") (eq .type \"FailureTarget\") }}{{ .status }} {{ end }}{{ end }}' 2>/dev/null || true)", " case \"$tracking\" in \"$app:\"*) ;; *) continue ;; esac", " case \"$failed\" in *True*) ;; *) continue ;; esac", " hook_name=${job#job.batch/}", " failed_hook_count=$((failed_hook_count + 1))", " if [ -z \"$failed_hooks\" ]; then failed_hooks=$hook_name; else failed_hooks=$failed_hooks,$hook_name; fi", " if [ \"$dry_run\" = false ]; then", " delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$job\" --wait=false 2>&1)", " delete_code=$?", " if [ \"$delete_code\" -eq 0 ]; then", " deleted_hook_count=$((deleted_hook_count + 1))", " if [ -z \"$deleted_hooks\" ]; then deleted_hooks=$hook_name; else deleted_hooks=$deleted_hooks,$hook_name; fi", " else", " if [ -z \"$hook_delete_errors\" ]; then hook_delete_errors=\"$hook_name:$delete_code\"; else hook_delete_errors=\"$hook_delete_errors,$hook_name:$delete_code\"; fi", " fi", " fi", "done", "if [ \"$dry_run\" = true ]; then", " output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)", "else", " output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" -o name 2>&1)", "fi", "code=$?", "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.operation.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)", "printf 'app\\t%s\\n' \"$app\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'before\\t%s\\n' \"$before\"", "printf 'after\\t%s\\n' \"$after\"", "printf 'operationPhase\\t%s\\n' \"$operation_phase\"", "printf 'terminatedOperation\\t%s\\n' \"$terminated_operation\"", "printf 'terminateExitCode\\t%s\\n' \"$terminate_code\"", "printf 'terminateOutput\\t%s\\n' \"$(printf '%s' \"$terminate_output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", "printf 'failedHookCount\\t%s\\n' \"$failed_hook_count\"", "printf 'failedHooks\\t%s\\n' \"$failed_hooks\"", "printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"", "printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"", "printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"", "printf 'patchExitCode\\t%s\\n' \"$code\"", "printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", "rm -f \"$patch_file\" \"$terminate_patch_file\"", "exit \"$code\"", ].join("\n"); const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); const fields = keyValueLinesFromText(statusText(result)); return { ok: isCommandSuccess(result), command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? "dry-run" : "confirmed-sync", mutation: !scoped.dryRun && isCommandSuccess(result), argoApplication: fields.app || spec.app, syncSource: { repoURL: spec.argoRepoUrl, targetRevision: spec.gitopsBranch, path: spec.runtimePath, }, before: fields.before || null, after: fields.after || null, operationRecovery: { operationPhase: fields.operationPhase || null, terminatedOperation: fields.terminatedOperation || "false", terminateExitCode: numericField(fields.terminateExitCode), terminateOutput: fields.terminateOutput || null, failedHookCount: numericField(fields.failedHookCount), failedHooks: commaListField(fields.failedHooks), deletedHookCount: numericField(fields.deletedHookCount), deletedHooks: commaListField(fields.deletedHooks), hookDeleteErrors: fields.hookDeleteErrors || null, }, patchExitCode: numericField(fields.patchExitCode), patchOutput: fields.patchOutput || null, result: compactRuntimeCommand(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, }; } function nodeRuntimeTriggerCurrent(scoped: ReturnType): Record { const spec = scoped.spec; printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" }); const head = resolveNodeRuntimeLaneHead(spec); const sourceCommit = head.sourceCommit; if (sourceCommit === null) { printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "failed" }); return { ok: false, command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, phase: "source-head", degradedReason: "node-runtime-source-head-unresolved", headProbe: compactRuntimeCommand(head.result), }; } const pipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit); printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun }); printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun }); const before = getNodeRuntimePipelineRun(spec, pipelineRun); printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null }); if (scoped.dryRun) { return { ok: true, command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: "dry-run", sourceCommit, pipelineRun, rerun: scoped.rerun, before, manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun), next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` }, }; } if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) { return { ok: true, command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: "confirmed-trigger", sourceCommit, pipelineRun, before, skipped: true, reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running", rerunAvailable: true, next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` }, }; } printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun }); const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false }); if (refresh.ok !== true) { printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "failed", sourceCommit, pipelineRun }); return { ok: false, command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, phase: "control-plane-apply", sourceCommit, pipelineRun, refresh, degradedReason: "node-runtime-control-plane-apply-before-trigger-failed", }; } printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun }); printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun }); const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun); const after = getNodeRuntimePipelineRun(spec, pipelineRun); const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True"); const createOk = isCommandSuccess(create) || createObserved; printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined }); return { ok: createOk, command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: "confirmed-trigger", mutation: createOk, sourceCommit, pipelineRun, rerun: scoped.rerun, before, refresh, create: compactRuntimeCommand(create), after, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined, degradedReason: createOk ? undefined : "node-runtime-pipelinerun-create-failed", next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` }, }; } function nodeRuntimeControlPlaneStatus(scoped: ReturnType): Record { const spec = scoped.spec; const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit"); const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run"); const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null; const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null; const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit)); const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60); const namespaceExists = namespace.exitCode === 0; const postgresObjects = namespaceExists ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60) : null; const localPostgresObjects = postgresObjects === null ? [] : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec)); const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60); const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60); const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60); const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u); const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun); const workloads = namespaceExists ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60) : null; const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); const bridge = externalPostgresBridgeStatus(spec, namespaceExists); const secrets = externalPostgresSecretStatus(spec, namespaceExists); const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0; const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready)); const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy"; const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True"; return { ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady, command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, mode: "node-scoped-runtime-status", mutation: false, node: scoped.node, lane: scoped.lane, sourceCommit, pipelineRun, expected: nodeRuntimeExpected(spec), sourceHead: head === null ? { ok: sourceCommit !== null, value: sourceCommit, source: "option" } : { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) }, controlPlane: { ready: controlPlaneReady, serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) }, pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) }, }, argo: { ready: argoReady, application: spec.app, repoURL, expectedRepoURL: spec.argoRepoUrl, targetRevision, path, syncRevision, syncStatus, health, result: compactRuntimeCommand(argo), }, pipelineRun: pipelineRunProbe, runtime: { namespace: spec.runtimeNamespace, namespaceExists, localPostgresObjects, localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0, workloadNames, workloadCount: workloadNames.length, workloadResult: workloads === null ? null : compactRuntimeCommand(workloads), externalPostgresBridge: bridge, externalPostgresSecrets: secrets, }, probes: { namespace: compactRuntimeCommand(namespace), postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects), }, degradedReason: controlPlaneReady ? runtimeReady ? argoReady ? pipelineRunReady ? undefined : "pipelinerun-not-succeeded" : "argo-not-synced-healthy" : namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing" : "control-plane-not-ready", next: { plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`, triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, }, }; } function nodeRuntimeRenderToken(): string { return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-"); } function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds); return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds); } function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean { return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId; } function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] { const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds)); const retryCount = Math.max(0, Math.floor(retries)); const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-"); return [ `yaml_registry=${shellQuote(registry)}`, `yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`, `yaml_fetch_retries=${shellQuote(String(retryCount))}`, `yaml_dependency_context=${shellQuote(safeContext)}`, "yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }", "yaml_npm_debug_log_tail() {", " yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"", " if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi", " yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"", " if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then", " echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2", " tail -n 80 \"$yaml_npm_log\" >&2 || true", " fi", "}", "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then", " mkdir -p node_modules/yaml", " yaml_tarball=\"${yaml_registry%/}/yaml/-/yaml-2.8.3.tgz\"", " yaml_tgz=\"$(mktemp)\"", " if command -v curl >/dev/null 2>&1 && command -v tar >/dev/null 2>&1; then", " if timeout \"$yaml_fetch_timeout\" curl -fsSL --retry \"$yaml_fetch_retries\" --connect-timeout 10 -o \"$yaml_tgz\" \"$yaml_tarball\"; then", " tar -xzf \"$yaml_tgz\" -C node_modules/yaml --strip-components=1", " yaml_dependency_log installed tarball", " else", " yaml_dependency_log tarball-failed tarball", " fi", " fi", " rm -f \"$yaml_tgz\"", "fi", "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then", " rm -rf node_modules/yaml", " if timeout \"$yaml_fetch_timeout\" bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then", " yaml_dependency_log installed bun", " else", " yaml_dependency_log bun-failed bun", " fi", "fi", "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then", " rm -rf node_modules/yaml", " command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }", " if npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then", " yaml_dependency_log installed npm", " else", " yaml_dependency_log npm-failed npm", " yaml_npm_debug_log_tail", " exit 34", " fi", "fi", "node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }", ]; } function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { const token = nodeRuntimeRenderToken(); const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`; const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`; const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); const script = [ "set -eu", runtimeLaneCicdRepoEnsureScript(spec), `source_commit=${shellQuote(sourceCommit)}`, `render_dir=${shellQuote(renderDir)}`, `worktree_dir=${shellQuote(worktreeDir)}`, `overlay_b64=${shellQuote(overlay)}`, "cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }", "trap cleanup_render_worktree EXIT", `test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`, "rm -rf \"$render_dir\" \"$worktree_dir\"", "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", "git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"", "git -C \"$worktree_dir\" checkout --detach \"$source_commit\"", "cd \"$worktree_dir\"", ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"), "node - \"$overlay_b64\" <<'NODE'", "const fs = require('fs');", "const YAML = require('yaml');", "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", "const path = 'deploy/deploy.yaml';", "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", "doc.nodes = doc.nodes || {};", "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", "doc.lanes = doc.lanes || {};", "const lane = doc.lanes[overlay.lane] || {};", "const downloadStack = {", " ...(lane.envRecipe?.downloadStack || {}),", " httpProxy: overlay.dockerProxyHttp,", " httpsProxy: overlay.dockerProxyHttps,", " noProxy: overlay.dockerNoProxyList,", "};", "doc.lanes[overlay.lane] = {", " ...lane,", " node: overlay.nodeId,", " sourceBranch: overlay.sourceBranch,", " gitopsBranch: overlay.gitopsBranch,", " namespace: overlay.runtimeNamespace,", " endpoint: overlay.publicApiUrl,", " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", " artifactCatalog: overlay.catalogPath,", " runtimePath: overlay.runtimePath,", " imageTagMode: 'full',", " sourceRepo: overlay.gitUrl,", " externalPostgres: overlay.externalPostgres,", " observability: overlay.observability,", " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", "};", "fs.writeFileSync(path, YAML.stringify(doc));", "NODE", "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", [ "node scripts/run-bun.mjs \"$render_script\"", `--lane ${shellQuote(spec.lane)}`, `--node ${shellQuote(spec.nodeId)}`, `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, `--catalog-path ${shellQuote(spec.catalogPath)}`, "--image-tag-mode full", `--source-revision ${shellQuote(sourceCommit)}`, `--source-repo ${shellQuote(spec.gitUrl)}`, `--source-branch ${shellQuote(spec.sourceBranch)}`, `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, `--git-read-url ${shellQuote(spec.gitReadUrl)}`, `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, `--registry-prefix ${shellQuote(spec.registryPrefix)}`, `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, `--out ${shellQuote(renderDir)}`, ].join(" "), ...nodeRuntimePipelinePostprocessScript(), ].join("\n"); return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" }; } function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { const token = nodeRuntimeRenderToken(); const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`; const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`; const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); const script = [ "set -eu", `source_url=${shellQuote(spec.gitUrl)}`, `source_branch=${shellQuote(spec.sourceBranch)}`, `source_commit=${shellQuote(sourceCommit)}`, `render_dir=${shellQuote(renderDir)}`, `worktree_dir=${shellQuote(worktreeDir)}`, `overlay_b64=${shellQuote(overlay)}`, `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", "rm -rf \"$render_dir\" \"$worktree_dir\"", "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", "echo \"phase=local-git-clone-worktree\" >&2", "run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"", "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"", "cd \"$worktree_dir\"", "echo \"phase=local-install-yaml\" >&2", ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"), "node - \"$overlay_b64\" <<'NODE'", "const fs = require('fs');", "const YAML = require('yaml');", "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", "const path = 'deploy/deploy.yaml';", "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", "doc.nodes = doc.nodes || {};", "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", "doc.lanes = doc.lanes || {};", "const lane = doc.lanes[overlay.lane] || {};", "const downloadStack = {", " ...(lane.envRecipe?.downloadStack || {}),", " httpProxy: overlay.dockerProxyHttp,", " httpsProxy: overlay.dockerProxyHttps,", " noProxy: overlay.dockerNoProxyList,", "};", "doc.lanes[overlay.lane] = {", " ...lane,", " node: overlay.nodeId,", " sourceBranch: overlay.sourceBranch,", " gitopsBranch: overlay.gitopsBranch,", " namespace: overlay.runtimeNamespace,", " endpoint: overlay.publicApiUrl,", " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", " artifactCatalog: overlay.catalogPath,", " runtimePath: overlay.runtimePath,", " imageTagMode: 'full',", " sourceRepo: overlay.gitUrl,", " externalPostgres: overlay.externalPostgres,", " observability: overlay.observability,", " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", "};", "fs.writeFileSync(path, YAML.stringify(doc));", "NODE", "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", "echo \"phase=local-gitops-render\" >&2", [ "node scripts/run-bun.mjs \"$render_script\"", `--lane ${shellQuote(spec.lane)}`, `--node ${shellQuote(spec.nodeId)}`, `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, `--catalog-path ${shellQuote(spec.catalogPath)}`, "--image-tag-mode full", `--source-revision ${shellQuote(sourceCommit)}`, `--source-repo ${shellQuote(spec.gitUrl)}`, `--source-branch ${shellQuote(spec.sourceBranch)}`, `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, `--git-read-url ${shellQuote(spec.gitReadUrl)}`, `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, `--registry-prefix ${shellQuote(spec.registryPrefix)}`, `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, `--out ${shellQuote(renderDir)}`, ].join(" "), ...nodeRuntimePipelinePostprocessScript(), ].join("\n"); return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" }; } function nodeRuntimePipelinePostprocessScript(): string[] { return [ "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'", "const fs = require('fs');", "const path = require('path');", "const vm = require('node:vm');", "const renderDir = process.argv[2];", "const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));", "const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');", "let text = fs.readFileSync(pipelinePath, 'utf8');", "let YAML = null;", "try { YAML = require('yaml'); } catch {}", "const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');", "const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;", "const yamlString = (value) => JSON.stringify(String(value));", "const proxyEnv = {", " HTTP_PROXY: overlay.proxyHttp,", " HTTPS_PROXY: overlay.proxyHttps,", " ALL_PROXY: overlay.proxyAll,", " NO_PROXY: overlay.noProxy,", " http_proxy: overlay.proxyHttp,", " https_proxy: overlay.proxyHttps,", " all_proxy: overlay.proxyAll,", " no_proxy: overlay.noProxy,", "};", "const dockerProxyEnv = {", " HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,", " HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,", " HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,", "};", "const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };", "function prepareSourceDependencyScript() {", " const registry = String(overlay.npmRegistry || 'https://registry.npmjs.org/');", " const timeoutSeconds = Math.max(15, Math.ceil(Number(overlay.npmFetchTimeoutMs || 120000) / 1000));", " const retryCount = Math.max(0, Math.floor(Number(overlay.npmRetries || 3)));", " return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"", "node <<'NODE_UNIDESK_YAML_DEPENDENCY'", "const { spawnSync } = require('node:child_process');", "const fs = require('node:fs');", "const os = require('node:os');", "const path = require('node:path');", "const registry = ${JSON.stringify(registry)};", "const timeoutMs = ${JSON.stringify(timeoutSeconds * 1000)};", "const timeoutSeconds = ${JSON.stringify(timeoutSeconds)};", "const retryCount = ${JSON.stringify(retryCount)};", "const dependency = 'yaml';", "const version = '2.8.3';", "function emit(status, extra = {}) { console.error(JSON.stringify({ event: 'prepare-source-dependencies', status, dependency, ...extra })); }", "function hasYaml() { try { require.resolve('yaml'); return true; } catch { return false; } }", "function run(command, args) {", " const result = spawnSync(command, args, { stdio: 'inherit', env: process.env, timeout: timeoutMs });", " if (result.error) console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'command-error', command, error: result.error.message }));", " return result.status === 0;", "}", "function tailNpmLog() {", " const dir = path.join(process.env.HOME || '/tmp', '.npm', '_logs');", " if (!fs.existsSync(dir)) return;", " const files = fs.readdirSync(dir).filter((name) => name.includes('debug') && name.endsWith('.log')).sort();", " const file = files[files.length - 1];", " if (!file) return;", " const full = path.join(dir, file);", " console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'npm-debug-log', path: full }));", " const lines = fs.readFileSync(full, 'utf8').split(String.fromCharCode(10));", " console.error(lines.slice(-80).join(String.fromCharCode(10)));", "}", "if (hasYaml()) { emit('cached'); process.exit(0); }", "fs.mkdirSync('node_modules/yaml', { recursive: true });", "let registryBase = registry;", "while (registryBase.endsWith('/')) registryBase = registryBase.slice(0, -1);", "const tarball = registryBase + '/yaml/-/yaml-' + version + '.tgz';", "const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hwlab-yaml-'));", "const tgz = path.join(tmpDir, 'yaml.tgz');", "if (run('curl', ['-fsSL', '--retry', String(retryCount), '--connect-timeout', '10', '--max-time', String(timeoutSeconds), '-o', tgz, tarball]) && run('tar', ['-xzf', tgz, '-C', 'node_modules/yaml', '--strip-components=1'])) emit('installed', { manager: 'tarball' });", "else emit('tarball-failed', { manager: 'tarball' });", "fs.rmSync(tmpDir, { recursive: true, force: true });", "if (!hasYaml()) {", " fs.rmSync('node_modules/yaml', { recursive: true, force: true });", " if (run('bun', ['add', '--no-save', '--ignore-scripts', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'bun' });", " else emit('bun-failed', { manager: 'bun' });", "}", "if (!hasYaml()) {", " fs.rmSync('node_modules/yaml', { recursive: true, force: true });", " if (run('npm', ['install', '--package-lock=false', '--no-save', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'npm' });", " else { emit('npm-failed', { manager: 'npm' }); tailNpmLog(); process.exit(34); }", "}", "if (!hasYaml()) { emit('failed', { reason: 'unresolved' }); process.exit(34); }", "NODE_UNIDESK_YAML_DEPENDENCY", "ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;", "}", "function validatePrepareSourceDependencyScript(script) {", " const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';", " let offset = 0;", " while (true) {", " const markerStart = script.indexOf(\"node <<'\" + marker + \"'\", offset);", " if (markerStart === -1) return;", " const bodyStart = script.indexOf('\\n', markerStart);", " if (bodyStart === -1) throw new Error('prepare-source dependency heredoc body missing newline');", " const bodyEnd = script.indexOf('\\n' + marker, bodyStart + 1);", " if (bodyEnd === -1) throw new Error('prepare-source dependency heredoc terminator missing');", " const body = script.slice(bodyStart + 1, bodyEnd);", " try { new vm.Script(body, { filename: 'NODE_UNIDESK_YAML_DEPENDENCY.js' }); } catch (error) { throw new Error(`generated prepare-source yaml dependency script is invalid: ${error.message}`); }", " offset = bodyEnd + marker.length + 1;", " }", "}", "function deployYamlOverlayScript() {", " const runtimeOverlay = JSON.stringify({", " nodeId: overlay.nodeId,", " lane: overlay.lane,", " sourceBranch: overlay.sourceBranch,", " gitopsBranch: overlay.gitopsBranch,", " gitopsRoot: overlay.gitopsRoot,", " runtimePath: overlay.runtimePath,", " runtimeRenderDir: overlay.runtimeRenderDir,", " runtimeNamespace: overlay.runtimeNamespace,", " catalogPath: overlay.catalogPath,", " gitUrl: overlay.gitUrl,", " publicWebUrl: overlay.publicWebUrl,", " publicApiUrl: overlay.publicApiUrl,", " externalPostgres: overlay.externalPostgres,", " observability: overlay.observability,", " runtimeImageRewrites: overlay.runtimeImageRewrites,", " dockerProxyHttp: overlay.dockerProxyHttp,", " dockerProxyHttps: overlay.dockerProxyHttps,", " dockerNoProxyList: overlay.dockerNoProxyList,", " npmRegistry: overlay.npmRegistry,", " npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,", " npmRetries: overlay.npmRetries,", " });", " return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'", "const fs = require('fs');", "const YAML = require('yaml');", "const overlay = ${runtimeOverlay};", "const file = 'deploy/deploy.yaml';", "if (!fs.existsSync(file)) {", " console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));", " process.exit(45);", "}", "const doc = YAML.parse(fs.readFileSync(file, 'utf8'));", "doc.nodes = doc.nodes || {};", "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", "doc.lanes = doc.lanes || {};", "const lane = doc.lanes[overlay.lane] || {};", "const envRecipe = lane.envRecipe || {};", "const downloadStack = {", " ...(envRecipe.downloadStack || {}),", " httpProxy: overlay.dockerProxyHttp,", " httpsProxy: overlay.dockerProxyHttps,", " noProxy: overlay.dockerNoProxyList,", "};", "if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;", "if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;", "doc.lanes[overlay.lane] = {", " ...lane,", " node: overlay.nodeId,", " sourceBranch: overlay.sourceBranch,", " gitopsBranch: overlay.gitopsBranch,", " namespace: overlay.runtimeNamespace,", " endpoint: overlay.publicApiUrl,", " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", " artifactCatalog: overlay.catalogPath,", " runtimePath: overlay.runtimePath,", " imageTagMode: 'full',", " sourceRepo: overlay.gitUrl,", " externalPostgres: overlay.externalPostgres,", " observability: overlay.observability,", " envRecipe: { ...envRecipe, downloadStack },", "};", "fs.writeFileSync(file, YAML.stringify(doc));", "console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));", "NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;", "}", "function runtimePathOverlayScript() {", " const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;", " const targetPath = String(overlay.runtimePath || '');", " if (!targetPath || sourcePath === targetPath) return '';", " return [", " `if [ ! -d ${shellSingle(targetPath)} ]; then`,", " ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,", " ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,", " ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,", " ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,", " `fi`,", " ].join('\\n');", "}", "function stepEnvBootstrapScript() {", " const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);", " if (entries.length === 0) return '';", " const lines = ['# unidesk-step-env-bootstrap'];", " for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);", " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');", " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');", " return lines.join('\\n');", "}", "function runtimeGitopsPostprocessScript() {", " const runtimeOverlay = JSON.stringify({", " gitopsRoot: overlay.gitopsRoot,", " runtimePath: overlay.runtimePath,", " runtimeRenderDir: overlay.runtimeRenderDir,", " runtimeNamespace: overlay.runtimeNamespace,", " externalPostgres: overlay.externalPostgres,", " publicExposure: overlay.publicExposure,", " observability: overlay.observability,", " runtimeImageRewrites: overlay.runtimeImageRewrites,", " gitReadUrl: overlay.gitReadUrl,", " publicWebUrl: overlay.publicWebUrl,", " publicApiUrl: overlay.publicApiUrl,", " });", " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'", "const fs = require('fs');", "const path = require('path');", "const crypto = require('crypto');", "const YAML = require('yaml');", "const overlay = ${runtimeOverlay};", "const runtimePath = String(overlay.runtimePath || '');", "const renderDir = String(overlay.runtimeRenderDir || '');", "const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';", "const candidates = [...new Set([", " runtimePath,", " renderDir,", " overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',", " legacyRuntimePath,", "].filter(Boolean))];", "const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;", "if (!runtimePath || !fs.existsSync(sourcePath)) {", " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));", " process.exit(47);", "}", "if (sourcePath !== runtimePath) {", " fs.rmSync(runtimePath, { recursive: true, force: true });", " fs.mkdirSync(path.dirname(runtimePath), { recursive: true });", " fs.cpSync(sourcePath, runtimePath, { recursive: true });", " fs.rmSync(sourcePath, { recursive: true, force: true });", "}", "for (const candidate of candidates) {", " if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });", "}", "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", "function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }", "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", "function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }", "function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }", "function yamlFiles(dir) {", " const files = [];", " for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {", " const file = path.join(dir, entry.name);", " if (entry.isDirectory()) files.push(...yamlFiles(file));", " else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);", " }", " return files;", "}", "function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", "function writeYamlDocuments(file, docs) { fs.writeFileSync(file, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\\\n---\\\\n') + '\\\\n'); }", "function podSpecFor(item) {", " if (!isObject(item) || !isObject(item.spec)) return null;", " if (item.kind === 'Pod') return item.spec;", " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;", " return null;", "}", "function templateMetadataFor(item) {", " if (!isObject(item) || !isObject(item.spec)) return null;", " if (item.kind === 'Pod') return item.metadata || null;", " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;", " if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;", " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;", " return null;", "}", "function stripMonitoringMetadata(metadata) {", " if (!isObject(metadata)) return false;", " let changed = false;", " if (isObject(metadata.labels) && metadata.labels['hwlab.pikastech.local/monitoring'] !== undefined && metadata.labels['hwlab.pikastech.local/monitoring'] !== 'disabled') {", " metadata.labels['hwlab.pikastech.local/monitoring'] = 'disabled';", " changed = true;", " }", " if (isObject(metadata.annotations) && metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'] !== undefined) {", " delete metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'];", " changed = true;", " }", " return changed;", "}", "function containerHasVolumeMount(container, name) { return isObject(container) && Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === name); }", "function removeMetricsSidecar(podSpec) {", " if (!isObject(podSpec)) return false;", " let changed = false;", " if (Array.isArray(podSpec.containers)) {", " const next = podSpec.containers.filter((container) => !(isObject(container) && (container.name === 'hwlab-metrics' || (Array.isArray(container.command) && container.command.includes('/metrics/metrics-sidecar.mjs')))));", " if (next.length !== podSpec.containers.length) { podSpec.containers = next; changed = true; }", " }", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container) || !Array.isArray(container.volumeMounts)) continue;", " const nextMounts = container.volumeMounts.filter((mount) => !(mount && mount.name === 'hwlab-metrics-sidecar'));", " if (nextMounts.length !== container.volumeMounts.length) { container.volumeMounts = nextMounts; changed = true; }", " }", " }", " if (Array.isArray(podSpec.volumes)) {", " const nextVolumes = podSpec.volumes.filter((volume) => !(volume && volume.name === 'hwlab-metrics-sidecar'));", " if (nextVolumes.length !== podSpec.volumes.length) { podSpec.volumes = nextVolumes; changed = true; }", " }", " return changed;", "}", "function envValue(container, name) {", " if (!isObject(container) || !Array.isArray(container.env)) return undefined;", " const item = container.env.find((env) => env && env.name === name);", " return item ? item.value : undefined;", "}", "function setEnvValue(container, name, value) {", " if (!isObject(container) || typeof value !== 'string') return false;", " container.env = Array.isArray(container.env) ? container.env : [];", " let item = container.env.find((env) => env && env.name === name);", " if (!item) { item = { name }; container.env.push(item); }", " const changed = item.value !== value || item.valueFrom !== undefined;", " item.value = value;", " delete item.valueFrom;", " return changed;", "}", "function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }", "function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }", "function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }", "function startupProbeFrom(probe) {", " const next = JSON.parse(JSON.stringify(probe));", " next.periodSeconds = 10;", " next.timeoutSeconds = Math.max(Number(next.timeoutSeconds || 0), 2);", " next.failureThreshold = 30;", " next.successThreshold = 1;", " delete next.initialDelaySeconds;", " return next;", "}", "function addEnvReuseStartupProbe(podSpec) {", " if (!isObject(podSpec) || !Array.isArray(podSpec.containers)) return false;", " let changed = false;", " for (const container of podSpec.containers) {", " if (!isObject(container) || !isEnvReuseContainer(container) || container.startupProbe) continue;", " const sourceProbe = container.readinessProbe || container.livenessProbe;", " if (!sourceProbe) continue;", " container.startupProbe = startupProbeFrom(sourceProbe);", " changed = true;", " }", " return changed;", "}", "function rewriteRuntimeImage(image) {", " if (typeof image !== 'string') return image;", " const match = (overlay.runtimeImageRewrites || []).find((item) => item && item.source === image);", " return match ? match.target : image;", "}", "function patchRuntimeImages(podSpec) {", " if (!isObject(podSpec)) return false;", " let changed = false;", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container) || typeof container.image !== 'string') continue;", " const nextImage = rewriteRuntimeImage(container.image);", " if (nextImage !== container.image) { container.image = nextImage; changed = true; }", " }", " }", " return changed;", "}", "function rewriteEnvValue(value) {", " if (typeof value !== 'string') return value;", " return value.replaceAll('http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git', overlay.gitReadUrl);", "}", "function patchGitReadUrlEnv(podSpec) {", " if (!isObject(podSpec)) return false;", " let changed = false;", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container) || !Array.isArray(container.env)) continue;", " for (const env of container.env) {", " if (!isObject(env) || typeof env.value !== 'string') continue;", " const nextValue = rewriteEnvValue(env.value);", " if (nextValue !== env.value) { env.value = nextValue; changed = true; }", " }", " }", " }", " return changed;", "}", "function patchRuntimeEnv(item, podSpec) {", " if (!isObject(podSpec)) return { publicEndpointChanged: false, dbSslModeChanged: false };", " let publicEndpointChanged = false;", " let dbSslModeChanged = false;", " const pg = overlay.externalPostgres;", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container)) continue;", " if (envValue(container, 'HWLAB_PUBLIC_ENDPOINT') !== undefined) publicEndpointChanged = setEnvValue(container, 'HWLAB_PUBLIC_ENDPOINT', expectedPublicEndpoint(item)) || publicEndpointChanged;", " if (pg && pg.sslmode && envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE') !== undefined) dbSslModeChanged = setEnvValue(container, 'HWLAB_CLOUD_DB_SSL_MODE', pg.sslmode) || dbSslModeChanged;", " }", " }", " return { publicEndpointChanged, dbSslModeChanged };", "}", "function patchRuntimeWorkloads() {", " let observabilityChanged = false;", " let startupProbeChanged = false;", " let imageRewriteChanged = false;", " let gitReadUrlChanged = false;", " let publicEndpointChanged = false;", " let dbSslModeChanged = false;", " for (const file of yamlFiles(runtimePath)) {", " if (path.basename(file) === 'kustomization.yaml') continue;", " const docs = readYamlDocuments(file);", " let changed = false;", " for (const doc of docs) {", " for (const item of listItems(doc).filter(Boolean)) {", " if (!isObject(item)) continue;", " if (overlay.observability && overlay.observability.prometheusOperator === false) {", " const metadataChanged = stripMonitoringMetadata(item.metadata);", " const templateChanged = stripMonitoringMetadata(templateMetadataFor(item));", " const sidecarChanged = removeMetricsSidecar(podSpecFor(item));", " const monitoringChanged = metadataChanged || templateChanged || sidecarChanged;", " changed = monitoringChanged || changed;", " observabilityChanged = observabilityChanged || monitoringChanged;", " }", " const probeChanged = addEnvReuseStartupProbe(podSpecFor(item));", " changed = probeChanged || changed;", " startupProbeChanged = startupProbeChanged || probeChanged;", " const imageChanged = patchRuntimeImages(podSpecFor(item));", " changed = imageChanged || changed;", " imageRewriteChanged = imageRewriteChanged || imageChanged;", " const gitUrlChanged = patchGitReadUrlEnv(podSpecFor(item));", " changed = gitUrlChanged || changed;", " gitReadUrlChanged = gitReadUrlChanged || gitUrlChanged;", " const envChanged = patchRuntimeEnv(item, podSpecFor(item));", " changed = envChanged.publicEndpointChanged || envChanged.dbSslModeChanged || changed;", " publicEndpointChanged = publicEndpointChanged || envChanged.publicEndpointChanged;", " dbSslModeChanged = dbSslModeChanged || envChanged.dbSslModeChanged;", " }", " }", " if (changed) writeYamlDocuments(file, docs);", " }", " return { observabilityChanged, startupProbeChanged, imageRewriteChanged, gitReadUrlChanged, publicEndpointChanged, dbSslModeChanged };", "}", "function patchKustomization() {", " const file = path.join(runtimePath, 'kustomization.yaml');", " if (!fs.existsSync(file)) return false;", " const doc = readYaml(file) || {};", " const resources = Array.isArray(doc.resources) ? doc.resources : [];", " const next = resources.filter((item) => !(overlay.observability && overlay.observability.prometheusOperator === false && item === 'observability.yaml'));", " let changed = false;", " if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); changed = true; }", " const observabilityFile = path.join(runtimePath, 'observability.yaml');", " if (overlay.observability && overlay.observability.prometheusOperator === false && fs.existsSync(observabilityFile)) { fs.rmSync(observabilityFile, { force: true }); changed = true; }", " return changed;", "}", "function patchExternalPostgres() {", " const pg = overlay.externalPostgres;", " if (!pg || !pg.serviceName) return false;", " const file = path.join(runtimePath, 'external-postgres.yaml');", " if (!fs.existsSync(file)) return false;", " const doc = readYaml(file);", " const items = listItems(doc).filter(Boolean);", " let changed = false;", " const endpointSliceName = String(pg.serviceName) + '-host';", " for (const item of items) {", " if (!item || typeof item !== 'object') continue;", " item.metadata = item.metadata || {};", " item.metadata.namespace = overlay.runtimeNamespace;", " item.metadata.labels = item.metadata.labels || {};", " item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;", " if (item.kind === 'Service') {", " item.metadata.name = pg.serviceName;", " item.spec = item.spec || {};", " item.spec.ports = [{ name: 'postgres', port: pg.port, targetPort: pg.port, protocol: 'TCP' }];", " delete item.spec.selector;", " changed = true;", " }", " if (item.kind === 'EndpointSlice') {", " item.metadata.name = endpointSliceName;", " item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;", " item.addressType = 'IPv4';", " item.ports = [{ name: 'postgres', port: pg.port, protocol: 'TCP' }];", " item.endpoints = [{ addresses: [pg.endpointAddress], conditions: { ready: true } }];", " changed = true;", " }", " }", " if (changed) writeYaml(file, normalizeList(items));", " return changed;", "}", "function patchHealthContract() {", " const file = path.join(runtimePath, 'health-contract.yaml');", " if (!fs.existsSync(file)) return false;", " const doc = readYaml(file);", " const items = listItems(doc).filter(Boolean);", " let changed = false;", " const pg = overlay.externalPostgres;", " for (const item of items) {", " if (!item || item.kind !== 'ConfigMap') continue;", " item.data = item.data || {};", " if (item.data.endpoint !== overlay.publicWebUrl) { item.data.endpoint = overlay.publicWebUrl; changed = true; }", " const cloudApi = 'GET /health/live through ' + overlay.publicApiUrl;", " if (item.data['cloud-api'] !== cloudApi) { item.data['cloud-api'] = cloudApi; changed = true; }", " const cloudWeb = 'GET /health/live on ' + overlay.publicWebUrl + '; consumes cloud-api only';", " if (item.data['cloud-web'] !== cloudWeb) { item.data['cloud-web'] = cloudWeb; changed = true; }", " if (pg && pg.sslmode && typeof item.data['cloud-api-db'] === 'string') {", " const next = item.data['cloud-api-db'].replace(/HWLAB_CLOUD_DB_SSL_MODE=[A-Za-z0-9_-]+/g, 'HWLAB_CLOUD_DB_SSL_MODE=' + pg.sslmode);", " if (next !== item.data['cloud-api-db']) { item.data['cloud-api-db'] = next; changed = true; }", " }", " }", " if (changed) writeYaml(file, normalizeList(items));", " return changed;", "}", "function renderPublicExposureFrpcToml(exposure) {", " return [", " 'serverAddr = ' + JSON.stringify(String(exposure.serverAddr)),", " 'serverPort = ' + Number(exposure.serverPort),", " 'loginFailExit = true',", " 'auth.token = \"{{ .Envs.HWLAB_FRP_TOKEN }}\"',", " '',", " '[[proxies]]',", " 'name = ' + JSON.stringify(String(exposure.webProxy.name)),", " 'type = \"tcp\"',", " 'localIP = ' + JSON.stringify(String(exposure.webProxy.localIP)),", " 'localPort = ' + Number(exposure.webProxy.localPort),", " 'remotePort = ' + Number(exposure.webProxy.remotePort),", " '',", " '[[proxies]]',", " 'name = ' + JSON.stringify(String(exposure.apiProxy.name)),", " 'type = \"tcp\"',", " 'localIP = ' + JSON.stringify(String(exposure.apiProxy.localIP)),", " 'localPort = ' + Number(exposure.apiProxy.localPort),", " 'remotePort = ' + Number(exposure.apiProxy.remotePort),", " '',", " ].join('\\\\n');", "}", "function setEnvFromSecret(container, name, secretName, secretKey) {", " if (!isObject(container)) return false;", " container.env = Array.isArray(container.env) ? container.env : [];", " let item = container.env.find((env) => env && env.name === name);", " if (!item) { item = { name }; container.env.push(item); }", " const nextValueFrom = { secretKeyRef: { name: secretName, key: secretKey } };", " const changed = item.value !== undefined || JSON.stringify(item.valueFrom || {}) !== JSON.stringify(nextValueFrom);", " delete item.value;", " item.valueFrom = nextValueFrom;", " return changed;", "}", "function patchPublicExposure() {", " const exposure = overlay.publicExposure;", " if (!exposure || !exposure.enabled) return { configured: false, changed: false };", " const file = path.join(runtimePath, 'node-frpc.yaml');", " if (!fs.existsSync(file)) {", " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));", " process.exit(49);", " }", " const docs = readYamlDocuments(file);", " const configName = String(overlay.runtimeNamespace) + '-frpc-config';", " const deploymentName = String(overlay.runtimeNamespace) + '-frpc';", " const configKey = String(exposure.secretKey || 'frpc.toml');", " const tokenKey = String(exposure.tokenKey || 'token');", " const toml = renderPublicExposureFrpcToml(exposure);", " let changed = false;", " let foundConfigMap = false;", " let foundDeployment = false;", " for (const doc of docs) {", " for (const item of listItems(doc).filter(Boolean)) {", " if (!isObject(item)) continue;", " item.metadata = item.metadata || {};", " if (item.kind === 'ConfigMap' && item.metadata.name === configName) {", " foundConfigMap = true;", " item.data = item.data || {};", " if (item.data[configKey] !== toml) { item.data[configKey] = toml; changed = true; }", " }", " if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {", " foundDeployment = true;", " item.spec = item.spec || {};", " const nextStrategy = { type: 'Recreate' };", " if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }", " const podSpec = podSpecFor(item);", " for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {", " if (!isObject(container)) continue;", " if (container.name === 'frpc' || String(container.image || '').includes('frpc')) changed = setEnvFromSecret(container, 'HWLAB_FRP_TOKEN', exposure.secretName, tokenKey) || changed;", " }", " }", " }", " }", " if (!foundConfigMap || !foundDeployment) {", " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'frpc-resource-missing', filePath: file, configName, deploymentName, foundConfigMap, foundDeployment }));", " process.exit(50);", " }", " if (changed) writeYamlDocuments(file, docs);", " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: true, applied: true, changed, filePath: file, hostname: exposure.hostname, serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, webProxy: exposure.webProxy.name, apiProxy: exposure.apiProxy.name, configSha256: crypto.createHash('sha256').update(toml).digest('hex') }));", " return { configured: true, changed, foundConfigMap, foundDeployment };", "}", "const kustomizationChanged = patchKustomization();", "const runtimeWorkloadsChanged = patchRuntimeWorkloads();", "const externalPostgresChanged = patchExternalPostgres();", "const healthContractChanged = patchHealthContract();", "const publicExposureChanged = patchPublicExposure();", "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, runtimeImageRewriteCount: (overlay.runtimeImageRewrites || []).length, kustomizationChanged, observabilityWorkloadsChanged: runtimeWorkloadsChanged.observabilityChanged, startupProbeChanged: runtimeWorkloadsChanged.startupProbeChanged, runtimeImageRewriteChanged: runtimeWorkloadsChanged.imageRewriteChanged, gitReadUrlChanged: runtimeWorkloadsChanged.gitReadUrlChanged, publicEndpointChanged: runtimeWorkloadsChanged.publicEndpointChanged, dbSslModeChanged: runtimeWorkloadsChanged.dbSslModeChanged, externalPostgresChanged, healthContractChanged, publicExposureChanged }));", "NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;", "}", "function runtimeGitopsVerifyScript() {", " const runtimeOverlay = JSON.stringify({", " runtimePath: overlay.runtimePath,", " runtimeNamespace: overlay.runtimeNamespace,", " externalPostgres: overlay.externalPostgres,", " publicExposure: overlay.publicExposure,", " observability: overlay.observability,", " runtimeImageRewrites: overlay.runtimeImageRewrites,", " gitReadUrl: overlay.gitReadUrl,", " publicWebUrl: overlay.publicWebUrl,", " publicApiUrl: overlay.publicApiUrl,", " });", " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'", "const fs = require('fs');", "const path = require('path');", "const YAML = require('yaml');", "const overlay = ${runtimeOverlay};", "const runtimePath = String(overlay.runtimePath || '');", "function fail(reason, extra = {}) {", " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));", " process.exit(48);", "}", "if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');", "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", "function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }", "function yamlFiles(dir) {", " const files = [];", " for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {", " const file = path.join(dir, entry.name);", " if (entry.isDirectory()) files.push(...yamlFiles(file));", " else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);", " }", " return files;", "}", "function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", "function allItemsFromFile(file) { return readYamlDocuments(file).flatMap((doc) => listItems(doc).filter(Boolean)); }", "function podSpecFor(item) {", " if (!isObject(item) || !isObject(item.spec)) return null;", " if (item.kind === 'Pod') return item.spec;", " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;", " return null;", "}", "function envValue(container, name) {", " if (!isObject(container) || !Array.isArray(container.env)) return undefined;", " const item = container.env.find((env) => env && env.name === name);", " return item ? item.value : undefined;", "}", "function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }", "function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }", "function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }", "function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }", "function workloadChecks() {", " const metricsRefs = [];", " const missingStartupProbes = [];", " const publicRuntimeImages = [];", " const staleGitReadUrls = [];", " const wrongPublicEndpoints = [];", " const wrongDbSslModes = [];", " const rewriteSources = new Set((overlay.runtimeImageRewrites || []).map((item) => item && item.source).filter(Boolean));", " for (const file of yamlFiles(runtimePath)) {", " if (path.basename(file) === 'kustomization.yaml') continue;", " for (const doc of readYamlDocuments(file)) {", " for (const item of listItems(doc).filter(Boolean)) {", " const podSpec = podSpecFor(item);", " if (!isObject(podSpec)) continue;", " for (const container of Array.isArray(podSpec.containers) ? podSpec.containers : []) {", " if (!isObject(container)) continue;", " if (container.name === 'hwlab-metrics' || (Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === 'hwlab-metrics-sidecar'))) metricsRefs.push(workloadRef(item, file, container));", " if (isEnvReuseContainer(container) && (container.readinessProbe || container.livenessProbe) && !container.startupProbe) missingStartupProbes.push(workloadRef(item, file, container));", " if (typeof container.image === 'string' && rewriteSources.has(container.image)) publicRuntimeImages.push({ ...workloadRef(item, file, container), image: container.image });", " if (Array.isArray(container.env) && container.env.some((env) => env && typeof env.value === 'string' && env.value.includes('git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git') && env.value !== overlay.gitReadUrl)) staleGitReadUrls.push(workloadRef(item, file, container));", " const publicEndpoint = envValue(container, 'HWLAB_PUBLIC_ENDPOINT');", " if (publicEndpoint !== undefined && publicEndpoint !== expectedPublicEndpoint(item)) wrongPublicEndpoints.push({ ...workloadRef(item, file, container), value: publicEndpoint, expected: expectedPublicEndpoint(item) });", " const dbSslMode = envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE');", " if (overlay.externalPostgres && overlay.externalPostgres.sslmode && dbSslMode !== undefined && dbSslMode !== overlay.externalPostgres.sslmode) wrongDbSslModes.push({ ...workloadRef(item, file, container), value: dbSslMode, expected: overlay.externalPostgres.sslmode });", " }", " if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));", " }", " }", " }", " return { metricsRefs, missingStartupProbes, publicRuntimeImages, staleGitReadUrls, wrongPublicEndpoints, wrongDbSslModes };", "}", "const checks = [];", "const workloadCheck = workloadChecks();", "const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');", "if (overlay.observability && overlay.observability.prometheusOperator === false) {", " if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');", " const resources = readYaml(kustomizationPath).resources || [];", " if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });", " if (workloadCheck.metricsRefs.length > 0) fail('observability-sidecar-still-rendered', { refs: workloadCheck.metricsRefs.slice(0, 12), count: workloadCheck.metricsRefs.length });", " checks.push('observability-disabled');", "}", "if (workloadCheck.missingStartupProbes.length > 0) fail('env-reuse-startup-probe-missing', { refs: workloadCheck.missingStartupProbes.slice(0, 12), count: workloadCheck.missingStartupProbes.length });", "checks.push('env-reuse-startup-probes');", "if (workloadCheck.publicRuntimeImages.length > 0) fail('runtime-image-rewrite-missing', { refs: workloadCheck.publicRuntimeImages.slice(0, 12), count: workloadCheck.publicRuntimeImages.length });", "if ((overlay.runtimeImageRewrites || []).length > 0) checks.push('runtime-image-rewrites');", "if (workloadCheck.staleGitReadUrls.length > 0) fail('runtime-git-read-url-stale', { refs: workloadCheck.staleGitReadUrls.slice(0, 12), count: workloadCheck.staleGitReadUrls.length, expected: overlay.gitReadUrl });", "checks.push('runtime-git-read-url');", "if (workloadCheck.wrongPublicEndpoints.length > 0) fail('runtime-public-endpoint-mismatch', { refs: workloadCheck.wrongPublicEndpoints.slice(0, 12), count: workloadCheck.wrongPublicEndpoints.length });", "checks.push('runtime-public-endpoint');", "if (workloadCheck.wrongDbSslModes.length > 0) fail('runtime-db-ssl-mode-mismatch', { refs: workloadCheck.wrongDbSslModes.slice(0, 12), count: workloadCheck.wrongDbSslModes.length });", "if (overlay.externalPostgres && overlay.externalPostgres.sslmode) checks.push('runtime-db-ssl-mode');", "const pg = overlay.externalPostgres;", "if (pg && pg.serviceName) {", " const file = path.join(runtimePath, 'external-postgres.yaml');", " if (!fs.existsSync(file)) fail('external-postgres-missing');", " const items = listItems(readYaml(file)).filter(Boolean);", " const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);", " const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');", " if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });", " if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });", " const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;", " const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;", " const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;", " if (Number(servicePort) !== Number(pg.port) || Number(endpointPort) !== Number(pg.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: pg.port });", " if (String(endpointAddress) !== String(pg.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: pg.endpointAddress });", " checks.push('external-postgres-bridge');", "}", "const exposure = overlay.publicExposure;", "if (exposure && exposure.enabled) {", " const file = path.join(runtimePath, 'node-frpc.yaml');", " if (!fs.existsSync(file)) fail('public-exposure-frpc-missing');", " const items = allItemsFromFile(file);", " const configName = String(overlay.runtimeNamespace) + '-frpc-config';", " const deploymentName = String(overlay.runtimeNamespace) + '-frpc';", " const configKey = String(exposure.secretKey || 'frpc.toml');", " const tokenKey = String(exposure.tokenKey || 'token');", " const configMap = items.find((item) => item && item.kind === 'ConfigMap' && item.metadata && item.metadata.name === configName);", " const deployment = items.find((item) => item && item.kind === 'Deployment' && item.metadata && item.metadata.name === deploymentName);", " if (!configMap) fail('public-exposure-frpc-configmap-missing', { expected: configName });", " if (!deployment) fail('public-exposure-frpc-deployment-missing', { expected: deploymentName });", " const toml = configMap.data && configMap.data[configKey];", " if (typeof toml !== 'string') fail('public-exposure-frpc-config-missing', { expectedKey: configKey });", " for (const expected of [String(exposure.serverAddr), String(exposure.serverPort), String(exposure.webProxy.name), String(exposure.webProxy.remotePort), String(exposure.apiProxy.name), String(exposure.apiProxy.remotePort), 'HWLAB_FRP_TOKEN']) {", " if (!toml.includes(expected)) fail('public-exposure-frpc-config-mismatch', { expected });", " }", " const podSpec = podSpecFor(deployment);", " const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];", " const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;", " if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });", " const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));", " const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;", " const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;", " if (!secretRef || secretRef.name !== exposure.secretName || secretRef.key !== tokenKey) fail('public-exposure-frpc-token-env-mismatch', { expectedSecret: exposure.secretName, expectedKey: tokenKey });", " checks.push('public-exposure-frpc');", "}", "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));", "NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;", "}", "function patchScript(script) {", " let result = String(script || '');", " if (result.includes('npm run gitops:ts:check')) {", " result = result.replace(/\\n[ \\t]*npm run gitops:ts:check\\n/g, '\\n echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-replaces-tsc-gate\"}\\' >&2\\n');", " }", " const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');", " if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {", " result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());", " }", " const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';", " if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", " result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);", " }", " const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';", " if (result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", " result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);", " }", " result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);", " result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", " result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", " result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", " result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", " result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);", " result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);", " const legacyRuntimePath = (() => {", " const runtimePath = String(overlay.runtimePath || '');", " const parts = runtimePath.split('/').filter(Boolean);", " if (parts.length < 2) return '';", " const leaf = parts.at(-1);", " const parent = parts.slice(0, -2).join('/');", " return parent && leaf ? `${parent}/${leaf}` : '';", " })();", " if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {", " result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);", " result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');", " result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');", " }", " const bootstrap = stepEnvBootstrapScript();", " if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {", " result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);", " }", " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {", " let next = match;", " if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", " if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", " return next;", " });", " result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {", " if (match.includes('--check')) return runtimeGitopsVerifyScript();", " return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;", " });", " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", " validatePrepareSourceDependencyScript(result);", " return result;", "}", "function patchManifestObject(doc) {", " if (!doc || typeof doc !== 'object') return false;", " if (doc.kind !== 'Pipeline' || !doc.spec) return false;", " const defaults = {", " 'git-url': overlay.gitUrl,", " 'git-read-url': overlay.gitReadUrl,", " 'git-write-url': overlay.gitWriteUrl,", " 'catalog-path': overlay.catalogPath,", " 'runtime-path': overlay.runtimePath,", " 'registry-prefix': overlay.registryPrefix,", " };", " for (const param of doc.spec?.params || []) {", " if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];", " }", " doc.metadata = doc.metadata || {};", " doc.metadata.annotations = doc.metadata.annotations || {};", " doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;", " doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;", " for (const task of doc.spec?.tasks || []) {", " for (const sidecar of task.taskSpec?.sidecars || []) {", " if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;", " }", " for (const step of task.taskSpec?.steps || []) {", " if (Array.isArray(step.env)) {", " for (const env of step.env) {", " if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];", " }", " }", " step.env = Array.isArray(step.env) ? step.env : [];", " const existingEnv = new Set(step.env.map((env) => env.name));", " for (const [name, value] of Object.entries(stepEnv)) {", " if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });", " }", " if (typeof step.script === 'string') step.script = patchScript(step.script);", " }", " }", " return true;", "}", "function patchStructuredPipeline() {", " try {", " const doc = JSON.parse(text);", " if (!patchManifestObject(doc)) return false;", " text = JSON.stringify(doc, null, 2) + '\\n';", " return true;", " } catch {}", " if (YAML) {", " try {", " const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);", " const changed = docs.some((doc) => patchManifestObject(doc));", " if (!changed) return false;", " text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';", " return true;", " } catch {}", " }", " return false;", "}", "const structured = patchStructuredPipeline();", "function replaceParamDefault(name, value) {", " const namePattern = escapeRegExp(name);", " text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);", " text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);", "}", "replaceParamDefault('git-url', overlay.gitUrl);", "replaceParamDefault('git-read-url', overlay.gitReadUrl);", "replaceParamDefault('git-write-url', overlay.gitWriteUrl);", "replaceParamDefault('catalog-path', overlay.catalogPath);", "replaceParamDefault('runtime-path', overlay.runtimePath);", "replaceParamDefault('registry-prefix', overlay.registryPrefix);", "text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);", "text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);", "if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);", "text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", "text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", "text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", "text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", "for (const [name, value] of Object.entries(stepEnv)) {", " if (value === undefined) continue;", " text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);", " text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);", "}", "const bootstrap = stepEnvBootstrapScript();", "if (bootstrap) {", " text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {", " const previous = fullText.slice(Math.max(0, offset - 500), offset);", " if (previous.includes('unidesk-step-env-bootstrap')) return match;", " return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;", " });", "}", "if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {", " text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);", "}", "const quotedRoot = JSON.stringify(overlay.gitopsRoot);", "const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');", "const replacements = [", " ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],", " ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],", "];", "let changed = false;", "for (const [needle, replacement] of replacements) {", " if (!text.includes(needle)) continue;", " text = text.split(needle).join(replacement);", " changed = true;", "}", "if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }", "if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }", "if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && !text.includes('NODE_UNIDESK_YAML_DEPENDENCY')) { throw new Error(`generated pipeline missing UniDesk yaml dependency install in ${pipelinePath}`); }", "if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }", "fs.writeFileSync(pipelinePath, text);", "function patchArgoYaml(filePath) {", " if (!YAML || !fs.existsSync(filePath)) return;", " const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);", " let changed = false;", " for (const doc of docs) {", " if (!doc || typeof doc !== 'object') continue;", " if (doc.kind === 'AppProject') {", " doc.spec = doc.spec || {};", " doc.spec.sourceRepos = [overlay.argoRepoUrl];", " changed = true;", " }", " if (doc.kind === 'Application') {", " doc.spec = doc.spec || {};", " doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };", " changed = true;", " }", " }", " if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", "}", "patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));", "patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));", "NODE", ]; } function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record { const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http); return { nodeId: spec.nodeId, lane: spec.lane, sourceBranch: spec.sourceBranch, gitopsBranch: spec.gitopsBranch, gitopsRoot: nodeRuntimeGitopsRoot(spec), runtimePath: spec.runtimePath, runtimeRenderDir: spec.runtimeRenderDir, runtimeNamespace: spec.runtimeNamespace, catalogPath: spec.catalogPath, tektonDir: spec.tektonDir, argoApplicationFile: spec.argoApplicationFile, argoRepoUrl: spec.argoRepoUrl, gitUrl: spec.gitUrl, gitReadUrl: spec.gitReadUrl, gitWriteUrl: spec.gitWriteUrl, networkProfileId: spec.networkProfileId, downloadProfileId: spec.downloadProfileId, gitSshProxyHost: gitSshProxy?.host, gitSshProxyPort: gitSshProxy?.port, proxyHttp: spec.networkProfile.proxy.http, proxyHttps: spec.networkProfile.proxy.https, proxyAll: spec.networkProfile.proxy.all, noProxy: spec.networkProfile.proxy.noProxy.join(","), dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http, dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https, dockerProxyAll: spec.networkProfile.dockerBuildProxy.all, dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","), dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy, npmRegistry: spec.downloadProfile.npm.registry, npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000, npmRetries: spec.downloadProfile.npm.retries, stepEnv: spec.stepEnv, observability: spec.observability, runtimeImageRewrites: spec.runtimeImageRewrites, registryPrefix: spec.registryPrefix, buildkitSidecarImage: spec.buildkit?.sidecarImage, publicWebUrl: spec.publicWebUrl, publicApiUrl: spec.publicApiUrl, publicExposure: spec.publicExposure === null ? undefined : { enabled: spec.publicExposure.enabled, mode: spec.publicExposure.mode, publicBaseUrl: spec.publicExposure.publicBaseUrl, hostname: spec.publicExposure.hostname, expectedA: spec.publicExposure.expectedA, serverAddr: spec.publicExposure.serverAddr, serverPort: spec.publicExposure.serverPort, secretName: spec.publicExposure.secretName, secretKey: spec.publicExposure.secretKey, tokenKey: spec.publicExposure.tokenKey, webProxy: spec.publicExposure.webProxy, apiProxy: spec.publicExposure.apiProxy, }, externalPostgres: spec.externalPostgres === undefined ? undefined : { enabled: true, serviceName: spec.externalPostgres.serviceName, endpointAddress: spec.externalPostgres.endpointAddress, port: spec.externalPostgres.port, sslmode: spec.externalPostgres.sslmode, }, }; } function httpProxyEndpoint(value: string): { host: string; port: number } | null { let parsed: URL; try { parsed = new URL(value); } catch { return null; } if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10); if (!Number.isInteger(port) || port <= 0) return null; return { host: parsed.hostname, port }; } function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] { return [ `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`, `${renderDir}/${spec.tektonDir}/rbac.yaml`, `${renderDir}/${spec.tektonDir}/pipeline.yaml`, `${renderDir}/argocd/project.yaml`, `${renderDir}/argocd/${spec.argoApplicationFile}`, ]; } function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { const files = nodeRuntimeControlPlaneFiles(spec, renderDir); const args = [ "kubectl", "apply", ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), ...files.flatMap((file) => ["-f", file]), ]; return runNodeK3sArgs(spec, args, timeoutSeconds); } function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir) .map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`) .join(""); const args = [ "kubectl", "apply", ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), "-f", "-", ]; return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest); } function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult { const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; const script = [ "set -eu", `render_dir=${shellQuote(renderDir)}`, `prefix=${shellQuote(prefix)}`, "case \"$render_dir\" in", " \"$prefix\"*) rm -rf \"$render_dir\" ;;", " *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;", "esac", ].join("\n"); return runNodeHostScript(spec, script, 60); } function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult { const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`; if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) { return { command: ["rm", "-rf", ""], cwd: repoRoot, exitCode: 44, stdout: "", stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`, signal: null, timedOut: false, }; } return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 }); } function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record { return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, namespace: "hwlab-ci", labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": spec.lane, "hwlab.pikastech.local/source-commit": sourceCommit, "hwlab.pikastech.local/trigger": "unidesk-node-cli", }, annotations: { "hwlab.pikastech.local/node": spec.nodeId, "hwlab.pikastech.local/source-branch": spec.sourceBranch, "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch, "hwlab.pikastech.local/runtime-path": spec.runtimePath, "hwlab.pikastech.local/network-profile": spec.networkProfileId, "hwlab.pikastech.local/download-profile": spec.downloadProfileId, }, }, spec: { pipelineRef: { name: spec.pipeline }, taskRunTemplate: { serviceAccountName: spec.serviceAccountName, podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, }, }, params: [ { name: "git-url", value: spec.gitUrl }, { name: "git-read-url", value: spec.gitReadUrl }, { name: "git-write-url", value: spec.gitWriteUrl }, { name: "source-branch", value: spec.sourceBranch }, { name: "gitops-branch", value: spec.gitopsBranch }, { name: "lane", value: spec.lane }, { name: "catalog-path", value: spec.catalogPath }, { name: "image-tag-mode", value: "full" }, { name: "runtime-path", value: spec.runtimePath }, { name: "revision", value: sourceCommit }, { name: "registry-prefix", value: spec.registryPrefix }, { name: "services", value: spec.serviceIds.join(",") }, { name: "base-image", value: spec.baseImage }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, ], }, }; } function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult { const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64"); const script = [ "set -eu", `manifest_b64=${shellQuote(manifestB64)}`, `pipeline_run=${shellQuote(pipelineRun)}`, `rerun=${rerun ? "true" : "false"}`, "manifest_path=\"/tmp/$pipeline_run.json\"", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", "existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", "if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then", " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "elif [ \"$existing_status\" = False ]; then", " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "fi", "if kubectl create -f \"$manifest_path\"; then", " :", "else", " code=$?", " if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then", " printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2", " else", " exit \"$code\"", " fi", "fi", "kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'", ].join("\n"); return runNodeK3sScript(spec, script, timeoutSeconds); } function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`); } function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record { const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60); if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) }; const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u); return { exists: true, name: pipelineRun, status: status || null, reason: reason || null, message: message || null, createdAt: createdAt || null, result: compactRuntimeCommand(result), }; } function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { const pg = spec.externalPostgres; if (pg === undefined) return null; const secretRoot = externalPostgresSecretSourceRoot(spec); const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey); const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey); const missing = [ ...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]), ...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]), ]; if (missing.length > 0) { if (dryRun) { return { ok: true, mode: "dry-run", mutation: false, skipped: true, reason: "external-postgres-secret-source-missing", missing, sourceRoot: secretRoot, valuesPrinted: false, next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, }; } return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, degradedReason: "external-postgres-secret-source-missing", missing, sourceRoot: secretRoot, valuesPrinted: false, next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, }; } const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds); if (!isCommandSuccess(setup)) { return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, phase: "namespace-authn-setup", setup: compactRuntimeCommand(setup), valuesPrinted: false, }; } const manifest = { apiVersion: "v1", kind: "List", items: [ { apiVersion: "v1", kind: "Secret", metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace }, type: "Opaque", stringData: { [pg.cloudApi.secretKey]: cloudApi.value }, }, { apiVersion: "v1", kind: "Secret", metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace }, type: "Opaque", stringData: { [pg.openfga.secretKey]: openfga.value }, }, ], }; const applyScript = [ "set -eu", [ "kubectl", "apply", "--server-side", "--force-conflicts", "--field-manager=unidesk-hwlab-node-external-postgres-secret", ...(dryRun ? ["--dry-run=server"] : []), "-f", "-", ].map(shellQuote).join(" "), ].join("\n"); const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest)); return { ok: isCommandSuccess(apply), mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: !dryRun && isCommandSuccess(apply), namespace: spec.runtimeNamespace, sourceRoot: secretRoot, secrets: [ { name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey }, { name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null }, ], setup: compactRuntimeCommand(setup), apply: compactRuntimeCommand(apply), valuesPrinted: false, }; } function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { if (spec.baseImageSource === undefined) return null; const script = [ "set -eu", `target=${shellQuote(spec.baseImage)}`, `source=${shellQuote(spec.baseImageSource)}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, "repo_tag=${target#*/}", "repo=${repo_tag%:*}", "tag=${repo_tag##*:}", "if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", "registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", "present=false", "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi", "action=none", "if [ \"$present\" = false ]; then", " action=seed", " if [ \"$dry_run\" = false ]; then", " docker image inspect \"$source\" >/dev/null", " docker tag \"$source\" \"$target\"", " docker push \"$target\" >/tmp/hwlab-node-base-image-push.out", " fi", "fi", "after=false", "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi", "printf 'target\\t%s\\n' \"$target\"", "printf 'source\\t%s\\n' \"$source\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'presentBefore\\t%s\\n' \"$present\"", "printf 'presentAfter\\t%s\\n' \"$after\"", "printf 'action\\t%s\\n' \"$action\"", "if [ \"$dry_run\" = true ] || [ \"$after\" = true ]; then exit 0; fi", "exit 1", ].join("\n"); const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); const fields = keyValueLinesFromText(statusText(result)); return { ok: isCommandSuccess(result), dryRun, target: fields.target ?? spec.baseImage, source: fields.source ?? spec.baseImageSource, presentBefore: fields.presentBefore === "true", presentAfter: fields.presentAfter === "true", action: fields.action ?? null, result: compactRuntimeCommand(result), }; } function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string { const pg = spec.externalPostgres; if (pg === undefined) return "true"; const authnKey = pg.openfga.authnKey ?? "authn-preshared-key"; return [ "set -eu", `namespace=${shellQuote(spec.runtimeNamespace)}`, `secret=${shellQuote(pg.openfga.secretName)}`, `authn_key=${shellQuote(authnKey)}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, "namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"", "if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi", "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -", "authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')", "if [ \"${authn_present:-0}\" -gt 0 ]; then", " action=kept", "elif [ \"$dry_run\" = true ]; then", " action=would-generate-authn-key", "else", " if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi", " kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null", " authn_value=", " action=generated-authn-key", "fi", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'authnKey\\t%s\\n' \"$authn_key\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'valuesPrinted\\tfalse\\n'", ].join("\n"); } function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string { const configRef = spec.externalPostgres?.configRef; if (configRef === undefined) return join(repoRoot, ".state", "secrets"); const configPath = rootPath(configRef); const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown; if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`); const secrets = (parsed as Record).secrets; if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`); const root = (secrets as Record).root; if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`); return root.startsWith("/") ? root : rootPath(root); } function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } { const sourcePath = join(secretRoot, sourceRef); if (!existsSync(sourcePath)) return { ok: false, sourcePath }; const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const value = values[key]; if (value === undefined || value.length === 0) return { ok: false, sourcePath }; return { ok: true, value, sourcePath }; } function parseEnvFile(text: string): Record { const values: Record = {}; for (const rawLine of text.split(/\r?\n/u)) { const line = rawLine.trim(); if (line.length === 0 || line.startsWith("#")) continue; const eq = line.indexOf("="); if (eq <= 0) continue; const key = line.slice(0, eq).trim(); const rawValue = line.slice(eq + 1).trim(); values[key] = rawValue.startsWith("'") && rawValue.endsWith("'") ? rawValue.slice(1, -1).replace(/'"'"'/gu, "'") : rawValue.startsWith("\"") && rawValue.endsWith("\"") ? rawValue.slice(1, -1) : rawValue; } return values; } function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean { if (!/postgres|pgsql|pgdata/iu.test(name)) return false; const externalServiceName = spec.externalPostgres?.serviceName; if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false; return true; } function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { const pg = spec.externalPostgres; if (pg === undefined) return { required: false, ready: true }; if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "name"], 60); const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60); const [address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u); return { required: true, ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && address === pg.endpointAddress && Number(port) === pg.port, serviceName: pg.serviceName, endpointAddress: address || null, expectedEndpointAddress: pg.endpointAddress, port: numericField(port), expectedPort: pg.port, service: compactRuntimeCommand(service), endpointSlice: compactRuntimeCommand(endpointSlice), }; } function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { const pg = spec.externalPostgres; if (pg === undefined) return { required: false, ready: true }; if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey); const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey); const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key"); return { required: true, ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present, valuesPrinted: false, cloudApi, openfga: { datastoreUri: openfgaUri, authnPresharedKey: openfgaAuthn, }, }; } function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record { const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60); return { secret, key, present: result.exitCode === 0 && result.stdout.trim().length > 0, valueBytesBase64: result.stdout.trim().length, exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500), valuesPrinted: false, }; } function startNodeDelegatedJob(options: ReturnType): Record { const commandArgs = [ "hwlab", "nodes", options.domain, ...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]), "--node", options.node, "--lane", options.lane, "--confirm", "--timeout-seconds", String(options.timeoutSeconds), "--wait", ]; const command = ["bun", "scripts/cli.ts", ...commandArgs]; const job = startJob( `hwlab_nodes_${options.lane}_${options.domain}_${options.action}`, command, `Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`, ); return { ok: true, command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`, node: options.node, lane: options.lane, mode: "async-job", reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking", job, statusCommand: `bun scripts/cli.ts job status ${job.id}`, waitCommand: command.join(" "), }; } function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType): Record { const rewritten = rewriteDelegatedNodeValue(value, scoped); const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record : { value: rewritten }; return { ...result, node: scoped.node, commandFamily: "hwlab nodes", }; } function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType): unknown { if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped); if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped)); if (typeof value !== "object" || value === null) return value; return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)])); } function rewriteDelegatedNodeString(value: string, scoped: ReturnType): string { const replaceCommand = (text: string, domain: DelegatedNodeDomain) => { const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); return text .replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`) .replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`); }; return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror"); } function parseSecretOptions(args: string[]): NodeSecretOptions { const [actionRaw] = args; if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") { throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-vNN-bootstrap-admin|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]"); } const node = requiredOption(args, "--node"); assertNodeId(node); const lane = requiredOption(args, "--lane"); assertLane(lane); const spec = runtimeSecretSpec({ node, lane }); const name = optionValue(args, "--name") ?? spec.openFgaSecret; const key = optionValue(args, "--key"); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run"); if (actionRaw === "cleanup-owned-postgres") { if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres"); if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key"); if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`); return { action: actionRaw, node, lane, name: spec.postgresSecret, preset: "owned-postgres-cleanup", confirm, dryRun: explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (actionRaw === "cleanup-obsolete") { if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup"); if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key"); if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`); return { action: actionRaw, node, lane, name, preset: "obsolete-secret-cleanup", confirm, dryRun: explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (name === spec.masterAdminApiKeySecret) { if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`); return { action: actionRaw, node, lane, name, key: key ?? MASTER_ADMIN_API_KEY_KEY, preset: "master-server-admin-api-key", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (name === spec.bootstrapAdminSecret) { if (key !== undefined && key !== BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY) throw new Error(`secret ${name} supports only key ${BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY}`); return { action: actionRaw, node, lane, name, key: key ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, preset: "bootstrap-admin", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (name === spec.codeAgentProviderSecret) { if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) { throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`); } return { action: actionRaw, node, lane, name, key, preset: "code-agent-provider", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } if (name === spec.cloudApiDbSecret) { if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`); if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { action: actionRaw, node, lane, name, key: key ?? spec.cloudApiDbKey, preset: "cloud-api-db", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900), }; } if (name !== spec.openFgaSecret) { throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.bootstrapAdminSecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`); } if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) { throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`); } if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { action: actionRaw, node, lane, name, key, preset: "openfga", confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), }; } function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec { const namespace = `hwlab-${input.lane}`; const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined; const externalPostgres = runtimeLaneSpec?.externalPostgres; const platformDb = externalPostgres !== undefined || /^v0*[3-9]\d*$/.test(input.lane); const platformPostgresService = externalPostgres?.serviceName ?? "g14-platform-postgres"; const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`; const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`; const platformPostgresEndpointSlice = `${platformPostgresService}-host`; const openFgaDbName = externalPostgres?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga"); const openFgaDbUser = externalPostgres?.openfga.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga"); const cloudApiDbName = externalPostgres?.database ?? `hwlab_${input.lane}`; const cloudApiDbUser = externalPostgres?.cloudApi.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`); return { node: input.node, lane: input.lane, namespace, platformDb, ...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }), ...(externalPostgres === undefined ? {} : { externalPostgres }), platformPostgresService, platformPostgresEndpointAddress: externalPostgres?.endpointAddress, platformPostgresEndpointSlice, postgresSecret: `${namespace}-postgres`, postgresStatefulSet: `${namespace}-postgres`, postgresAdminUser: `hwlab_${input.lane}`, openFgaSecret: externalPostgres?.openfga.secretName ?? `${namespace}-openfga`, openFgaDbName, openFgaDbUser, openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`, bootstrapAdminSecret: `${namespace}-bootstrap-admin`, bootstrapAdminPasswordHashKey: BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE, bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET, cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? `hwlab-cloud-api-${input.lane}-db`, cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? CLOUD_API_DB_KEY, cloudApiDbName, cloudApiDbUser, cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, cloudApiDeployment: "hwlab-cloud-api", obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`, obsoleteHwpodDbName: `hwpod_${input.lane}`, obsoleteHwpodDbUser: `hwpod_${input.lane}_app`, codeAgentProviderSecret: `${namespace}-code-agent-provider`, codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE, codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET, fieldManager: `unidesk-hwlab-node-${input.lane}-secret`, }; } function runNodeSecret(options: NodeSecretOptions): Record { const spec = runtimeSecretSpec(options); if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec); if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) { return runExternalPostgresSecretEnsure(options, spec); } const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun ? readMasterAdminApiKey().key : ""; const script = options.preset === "openfga" ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec) : options.preset === "master-server-admin-api-key" ? masterAdminApiKeySecretScript(options, spec) : options.preset === "bootstrap-admin" ? bootstrapAdminSecretScript(options, spec) : options.preset === "cloud-api-db" ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec) : options.preset === "owned-postgres-cleanup" ? ownedPostgresCleanupScript(options, spec) : codeAgentProviderSecretScript(options, spec); const result = runTransScript(options.node, script, input, options.timeoutSeconds); const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec); const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0; const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0; const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true; const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true; return { ok, command: `hwlab nodes secret ${options.action}`, node: options.node, lane: options.lane, namespace: spec.namespace, secret: options.name, key: options.key ?? null, preset: options.preset, mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure", status, mutation: status.mutation === true, result: compactCommandResult(result), valuesRedacted: true, next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec), }; } function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { if (options.action === "cleanup-owned-postgres") { return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` }; } if (options.action === "cleanup-obsolete") { return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` }; } if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) { return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; } if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) { return { status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`, controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`, }; } return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; } function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { const runtimeLaneSpec = spec.runtimeLaneSpec; if (runtimeLaneSpec === undefined) { return { ok: false, command: `hwlab nodes secret ${options.action}`, node: options.node, lane: options.lane, namespace: spec.namespace, secret: options.name, key: options.key ?? null, preset: options.preset, mode: options.dryRun ? "dry-run" : "confirmed-ensure", mutation: false, degradedReason: "external-postgres-runtime-lane-spec-missing", valuesRedacted: true, }; } const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds); const shouldReadStatus = sync !== null && sync.ok === true; const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null; const status = statusResult === null ? null : secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec); const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true); return { ok, command: `hwlab nodes secret ${options.action}`, node: options.node, lane: options.lane, namespace: spec.namespace, secret: options.name, key: options.key ?? null, preset: options.preset, mode: options.dryRun ? "dry-run" : "confirmed-ensure", status: status ?? sync, externalPostgresSecretSync: sync, mutation: sync?.mutation === true, result: statusResult === null ? null : compactCommandResult(statusResult), valuesRedacted: true, next: ok ? undefined : nextSecretCommand(options, spec), }; } function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record { return { mode: exposure.mode, publicBaseUrl: exposure.publicBaseUrl, hostname: exposure.hostname, expectedA: exposure.expectedA, frpc: { serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, tokenSourceRef: exposure.tokenSourceRef, tokenSourceKey: exposure.tokenSourceKey, secretName: exposure.secretName, secretKey: exposure.secretKey, tokenKey: exposure.tokenKey, webProxy: exposure.webProxy, apiProxy: exposure.apiProxy, }, caddy: { route: exposure.caddyRoute, configPath: exposure.caddyConfigPath, serviceName: exposure.caddyServiceName, tls: exposure.caddyTls, responseHeaderTimeoutSeconds: exposure.responseHeaderTimeoutSeconds, }, }; } function runNodePublicExposure(options: NodePublicExposureOptions): Record { const exposure = options.spec.publicExposure; if (exposure === null || !exposure.enabled) { return { ok: false, command: "hwlab nodes control-plane public-exposure", node: options.node, lane: options.lane, configPath: hwlabRuntimeLaneConfigPath(), error: "publicExposure is not configured for this node/lane target", mutation: false, }; } const source = readPublicExposureTokenSource(exposure); if (!source.ok) { return { ok: false, command: "hwlab nodes control-plane public-exposure", node: options.node, lane: options.lane, mode: options.dryRun ? "dry-run" : "confirmed", configPath: hwlabRuntimeLaneConfigPath(), publicExposure: publicExposureSummary(exposure), source, mutation: false, valuesRedacted: true, next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=` }, }; } const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds); let secretFields = keyValueLinesFromText(statusText(secretApplyResult)); let secretResult = secretApplyResult; if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") { const restartResult = runPublicExposureFrpcRecreate(options); secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) }; secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult); } const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds); const caddyFields = keyValueLinesFromText(statusText(caddyResult)); const secretStatus = publicExposureSecretStatus(secretFields, secretResult); const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult); const ok = secretStatus.ok === true && caddyStatus.ok === true; return { ok, command: "hwlab nodes control-plane public-exposure", node: options.node, lane: options.lane, mode: options.dryRun ? "dry-run" : "confirmed", mutation: !options.dryRun && (secretFields.mutation === "true" || caddyFields.mutation === "true"), configPath: hwlabRuntimeLaneConfigPath(), publicExposure: publicExposureSummary(exposure), source: { ok: source.ok, path: source.path, key: exposure.tokenSourceKey, bytes: source.value?.length ?? 0, fingerprint: source.fingerprint, valueRedacted: true, }, secret: secretStatus, caddy: caddyStatus, valuesRedacted: true, next: ok ? { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${options.node} --lane ${options.lane} --confirm`, status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`, } : { retry: `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${options.node} --lane ${options.lane} --confirm` }, }; } function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } { const checkedPaths = publicExposureTokenSourcePaths(exposure); const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef); if (!existsSync(path)) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-source-missing" }; const values = parseEnvFile(readFileSync(path, "utf8")); const value = values[exposure.tokenSourceKey]; if (value === undefined || value.length === 0) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-key-missing" }; return { ok: true, path, checkedPaths, key: exposure.tokenSourceKey, value, fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`, }; } function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec): string[] { const paths = [join(repoRoot, ".state", "secrets", exposure.tokenSourceRef)]; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", exposure.tokenSourceRef)); return [...new Set(paths)]; } function publicExposureSecretStatus(fields: Record, result: CommandResult): Record { const dryRun = fields.dryRun === "true"; return { ok: result.exitCode === 0 && (dryRun || ( fields.afterSecretExists === "yes" && fields.strategyPatchExitCode === "0" && fields.rolloutRestartExitCode === "0" && fields.rolloutStatusExitCode === "0" )), dryRun, mutation: fields.mutation === "true", namespace: fields.namespace || null, secret: fields.secret || null, deployment: fields.deployment || null, beforeExists: fields.beforeSecretExists === "yes", afterExists: fields.afterSecretExists === "yes", tokenBytes: numericField(fields.tokenBytes), applyExitCode: numericField(fields.applyExitCode), restartMode: fields.restartMode || null, strategyPatchExitCode: numericField(fields.strategyPatchExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), scaleDownExitCode: numericField(fields.scaleDownExitCode), scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode), scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount), scaleUpExitCode: numericField(fields.scaleUpExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), readyReplicas: numericField(fields.readyReplicas), availableReplicas: numericField(fields.availableReplicas), desiredReplicas: numericField(fields.desiredReplicas), strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null, restartErrorPreview: fields.restartErrorPreview || null, scaleDownErrorPreview: fields.scaleDownErrorPreview || null, scaleUpErrorPreview: fields.scaleUpErrorPreview || null, readyErrorPreview: fields.readyErrorPreview || null, exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), }; } function publicExposureCaddyStatus(fields: Record, result: CommandResult): Record { const dryRun = fields.dryRun === "true"; return { ok: result.exitCode === 0 && fields.afterBlockPresent === "yes" && fields.validateExitCode === "0" && (dryRun || ( fields.active === "active" && fields.localWebStatus === "200" && fields.localApiStatus === "200" )), dryRun, mutation: fields.mutation === "true", hostname: fields.hostname || null, configPath: fields.configPath || null, beforeBlockPresent: fields.beforeBlockPresent === "yes", afterBlockPresent: fields.afterBlockPresent === "yes", staleBlocksRemoved: numericField(fields.staleBlocksRemoved), pythonExitCode: numericField(fields.pythonExitCode), validateMode: fields.validateMode || null, validateExitCode: numericField(fields.validateExitCode), reloadExitCode: numericField(fields.reloadExitCode), active: fields.active || null, localWebStatus: fields.localWebStatus || null, localApiStatus: fields.localApiStatus || null, validateErrorPreview: fields.validateErrorPreview || null, localWebErrorPreview: fields.localWebErrorPreview || null, localApiErrorPreview: fields.localApiErrorPreview || null, exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), }; } function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string { const deployment = `${options.spec.runtimeNamespace}-frpc`; return [ "set +e", `namespace=${shellQuote(options.spec.runtimeNamespace)}`, `secret=${shellQuote(exposure.secretName)}`, `deployment=${shellQuote(deployment)}`, `token_key=${shellQuote(exposure.tokenKey)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "token=$(cat)", "before_secret_exists=no", "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes", "token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')", "apply_exit=", "restart_mode=recreate-rollout-restart", "strategy_patch_exit=", "rollout_restart_exit=", "scale_down_exit=", "scale_down_wait_exit=", "scale_down_wait_pod_count=", "scale_up_exit=", "rollout_status_exit=", "ready_replicas=", "available_replicas=", "desired_replicas=", "mutation=false", "if [ \"$dry_run\" = false ]; then", " tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)", " token_b64=$(printf '%s' \"$token\" | base64 | tr -d '\\n')", " cat >\"$tmp\" </tmp/hwlab-public-frpc-secret.apply.out 2>/tmp/hwlab-public-frpc-secret.apply.err", " apply_exit=$?", " rm -f \"$tmp\"", " if [ \"$apply_exit\" = 0 ]; then mutation=true; fi", "fi", "after_secret_exists=no", "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'deployment\\t%s\\n' \"$deployment\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", "printf 'tokenBytes\\t%s\\n' \"$token_bytes\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'restartMode\\t%s\\n' \"$restart_mode\"", "printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"", "printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"", "printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"", "printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", "printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"", "printf 'availableReplicas\\t%s\\n' \"$available_replicas\"", "printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"", "if [ \"$dry_run\" = true ]; then exit 0; fi", "[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]", ].join("\n"); } function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult { const namespace = options.spec.runtimeNamespace; const deployment = `${namespace}-frpc`; const fields: Record = { deployment, restartMode: "recreate-rollout-restart", strategyPatchExitCode: "", rolloutRestartExitCode: "", scaleDownExitCode: "", scaleDownWaitExitCode: "", scaleDownWaitPodCount: "", scaleUpExitCode: "", rolloutStatusExitCode: "", readyReplicas: "", availableReplicas: "", desiredReplicas: "", strategyPatchErrorPreview: "", restartErrorPreview: "", scaleDownErrorPreview: "", scaleUpErrorPreview: "", readyErrorPreview: "", }; const stdoutParts: string[] = []; const stderrParts: string[] = []; const addResult = (result: CommandResult): void => { if (result.stdout.trim()) stdoutParts.push(result.stdout.trim()); if (result.stderr.trim()) stderrParts.push(result.stderr.trim()); }; const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55); const strategyPatch = runTransScript(options.node, [ "set +e", `namespace=${shellQuote(namespace)}`, `deployment=${shellQuote(deployment)}`, "kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err", "code=$?", "printf 'strategyPatchExitCode\\t%s\\n' \"$code\"", "printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' /tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err", "code=$?", "printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"", "printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' /tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err", "code=$?", "values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)", "desired=$(printf '%s' \"$values\" | cut -f1)", "ready=$(printf '%s' \"$values\" | cut -f2)", "available=$(printf '%s' \"$values\" | cut -f3)", "printf 'desiredReplicas\\t%s\\n' \"$desired\"", "printf 'readyReplicas\\t%s\\n' \"$ready\"", "printf 'availableReplicas\\t%s\\n' \"$available\"", "printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' `${key}\t${value}`).join("\n") + "\n"; return { command: [transPath(), `${options.node}:k3s`, "script", "--", ""], cwd: repoRoot, exitCode: ok ? 0 : 1, stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n", stderr: stderrParts.join("\n"), signal: null, timedOut: false, }; } function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult { return { command: [...first.command, "&&", ...second.command], cwd: repoRoot, exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode, stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n", stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"), signal: second.signal ?? first.signal, timedOut: first.timedOut || second.timedOut, }; } function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string { const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64"); const marker = `unidesk managed ${exposure.hostname}`; return [ "set +e", `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `hostname=${shellQuote(exposure.hostname)}`, `config_path=${shellQuote(exposure.caddyConfigPath)}`, `service=${shellQuote(exposure.caddyServiceName)}`, `marker=${shellQuote(marker)}`, `block_b64=${shellQuote(blockB64)}`, `expected_a=${shellQuote(exposure.expectedA)}`, `web_port=${shellQuote(String(exposure.webProxy.remotePort))}`, `api_port=${shellQuote(String(exposure.apiProxy.remotePort))}`, "rm -f /tmp/hwlab-public-caddy-validate.out /tmp/hwlab-public-caddy-validate.err /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-web.err /tmp/hwlab-public-caddy-api.err", "tmp=$(mktemp -d)", "block=\"$tmp/block\"", "next=\"$tmp/Caddyfile\"", "printf '%s' \"$block_b64\" | base64 -d >\"$block\"", "before_present=no", "if [ -f \"$config_path\" ] && grep -Fq \"# BEGIN $marker\" \"$config_path\"; then before_present=yes; fi", "if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi", "stale_blocks_removed=$(python3 - \"$next\" \"$block\" \"$marker\" \"$web_port\" \"$api_port\" 2>/tmp/hwlab-public-caddy-python.err <<'PY'", "import pathlib, re, sys", "config = pathlib.Path(sys.argv[1])", "block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')", "marker = sys.argv[3]", "web_port = sys.argv[4]", "api_port = sys.argv[5]", "current_name = marker[len('unidesk managed '):] if marker.startswith('unidesk managed ') else marker", "text = config.read_text(encoding='utf-8') if config.exists() else ''", "begin = f'# BEGIN {marker}'", "end = f'# END {marker}'", "managed = f'{begin}\\n{block.rstrip()}\\n{end}\\n'", "stale_removed = [0]", "pattern = re.compile(r'(?ms)^# BEGIN unidesk managed (?P[^\\n]+)\\n(?P.*?)\\n# END unidesk managed (?P=name)\\n*')", "def keep(match):", " name = match.group('name')", " body = match.group('body')", " if name != current_name and (f'127.0.0.1:{web_port}' in body or f'127.0.0.1:{api_port}' in body):", " stale_removed[0] += 1", " return ''", " return match.group(0)", "text = pattern.sub(keep, text)", "if begin in text and end in text:", " start = text.index(begin)", " stop = text.index(end, start) + len(end)", " text = text[:start].rstrip() + '\\n\\n' + managed.rstrip() + '\\n' + text[stop:].lstrip('\\n')", "else:", " text = text.rstrip() + '\\n\\n' + managed", "config.write_text(text, encoding='utf-8')", "print(stale_removed[0])", "PY", ")", "python_exit=$?", "validate_exit=1", "validate_mode=validate", "if [ \"$python_exit\" != 0 ]; then", " validate_mode=python", " validate_exit=$python_exit", "else", " if [ \"$dry_run\" = true ]; then", " validate_mode=adapt", " caddy adapt --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err", " else", " sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err", " fi", " validate_exit=$?", "fi", "reload_exit=", "mutation=false", "if [ \"$dry_run\" = false ] && [ \"$validate_exit\" = 0 ]; then", " sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/hwlab-public-caddy-install.out 2>/tmp/hwlab-public-caddy-install.err", " install_exit=$?", " if [ \"$install_exit\" = 0 ]; then", " mutation=true", " sudo systemctl reload \"$service\" >/tmp/hwlab-public-caddy-reload.out 2>/tmp/hwlab-public-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/hwlab-public-caddy-reload.out 2>>/tmp/hwlab-public-caddy-reload.err", " reload_exit=$?", " else", " reload_exit=$install_exit", " fi", "fi", "active=$(systemctl is-active \"$service\" 2>/dev/null || true)", "after_present=no", "if [ \"$dry_run\" = true ]; then grep -Fq \"# BEGIN $marker\" \"$next\" && after_present=yes; else grep -Fq \"# BEGIN $marker\" \"$config_path\" && after_present=yes; fi", "local_web_status=", "local_api_status=", "if [ \"$dry_run\" = false ]; then", " local_web_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-web.err || true)", " local_api_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/health/live\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-api.err || true)", "fi", "validate_err_preview=$(cat /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-validate.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)", "local_web_err_preview=$([ -f /tmp/hwlab-public-caddy-web.err ] && tr '\\n' ';' /dev/null | cut -c1-1000 || true)", "local_api_err_preview=$([ -f /tmp/hwlab-public-caddy-api.err ] && tr '\\n' ';' /dev/null | cut -c1-1000 || true)", "printf 'hostname\\t%s\\n' \"$hostname\"", "printf 'expectedA\\t%s\\n' \"$expected_a\"", "printf 'configPath\\t%s\\n' \"$config_path\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeBlockPresent\\t%s\\n' \"$before_present\"", "printf 'afterBlockPresent\\t%s\\n' \"$after_present\"", "printf 'staleBlocksRemoved\\t%s\\n' \"$stale_blocks_removed\"", "printf 'pythonExitCode\\t%s\\n' \"$python_exit\"", "printf 'validateMode\\t%s\\n' \"$validate_mode\"", "printf 'validateExitCode\\t%s\\n' \"$validate_exit\"", "printf 'reloadExitCode\\t%s\\n' \"$reload_exit\"", "printf 'active\\t%s\\n' \"$active\"", "printf 'localWebStatus\\t%s\\n' \"$local_web_status\"", "printf 'localApiStatus\\t%s\\n' \"$local_api_status\"", "printf 'validateErrorPreview\\t%s\\n' \"$validate_err_preview\"", "printf 'localWebErrorPreview\\t%s\\n' \"$local_web_err_preview\"", "printf 'localApiErrorPreview\\t%s\\n' \"$local_api_err_preview\"", "rm -rf \"$tmp\"", "[ \"$validate_exit\" = 0 ] && [ \"$after_present\" = yes ]", ].join("\n"); } function publicExposureCaddyBlock(exposure: HwlabRuntimePublicExposureSpec): string { const tlsLines = exposure.caddyTls === "internal" ? " tls internal\n" : ""; return `${exposure.hostname} { ${tlsLines} @api path /health* /api* /v1* /openapi* /docs* /swagger* reverse_proxy @api 127.0.0.1:${exposure.apiProxy.remotePort} { transport http { response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s } } reverse_proxy 127.0.0.1:${exposure.webProxy.remotePort} { transport http { response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s } } }`; } function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { return runCommand([transPath(), `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { return runCommand([transPath(), node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds); const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec); const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true }; const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds); const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec); const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false); const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true; return { ok, command: `hwlab nodes secret ${options.action}`, node: options.node, lane: options.lane, namespace: spec.namespace, secret: options.name, key: null, preset: options.preset, mode: options.dryRun ? "dry-run" : "confirmed-delete", status: { ok, preset: "obsolete-hwpod-db-cleanup", dryRun: options.dryRun, mutation, kubernetesSecret: kubernetesStatus, platformDatabase: platformDbStatus, hostMutationSkipped: !options.dryRun && hostOptions.dryRun, summary: ok ? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove` : `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`, }, mutation, result: { kubernetesSecret: compactCommandResult(kubernetesResult), platformDatabase: compactCommandResult(platformDbResult), }, valuesRedacted: true, next: ok ? undefined : nextSecretCommand(options, spec), }; } function runNodeEndpointBridge(options: ReturnType): Record { if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm"); const dryRun = options.dryRun || !options.confirm; const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds); const fields = keyValueLinesFromText(statusText(result)); const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes"; const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes"; const afterExcluded = fields.afterEndpointResourcesExcluded === "yes"; const afterIgnored = fields.afterEndpointsIgnoreUpdates === "yes" || fields.afterEndpointSliceIgnoreUpdates === "yes"; const beforeExtraEndpointSlices = splitWhitespaceField(fields.beforeExtraEndpointSliceNames); const afterExtraEndpointSlices = splitWhitespaceField(fields.afterExtraEndpointSliceNames); const beforeLegacyEndpoints = fields.beforeLegacyEndpointsExists === "yes"; const afterLegacyEndpoints = fields.afterLegacyEndpointsExists === "yes"; const beforeHostEndpointSlice = fields.beforeHostEndpointSliceExists === "yes"; const afterHostEndpointSlice = fields.afterHostEndpointSliceExists === "yes"; const bridgeReady = !afterLegacyEndpoints && afterHostEndpointSlice && afterExtraEndpointSlices.length === 0; const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored && bridgeReady; return { ok: dryRun ? result.exitCode === 0 : ok, command: "hwlab nodes control-plane allow-endpoint-bridge", node: options.node, lane: options.lane, namespace: "argocd", application: fields.application || `hwlab-node-${options.lane}`, mode: dryRun ? "dry-run" : "confirmed-control-plane-update", status: { action: fields.action || null, dryRun, mutation: fields.mutation === "true", before: { endpointResourcesExcluded: beforeExcluded, endpointsIgnoreUpdates: fields.beforeEndpointsIgnoreUpdates === "yes", endpointSliceIgnoreUpdates: fields.beforeEndpointSliceIgnoreUpdates === "yes", legacyEndpointsExist: beforeLegacyEndpoints, hostEndpointSliceExists: beforeHostEndpointSlice, extraEndpointSlices: beforeExtraEndpointSlices, }, after: { endpointResourcesExcluded: afterExcluded, endpointsIgnoreUpdates: fields.afterEndpointsIgnoreUpdates === "yes", endpointSliceIgnoreUpdates: fields.afterEndpointSliceIgnoreUpdates === "yes", legacyEndpointsExist: afterLegacyEndpoints, hostEndpointSliceExists: afterHostEndpointSlice, extraEndpointSlices: afterExtraEndpointSlices, }, runtimeNamespace: fields.runtimeNamespace || `hwlab-${options.lane}`, platformService: fields.platformService || "g14-platform-postgres", hostEndpointSlice: fields.hostEndpointSlice || "g14-platform-postgres-host", patchExitCode: numericField(fields.patchExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), deleteLegacyEndpointsExitCode: numericField(fields.deleteLegacyEndpointsExitCode), deleteExtraEndpointSlicesExitCode: numericField(fields.deleteExtraEndpointSlicesExitCode), refreshExitCode: numericField(fields.refreshExitCode), exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), summary: !afterExcluded && !afterIgnored && bridgeReady ? "Argo tracks HWLAB external Postgres EndpointSlice and no legacy Endpoints remain" : "Argo endpoint bridge is not in final Service plus EndpointSlice shape", }, result: compactCommandResult(result), }; } function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string { const application = `hwlab-node-${options.lane}`; const runtimeNamespace = `hwlab-${options.lane}`; return [ "set +e", "namespace=argocd", `runtime_namespace=${shellQuote(runtimeNamespace)}`, "configmap=argocd-cm", `application=${shellQuote(application)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "platform_service=g14-platform-postgres", "host_endpointslice=g14-platform-postgres-host", "preset=endpoint-bridge-resource-tracking", "cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", "cm_has_key() { value=$(cm_data \"$1\"); [ -n \"$value\" ] && [ \"$value\" != \"\" ] && printf yes || printf no; }", "endpoint_resources_excluded() { exclusions=$(cm_data resource.exclusions); printf '%s' \"$exclusions\" | grep -Eq '(^|[[:space:]])(Endpoints|EndpointSlice)([[:space:]]|$)' && printf yes || printf no; }", "resource_exists() { kubectl -n \"$runtime_namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "extra_endpoint_slices() { kubectl -n \"$runtime_namespace\" get endpointslice -l \"kubernetes.io/service-name=$platform_service\" -o name 2>/dev/null | sed \"/\\/$host_endpointslice$/d\" | tr '\\n' ' ' | sed 's/[[:space:]]*$//'; }", "wait_runtime_bridge_clean() {", " for _ in $(seq 1 30); do", " current_legacy=$(resource_exists endpoints \"$platform_service\")", " current_extra=$(extra_endpoint_slices)", " current_host=$(resource_exists endpointslice \"$host_endpointslice\")", " if [ \"$current_legacy\" != yes ] && [ -z \"$current_extra\" ] && [ \"$current_host\" = yes ]; then return 0; fi", " sleep 2", " done", " return 1", "}", "before_endpoint_resources_excluded=$(endpoint_resources_excluded)", "before_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')", "before_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')", "before_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")", "before_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")", "before_extra_endpoint_slice_names=$(extra_endpoint_slices)", "needs_argo_update=false", "if [ \"$before_endpoint_resources_excluded\" = yes ] || [ \"$before_endpoints_ignore_updates\" = yes ] || [ \"$before_endpoint_slice_ignore_updates\" = yes ]; then needs_argo_update=true; fi", "needs_runtime_cleanup=false", "if [ \"$before_legacy_endpoints_exists\" = yes ] || [ -n \"$before_extra_endpoint_slice_names\" ]; then needs_runtime_cleanup=true; fi", "action=observed", "mutation=false", "patch_exit=", "rollout_restart_exit=", "rollout_status_exit=", "delete_legacy_endpoints_exit=", "delete_extra_endpointslices_exit=", "refresh_exit=", "if [ \"$dry_run\" = true ]; then", " if [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-old-endpoint-exclusions-and-legacy-endpoints", " elif [ \"$needs_argo_update\" = true ]; then action=would-remove-old-endpoint-exclusions", " elif [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-legacy-endpoints", " else action=kept; fi", "else", " if [ \"$needs_argo_update\" = true ]; then", " patch_file=$(mktemp /tmp/hwlab-argocd-endpoint-bridge.XXXXXX.json)", " python3 - <<'PY' >\"$patch_file\"", "import json", "desired = '''### Internal Kubernetes resources excluded to reduce watch volume", "- apiGroups:", " - coordination.k8s.io", " kinds:", " - Lease", "### Internal Kubernetes Authz/Authn resources excluded to reduce watched events", "- apiGroups:", " - authentication.k8s.io", " - authorization.k8s.io", " kinds:", " - SelfSubjectReview", " - TokenReview", " - LocalSubjectAccessReview", " - SelfSubjectAccessReview", " - SelfSubjectRulesReview", " - SubjectAccessReview", "### Intermediate Certificate Request excluded to reduce watched events", "- apiGroups:", " - certificates.k8s.io", " kinds:", " - CertificateSigningRequest", "- apiGroups:", " - cert-manager.io", " kinds:", " - CertificateRequest", "### Cilium internal resources excluded to reduce UI clutter", "- apiGroups:", " - cilium.io", " kinds:", " - CiliumIdentity", " - CiliumEndpoint", " - CiliumEndpointSlice", "### Kyverno intermediate and reporting resources excluded to reduce watched events", "- apiGroups:", " - kyverno.io", " - reports.kyverno.io", " - wgpolicyk8s.io", " kinds:", " - PolicyReport", " - ClusterPolicyReport", " - EphemeralReport", " - ClusterEphemeralReport", " - AdmissionReport", " - ClusterAdmissionReport", " - BackgroundScanReport", " - ClusterBackgroundScanReport", " - UpdateRequest", "'''", "print(json.dumps({", " 'data': {", " 'resource.exclusions': desired,", " 'resource.customizations.ignoreResourceUpdates.Endpoints': None,", " 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice': None,", " }", "}))", "PY", " kubectl -n \"$namespace\" patch configmap \"$configmap\" --type merge --patch-file \"$patch_file\" >/tmp/hwlab-argocd-endpoint-bridge-patch.out 2>/tmp/hwlab-argocd-endpoint-bridge-patch.err", " patch_exit=$?", " rm -f \"$patch_file\"", " if [ \"$patch_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout restart statefulset/argocd-application-controller >/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.err", " rollout_restart_exit=$?", " if [ \"$rollout_restart_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status statefulset/argocd-application-controller --timeout=180s >/tmp/hwlab-argocd-endpoint-bridge-rollout-status.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-status.err", " rollout_status_exit=$?", " fi", " fi", " fi", " if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then action=patch-failed", " elif [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", " else", " if [ \"$needs_runtime_cleanup\" = true ]; then", " kubectl -n \"$runtime_namespace\" delete endpoints \"$platform_service\" --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpoints-delete.out 2>/tmp/hwlab-platform-postgres-endpoints-delete.err", " delete_legacy_endpoints_exit=$?", " wait_runtime_bridge_clean", " remaining_extra=$(extra_endpoint_slices)", " if [ -n \"$remaining_extra\" ]; then", " kubectl -n \"$runtime_namespace\" delete $remaining_extra --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpointslices-delete.out 2>/tmp/hwlab-platform-postgres-endpointslices-delete.err", " delete_extra_endpointslices_exit=$?", " wait_runtime_bridge_clean", " fi", " fi", " if [ \"$needs_argo_update\" = true ] || [ \"$needs_runtime_cleanup\" = true ]; then", " kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/hwlab-argocd-endpoint-bridge-refresh.out 2>/tmp/hwlab-argocd-endpoint-bridge-refresh.err", " refresh_exit=$?", " fi", " if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then action=delete-legacy-endpoints-failed", " elif [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then action=delete-extra-endpointslices-failed", " elif [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then action=refresh-failed", " elif [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=removed-old-endpoint-exclusions-and-legacy-endpoints; mutation=true", " elif [ \"$needs_argo_update\" = true ]; then action=removed-old-endpoint-exclusions; mutation=true", " elif [ \"$needs_runtime_cleanup\" = true ]; then action=removed-legacy-endpoints; mutation=true", " else action=kept; fi", " fi", "fi", "after_endpoint_resources_excluded=$(endpoint_resources_excluded)", "after_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')", "after_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')", "after_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")", "after_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")", "after_extra_endpoint_slice_names=$(extra_endpoint_slices)", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"", "printf 'configMap\\t%s\\n' \"$configmap\"", "printf 'application\\t%s\\n' \"$application\"", "printf 'platformService\\t%s\\n' \"$platform_service\"", "printf 'hostEndpointSlice\\t%s\\n' \"$host_endpointslice\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeEndpointResourcesExcluded\\t%s\\n' \"$before_endpoint_resources_excluded\"", "printf 'beforeEndpointsIgnoreUpdates\\t%s\\n' \"$before_endpoints_ignore_updates\"", "printf 'beforeEndpointSliceIgnoreUpdates\\t%s\\n' \"$before_endpoint_slice_ignore_updates\"", "printf 'beforeLegacyEndpointsExists\\t%s\\n' \"$before_legacy_endpoints_exists\"", "printf 'beforeHostEndpointSliceExists\\t%s\\n' \"$before_host_endpointslice_exists\"", "printf 'beforeExtraEndpointSliceNames\\t%s\\n' \"$before_extra_endpoint_slice_names\"", "printf 'afterEndpointResourcesExcluded\\t%s\\n' \"$after_endpoint_resources_excluded\"", "printf 'afterEndpointsIgnoreUpdates\\t%s\\n' \"$after_endpoints_ignore_updates\"", "printf 'afterEndpointSliceIgnoreUpdates\\t%s\\n' \"$after_endpoint_slice_ignore_updates\"", "printf 'afterLegacyEndpointsExists\\t%s\\n' \"$after_legacy_endpoints_exists\"", "printf 'afterHostEndpointSliceExists\\t%s\\n' \"$after_host_endpointslice_exists\"", "printf 'afterExtraEndpointSliceNames\\t%s\\n' \"$after_extra_endpoint_slice_names\"", "printf 'patchExitCode\\t%s\\n' \"$patch_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", "printf 'deleteLegacyEndpointsExitCode\\t%s\\n' \"$delete_legacy_endpoints_exit\"", "printf 'deleteExtraEndpointSlicesExitCode\\t%s\\n' \"$delete_extra_endpointslices_exit\"", "printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"", "if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi", "if [ \"$dry_run\" != true ] && { [ \"$after_legacy_endpoints_exists\" = yes ] || [ -n \"$after_extra_endpoint_slice_names\" ] || [ \"$after_host_endpointslice_exists\" != yes ]; }; then exit 47; fi", "if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi", "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", "if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then exit \"$delete_legacy_endpoints_exit\"; fi", "if [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then exit \"$delete_extra_endpointslices_exit\"; fi", "if [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then exit \"$refresh_exit\"; fi", ].join("\n"); } function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const pvc = `data-${spec.postgresSecret}-0`; const platformService = spec.platformPostgresService; const postgresService = spec.postgresSecret; const postgresConfigMap = `${spec.postgresSecret}-init`; return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `postgres_secret=${shellQuote(spec.postgresSecret)}`, `postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`, `postgres_service=${shellQuote(postgresService)}`, `postgres_configmap=${shellQuote(postgresConfigMap)}`, `pvc=${shellQuote(pvc)}`, `platform_service=${shellQuote(platformService)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "preset=owned-postgres-cleanup", "exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }", "pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }", "phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }", "before_secret_exists=$(exists_flag secret \"$postgres_secret\")", "before_pvc_exists=$(exists_flag pvc \"$pvc\")", "before_pvc_phase=$(phase_of_pvc)", "before_pv=$(pv_name)", "before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")", "before_service_exists=$(exists_flag service \"$postgres_service\")", "before_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")", "platform_service_exists=$(exists_flag service \"$platform_service\")", "action=observed", "mutation=false", "delete_statefulset_exit=", "delete_service_exit=", "delete_configmap_exit=", "delete_secret_exit=", "delete_pvc_exit=", "before_any_owned=false", "for flag in \"$before_statefulset_exists\" \"$before_service_exists\" \"$before_configmap_exists\" \"$before_secret_exists\" \"$before_pvc_exists\"; do", " if [ \"$flag\" = yes ]; then before_any_owned=true; fi", "done", "if [ \"$dry_run\" = true ]; then", " if [ \"$before_any_owned\" = true ]; then action=would-delete; else action=already-absent; fi", "else", " kubectl -n \"$namespace\" delete statefulset \"$postgres_statefulset\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-statefulset-delete.out 2>/tmp/hwlab-owned-postgres-statefulset-delete.err", " delete_statefulset_exit=$?", " kubectl -n \"$namespace\" delete service \"$postgres_service\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-service-delete.out 2>/tmp/hwlab-owned-postgres-service-delete.err", " delete_service_exit=$?", " kubectl -n \"$namespace\" delete configmap \"$postgres_configmap\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-configmap-delete.out 2>/tmp/hwlab-owned-postgres-configmap-delete.err", " delete_configmap_exit=$?", " kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err", " delete_secret_exit=$?", " kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err", " delete_pvc_exit=$?", " for _ in $(seq 1 30); do", " current_statefulset=$(exists_flag statefulset \"$postgres_statefulset\")", " current_service=$(exists_flag service \"$postgres_service\")", " current_configmap=$(exists_flag configmap \"$postgres_configmap\")", " current_secret=$(exists_flag secret \"$postgres_secret\")", " current_pvc=$(exists_flag pvc \"$pvc\")", " if [ \"$current_statefulset\" != yes ] && [ \"$current_service\" != yes ] && [ \"$current_configmap\" != yes ] && [ \"$current_secret\" != yes ] && [ \"$current_pvc\" != yes ]; then break; fi", " sleep 2", " done", " if [ \"$delete_statefulset_exit\" -eq 0 ] && [ \"$delete_service_exit\" -eq 0 ] && [ \"$delete_configmap_exit\" -eq 0 ] && [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then", " if [ \"$before_any_owned\" = true ]; then action=deleted; mutation=true; else action=already-absent; fi", " else", " action=delete-failed", " fi", "fi", "after_secret_exists=$(exists_flag secret \"$postgres_secret\")", "after_pvc_exists=$(exists_flag pvc \"$pvc\")", "after_pvc_phase=$(phase_of_pvc)", "after_pv=$(pv_name)", "after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")", "after_service_exists=$(exists_flag service \"$postgres_service\")", "after_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$postgres_secret\"", "printf 'statefulSet\\t%s\\n' \"$postgres_statefulset\"", "printf 'service\\t%s\\n' \"$postgres_service\"", "printf 'configMap\\t%s\\n' \"$postgres_configmap\"", "printf 'pvc\\t%s\\n' \"$pvc\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", "printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"", "printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"", "printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"", "printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"", "printf 'beforeServiceExists\\t%s\\n' \"$before_service_exists\"", "printf 'beforeConfigMapExists\\t%s\\n' \"$before_configmap_exists\"", "printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", "printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"", "printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"", "printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"", "printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"", "printf 'afterServiceExists\\t%s\\n' \"$after_service_exists\"", "printf 'afterConfigMapExists\\t%s\\n' \"$after_configmap_exists\"", "printf 'deleteStatefulSetExitCode\\t%s\\n' \"$delete_statefulset_exit\"", "printf 'deleteServiceExitCode\\t%s\\n' \"$delete_service_exit\"", "printf 'deleteConfigMapExitCode\\t%s\\n' \"$delete_configmap_exit\"", "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", "printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"", "if [ \"$platform_service_exists\" != yes ]; then exit 44; fi", "if [ \"$after_statefulset_exists\" = yes ] || [ \"$after_service_exists\" = yes ] || [ \"$after_configmap_exists\" = yes ] || [ \"$after_secret_exists\" = yes ] || [ \"$after_pvc_exists\" = yes ]; then exit 45; fi", "if [ -n \"$delete_statefulset_exit\" ] && [ \"$delete_statefulset_exit\" != 0 ]; then exit \"$delete_statefulset_exit\"; fi", "if [ -n \"$delete_service_exit\" ] && [ \"$delete_service_exit\" != 0 ]; then exit \"$delete_service_exit\"; fi", "if [ -n \"$delete_configmap_exit\" ] && [ \"$delete_configmap_exit\" != 0 ]; then exit \"$delete_configmap_exit\"; fi", "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", "if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi", ].join("\n"); } function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `secret=${shellQuote(options.name)}`, `expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "preset=obsolete-secret-cleanup", "exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }", "workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)", "workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)", "workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)", "before_exists=$(exists_flag)", "action=observed", "mutation=false", "delete_secret_exit=", "if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi", "if [ \"$action\" = observed ]; then", " if [ \"$workload_refs_present\" = yes ]; then", " action=blocked-referenced", " elif [ \"$dry_run\" = true ]; then", " if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi", " else", " kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err", " delete_secret_exit=$?", " for _ in $(seq 1 15); do", " current_exists=$(exists_flag)", " if [ \"$current_exists\" != yes ]; then break; fi", " sleep 1", " done", " if [ \"$delete_secret_exit\" -eq 0 ]; then", " if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi", " else", " action=delete-failed", " fi", " fi", "fi", "after_exists=$(exists_flag)", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_exists\"", "printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"", "printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"", "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", "if [ \"$action\" = unsupported-secret ]; then exit 43; fi", "if [ \"$workload_refs_present\" = yes ]; then exit 46; fi", "if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi", "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", ].join("\n"); } function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `db_name=${shellQuote(spec.obsoleteHwpodDbName)}`, `db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "preset=obsolete-platform-db-cleanup", "database_exists_flag() {", " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)", " code=$?", " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", " [ \"$output\" = t ] && printf yes || printf no", "}", "role_exists_flag() {", " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)", " code=$?", " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", " [ \"$output\" = t ] && printf yes || printf no", "}", "before_database_exists=$(database_exists_flag)", "before_database_probe_exit=$?", "before_role_exists=$(role_exists_flag)", "before_role_probe_exit=$?", "action=observed", "mutation=false", "drop_database_exit=", "drop_role_exit=", "before_any=false", "if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi", "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then", " action=probe-failed", "elif [ \"$dry_run\" = true ]; then", " if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi", "else", " if [ \"$before_database_exists\" = yes ]; then", " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'", "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();", "DROP DATABASE IF EXISTS :\"db_name\";", "SQL", " drop_database_exit=$?", " else", " drop_database_exit=0", " fi", " if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then", " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'", "DROP ROLE IF EXISTS :\"db_user\";", "SQL", " drop_role_exit=$?", " elif [ \"$drop_database_exit\" -eq 0 ]; then", " drop_role_exit=0", " else", " drop_role_exit=", " fi", " if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then", " if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi", " else", " action=drop-failed", " fi", "fi", "after_database_exists=$(database_exists_flag)", "after_database_probe_exit=$?", "after_role_exists=$(role_exists_flag)", "after_role_probe_exit=$?", "printf 'database\\t%s\\n' \"$db_name\"", "printf 'role\\t%s\\n' \"$db_user\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"", "printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"", "printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"", "printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"", "printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"", "printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"", "printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"", "printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"", "printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"", "printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"", "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi", "if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi", "if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi", "if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi", ].join("\n"); } function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const isOpenFga = options.preset === "openfga"; const platformEndpointSlice = spec.platformPostgresEndpointSlice; const expectedUriHost = spec.platformPostgresEndpointAddress ?? (isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost); const databaseUrlKey = isOpenFga ? spec.externalPostgres?.openfga.secretKey ?? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey; return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`, `database_url_key=${shellQuote(databaseUrlKey)}`, `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, `legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`, `platform_service=${shellQuote(spec.platformPostgresService)}`, `platform_endpointslice=${shellQuote(platformEndpointSlice)}`, `platform_host=${shellQuote(spec.platformPostgresService)}`, `platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`, `platform_endpoint_address=${shellQuote(spec.platformPostgresEndpointAddress ?? "")}`, `db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`, `db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`, `db_host=${shellQuote(expectedUriHost)}`, `selected_key=${shellQuote(options.key ?? "")}`, `preset=${shellQuote(options.preset)}`, "dry_run=true", "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", "resource_exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "endpointslice_exists_flag() { kubectl -n \"$namespace\" get endpointslice \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "uri_has_platform_host=no", "uri_has_db_name=no", "uri_has_db_user=no", "uri_matches_expected() {", " uri=$1", " uri_has_platform_host=no", " uri_has_db_name=no", " uri_has_db_user=no", " case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*|*\"@$db_host:\"*|*\"@$db_host/\"*) uri_has_platform_host=yes ;; esac", " if [ -n \"$platform_endpoint_address\" ]; then", " case \"$uri\" in *\"@$platform_endpoint_address:\"*|*\"@$platform_endpoint_address/\"*) uri_has_platform_host=yes ;; esac", " fi", " case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac", " case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac", "}", "exists=$(secret_exists_flag \"$name\")", "legacy_postgres_exists=$(secret_exists_flag \"$legacy_postgres_secret\")", "uri_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", "uri_present=$([ -n \"$uri_b64\" ] && printf yes || printf no)", "uri_bytes=$(decoded_length \"$uri_b64\")", "uri_value=$(decoded_value \"$uri_b64\")", "authn_b64=$(secret_b64_key \"$name\" \"$authn_key\")", "authn_present=$([ -n \"$authn_b64\" ] && printf yes || printf no)", "authn_bytes=$(decoded_length \"$authn_b64\")", "pg_password_b64=$(secret_b64_key \"$name\" \"$postgres_password_key\")", "pg_password_present=$([ -n \"$pg_password_b64\" ] && printf yes || printf no)", "pg_password_bytes=$(decoded_length \"$pg_password_b64\")", "platform_service_exists=$(resource_exists_flag service \"$platform_service\")", "platform_endpoints_exists=$(resource_exists_flag endpoints \"$platform_service\")", "platform_endpointslice_exists=$(endpointslice_exists_flag \"$platform_endpointslice\")", "uri_matches_expected \"$uri_value\"", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$name\"", "printf 'key\\t%s\\n' \"$database_url_key\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\tobserved\\n'", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\tfalse\\n'", "printf 'platformDbMode\\ttrue\\n'", "printf 'afterExists\\t%s\\n' \"$exists\"", "printf 'afterDatabaseUrlPresent\\t%s\\n' \"$uri_present\"", "printf 'afterDatabaseUrlBytes\\t%s\\n' \"$uri_bytes\"", "printf 'afterDatastoreUriPresent\\t%s\\n' \"$uri_present\"", "printf 'afterDatastoreUriBytes\\t%s\\n' \"$uri_bytes\"", "printf 'afterAuthnPresent\\t%s\\n' \"$authn_present\"", "printf 'afterAuthnBytes\\t%s\\n' \"$authn_bytes\"", "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$pg_password_present\"", "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$pg_password_bytes\"", "printf 'legacyPostgresSecret\\t%s\\n' \"$legacy_postgres_secret\"", "printf 'legacyPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"", "printf 'afterPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"", "printf 'platformService\\t%s\\n' \"$platform_service\"", "printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"", "printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"", "printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"", "printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"", "printf 'platformEndpointAddress\\t%s\\n' \"$platform_endpoint_address\"", "printf 'dbName\\t%s\\n' \"$db_name\"", "printf 'dbUser\\t%s\\n' \"$db_user\"", "printf 'dbHost\\t%s\\n' \"$db_host\"", "printf 'dbHostMatchesPlatform\\t%s\\n' \"$uri_has_platform_host\"", "printf 'dbNameMatchesExpected\\t%s\\n' \"$uri_has_db_name\"", "printf 'dbUserMatchesExpected\\t%s\\n' \"$uri_has_db_user\"", "uri_value=", "if [ \"$platform_service_exists\" != yes ]; then exit 44; fi", ].join("\n"); } function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `openfga_secret=${shellQuote(spec.openFgaSecret)}`, `postgres_secret=${shellQuote(spec.postgresSecret)}`, `postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`, `postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`, `selected_key=${shellQuote(options.key ?? "")}`, `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, `datastore_uri_key=${shellQuote(OPENFGA_DATASTORE_URI_KEY)}`, `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, `db_name=${shellQuote(spec.openFgaDbName)}`, `db_user=${shellQuote(spec.openFgaDbUser)}`, `db_host=${shellQuote(spec.openFgaDbHost)}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, "preset=openfga", "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", "probe_db() {", " role_result=unknown", " database_result=unknown", " probe_exit=missing-postgres-admin-secret", " if [ -n \"$postgres_admin_password\" ]; then", " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", " role_exit=$?", " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", " database_exit=$?", " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", " fi", "}", "before_exists=$(secret_exists_flag \"$openfga_secret\")", "before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", "before_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", "before_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", "before_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", "before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)", "before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)", "before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)", "before_authn_bytes=$(decoded_length \"$before_authn_b64\")", "before_uri_bytes=$(decoded_length \"$before_uri_b64\")", "before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")", "authn_value=$(decoded_value \"$before_authn_b64\")", "datastore_uri=$(decoded_value \"$before_uri_b64\")", "pg_password=$(decoded_value \"$before_pg_password_b64\")", "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", "probe_db", "db_role_exists_before=$role_result", "db_database_exists_before=$database_result", "db_probe_exit_before=$probe_exit", "action=observed", "mutation=false", "postgres_secret_exit=", "postgres_rollout_exit=", "apply_exit=", "db_ensure_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_secret=false", " [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true", " [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true", " [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true", " missing_db=false", " [ \"$db_role_exists_before\" = t ] || missing_db=true", " [ \"$db_database_exists_before\" = t ] || missing_db=true", " if [ \"$dry_run\" = true ]; then", " if [ \"$before_postgres_exists\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", " elif ! command -v openssl >/dev/null 2>&1; then", " action=openssl-missing; apply_exit=127", " else", " [ -n \"$postgres_admin_password\" ] || postgres_admin_password=$(openssl rand -hex 24)", " kubectl -n \"$namespace\" create secret generic \"$postgres_secret\" --from-literal=\"POSTGRES_PASSWORD=$postgres_admin_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " postgres_secret_exit=$?", " if [ \"$postgres_secret_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status \"statefulset/$postgres_statefulset\" --timeout=120s >/tmp/hwlab-postgres-rollout.out 2>/tmp/hwlab-postgres-rollout.err", " postgres_rollout_exit=$?", " fi", " if [ \"$postgres_secret_exit\" -ne 0 ]; then action=postgres-secret-failed; apply_exit=$postgres_secret_exit", " elif [ \"$postgres_rollout_exit\" -ne 0 ]; then action=postgres-rollout-failed; apply_exit=$postgres_rollout_exit", " else", " [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)", " [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)", " [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"", " kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " apply_exit=$?", " if [ \"$apply_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'", "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", "\\gexec", "ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';", "SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')", "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')", "\\gexec", "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", "SQL", " db_ensure_exit=$?", " if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi", " else action=apply-failed; fi", " fi", " fi", "fi", "after_exists=$(secret_exists_flag \"$openfga_secret\")", "after_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", "after_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", "after_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", "after_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", "after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)", "after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)", "after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)", "after_authn_bytes=$(decoded_length \"$after_authn_b64\")", "after_uri_bytes=$(decoded_length \"$after_uri_b64\")", "after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")", "probe_db", "db_role_exists_after=$role_result", "db_database_exists_after=$database_result", "db_probe_exit_after=$probe_exit", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$openfga_secret\"", "printf 'key\\t%s\\n' \"$selected_key\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeExists\\t%s\\n' \"$before_exists\"", "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterPostgresSecretExists\\t%s\\n' \"$after_postgres_exists\"", "printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"", "printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"", "printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"", "printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"", "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"", "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"", "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", "printf 'postgresSecretExitCode\\t%s\\n' \"$postgres_secret_exit\"", "printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", "authn_value= datastore_uri= pg_password= postgres_admin_password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", ].join("\n"); } function masterAdminApiKeySecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(spec.masterAdminApiKeySecret)}`, `api_key_name=${shellQuote(MASTER_ADMIN_API_KEY_KEY)}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, "preset=master-server-admin-api-key", "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-12; value=; fi; }", "before_exists=$(secret_exists_flag)", "before_api_key_b64=$(secret_b64_key \"$api_key_name\")", "before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)", "before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")", "action=observed", "mutation=false", "apply_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_secret=false", " [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true", " if [ \"$dry_run\" = true ]; then", " if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi", " else", " api_key=$(cat)", " case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac", " if [ -z \"$apply_exit\" ]; then", " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " apply_exit=$?", " if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi", " fi", " api_key=", " fi", "fi", "after_exists=$(secret_exists_flag)", "after_api_key_b64=$(secret_b64_key \"$api_key_name\")", "after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)", "after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")", "after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$name\"", "printf 'key\\t%s\\n' \"$api_key_name\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"", "printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"", "printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", ].join("\n"); } function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(spec.bootstrapAdminSecret)}`, `source_namespace=${shellQuote(spec.bootstrapAdminSourceNamespace)}`, `source_name=${shellQuote(spec.bootstrapAdminSourceSecret)}`, `password_hash_key=${shellQuote(spec.bootstrapAdminPasswordHashKey)}`, `cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, "preset=bootstrap-admin", "secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "before_exists=$(secret_exists_flag \"$namespace\" \"$name\")", "before_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")", "source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")", "source_hash_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$password_hash_key\")", "before_hash_present=$([ -n \"$before_hash_b64\" ] && printf yes || printf no)", "source_hash_present=$([ -n \"$source_hash_b64\" ] && printf yes || printf no)", "before_hash_bytes=$(decoded_length \"$before_hash_b64\")", "source_hash_bytes=$(decoded_length \"$source_hash_b64\")", "action=observed", "mutation=false", "apply_exit=", "rollout_restart_exit=", "rollout_status_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_target=false", " [ \"$before_exists\" = yes ] && [ \"$before_hash_bytes\" -gt 0 ] || missing_target=true", " missing_source=false", " [ \"$source_exists\" = yes ] && [ \"$source_hash_bytes\" -gt 0 ] || missing_source=true", " if [ \"$missing_source\" = true ]; then", " action=source-missing-password-hash", " apply_exit=44", " elif [ \"$dry_run\" = true ]; then", " if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi", " elif [ \"$missing_target\" = false ]; then", " action=kept", " else", " cat </tmp/hwlab-bootstrap-admin-rollout-restart.out 2>/tmp/hwlab-bootstrap-admin-rollout-restart.err", " rollout_restart_exit=$?", " if [ \"$rollout_restart_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-bootstrap-admin-rollout-status.out 2>/tmp/hwlab-bootstrap-admin-rollout-status.err", " rollout_status_exit=$?", " fi", " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", " else action=copied-from-source; mutation=true; fi", " else action=apply-failed; fi", " fi", "fi", "after_exists=$(secret_exists_flag \"$namespace\" \"$name\")", "after_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")", "after_hash_present=$([ -n \"$after_hash_b64\" ] && printf yes || printf no)", "after_hash_bytes=$(decoded_length \"$after_hash_b64\")", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$name\"", "printf 'key\\t%s\\n' \"$password_hash_key\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'sourceNamespace\\t%s\\n' \"$source_namespace\"", "printf 'sourceSecret\\t%s\\n' \"$source_name\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeExists\\t%s\\n' \"$before_exists\"", "printf 'beforePasswordHashPresent\\t%s\\n' \"$before_hash_present\"", "printf 'beforePasswordHashBytes\\t%s\\n' \"$before_hash_bytes\"", "printf 'sourceExists\\t%s\\n' \"$source_exists\"", "printf 'sourcePasswordHashPresent\\t%s\\n' \"$source_hash_present\"", "printf 'sourcePasswordHashBytes\\t%s\\n' \"$source_hash_bytes\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterPasswordHashPresent\\t%s\\n' \"$after_hash_present\"", "printf 'afterPasswordHashBytes\\t%s\\n' \"$after_hash_bytes\"", "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", ].join("\n"); } function codeAgentProviderSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(spec.codeAgentProviderSecret)}`, `source_namespace=${shellQuote(spec.codeAgentProviderSourceNamespace)}`, `source_name=${shellQuote(spec.codeAgentProviderSourceSecret)}`, `openai_key=${shellQuote(CODE_AGENT_PROVIDER_OPENAI_KEY)}`, `opencode_key=${shellQuote(CODE_AGENT_PROVIDER_OPENCODE_KEY)}`, `selected_key=${shellQuote(options.key ?? "")}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, `field_manager=${shellQuote(spec.fieldManager)}`, "preset=code-agent-provider", "secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "before_exists=$(secret_exists_flag \"$namespace\" \"$name\")", "before_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")", "before_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")", "source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")", "source_openai_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$openai_key\")", "source_opencode_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$opencode_key\")", "before_openai_present=$([ -n \"$before_openai_b64\" ] && printf yes || printf no)", "before_opencode_present=$([ -n \"$before_opencode_b64\" ] && printf yes || printf no)", "source_openai_present=$([ -n \"$source_openai_b64\" ] && printf yes || printf no)", "source_opencode_present=$([ -n \"$source_opencode_b64\" ] && printf yes || printf no)", "before_openai_bytes=$(decoded_length \"$before_openai_b64\")", "before_opencode_bytes=$(decoded_length \"$before_opencode_b64\")", "source_openai_bytes=$(decoded_length \"$source_openai_b64\")", "source_opencode_bytes=$(decoded_length \"$source_opencode_b64\")", "action=observed", "mutation=false", "apply_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_target=false", " [ \"$before_exists\" = yes ] && { [ \"$before_openai_bytes\" -gt 0 ] || [ \"$before_opencode_bytes\" -gt 0 ]; } || missing_target=true", " missing_source=false", " [ \"$source_exists\" = yes ] && { [ \"$source_openai_bytes\" -gt 0 ] || [ \"$source_opencode_bytes\" -gt 0 ]; } || missing_source=true", " if [ \"$missing_source\" = true ]; then", " action=source-missing-provider-key", " apply_exit=44", " elif [ \"$dry_run\" = true ]; then", " if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi", " elif [ \"$missing_target\" = false ]; then", " action=kept", " else", " cat </dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", "probe_db() {", " role_result=unknown", " database_result=unknown", " probe_exit=missing-postgres-admin-secret", " if [ -n \"$postgres_admin_password\" ]; then", " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", " role_exit=$?", " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", " database_exit=$?", " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", " fi", "}", "before_exists=$(secret_exists_flag \"$name\")", "before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", "before_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", "before_url_present=$([ -n \"$before_url_b64\" ] && printf yes || printf no)", "before_url_bytes=$(decoded_length \"$before_url_b64\")", "database_url=$(decoded_value \"$before_url_b64\")", "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", "postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)", "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", "probe_db", "db_role_exists_before=$role_result", "db_database_exists_before=$database_result", "db_probe_exit_before=$probe_exit", "action=observed", "mutation=false", "apply_exit=", "db_ensure_exit=", "rollout_restart_exit=", "rollout_status_exit=", "if [ \"$action_request\" = ensure ]; then", " missing_secret=false", " [ \"$before_url_present\" = yes ] && [ \"$before_url_bytes\" -gt 0 ] || missing_secret=true", " missing_db=false", " [ \"$db_role_exists_before\" = t ] || missing_db=true", " [ \"$db_database_exists_before\" = t ] || missing_db=true", " if [ \"$dry_run\" = true ]; then", " if [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", " elif [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ -z \"$postgres_admin_password\" ]; then", " action=postgres-admin-secret-missing", " apply_exit=44", " elif [ \"$missing_secret\" = false ] && [ \"$missing_db\" = false ]; then", " action=kept", " else", " [ -n \"$database_url\" ] || database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"", " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$database_url_key=$database_url\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " apply_exit=$?", " if [ \"$apply_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$postgres_admin_password\" >/tmp/hwlab-cloud-api-db-psql.out 2>/tmp/hwlab-cloud-api-db-psql.err <<'SQL'", "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", "\\gexec", "ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';", "SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')", "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')", "\\gexec", "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", "SQL", " db_ensure_exit=$?", " if [ \"$db_ensure_exit\" -eq 0 ]; then", " if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then", " kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-cloud-api-rollout-restart.out 2>/tmp/hwlab-cloud-api-rollout-restart.err", " rollout_restart_exit=$?", " if [ \"$rollout_restart_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-cloud-api-rollout-status.out 2>/tmp/hwlab-cloud-api-rollout-status.err", " rollout_status_exit=$?", " fi", " fi", " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", " else action=ensured; mutation=true; fi", " else action=db-ensure-failed; fi", " else action=apply-failed; fi", " fi", "fi", "after_exists=$(secret_exists_flag \"$name\")", "after_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", "after_url_present=$([ -n \"$after_url_b64\" ] && printf yes || printf no)", "after_url_bytes=$(decoded_length \"$after_url_b64\")", "probe_db", "db_role_exists_after=$role_result", "db_database_exists_after=$database_result", "db_probe_exit_after=$probe_exit", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$name\"", "printf 'key\\t%s\\n' \"$database_url_key\"", "printf 'preset\\t%s\\n' \"$preset\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeExists\\t%s\\n' \"$before_exists\"", "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", "printf 'beforeDatabaseUrlPresent\\t%s\\n' \"$before_url_present\"", "printf 'beforeDatabaseUrlBytes\\t%s\\n' \"$before_url_bytes\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterDatabaseUrlPresent\\t%s\\n' \"$after_url_present\"", "printf 'afterDatabaseUrlBytes\\t%s\\n' \"$after_url_bytes\"", "printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"", "printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"", "printf 'dbName\\t%s\\n' \"$db_name\"", "printf 'dbUser\\t%s\\n' \"$db_user\"", "printf 'dbHost\\t%s\\n' \"$db_host\"", "printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"", "printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"", "printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"", "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", "database_url= postgres_admin_password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", ].join("\n"); } function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { const fields = keyValueLinesFromText(text); if (fields.preset === "owned-postgres-cleanup") { const absent = fields.afterStatefulSetExists !== "yes" && fields.afterServiceExists !== "yes" && fields.afterConfigMapExists !== "yes" && fields.afterSecretExists !== "yes" && fields.afterPvcExists !== "yes"; const platformServiceReady = fields.platformServiceExists === "yes"; return { ok: commandOk && absent && platformServiceReady, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.postgresSecret, statefulSet: fields.statefulSet || spec.postgresStatefulSet, service: fields.service || spec.postgresSecret, configMap: fields.configMap || `${spec.postgresSecret}-init`, pvc: fields.pvc || `data-${spec.postgresSecret}-0`, preset: "owned-postgres-cleanup", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", before: { statefulSetExists: fields.beforeStatefulSetExists === "yes", serviceExists: fields.beforeServiceExists === "yes", configMapExists: fields.beforeConfigMapExists === "yes", secretExists: fields.beforeSecretExists === "yes", pvcExists: fields.beforePvcExists === "yes", pvcPhase: fields.beforePvcPhase || null, persistentVolume: fields.beforePersistentVolume || null, }, after: { statefulSetExists: fields.afterStatefulSetExists === "yes", serviceExists: fields.afterServiceExists === "yes", configMapExists: fields.afterConfigMapExists === "yes", secretExists: fields.afterSecretExists === "yes", pvcExists: fields.afterPvcExists === "yes", pvcPhase: fields.afterPvcPhase || null, persistentVolume: fields.afterPersistentVolume || null, }, platformService: { name: "g14-platform-postgres", exists: platformServiceReady, }, deleteStatefulSetExitCode: numericField(fields.deleteStatefulSetExitCode), deleteServiceExitCode: numericField(fields.deleteServiceExitCode), deleteConfigMapExitCode: numericField(fields.deleteConfigMapExitCode), deleteSecretExitCode: numericField(fields.deleteSecretExitCode), deletePvcExitCode: numericField(fields.deletePvcExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: absent ? `${fields.statefulSet || spec.postgresStatefulSet}, ${fields.service || spec.postgresSecret}, ${fields.configMap || `${spec.postgresSecret}-init`}, ${fields.secret || spec.postgresSecret}, and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent` : `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`, }; } if (fields.preset === "obsolete-secret-cleanup") { const absent = fields.afterSecretExists !== "yes"; const refsAbsent = fields.workloadRefsPresent !== "yes"; const dryRun = fields.dryRun === "true"; return { ok: commandOk && refsAbsent && (dryRun || absent), namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.obsoleteHwpodDbSecret, preset: "obsolete-secret-cleanup", action: fields.action || null, dryRun, mutation: fields.mutation === "true", before: { secretExists: fields.beforeSecretExists === "yes", }, after: { secretExists: fields.afterSecretExists === "yes", }, workloadRefs: { present: fields.workloadRefsPresent === "yes", preview: fields.workloadRefsPreview || "", }, deleteSecretExitCode: numericField(fields.deleteSecretExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: refsAbsent && (dryRun || absent) ? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}` : `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`, }; } if (fields.preset === "master-server-admin-api-key") { const afterBytes = numericField(fields.afterApiKeyBytes); const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.masterAdminApiKeySecret, preset: "master-server-admin-api-key", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } }, applyExitCode: numericField(fields.applyExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} exists` : `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} missing`, }; } if (fields.preset === "bootstrap-admin") { const beforeHashBytes = numericField(fields.beforePasswordHashBytes); const sourceHashBytes = numericField(fields.sourcePasswordHashBytes); const afterHashBytes = numericField(fields.afterPasswordHashBytes); const healthy = fields.afterExists === "yes" && fields.afterPasswordHashPresent === "yes" && typeof afterHashBytes === "number" && afterHashBytes > 0; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.bootstrapAdminSecret, key: fields.key || spec.bootstrapAdminPasswordHashKey, preset: "bootstrap-admin", source: { namespace: fields.sourceNamespace || spec.bootstrapAdminSourceNamespace, secret: fields.sourceSecret || spec.bootstrapAdminSourceSecret, exists: fields.sourceExists === "yes", passwordHash: { keyPresent: fields.sourcePasswordHashPresent === "yes", valueBytes: sourceHashBytes }, }, action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", before: { exists: fields.beforeExists === "yes", passwordHash: { keyPresent: fields.beforePasswordHashPresent === "yes", valueBytes: beforeHashBytes }, }, after: { exists: fields.afterExists === "yes", passwordHash: { keyPresent: fields.afterPasswordHashPresent === "yes", valueBytes: afterHashBytes }, }, cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, applyExitCode: numericField(fields.applyExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} exists` : `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} missing`, }; } if (fields.preset === "code-agent-provider") { const beforeOpenaiBytes = numericField(fields.beforeOpenaiBytes); const beforeOpencodeBytes = numericField(fields.beforeOpencodeBytes); const sourceOpenaiBytes = numericField(fields.sourceOpenaiBytes); const sourceOpencodeBytes = numericField(fields.sourceOpencodeBytes); const afterOpenaiBytes = numericField(fields.afterOpenaiBytes); const afterOpencodeBytes = numericField(fields.afterOpencodeBytes); const openaiReady = fields.afterOpenaiPresent === "yes" && typeof afterOpenaiBytes === "number" && afterOpenaiBytes > 0; const opencodeReady = fields.afterOpencodePresent === "yes" && typeof afterOpencodeBytes === "number" && afterOpencodeBytes > 0; const healthy = fields.afterExists === "yes" && (openaiReady || opencodeReady); return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.codeAgentProviderSecret, preset: "code-agent-provider", source: { namespace: fields.sourceNamespace || spec.codeAgentProviderSourceNamespace, secret: fields.sourceSecret || spec.codeAgentProviderSourceSecret, exists: fields.sourceExists === "yes", openaiApiKey: { keyPresent: fields.sourceOpenaiPresent === "yes", valueBytes: sourceOpenaiBytes }, opencodeApiKey: { keyPresent: fields.sourceOpencodePresent === "yes", valueBytes: sourceOpencodeBytes }, }, selectedKey: fields.selectedKey || null, action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", before: { exists: fields.beforeExists === "yes", openaiApiKey: { keyPresent: fields.beforeOpenaiPresent === "yes", valueBytes: beforeOpenaiBytes }, opencodeApiKey: { keyPresent: fields.beforeOpencodePresent === "yes", valueBytes: beforeOpencodeBytes }, }, after: { exists: fields.afterExists === "yes", openaiApiKey: { keyPresent: fields.afterOpenaiPresent === "yes", valueBytes: afterOpenaiBytes }, opencodeApiKey: { keyPresent: fields.afterOpencodePresent === "yes", valueBytes: afterOpencodeBytes }, requiredAnyProviderKeyPresent: openaiReady || opencodeReady, }, applyExitCode: numericField(fields.applyExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`, }; } if (fields.preset === "cloud-api-db") { const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes); const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes); if (fields.platformDbMode === "true") { const keysHealthy = fields.afterExists === "yes" && fields.afterDatabaseUrlPresent === "yes" && typeof afterUrlBytes === "number" && afterUrlBytes > 0; const platformBridgeHealthy = fields.platformServiceExists === "yes" && fields.platformEndpointsExists !== "yes" && fields.platformEndpointSliceExists === "yes"; const uriHealthy = fields.dbHostMatchesPlatform === "yes" && fields.dbNameMatchesExpected === "yes" && fields.dbUserMatchesExpected === "yes"; const healthy = keysHealthy && platformBridgeHealthy && uriHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.cloudApiDbSecret, key: fields.key || spec.cloudApiDbKey, preset: "cloud-api-db", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", platformDbMode: true, after: { exists: fields.afterExists === "yes", databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes }, }, legacyPostgresSecret: { name: fields.legacyPostgresSecret || spec.postgresSecret, exists: fields.legacyPostgresSecretExists === "yes", }, platformService: { name: fields.platformService || spec.platformPostgresService, exists: fields.platformServiceExists === "yes", endpointsExist: fields.platformEndpointsExists === "yes", legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes", endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`, endpointSliceExists: fields.platformEndpointSliceExists === "yes", }, dbName: fields.dbName || spec.cloudApiDbName, dbUser: fields.dbUser || spec.cloudApiDbUser, dbHost: fields.dbHost || spec.cloudApiDbHost, dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes", dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes", dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes", exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`, }; } const keysHealthy = fields.afterExists === "yes" && fields.afterDatabaseUrlPresent === "yes" && typeof afterUrlBytes === "number" && afterUrlBytes > 0; const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; const healthy = keysHealthy && databaseHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.cloudApiDbSecret, key: fields.key || spec.cloudApiDbKey, preset: "cloud-api-db", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", before: { exists: fields.beforeExists === "yes", postgresSecretExists: fields.beforePostgresSecretExists === "yes", databaseUrl: { keyPresent: fields.beforeDatabaseUrlPresent === "yes", valueBytes: beforeUrlBytes }, database: { roleExists: fields.dbRoleExistsBefore || "unknown", databaseExists: fields.dbDatabaseExistsBefore || "unknown", probeExitCode: fields.dbProbeExitCodeBefore || null, }, }, after: { exists: fields.afterExists === "yes", databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes }, database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null, }, }, postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes", postgresSecret: fields.postgresSecret || spec.postgresSecret, dbName: fields.dbName || spec.cloudApiDbName, dbUser: fields.dbUser || spec.cloudApiDbUser, dbHost: fields.dbHost || spec.cloudApiDbHost, cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, applyExitCode: numericField(fields.applyExitCode), dbEnsureExitCode: numericField(fields.dbEnsureExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`, }; } const afterAuthnBytes = numericField(fields.afterAuthnBytes); const afterUriBytes = numericField(fields.afterDatastoreUriBytes); const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes); if (fields.platformDbMode === "true") { const keysHealthy = fields.afterExists === "yes" && fields.afterAuthnPresent === "yes" && fields.afterDatastoreUriPresent === "yes" && typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && typeof afterUriBytes === "number" && afterUriBytes > 0; const platformBridgeHealthy = fields.platformServiceExists === "yes" && fields.platformEndpointsExists !== "yes" && fields.platformEndpointSliceExists === "yes"; const uriHealthy = fields.dbHostMatchesPlatform === "yes" && fields.dbNameMatchesExpected === "yes" && fields.dbUserMatchesExpected === "yes"; const healthy = keysHealthy && platformBridgeHealthy && uriHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.openFgaSecret, preset: fields.preset || "openfga", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", platformDbMode: true, after: { exists: fields.afterExists === "yes", authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, }, legacyPostgresSecret: { name: fields.legacyPostgresSecret || spec.postgresSecret, exists: fields.legacyPostgresSecretExists === "yes", }, platformService: { name: fields.platformService || spec.platformPostgresService, exists: fields.platformServiceExists === "yes", endpointsExist: fields.platformEndpointsExists === "yes", legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes", endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`, endpointSliceExists: fields.platformEndpointSliceExists === "yes", }, dbName: fields.dbName || spec.openFgaDbName, dbUser: fields.dbUser || spec.openFgaDbUser, dbHost: fields.dbHost || spec.openFgaDbHost, dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes", dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes", dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes", exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.platformService || spec.platformPostgresService}` : `${fields.secret || spec.openFgaSecret} datastore-uri is not aligned to platform DB`, }; } const keysHealthy = fields.afterExists === "yes" && fields.afterPostgresSecretExists === "yes" && fields.afterAuthnPresent === "yes" && fields.afterDatastoreUriPresent === "yes" && fields.afterPostgresPasswordPresent === "yes" && typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && typeof afterUriBytes === "number" && afterUriBytes > 0 && typeof afterPasswordBytes === "number" && afterPasswordBytes > 0; const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; const healthy = keysHealthy && databaseHealthy; return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, secret: fields.secret || spec.openFgaSecret, preset: fields.preset || "openfga", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", after: { exists: fields.afterExists === "yes", postgresSecretExists: fields.afterPostgresSecretExists === "yes", authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null }, }, postgresSecretExitCode: numericField(fields.postgresSecretExitCode), postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode), applyExitCode: numericField(fields.applyExitCode), dbEnsureExitCode: numericField(fields.dbEnsureExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy ? `${fields.secret || spec.openFgaSecret} keys and Postgres database exist` : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`, }; } function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { const fields = keyValueLinesFromText(text); const dryRun = fields.dryRun === "true"; const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown"; const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown"; const probesOk = fields.beforeDatabaseExists !== "unknown" && fields.beforeRoleExists !== "unknown" && fields.afterDatabaseExists !== "unknown" && fields.afterRoleExists !== "unknown"; return { ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)), database: fields.database || spec.obsoleteHwpodDbName, role: fields.role || spec.obsoleteHwpodDbUser, preset: "obsolete-platform-db-cleanup", action: fields.action || null, dryRun, mutation: fields.mutation === "true", before: { databaseExists: fields.beforeDatabaseExists === "yes", roleExists: fields.beforeRoleExists === "yes", }, after: { databaseExists: fields.afterDatabaseExists === "yes", roleExists: fields.afterRoleExists === "yes", }, beforeProbeExitCode: { database: numericField(fields.beforeDatabaseProbeExitCode), role: numericField(fields.beforeRoleProbeExitCode), }, afterProbeExitCode: { database: numericField(fields.afterDatabaseProbeExitCode), role: numericField(fields.afterRoleProbeExitCode), }, dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode), dropRoleExitCode: numericField(fields.dropRoleExitCode), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: probesOk && (dryRun || (databaseAbsent && roleAbsent)) ? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}` : `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`, }; } export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record { return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane })); } function readMasterAdminApiKey(): { key: string; source: string } { if (!existsSync(MASTER_ADMIN_API_KEY_ENV)) throw new Error(`HWLAB_API_KEY source missing: ${MASTER_ADMIN_API_KEY_ENV}`); const content = readFileSync(MASTER_ADMIN_API_KEY_ENV, "utf8"); const match = content.match(/^HWLAB_API_KEY=(.+)$/m); const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, ""); if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${MASTER_ADMIN_API_KEY_ENV}`); return { key: raw, source: MASTER_ADMIN_API_KEY_ENV }; } function optionValue(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index === -1) return undefined; const value = args[index + 1]; if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); return value; } function requiredOption(args: string[], name: string): string { const value = optionValue(args, name); if (value === undefined) throw new Error(`${name} is required`); return value; } function stripOption(args: string[], name: string): string[] { return stripOptions(args, [name]); } function stripOptions(args: string[], names: readonly string[]): string[] { const remove = new Set(names); const without: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; if (remove.has(arg)) { if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1; continue; } without.push(arg); } return without; } function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const raw = optionValue(args, name); if (raw === undefined) return defaultValue; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return Math.min(value, maxValue); } function assertLane(value: string): void { if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`); } function assertNodeId(value: string): void { if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`); } function shellQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; } function statusText(result: CommandResult): string { return result.stdout || result.stderr; } function keyValueLinesFromText(text: string): Record { const fields: Record = {}; for (const line of text.split(/\r?\n/u)) { const index = line.indexOf("\t"); if (index <= 0) continue; fields[line.slice(0, index)] = line.slice(index + 1); } return fields; } function numericField(value: string | undefined): number | null { if (value === undefined || value === "") return null; const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function commaListField(value: string | undefined): string[] { if (!value) return []; return value.split(",").map((item) => item.trim()).filter(Boolean); } function splitWhitespaceField(value: string | undefined): string[] { if (!value) return []; return value.split(/\s+/u).filter(Boolean); } function compactCommandResult(result: CommandResult): Record { return { command: compactCommand(result.command), exitCode: result.exitCode, stdoutBytes: result.stdout.length, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), timedOut: result.timedOut, }; } function compactCommand(command: string[]): string[] { const scriptIndex = command.indexOf("--"); if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "