mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7ba28c8f40ef84acd88ee74c00946a1b41e07749
pikasTech-unidesk/scripts/src/hwlab-node.ts
T
Lyon 5727fe275c fix: support D601 v03 PK01 postgres secrets (#324) 
Co-authored-by: Codex <codex@noreply.local>
2026-06-13 14:25:33 +08:00

5437 lines
293 KiB
TypeScript
Raw Blame History

import { createHash } from "node:crypto";
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { repoRoot, rootPath, type Config } from "./config";
import { runCommand, type CommandResult } from "./command";
import { startJob } from "./jobs";
import { runHwlabG14Command } from "./hwlab-g14";
import { hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete";
type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
type DelegatedNodeDomain = "control-plane" | "git-mirror";
type NodeRuntimeRenderLocation = "node-host" | "local";
interface NodeRuntimeRenderResult {
readonly result: CommandResult;
readonly renderDir: string;
readonly worktreeDir: string;
readonly location: NodeRuntimeRenderLocation;
}
interface NodeSecretOptions {
action: SecretAction;
node: string;
lane: string;
name: string;
key?: string;
preset: SecretPreset;
dryRun: boolean;
confirm: boolean;
timeoutSeconds: number;
}
interface NodePublicExposureOptions {
action: "public-exposure";
node: string;
lane: HwlabRuntimeLane;
dryRun: boolean;
confirm: boolean;
timeoutSeconds: number;
spec: HwlabRuntimeLaneSpec;
}
interface RuntimeSecretSpec {
node: string;
lane: string;
namespace: string;
platformDb: boolean;
runtimeLaneSpec?: HwlabRuntimeLaneSpec;
externalPostgres?: NonNullable<HwlabRuntimeLaneSpec["externalPostgres"]>;
platformPostgresService: string;
platformPostgresEndpointAddress?: string;
platformPostgresEndpointSlice: string;
postgresSecret: string;
postgresStatefulSet: string;
postgresAdminUser: string;
openFgaSecret: string;
openFgaDbName: string;
openFgaDbUser: string;
openFgaDbHost: string;
masterAdminApiKeySecret: string;
bootstrapAdminSecret: string;
bootstrapAdminPasswordHashKey: string;
bootstrapAdminSourceNamespace: string;
bootstrapAdminSourceSecret: string;
cloudApiDbSecret: string;
cloudApiDbKey: string;
cloudApiDbName: string;
cloudApiDbUser: string;
cloudApiDbHost: string;
cloudApiDeployment: string;
obsoleteHwpodDbSecret: string;
obsoleteHwpodDbName: string;
obsoleteHwpodDbUser: string;
codeAgentProviderSecret: string;
codeAgentProviderSourceNamespace: string;
codeAgentProviderSourceSecret: string;
fieldManager: string;
}
const MASTER_ADMIN_API_KEY_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env";
const MASTER_ADMIN_API_KEY_KEY = "api-key";
const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash";
const BOOTSTRAP_ADMIN_SOURCE_NAMESPACE = "hwlab-v02";
const BOOTSTRAP_ADMIN_SOURCE_SECRET = "hwlab-v02-bootstrap-admin";
const OPENFGA_AUTHN_KEY = "authn-preshared-key";
const OPENFGA_DATASTORE_URI_KEY = "datastore-uri";
const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password";
const CLOUD_API_DB_KEY = "database-url";
const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key";
const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key";
const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown>> {
if (args.length === 0) return hwlabNodeHelp();
const [domain] = args;
if (domain === "control-plane" && args[1] === "infra") {
if (args.length === 2 || args.includes("--help") || args.includes("-h") || args[2] === "help") return hwlabNodeControlPlaneInfraHelp();
return runHwlabNodeControlPlaneInfra(args.slice(2));
}
if (args.includes("--help") || args.includes("-h")) return hwlabNodeHelp();
if (domain === "control-plane" || domain === "git-mirror") {
return runNodeDelegatedDomain(_config, domain, args.slice(1));
}
if (domain !== "secret") {
return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret" };
}
const options = parseSecretOptions(args.slice(1));
return runNodeSecret(options);
}
export function hwlabNodeHelp(): Record<string, unknown> {
return {
ok: true,
command: "hwlab nodes",
description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.",
configPath: hwlabRuntimeLaneConfigPath(),
examples: [
"bun scripts/cli.ts hwlab nodes control-plane infra plan --node D601 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane infra status --node D601 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane plan --node D601 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
"bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-bootstrap-admin",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-bootstrap-admin --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
],
};
}
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown>> {
const scoped = parseNodeScopedDelegatedOptions(domain, args);
const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane);
if (domain === "control-plane" && scoped.action === "public-exposure") {
return runNodePublicExposure({
action: "public-exposure",
node: scoped.node,
lane: scoped.lane,
dryRun: scoped.dryRun || !scoped.confirm,
confirm: scoped.confirm,
timeoutSeconds: scoped.timeoutSeconds,
spec: scoped.spec,
});
}
if (domain === "control-plane" && scoped.action === "plan") {
return nodeRuntimeControlPlanePlan(scoped);
}
if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) {
if (scoped.action === "status") return nodeRuntimeControlPlaneStatus(scoped);
if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration") {
if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped);
return nodeRuntimeControlPlaneRun(scoped);
}
return nodeRuntimeUnsupportedAction(scoped);
}
if (domain === "git-mirror" && scoped.node !== defaultSpec.nodeId) {
return {
ok: false,
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mutation: false,
degradedReason: "node-scoped-git-mirror-action-not-enabled",
message: "D601 runtime GitOps is declared in UniDesk YAML and triggered by node-scoped control-plane commands; this command intentionally does not delegate D601 git-mirror actions to G14.",
expected: nodeRuntimeExpected(scoped.spec),
};
}
if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") {
return runNodeEndpointBridge(scoped);
}
if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) {
return startNodeDelegatedJob(scoped);
}
if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) {
return startNodeDelegatedJob(scoped);
}
const delegatedArgs = stripOption(args, "--node");
const result = await runHwlabG14Command(config, [domain, ...delegatedArgs]);
return rewriteDelegatedNodeResult(result, scoped);
}
function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): {
domain: DelegatedNodeDomain;
action: string;
node: string;
lane: HwlabRuntimeLane;
confirm: boolean;
dryRun: boolean;
wait: boolean;
rerun: boolean;
allowLiveDbRead: boolean;
timeoutSeconds: number;
originalArgs: string[];
spec: HwlabRuntimeLaneSpec;
} {
const [actionRaw] = args;
if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`);
const node = requiredOption(args, "--node");
assertNodeId(node);
const laneRaw = requiredOption(args, "--lane");
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`);
const spec = hwlabRuntimeLaneSpecForNode(laneRaw, node);
const confirm = args.includes("--confirm");
const dryRun = args.includes("--dry-run");
if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`);
return {
domain,
action: actionRaw,
node,
lane: laneRaw,
confirm,
dryRun,
wait: args.includes("--wait"),
rerun: args.includes("--rerun"),
allowLiveDbRead: args.includes("--allow-live-db-read"),
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600),
originalArgs: [...args],
spec,
};
}
function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
return {
configPath: hwlabRuntimeLaneConfigPath(),
node: spec.nodeId,
nodeRoute: spec.nodeRoute,
nodeKubeRoute: spec.nodeKubeRoute,
lane: spec.lane,
sourceBranch: spec.sourceBranch,
workspace: spec.workspace,
cicdRepo: spec.cicdRepo,
git: {
sourceUrl: spec.gitUrl,
readUrl: spec.gitReadUrl,
writeUrl: spec.gitWriteUrl,
},
argo: {
repoURL: spec.argoRepoUrl,
},
gitopsBranch: spec.gitopsBranch,
catalogPath: spec.catalogPath,
runtimePath: spec.runtimePath,
runtimeNamespace: spec.runtimeNamespace,
renderDir: spec.runtimeRenderDir,
pipeline: spec.pipeline,
pipelineRunPrefix: spec.pipelineRunPrefix,
serviceAccount: spec.serviceAccountName,
argoApplication: spec.app,
registryPrefix: spec.registryPrefix,
baseImage: {
image: spec.baseImage,
sourceImage: spec.baseImageSource ?? null,
},
buildkit: spec.buildkit === undefined ? null : {
sidecarImage: spec.buildkit.sidecarImage,
},
dockerBuildProxy: {
http: spec.networkProfile.dockerBuildProxy.http,
https: spec.networkProfile.dockerBuildProxy.https,
all: spec.networkProfile.dockerBuildProxy.all,
noProxy: spec.networkProfile.dockerBuildProxy.noProxy,
},
stepEnv: spec.stepEnv,
public: {
webUrl: spec.publicWebUrl,
apiUrl: spec.publicApiUrl,
},
publicExposure: spec.publicExposure === null ? null : publicExposureSummary(spec.publicExposure),
downloadProfile: {
id: spec.downloadProfileId,
git: spec.downloadProfile.git,
npm: spec.downloadProfile.npm,
},
observability: spec.observability,
runtimeImageRewrites: spec.runtimeImageRewrites,
externalPostgres: spec.externalPostgres === undefined ? null : {
provider: spec.externalPostgres.provider,
configRef: spec.externalPostgres.configRef,
serviceName: spec.externalPostgres.serviceName,
endpointAddress: spec.externalPostgres.endpointAddress,
port: spec.externalPostgres.port,
sslmode: spec.externalPostgres.sslmode,
database: spec.externalPostgres.database,
cloudApi: {
secretName: spec.externalPostgres.cloudApi.secretName,
secretKey: spec.externalPostgres.cloudApi.secretKey,
sourceRef: spec.externalPostgres.cloudApi.sourceRef,
envKey: spec.externalPostgres.cloudApi.envKey,
role: spec.externalPostgres.cloudApi.role,
},
openfga: {
secretName: spec.externalPostgres.openfga.secretName,
secretKey: spec.externalPostgres.openfga.secretKey,
sourceRef: spec.externalPostgres.openfga.sourceRef,
envKey: spec.externalPostgres.openfga.envKey,
authnKey: spec.externalPostgres.openfga.authnKey ?? null,
role: spec.externalPostgres.openfga.role,
schema: spec.externalPostgres.openfga.schema ?? null,
},
valuesPrinted: false,
},
localPostgres: {
shouldRender: spec.externalPostgres === undefined,
expectedAbsent: spec.externalPostgres !== undefined,
},
};
}
function nodeRuntimeControlPlanePlan(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
return {
ok: true,
command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
mode: "plan",
mutation: false,
node: scoped.node,
lane: scoped.lane,
expected: nodeRuntimeExpected(scoped.spec),
checks: {
nodeScopedTargetConfigured: true,
externalPostgresDeclared: scoped.spec.externalPostgres !== undefined,
secretValuesPrinted: false,
runtimeNamespace: scoped.spec.runtimeNamespace,
localPostgresExpectedAbsent: scoped.spec.externalPostgres !== undefined,
publicExposureDeclared: scoped.spec.publicExposure !== null,
},
next: {
infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
platformDbStatus: scoped.spec.externalPostgres === undefined
? null
: `bun scripts/cli.ts platform-db postgres status --config ${scoped.spec.externalPostgres.configRef}`,
publicExposure: scoped.spec.publicExposure === null ? null : `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${scoped.node} --lane ${scoped.lane} --confirm`,
},
};
}
function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
return {
exitCode: result.exitCode,
stdoutBytes: Buffer.byteLength(result.stdout),
stderrBytes: Buffer.byteLength(result.stderr),
stdoutTail: result.stdout.trim().slice(-2000),
stderrTail: result.stderr.trim().slice(-2000),
timedOut: result.timedOut,
};
}
function nodeRuntimeUnsupportedAction(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
return {
ok: false,
command: `hwlab nodes control-plane ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mutation: false,
degradedReason: "unsupported-node-scoped-runtime-action",
message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/runtime-migration",
expected: nodeRuntimeExpected(scoped.spec),
};
}
function transPath(): string {
return join(repoRoot, "scripts", "trans");
}
function runNodeHostScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult {
return runCommand([transPath(), spec.nodeRoute, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}
function runNodeHostScriptAsync(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, label: string): CommandResult {
const token = nodeRuntimeRenderToken();
const stateDir = `/tmp/unidesk-hwlab-node-runtime/${label}-${token}`;
const statusPath = `${stateDir}/status.json`;
const stdoutPath = `${stateDir}/stdout.log`;
const stderrPath = `${stateDir}/stderr.log`;
const runnerPath = `${stateDir}/run.sh`;
const startScript = [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
`status_path=${shellQuote(statusPath)}`,
`stdout_path=${shellQuote(stdoutPath)}`,
`stderr_path=${shellQuote(stderrPath)}`,
`runner_path=${shellQuote(runnerPath)}`,
"rm -rf \"$state_dir\"",
"mkdir -p \"$state_dir\"",
"cat > \"$runner_path\" <<'UNIDESK_REMOTE_RUNNER'",
script,
"UNIDESK_REMOTE_RUNNER",
"chmod 700 \"$runner_path\"",
"python3 - \"$status_path\" <<'PY'",
"import datetime, json, sys",
"json.dump({'state':'running','ok':None,'pid':None,'startedAt':datetime.datetime.utcnow().isoformat()+'Z','finishedAt':None,'exitCode':None}, open(sys.argv[1], 'w'), indent=2)",
"PY",
"(",
" set +e",
" sh \"$runner_path\" >\"$stdout_path\" 2>\"$stderr_path\"",
" code=$?",
" python3 - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'PY'",
"import datetime, json, pathlib, sys",
"status_path, stdout_path, stderr_path, code = sys.argv[1:5]",
"def tail(path):",
" try:",
" text = pathlib.Path(path).read_text(errors='replace')",
" except FileNotFoundError:",
" return ''",
" return text[-12000:]",
"payload = {'state':'finished','ok':code == '0','pid':None,'startedAt':None,'finishedAt':datetime.datetime.utcnow().isoformat()+'Z','exitCode':int(code),'stdout':tail(stdout_path),'stderr':tail(stderr_path)}",
"try:",
" current = json.load(open(status_path))",
" payload['startedAt'] = current.get('startedAt')",
"except Exception:",
" pass",
"json.dump(payload, open(status_path, 'w'), indent=2)",
"PY",
") >/dev/null 2>&1 &",
"pid=$!",
"python3 - \"$status_path\" \"$pid\" <<'PY'",
"import json, sys",
"path, pid = sys.argv[1:3]",
"payload = json.load(open(path))",
"payload['pid'] = int(pid)",
"json.dump(payload, open(path, 'w'), indent=2)",
"PY",
"printf '{\"ok\":true,\"state\":\"started\",\"pid\":%s,\"statusPath\":\"%s\"}\\n' \"$pid\" \"$status_path\"",
].join("\n");
const start = runNodeHostScript(spec, startScript, 55);
if (!isCommandSuccess(start)) return start;
const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000;
let lastStatus: CommandResult | null = null;
let payload: Record<string, unknown> = {};
while (Date.now() <= deadline) {
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
const status = runNodeHostScript(spec, `cat ${shellQuote(statusPath)} 2>/dev/null || printf '{"state":"missing","ok":false,"exitCode":127,"stdout":"","stderr":"missing async status"}\\n'`, 55);
lastStatus = status;
payload = parseJsonObject(status.stdout.trim());
if (payload.state === "finished" || payload.ok === true || payload.ok === false) {
return commandResultFromAsync(spec, payload, statusPath, false);
}
}
const kill = runNodeHostScript(spec, [
"set +e",
`status_path=${shellQuote(statusPath)}`,
"pid=$(python3 - \"$status_path\" <<'PY' 2>/dev/null",
"import json, sys",
"try: print(json.load(open(sys.argv[1])).get('pid') or '')",
"except Exception: print('')",
"PY",
")",
"if [ -n \"$pid\" ]; then kill \"$pid\" 2>/dev/null || true; fi",
"cat \"$status_path\" 2>/dev/null || true",
].join("\n"), 55);
return {
command: [transPath(), spec.nodeRoute, "script", "--", "<remote async script>"],
cwd: repoRoot,
exitCode: 124,
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
stderr: [
`remote async command timed out after ${timeoutSeconds}s; statusPath=${statusPath}`,
typeof payload.stderr === "string" ? payload.stderr : "",
lastStatus?.stderr.trim() ?? "",
kill.stderr.trim(),
].filter(Boolean).join("\n"),
signal: null,
timedOut: true,
};
}
function parseJsonObject(text: string): Record<string, unknown> {
try {
const parsed = JSON.parse(text) as unknown;
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
} catch {
return {};
}
}
function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record<string, unknown>, statusPath: string, timedOut: boolean): CommandResult {
return {
command: [transPath(), spec.nodeRoute, "script", "--", "<remote async script>"],
cwd: repoRoot,
exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1,
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`,
signal: null,
timedOut,
};
}
function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult {
return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 });
}
function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult {
return runCommand([transPath(), spec.nodeKubeRoute, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}
function isCommandSuccess(result: CommandResult): boolean {
return result.exitCode === 0 && !result.timedOut;
}
function shortSha(value: string): string {
return value.slice(0, 12).toLowerCase();
}
function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
}
function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
const suffix = `/${spec.runtimeRenderDir}`;
if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length);
return spec.gitopsRoot;
}
function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 });
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
const match = /[0-9a-f]{40}/iu.exec(statusText(result));
return { sourceCommit: match?.[0].toLowerCase() ?? null, result };
}
function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
const gitRetries = Math.max(1, spec.downloadProfile.git.retries);
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
return [
`cicd_repo=${shellQuote(spec.cicdRepo)}`,
`cicd_url=${shellQuote(spec.gitUrl)}`,
`cicd_branch=${shellQuote(spec.sourceBranch)}`,
`cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`,
`git_retries=${shellQuote(String(gitRetries))}`,
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
"retry_git() {",
" op=$1",
" shift",
" attempt=1",
" while [ \"$attempt\" -le \"$git_retries\" ]; do",
" echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2",
" run_git \"$@\"",
" code=$?",
" if [ \"$code\" -eq 0 ]; then return 0; fi",
" echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2",
" attempt=$((attempt + 1))",
" done",
" return 1",
"}",
"mkdir -p \"$(dirname \"$cicd_repo\")\"",
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
" :",
"elif [ -e \"$cicd_repo\" ]; then",
" echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
" exit 41",
"else",
" retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"",
"fi",
"git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"",
"git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
"if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then",
" rm -rf \"$cicd_repo\"",
" retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"",
" git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
" retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune",
"fi",
].join("\n");
}
function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped);
if (scoped.action === "sync") return nodeRuntimeSync(scoped);
if (scoped.action === "apply") return nodeRuntimeApply(scoped);
if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped);
if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped);
return nodeRuntimeUnsupportedAction(scoped);
}
function nodeRuntimeMigration(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
if (scoped.allowLiveDbRead && scoped.confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-migration requires --dry-run or --confirm");
const head = resolveNodeRuntimeLaneHead(spec);
const sourceCommit = head.sourceCommit;
if (sourceCommit === null) {
return {
ok: false,
command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "source-head",
mutation: false,
degradedReason: "node-runtime-source-head-unresolved",
headProbe: compactRuntimeCommand(head.result),
};
}
const reportPath = `/tmp/hwlab-${scoped.node.toLowerCase()}-${scoped.lane}-runtime-migration-${shortSha(sourceCommit)}.json`;
const migrationArgs = scoped.dryRun
? [
...(scoped.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
"--report",
reportPath,
]
: [
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--report",
reportPath,
];
const result = runNodeK3sArgs(spec, [
"kubectl",
"exec",
"-n",
spec.runtimeNamespace,
"deployment/hwlab-cloud-api",
"-c",
"hwlab-cloud-api",
"--",
"sh",
"-lc",
`cd /workspace/hwlab-boot/repo && exec bun cmd/hwlab-cloud-api/migrate.ts ${migrationArgs.map(shellQuote).join(" ")}`,
], scoped.timeoutSeconds);
const ok = isCommandSuccess(result);
return {
ok,
command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? scoped.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
sourceCommit,
runtimeNamespace: spec.runtimeNamespace,
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
workingDirectory: "/workspace/hwlab-boot/repo",
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
reportPath,
mutation: !scoped.dryRun && ok,
result: compactRuntimeCommand(result),
degradedReason: ok ? undefined : "node-runtime-migration-failed",
next: scoped.dryRun
? {
liveReadDryRun: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --allow-live-db-read --dry-run`,
apply: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --confirm`,
}
: {
health: `curl -fsS --max-time 20 ${spec.publicApiUrl}/health/live`,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
},
};
}
function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const head = resolveNodeRuntimeLaneHead(spec);
const sourceCommit = head.sourceCommit;
if (sourceCommit === null) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "source-head",
degradedReason: "node-runtime-source-head-unresolved",
headProbe: compactRuntimeCommand(head.result),
};
}
const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds);
if (secrets !== null && secrets.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
phase: "external-postgres-secret-sync",
sourceCommit,
secrets,
degradedReason: "external-postgres-secret-sync-failed",
};
}
const baseImage = ensureNodeBaseImage(spec, scoped.dryRun, scoped.timeoutSeconds);
if (baseImage !== null && baseImage.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
phase: "base-image-seed",
sourceCommit,
secrets,
baseImage,
degradedReason: "node-runtime-base-image-seed-failed",
};
}
const render = renderNodeRuntimeControlPlane(spec, sourceCommit, scoped.timeoutSeconds);
if (!isCommandSuccess(render.result)) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
phase: "source-render",
sourceCommit,
secrets,
baseImage,
renderDir: render.renderDir,
renderLocation: render.location,
render: compactRuntimeCommand(render.result),
degradedReason: "node-runtime-control-plane-render-failed",
};
}
const apply = render.location === "local"
? applyLocalNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds)
: applyNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds);
const cleanup = render.location === "local"
? cleanupLocalNodeRuntimeRenderDir(spec, render)
: cleanupNodeRuntimeRenderDir(spec, render.renderDir);
return {
ok: isCommandSuccess(apply),
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
mutation: !scoped.dryRun && isCommandSuccess(apply),
sourceCommit,
expected: nodeRuntimeExpected(spec),
secrets,
baseImage,
renderDir: render.renderDir,
renderLocation: render.location,
render: compactRuntimeCommand(render.result),
apply: compactRuntimeCommand(apply),
cleanupRenderDir: compactRuntimeCommand(cleanup),
degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed",
next: scoped.dryRun
? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` }
: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
function nodeRuntimeRefresh(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const script = [
"set +e",
`app=${shellQuote(spec.app)}`,
"argo_namespace=argocd",
`dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
"before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
"if [ \"$dry_run\" = true ]; then",
" output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)",
"else",
" output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)",
"fi",
"code=$?",
"after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
"printf 'app\\t%s\\n' \"$app\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'before\\t%s\\n' \"$before\"",
"printf 'after\\t%s\\n' \"$after\"",
"printf 'annotateExitCode\\t%s\\n' \"$code\"",
"printf 'annotateOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
"exit \"$code\"",
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const fields = keyValueLinesFromText(statusText(result));
return {
ok: isCommandSuccess(result),
command: `hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-refresh",
mutation: !scoped.dryRun && isCommandSuccess(result),
argoApplication: fields.app || spec.app,
before: fields.before || null,
after: fields.after || null,
annotateExitCode: numericField(fields.annotateExitCode),
result: compactRuntimeCommand(result),
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const operation = {
operation: {
initiatedBy: { username: "unidesk-hwlab-node-cli" },
retry: { limit: 1 },
sync: {
prune: true,
revision: spec.gitopsBranch,
source: {
repoURL: spec.argoRepoUrl,
targetRevision: spec.gitopsBranch,
path: spec.runtimePath,
},
syncOptions: ["CreateNamespace=true"],
syncStrategy: { hook: {} },
},
},
};
const patchB64 = Buffer.from(JSON.stringify(operation), "utf8").toString("base64");
const script = [
"set +e",
`app=${shellQuote(spec.app)}`,
"argo_namespace=argocd",
`runtime_namespace=${shellQuote(spec.runtimeNamespace)}`,
`dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
`patch_b64=${shellQuote(patchB64)}`,
"terminate_patch_file=$(mktemp /tmp/hwlab-node-argocd-terminate.XXXXXX.json)",
"patch_file=$(mktemp /tmp/hwlab-node-argocd-sync.XXXXXX.json)",
"printf '{\"operation\":null}' > \"$terminate_patch_file\"",
"printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"",
"before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
"operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)",
"terminate_code=0",
"terminate_output=",
"terminated_operation=false",
"if [ \"$operation_phase\" = Running ]; then",
" if [ \"$dry_run\" = true ]; then",
" terminated_operation=would-terminate",
" else",
" terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)",
" terminate_code=$?",
" if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi",
" sleep 2",
" fi",
"fi",
"failed_hook_count=0",
"deleted_hook_count=0",
"failed_hooks=",
"deleted_hooks=",
"hook_delete_errors=",
"for job in $(kubectl -n \"$runtime_namespace\" get job -o name 2>/dev/null || true); do",
" tracking=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ index .metadata.annotations \"argocd.argoproj.io/tracking-id\" }}' 2>/dev/null || true)",
" failed=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ range .status.conditions }}{{ if or (eq .type \"Failed\") (eq .type \"FailureTarget\") }}{{ .status }} {{ end }}{{ end }}' 2>/dev/null || true)",
" case \"$tracking\" in \"$app:\"*) ;; *) continue ;; esac",
" case \"$failed\" in *True*) ;; *) continue ;; esac",
" hook_name=${job#job.batch/}",
" failed_hook_count=$((failed_hook_count + 1))",
" if [ -z \"$failed_hooks\" ]; then failed_hooks=$hook_name; else failed_hooks=$failed_hooks,$hook_name; fi",
" if [ \"$dry_run\" = false ]; then",
" delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$job\" --wait=false 2>&1)",
" delete_code=$?",
" if [ \"$delete_code\" -eq 0 ]; then",
" deleted_hook_count=$((deleted_hook_count + 1))",
" if [ -z \"$deleted_hooks\" ]; then deleted_hooks=$hook_name; else deleted_hooks=$deleted_hooks,$hook_name; fi",
" else",
" if [ -z \"$hook_delete_errors\" ]; then hook_delete_errors=\"$hook_name:$delete_code\"; else hook_delete_errors=\"$hook_delete_errors,$hook_name:$delete_code\"; fi",
" fi",
" fi",
"done",
"if [ \"$dry_run\" = true ]; then",
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)",
"else",
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" -o name 2>&1)",
"fi",
"code=$?",
"after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.operation.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
"printf 'app\\t%s\\n' \"$app\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'before\\t%s\\n' \"$before\"",
"printf 'after\\t%s\\n' \"$after\"",
"printf 'operationPhase\\t%s\\n' \"$operation_phase\"",
"printf 'terminatedOperation\\t%s\\n' \"$terminated_operation\"",
"printf 'terminateExitCode\\t%s\\n' \"$terminate_code\"",
"printf 'terminateOutput\\t%s\\n' \"$(printf '%s' \"$terminate_output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
"printf 'failedHookCount\\t%s\\n' \"$failed_hook_count\"",
"printf 'failedHooks\\t%s\\n' \"$failed_hooks\"",
"printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"",
"printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"",
"printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"",
"printf 'patchExitCode\\t%s\\n' \"$code\"",
"printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
"rm -f \"$patch_file\" \"$terminate_patch_file\"",
"exit \"$code\"",
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const fields = keyValueLinesFromText(statusText(result));
return {
ok: isCommandSuccess(result),
command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
mutation: !scoped.dryRun && isCommandSuccess(result),
argoApplication: fields.app || spec.app,
syncSource: {
repoURL: spec.argoRepoUrl,
targetRevision: spec.gitopsBranch,
path: spec.runtimePath,
},
before: fields.before || null,
after: fields.after || null,
operationRecovery: {
operationPhase: fields.operationPhase || null,
terminatedOperation: fields.terminatedOperation || "false",
terminateExitCode: numericField(fields.terminateExitCode),
terminateOutput: fields.terminateOutput || null,
failedHookCount: numericField(fields.failedHookCount),
failedHooks: commaListField(fields.failedHooks),
deletedHookCount: numericField(fields.deletedHookCount),
deletedHooks: commaListField(fields.deletedHooks),
hookDeleteErrors: fields.hookDeleteErrors || null,
},
patchExitCode: numericField(fields.patchExitCode),
patchOutput: fields.patchOutput || null,
result: compactRuntimeCommand(result),
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
const head = resolveNodeRuntimeLaneHead(spec);
const sourceCommit = head.sourceCommit;
if (sourceCommit === null) {
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "failed" });
return {
ok: false,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "source-head",
degradedReason: "node-runtime-source-head-unresolved",
headProbe: compactRuntimeCommand(head.result),
};
}
const pipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun });
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun });
const before = getNodeRuntimePipelineRun(spec, pipelineRun);
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null });
if (scoped.dryRun) {
return {
ok: true,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "dry-run",
sourceCommit,
pipelineRun,
rerun: scoped.rerun,
before,
manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) {
return {
ok: true,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "confirmed-trigger",
sourceCommit,
pipelineRun,
before,
skipped: true,
reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
rerunAvailable: true,
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` },
};
}
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun });
const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false });
if (refresh.ok !== true) {
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "failed", sourceCommit, pipelineRun });
return {
ok: false,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "control-plane-apply",
sourceCommit,
pipelineRun,
refresh,
degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
};
}
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun });
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun });
const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun);
const after = getNodeRuntimePipelineRun(spec, pipelineRun);
const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True");
const createOk = isCommandSuccess(create) || createObserved;
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
return {
ok: createOk,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "confirmed-trigger",
mutation: createOk,
sourceCommit,
pipelineRun,
rerun: scoped.rerun,
before,
refresh,
create: compactRuntimeCommand(create),
after,
createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined,
degradedReason: createOk ? undefined : "node-runtime-pipelinerun-create-failed",
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` },
};
}
function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run");
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit));
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60);
const namespaceExists = namespace.exitCode === 0;
const postgresObjects = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60)
: null;
const localPostgresObjects = postgresObjects === null
? []
: postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60);
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60);
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
const workloads = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
: null;
const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
const secrets = externalPostgresSecretStatus(spec, namespaceExists);
const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0;
const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready));
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
return {
ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady,
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
mode: "node-scoped-runtime-status",
mutation: false,
node: scoped.node,
lane: scoped.lane,
sourceCommit,
pipelineRun,
expected: nodeRuntimeExpected(spec),
sourceHead: head === null
? { ok: sourceCommit !== null, value: sourceCommit, source: "option" }
: { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) },
controlPlane: {
ready: controlPlaneReady,
serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) },
pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) },
},
argo: {
ready: argoReady,
application: spec.app,
repoURL,
expectedRepoURL: spec.argoRepoUrl,
targetRevision,
path,
syncRevision,
syncStatus,
health,
result: compactRuntimeCommand(argo),
},
pipelineRun: pipelineRunProbe,
runtime: {
namespace: spec.runtimeNamespace,
namespaceExists,
localPostgresObjects,
localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0,
workloadNames,
workloadCount: workloadNames.length,
workloadResult: workloads === null ? null : compactRuntimeCommand(workloads),
externalPostgresBridge: bridge,
externalPostgresSecrets: secrets,
},
probes: {
namespace: compactRuntimeCommand(namespace),
postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects),
},
degradedReason: controlPlaneReady
? runtimeReady
? argoReady
? pipelineRunReady ? undefined : "pipelinerun-not-succeeded"
: "argo-not-synced-healthy"
: namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing"
: "control-plane-not-ready",
next: {
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
},
};
}
function nodeRuntimeRenderToken(): string {
return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
}
function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds);
return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds);
}
function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean {
return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
}
function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
const retryCount = Math.max(0, Math.floor(retries));
const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-");
return [
`yaml_registry=${shellQuote(registry)}`,
`yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`,
`yaml_fetch_retries=${shellQuote(String(retryCount))}`,
`yaml_dependency_context=${shellQuote(safeContext)}`,
"yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }",
"yaml_npm_debug_log_tail() {",
" yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"",
" if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi",
" yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"",
" if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then",
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2",
" tail -n 80 \"$yaml_npm_log\" >&2 || true",
" fi",
"}",
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
" mkdir -p node_modules/yaml",
" yaml_tarball=\"${yaml_registry%/}/yaml/-/yaml-2.8.3.tgz\"",
" yaml_tgz=\"$(mktemp)\"",
" if command -v curl >/dev/null 2>&1 && command -v tar >/dev/null 2>&1; then",
" if timeout \"$yaml_fetch_timeout\" curl -fsSL --retry \"$yaml_fetch_retries\" --connect-timeout 10 -o \"$yaml_tgz\" \"$yaml_tarball\"; then",
" tar -xzf \"$yaml_tgz\" -C node_modules/yaml --strip-components=1",
" yaml_dependency_log installed tarball",
" else",
" yaml_dependency_log tarball-failed tarball",
" fi",
" fi",
" rm -f \"$yaml_tgz\"",
"fi",
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then",
" rm -rf node_modules/yaml",
" if timeout \"$yaml_fetch_timeout\" bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then",
" yaml_dependency_log installed bun",
" else",
" yaml_dependency_log bun-failed bun",
" fi",
"fi",
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
" rm -rf node_modules/yaml",
" command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }",
" if npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then",
" yaml_dependency_log installed npm",
" else",
" yaml_dependency_log npm-failed npm",
" yaml_npm_debug_log_tail",
" exit 34",
" fi",
"fi",
"node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }",
];
}
function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
const token = nodeRuntimeRenderToken();
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
const script = [
"set -eu",
runtimeLaneCicdRepoEnsureScript(spec),
`source_commit=${shellQuote(sourceCommit)}`,
`render_dir=${shellQuote(renderDir)}`,
`worktree_dir=${shellQuote(worktreeDir)}`,
`overlay_b64=${shellQuote(overlay)}`,
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
"trap cleanup_render_worktree EXIT",
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`,
"rm -rf \"$render_dir\" \"$worktree_dir\"",
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
"cd \"$worktree_dir\"",
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"),
"node - \"$overlay_b64\" <<'NODE'",
"const fs = require('fs');",
"const YAML = require('yaml');",
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
"const path = 'deploy/deploy.yaml';",
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
"doc.nodes = doc.nodes || {};",
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
"doc.lanes = doc.lanes || {};",
"const lane = doc.lanes[overlay.lane] || {};",
"const downloadStack = {",
" ...(lane.envRecipe?.downloadStack || {}),",
" httpProxy: overlay.dockerProxyHttp,",
" httpsProxy: overlay.dockerProxyHttps,",
" noProxy: overlay.dockerNoProxyList,",
"};",
"doc.lanes[overlay.lane] = {",
" ...lane,",
" node: overlay.nodeId,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" namespace: overlay.runtimeNamespace,",
" endpoint: overlay.publicApiUrl,",
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
" artifactCatalog: overlay.catalogPath,",
" runtimePath: overlay.runtimePath,",
" imageTagMode: 'full',",
" sourceRepo: overlay.gitUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
"};",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
[
"node scripts/run-bun.mjs \"$render_script\"",
`--lane ${shellQuote(spec.lane)}`,
`--node ${shellQuote(spec.nodeId)}`,
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
`--catalog-path ${shellQuote(spec.catalogPath)}`,
"--image-tag-mode full",
`--source-revision ${shellQuote(sourceCommit)}`,
`--source-repo ${shellQuote(spec.gitUrl)}`,
`--source-branch ${shellQuote(spec.sourceBranch)}`,
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
`--out ${shellQuote(renderDir)}`,
].join(" "),
...nodeRuntimePipelinePostprocessScript(),
].join("\n");
return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
}
function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
const token = nodeRuntimeRenderToken();
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
const script = [
"set -eu",
`source_url=${shellQuote(spec.gitUrl)}`,
`source_branch=${shellQuote(spec.sourceBranch)}`,
`source_commit=${shellQuote(sourceCommit)}`,
`render_dir=${shellQuote(renderDir)}`,
`worktree_dir=${shellQuote(worktreeDir)}`,
`overlay_b64=${shellQuote(overlay)}`,
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
"rm -rf \"$render_dir\" \"$worktree_dir\"",
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
"echo \"phase=local-git-clone-worktree\" >&2",
"run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
"cd \"$worktree_dir\"",
"echo \"phase=local-install-yaml\" >&2",
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
"node - \"$overlay_b64\" <<'NODE'",
"const fs = require('fs');",
"const YAML = require('yaml');",
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
"const path = 'deploy/deploy.yaml';",
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
"doc.nodes = doc.nodes || {};",
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
"doc.lanes = doc.lanes || {};",
"const lane = doc.lanes[overlay.lane] || {};",
"const downloadStack = {",
" ...(lane.envRecipe?.downloadStack || {}),",
" httpProxy: overlay.dockerProxyHttp,",
" httpsProxy: overlay.dockerProxyHttps,",
" noProxy: overlay.dockerNoProxyList,",
"};",
"doc.lanes[overlay.lane] = {",
" ...lane,",
" node: overlay.nodeId,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" namespace: overlay.runtimeNamespace,",
" endpoint: overlay.publicApiUrl,",
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
" artifactCatalog: overlay.catalogPath,",
" runtimePath: overlay.runtimePath,",
" imageTagMode: 'full',",
" sourceRepo: overlay.gitUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
"};",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
"echo \"phase=local-gitops-render\" >&2",
[
"node scripts/run-bun.mjs \"$render_script\"",
`--lane ${shellQuote(spec.lane)}`,
`--node ${shellQuote(spec.nodeId)}`,
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
`--catalog-path ${shellQuote(spec.catalogPath)}`,
"--image-tag-mode full",
`--source-revision ${shellQuote(sourceCommit)}`,
`--source-repo ${shellQuote(spec.gitUrl)}`,
`--source-branch ${shellQuote(spec.sourceBranch)}`,
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
`--out ${shellQuote(renderDir)}`,
].join(" "),
...nodeRuntimePipelinePostprocessScript(),
].join("\n");
return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" };
}
function nodeRuntimePipelinePostprocessScript(): string[] {
return [
"node - \"$render_dir\" \"$overlay_b64\" <<'NODE'",
"const fs = require('fs');",
"const path = require('path');",
"const vm = require('node:vm');",
"const renderDir = process.argv[2];",
"const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
"const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');",
"let text = fs.readFileSync(pipelinePath, 'utf8');",
"let YAML = null;",
"try { YAML = require('yaml'); } catch {}",
"const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');",
"const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;",
"const yamlString = (value) => JSON.stringify(String(value));",
"const proxyEnv = {",
" HTTP_PROXY: overlay.proxyHttp,",
" HTTPS_PROXY: overlay.proxyHttps,",
" ALL_PROXY: overlay.proxyAll,",
" NO_PROXY: overlay.noProxy,",
" http_proxy: overlay.proxyHttp,",
" https_proxy: overlay.proxyHttps,",
" all_proxy: overlay.proxyAll,",
" no_proxy: overlay.noProxy,",
"};",
"const dockerProxyEnv = {",
" HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,",
" HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,",
" HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,",
"};",
"const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };",
"function prepareSourceDependencyScript() {",
" const registry = String(overlay.npmRegistry || 'https://registry.npmjs.org/');",
" const timeoutSeconds = Math.max(15, Math.ceil(Number(overlay.npmFetchTimeoutMs || 120000) / 1000));",
" const retryCount = Math.max(0, Math.floor(Number(overlay.npmRetries || 3)));",
" return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"",
"node <<'NODE_UNIDESK_YAML_DEPENDENCY'",
"const { spawnSync } = require('node:child_process');",
"const fs = require('node:fs');",
"const os = require('node:os');",
"const path = require('node:path');",
"const registry = ${JSON.stringify(registry)};",
"const timeoutMs = ${JSON.stringify(timeoutSeconds * 1000)};",
"const timeoutSeconds = ${JSON.stringify(timeoutSeconds)};",
"const retryCount = ${JSON.stringify(retryCount)};",
"const dependency = 'yaml';",
"const version = '2.8.3';",
"function emit(status, extra = {}) { console.error(JSON.stringify({ event: 'prepare-source-dependencies', status, dependency, ...extra })); }",
"function hasYaml() { try { require.resolve('yaml'); return true; } catch { return false; } }",
"function run(command, args) {",
" const result = spawnSync(command, args, { stdio: 'inherit', env: process.env, timeout: timeoutMs });",
" if (result.error) console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'command-error', command, error: result.error.message }));",
" return result.status === 0;",
"}",
"function tailNpmLog() {",
" const dir = path.join(process.env.HOME || '/tmp', '.npm', '_logs');",
" if (!fs.existsSync(dir)) return;",
" const files = fs.readdirSync(dir).filter((name) => name.includes('debug') && name.endsWith('.log')).sort();",
" const file = files[files.length - 1];",
" if (!file) return;",
" const full = path.join(dir, file);",
" console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'npm-debug-log', path: full }));",
" const lines = fs.readFileSync(full, 'utf8').split(String.fromCharCode(10));",
" console.error(lines.slice(-80).join(String.fromCharCode(10)));",
"}",
"if (hasYaml()) { emit('cached'); process.exit(0); }",
"fs.mkdirSync('node_modules/yaml', { recursive: true });",
"let registryBase = registry;",
"while (registryBase.endsWith('/')) registryBase = registryBase.slice(0, -1);",
"const tarball = registryBase + '/yaml/-/yaml-' + version + '.tgz';",
"const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hwlab-yaml-'));",
"const tgz = path.join(tmpDir, 'yaml.tgz');",
"if (run('curl', ['-fsSL', '--retry', String(retryCount), '--connect-timeout', '10', '--max-time', String(timeoutSeconds), '-o', tgz, tarball]) && run('tar', ['-xzf', tgz, '-C', 'node_modules/yaml', '--strip-components=1'])) emit('installed', { manager: 'tarball' });",
"else emit('tarball-failed', { manager: 'tarball' });",
"fs.rmSync(tmpDir, { recursive: true, force: true });",
"if (!hasYaml()) {",
" fs.rmSync('node_modules/yaml', { recursive: true, force: true });",
" if (run('bun', ['add', '--no-save', '--ignore-scripts', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'bun' });",
" else emit('bun-failed', { manager: 'bun' });",
"}",
"if (!hasYaml()) {",
" fs.rmSync('node_modules/yaml', { recursive: true, force: true });",
" if (run('npm', ['install', '--package-lock=false', '--no-save', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'npm' });",
" else { emit('npm-failed', { manager: 'npm' }); tailNpmLog(); process.exit(34); }",
"}",
"if (!hasYaml()) { emit('failed', { reason: 'unresolved' }); process.exit(34); }",
"NODE_UNIDESK_YAML_DEPENDENCY",
"ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;",
"}",
"function validatePrepareSourceDependencyScript(script) {",
" const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';",
" let offset = 0;",
" while (true) {",
" const markerStart = script.indexOf(\"node <<'\" + marker + \"'\", offset);",
" if (markerStart === -1) return;",
" const bodyStart = script.indexOf('\\n', markerStart);",
" if (bodyStart === -1) throw new Error('prepare-source dependency heredoc body missing newline');",
" const bodyEnd = script.indexOf('\\n' + marker, bodyStart + 1);",
" if (bodyEnd === -1) throw new Error('prepare-source dependency heredoc terminator missing');",
" const body = script.slice(bodyStart + 1, bodyEnd);",
" try { new vm.Script(body, { filename: 'NODE_UNIDESK_YAML_DEPENDENCY.js' }); } catch (error) { throw new Error(`generated prepare-source yaml dependency script is invalid: ${error.message}`); }",
" offset = bodyEnd + marker.length + 1;",
" }",
"}",
"function deployYamlOverlayScript() {",
" const runtimeOverlay = JSON.stringify({",
" nodeId: overlay.nodeId,",
" lane: overlay.lane,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" gitopsRoot: overlay.gitopsRoot,",
" runtimePath: overlay.runtimePath,",
" runtimeRenderDir: overlay.runtimeRenderDir,",
" runtimeNamespace: overlay.runtimeNamespace,",
" catalogPath: overlay.catalogPath,",
" gitUrl: overlay.gitUrl,",
" publicWebUrl: overlay.publicWebUrl,",
" publicApiUrl: overlay.publicApiUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
" dockerProxyHttp: overlay.dockerProxyHttp,",
" dockerProxyHttps: overlay.dockerProxyHttps,",
" dockerNoProxyList: overlay.dockerNoProxyList,",
" npmRegistry: overlay.npmRegistry,",
" npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,",
" npmRetries: overlay.npmRetries,",
" });",
" return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'",
"const fs = require('fs');",
"const YAML = require('yaml');",
"const overlay = ${runtimeOverlay};",
"const file = 'deploy/deploy.yaml';",
"if (!fs.existsSync(file)) {",
" console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));",
" process.exit(45);",
"}",
"const doc = YAML.parse(fs.readFileSync(file, 'utf8'));",
"doc.nodes = doc.nodes || {};",
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
"doc.lanes = doc.lanes || {};",
"const lane = doc.lanes[overlay.lane] || {};",
"const envRecipe = lane.envRecipe || {};",
"const downloadStack = {",
" ...(envRecipe.downloadStack || {}),",
" httpProxy: overlay.dockerProxyHttp,",
" httpsProxy: overlay.dockerProxyHttps,",
" noProxy: overlay.dockerNoProxyList,",
"};",
"if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;",
"if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;",
"doc.lanes[overlay.lane] = {",
" ...lane,",
" node: overlay.nodeId,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" namespace: overlay.runtimeNamespace,",
" endpoint: overlay.publicApiUrl,",
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
" artifactCatalog: overlay.catalogPath,",
" runtimePath: overlay.runtimePath,",
" imageTagMode: 'full',",
" sourceRepo: overlay.gitUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" envRecipe: { ...envRecipe, downloadStack },",
"};",
"fs.writeFileSync(file, YAML.stringify(doc));",
"console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));",
"NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;",
"}",
"function runtimePathOverlayScript() {",
" const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;",
" const targetPath = String(overlay.runtimePath || '');",
" if (!targetPath || sourcePath === targetPath) return '';",
" return [",
" `if [ ! -d ${shellSingle(targetPath)} ]; then`,",
" ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,",
" ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,",
" ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,",
" ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,",
" `fi`,",
" ].join('\\n');",
"}",
"function stepEnvBootstrapScript() {",
" const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);",
" if (entries.length === 0) return '';",
" const lines = ['# unidesk-step-env-bootstrap'];",
" for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);",
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');",
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');",
" return lines.join('\\n');",
"}",
"function runtimeGitopsPostprocessScript() {",
" const runtimeOverlay = JSON.stringify({",
" gitopsRoot: overlay.gitopsRoot,",
" runtimePath: overlay.runtimePath,",
" runtimeRenderDir: overlay.runtimeRenderDir,",
" runtimeNamespace: overlay.runtimeNamespace,",
" externalPostgres: overlay.externalPostgres,",
" publicExposure: overlay.publicExposure,",
" observability: overlay.observability,",
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
" gitReadUrl: overlay.gitReadUrl,",
" publicWebUrl: overlay.publicWebUrl,",
" publicApiUrl: overlay.publicApiUrl,",
" });",
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'",
"const fs = require('fs');",
"const path = require('path');",
"const crypto = require('crypto');",
"const YAML = require('yaml');",
"const overlay = ${runtimeOverlay};",
"const runtimePath = String(overlay.runtimePath || '');",
"const renderDir = String(overlay.runtimeRenderDir || '');",
"const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';",
"const candidates = [...new Set([",
" runtimePath,",
" renderDir,",
" overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',",
" legacyRuntimePath,",
"].filter(Boolean))];",
"const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;",
"if (!runtimePath || !fs.existsSync(sourcePath)) {",
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));",
" process.exit(47);",
"}",
"if (sourcePath !== runtimePath) {",
" fs.rmSync(runtimePath, { recursive: true, force: true });",
" fs.mkdirSync(path.dirname(runtimePath), { recursive: true });",
" fs.cpSync(sourcePath, runtimePath, { recursive: true });",
" fs.rmSync(sourcePath, { recursive: true, force: true });",
"}",
"for (const candidate of candidates) {",
" if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });",
"}",
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
"function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }",
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
"function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }",
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
"function yamlFiles(dir) {",
" const files = [];",
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
" const file = path.join(dir, entry.name);",
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
" }",
" return files;",
"}",
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
"function writeYamlDocuments(file, docs) { fs.writeFileSync(file, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\\\n---\\\\n') + '\\\\n'); }",
"function podSpecFor(item) {",
" if (!isObject(item) || !isObject(item.spec)) return null;",
" if (item.kind === 'Pod') return item.spec;",
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
" return null;",
"}",
"function templateMetadataFor(item) {",
" if (!isObject(item) || !isObject(item.spec)) return null;",
" if (item.kind === 'Pod') return item.metadata || null;",
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;",
" if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;",
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;",
" return null;",
"}",
"function stripMonitoringMetadata(metadata) {",
" if (!isObject(metadata)) return false;",
" let changed = false;",
" if (isObject(metadata.labels) && metadata.labels['hwlab.pikastech.local/monitoring'] !== undefined && metadata.labels['hwlab.pikastech.local/monitoring'] !== 'disabled') {",
" metadata.labels['hwlab.pikastech.local/monitoring'] = 'disabled';",
" changed = true;",
" }",
" if (isObject(metadata.annotations) && metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'] !== undefined) {",
" delete metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'];",
" changed = true;",
" }",
" return changed;",
"}",
"function containerHasVolumeMount(container, name) { return isObject(container) && Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === name); }",
"function removeMetricsSidecar(podSpec) {",
" if (!isObject(podSpec)) return false;",
" let changed = false;",
" if (Array.isArray(podSpec.containers)) {",
" const next = podSpec.containers.filter((container) => !(isObject(container) && (container.name === 'hwlab-metrics' || (Array.isArray(container.command) && container.command.includes('/metrics/metrics-sidecar.mjs')))));",
" if (next.length !== podSpec.containers.length) { podSpec.containers = next; changed = true; }",
" }",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container) || !Array.isArray(container.volumeMounts)) continue;",
" const nextMounts = container.volumeMounts.filter((mount) => !(mount && mount.name === 'hwlab-metrics-sidecar'));",
" if (nextMounts.length !== container.volumeMounts.length) { container.volumeMounts = nextMounts; changed = true; }",
" }",
" }",
" if (Array.isArray(podSpec.volumes)) {",
" const nextVolumes = podSpec.volumes.filter((volume) => !(volume && volume.name === 'hwlab-metrics-sidecar'));",
" if (nextVolumes.length !== podSpec.volumes.length) { podSpec.volumes = nextVolumes; changed = true; }",
" }",
" return changed;",
"}",
"function envValue(container, name) {",
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
" const item = container.env.find((env) => env && env.name === name);",
" return item ? item.value : undefined;",
"}",
"function setEnvValue(container, name, value) {",
" if (!isObject(container) || typeof value !== 'string') return false;",
" container.env = Array.isArray(container.env) ? container.env : [];",
" let item = container.env.find((env) => env && env.name === name);",
" if (!item) { item = { name }; container.env.push(item); }",
" const changed = item.value !== value || item.valueFrom !== undefined;",
" item.value = value;",
" delete item.valueFrom;",
" return changed;",
"}",
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
"function startupProbeFrom(probe) {",
" const next = JSON.parse(JSON.stringify(probe));",
" next.periodSeconds = 10;",
" next.timeoutSeconds = Math.max(Number(next.timeoutSeconds || 0), 2);",
" next.failureThreshold = 30;",
" next.successThreshold = 1;",
" delete next.initialDelaySeconds;",
" return next;",
"}",
"function addEnvReuseStartupProbe(podSpec) {",
" if (!isObject(podSpec) || !Array.isArray(podSpec.containers)) return false;",
" let changed = false;",
" for (const container of podSpec.containers) {",
" if (!isObject(container) || !isEnvReuseContainer(container) || container.startupProbe) continue;",
" const sourceProbe = container.readinessProbe || container.livenessProbe;",
" if (!sourceProbe) continue;",
" container.startupProbe = startupProbeFrom(sourceProbe);",
" changed = true;",
" }",
" return changed;",
"}",
"function rewriteRuntimeImage(image) {",
" if (typeof image !== 'string') return image;",
" const match = (overlay.runtimeImageRewrites || []).find((item) => item && item.source === image);",
" return match ? match.target : image;",
"}",
"function patchRuntimeImages(podSpec) {",
" if (!isObject(podSpec)) return false;",
" let changed = false;",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container) || typeof container.image !== 'string') continue;",
" const nextImage = rewriteRuntimeImage(container.image);",
" if (nextImage !== container.image) { container.image = nextImage; changed = true; }",
" }",
" }",
" return changed;",
"}",
"function rewriteEnvValue(value) {",
" if (typeof value !== 'string') return value;",
" return value.replaceAll('http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git', overlay.gitReadUrl);",
"}",
"function patchGitReadUrlEnv(podSpec) {",
" if (!isObject(podSpec)) return false;",
" let changed = false;",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container) || !Array.isArray(container.env)) continue;",
" for (const env of container.env) {",
" if (!isObject(env) || typeof env.value !== 'string') continue;",
" const nextValue = rewriteEnvValue(env.value);",
" if (nextValue !== env.value) { env.value = nextValue; changed = true; }",
" }",
" }",
" }",
" return changed;",
"}",
"function patchRuntimeEnv(item, podSpec) {",
" if (!isObject(podSpec)) return { publicEndpointChanged: false, dbSslModeChanged: false };",
" let publicEndpointChanged = false;",
" let dbSslModeChanged = false;",
" const pg = overlay.externalPostgres;",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container)) continue;",
" if (envValue(container, 'HWLAB_PUBLIC_ENDPOINT') !== undefined) publicEndpointChanged = setEnvValue(container, 'HWLAB_PUBLIC_ENDPOINT', expectedPublicEndpoint(item)) || publicEndpointChanged;",
" if (pg && pg.sslmode && envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE') !== undefined) dbSslModeChanged = setEnvValue(container, 'HWLAB_CLOUD_DB_SSL_MODE', pg.sslmode) || dbSslModeChanged;",
" }",
" }",
" return { publicEndpointChanged, dbSslModeChanged };",
"}",
"function patchRuntimeWorkloads() {",
" let observabilityChanged = false;",
" let startupProbeChanged = false;",
" let imageRewriteChanged = false;",
" let gitReadUrlChanged = false;",
" let publicEndpointChanged = false;",
" let dbSslModeChanged = false;",
" for (const file of yamlFiles(runtimePath)) {",
" if (path.basename(file) === 'kustomization.yaml') continue;",
" const docs = readYamlDocuments(file);",
" let changed = false;",
" for (const doc of docs) {",
" for (const item of listItems(doc).filter(Boolean)) {",
" if (!isObject(item)) continue;",
" if (overlay.observability && overlay.observability.prometheusOperator === false) {",
" const metadataChanged = stripMonitoringMetadata(item.metadata);",
" const templateChanged = stripMonitoringMetadata(templateMetadataFor(item));",
" const sidecarChanged = removeMetricsSidecar(podSpecFor(item));",
" const monitoringChanged = metadataChanged || templateChanged || sidecarChanged;",
" changed = monitoringChanged || changed;",
" observabilityChanged = observabilityChanged || monitoringChanged;",
" }",
" const probeChanged = addEnvReuseStartupProbe(podSpecFor(item));",
" changed = probeChanged || changed;",
" startupProbeChanged = startupProbeChanged || probeChanged;",
" const imageChanged = patchRuntimeImages(podSpecFor(item));",
" changed = imageChanged || changed;",
" imageRewriteChanged = imageRewriteChanged || imageChanged;",
" const gitUrlChanged = patchGitReadUrlEnv(podSpecFor(item));",
" changed = gitUrlChanged || changed;",
" gitReadUrlChanged = gitReadUrlChanged || gitUrlChanged;",
" const envChanged = patchRuntimeEnv(item, podSpecFor(item));",
" changed = envChanged.publicEndpointChanged || envChanged.dbSslModeChanged || changed;",
" publicEndpointChanged = publicEndpointChanged || envChanged.publicEndpointChanged;",
" dbSslModeChanged = dbSslModeChanged || envChanged.dbSslModeChanged;",
" }",
" }",
" if (changed) writeYamlDocuments(file, docs);",
" }",
" return { observabilityChanged, startupProbeChanged, imageRewriteChanged, gitReadUrlChanged, publicEndpointChanged, dbSslModeChanged };",
"}",
"function patchKustomization() {",
" const file = path.join(runtimePath, 'kustomization.yaml');",
" if (!fs.existsSync(file)) return false;",
" const doc = readYaml(file) || {};",
" const resources = Array.isArray(doc.resources) ? doc.resources : [];",
" const next = resources.filter((item) => !(overlay.observability && overlay.observability.prometheusOperator === false && item === 'observability.yaml'));",
" let changed = false;",
" if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); changed = true; }",
" const observabilityFile = path.join(runtimePath, 'observability.yaml');",
" if (overlay.observability && overlay.observability.prometheusOperator === false && fs.existsSync(observabilityFile)) { fs.rmSync(observabilityFile, { force: true }); changed = true; }",
" return changed;",
"}",
"function patchExternalPostgres() {",
" const pg = overlay.externalPostgres;",
" if (!pg || !pg.serviceName) return false;",
" const file = path.join(runtimePath, 'external-postgres.yaml');",
" if (!fs.existsSync(file)) return false;",
" const doc = readYaml(file);",
" const items = listItems(doc).filter(Boolean);",
" let changed = false;",
" const endpointSliceName = String(pg.serviceName) + '-host';",
" for (const item of items) {",
" if (!item || typeof item !== 'object') continue;",
" item.metadata = item.metadata || {};",
" item.metadata.namespace = overlay.runtimeNamespace;",
" item.metadata.labels = item.metadata.labels || {};",
" item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;",
" if (item.kind === 'Service') {",
" item.metadata.name = pg.serviceName;",
" item.spec = item.spec || {};",
" item.spec.ports = [{ name: 'postgres', port: pg.port, targetPort: pg.port, protocol: 'TCP' }];",
" delete item.spec.selector;",
" changed = true;",
" }",
" if (item.kind === 'EndpointSlice') {",
" item.metadata.name = endpointSliceName;",
" item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;",
" item.addressType = 'IPv4';",
" item.ports = [{ name: 'postgres', port: pg.port, protocol: 'TCP' }];",
" item.endpoints = [{ addresses: [pg.endpointAddress], conditions: { ready: true } }];",
" changed = true;",
" }",
" }",
" if (changed) writeYaml(file, normalizeList(items));",
" return changed;",
"}",
"function patchHealthContract() {",
" const file = path.join(runtimePath, 'health-contract.yaml');",
" if (!fs.existsSync(file)) return false;",
" const doc = readYaml(file);",
" const items = listItems(doc).filter(Boolean);",
" let changed = false;",
" const pg = overlay.externalPostgres;",
" for (const item of items) {",
" if (!item || item.kind !== 'ConfigMap') continue;",
" item.data = item.data || {};",
" if (item.data.endpoint !== overlay.publicWebUrl) { item.data.endpoint = overlay.publicWebUrl; changed = true; }",
" const cloudApi = 'GET /health/live through ' + overlay.publicApiUrl;",
" if (item.data['cloud-api'] !== cloudApi) { item.data['cloud-api'] = cloudApi; changed = true; }",
" const cloudWeb = 'GET /health/live on ' + overlay.publicWebUrl + '; consumes cloud-api only';",
" if (item.data['cloud-web'] !== cloudWeb) { item.data['cloud-web'] = cloudWeb; changed = true; }",
" if (pg && pg.sslmode && typeof item.data['cloud-api-db'] === 'string') {",
" const next = item.data['cloud-api-db'].replace(/HWLAB_CLOUD_DB_SSL_MODE=[A-Za-z0-9_-]+/g, 'HWLAB_CLOUD_DB_SSL_MODE=' + pg.sslmode);",
" if (next !== item.data['cloud-api-db']) { item.data['cloud-api-db'] = next; changed = true; }",
" }",
" }",
" if (changed) writeYaml(file, normalizeList(items));",
" return changed;",
"}",
"function renderPublicExposureFrpcToml(exposure) {",
" return [",
" 'serverAddr = ' + JSON.stringify(String(exposure.serverAddr)),",
" 'serverPort = ' + Number(exposure.serverPort),",
" 'loginFailExit = true',",
" 'auth.token = \"{{ .Envs.HWLAB_FRP_TOKEN }}\"',",
" '',",
" '[[proxies]]',",
" 'name = ' + JSON.stringify(String(exposure.webProxy.name)),",
" 'type = \"tcp\"',",
" 'localIP = ' + JSON.stringify(String(exposure.webProxy.localIP)),",
" 'localPort = ' + Number(exposure.webProxy.localPort),",
" 'remotePort = ' + Number(exposure.webProxy.remotePort),",
" '',",
" '[[proxies]]',",
" 'name = ' + JSON.stringify(String(exposure.apiProxy.name)),",
" 'type = \"tcp\"',",
" 'localIP = ' + JSON.stringify(String(exposure.apiProxy.localIP)),",
" 'localPort = ' + Number(exposure.apiProxy.localPort),",
" 'remotePort = ' + Number(exposure.apiProxy.remotePort),",
" '',",
" ].join('\\\\n');",
"}",
"function setEnvFromSecret(container, name, secretName, secretKey) {",
" if (!isObject(container)) return false;",
" container.env = Array.isArray(container.env) ? container.env : [];",
" let item = container.env.find((env) => env && env.name === name);",
" if (!item) { item = { name }; container.env.push(item); }",
" const nextValueFrom = { secretKeyRef: { name: secretName, key: secretKey } };",
" const changed = item.value !== undefined || JSON.stringify(item.valueFrom || {}) !== JSON.stringify(nextValueFrom);",
" delete item.value;",
" item.valueFrom = nextValueFrom;",
" return changed;",
"}",
"function patchPublicExposure() {",
" const exposure = overlay.publicExposure;",
" if (!exposure || !exposure.enabled) return { configured: false, changed: false };",
" const file = path.join(runtimePath, 'node-frpc.yaml');",
" if (!fs.existsSync(file)) {",
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));",
" process.exit(49);",
" }",
" const docs = readYamlDocuments(file);",
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
" const configKey = String(exposure.secretKey || 'frpc.toml');",
" const tokenKey = String(exposure.tokenKey || 'token');",
" const toml = renderPublicExposureFrpcToml(exposure);",
" let changed = false;",
" let foundConfigMap = false;",
" let foundDeployment = false;",
" for (const doc of docs) {",
" for (const item of listItems(doc).filter(Boolean)) {",
" if (!isObject(item)) continue;",
" item.metadata = item.metadata || {};",
" if (item.kind === 'ConfigMap' && item.metadata.name === configName) {",
" foundConfigMap = true;",
" item.data = item.data || {};",
" if (item.data[configKey] !== toml) { item.data[configKey] = toml; changed = true; }",
" }",
" if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {",
" foundDeployment = true;",
" item.spec = item.spec || {};",
" const nextStrategy = { type: 'Recreate' };",
" if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }",
" const podSpec = podSpecFor(item);",
" for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {",
" if (!isObject(container)) continue;",
" if (container.name === 'frpc' || String(container.image || '').includes('frpc')) changed = setEnvFromSecret(container, 'HWLAB_FRP_TOKEN', exposure.secretName, tokenKey) || changed;",
" }",
" }",
" }",
" }",
" if (!foundConfigMap || !foundDeployment) {",
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'frpc-resource-missing', filePath: file, configName, deploymentName, foundConfigMap, foundDeployment }));",
" process.exit(50);",
" }",
" if (changed) writeYamlDocuments(file, docs);",
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: true, applied: true, changed, filePath: file, hostname: exposure.hostname, serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, webProxy: exposure.webProxy.name, apiProxy: exposure.apiProxy.name, configSha256: crypto.createHash('sha256').update(toml).digest('hex') }));",
" return { configured: true, changed, foundConfigMap, foundDeployment };",
"}",
"const kustomizationChanged = patchKustomization();",
"const runtimeWorkloadsChanged = patchRuntimeWorkloads();",
"const externalPostgresChanged = patchExternalPostgres();",
"const healthContractChanged = patchHealthContract();",
"const publicExposureChanged = patchPublicExposure();",
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, runtimeImageRewriteCount: (overlay.runtimeImageRewrites || []).length, kustomizationChanged, observabilityWorkloadsChanged: runtimeWorkloadsChanged.observabilityChanged, startupProbeChanged: runtimeWorkloadsChanged.startupProbeChanged, runtimeImageRewriteChanged: runtimeWorkloadsChanged.imageRewriteChanged, gitReadUrlChanged: runtimeWorkloadsChanged.gitReadUrlChanged, publicEndpointChanged: runtimeWorkloadsChanged.publicEndpointChanged, dbSslModeChanged: runtimeWorkloadsChanged.dbSslModeChanged, externalPostgresChanged, healthContractChanged, publicExposureChanged }));",
"NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;",
"}",
"function runtimeGitopsVerifyScript() {",
" const runtimeOverlay = JSON.stringify({",
" runtimePath: overlay.runtimePath,",
" runtimeNamespace: overlay.runtimeNamespace,",
" externalPostgres: overlay.externalPostgres,",
" publicExposure: overlay.publicExposure,",
" observability: overlay.observability,",
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
" gitReadUrl: overlay.gitReadUrl,",
" publicWebUrl: overlay.publicWebUrl,",
" publicApiUrl: overlay.publicApiUrl,",
" });",
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'",
"const fs = require('fs');",
"const path = require('path');",
"const YAML = require('yaml');",
"const overlay = ${runtimeOverlay};",
"const runtimePath = String(overlay.runtimePath || '');",
"function fail(reason, extra = {}) {",
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));",
" process.exit(48);",
"}",
"if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');",
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
"function yamlFiles(dir) {",
" const files = [];",
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
" const file = path.join(dir, entry.name);",
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
" }",
" return files;",
"}",
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
"function allItemsFromFile(file) { return readYamlDocuments(file).flatMap((doc) => listItems(doc).filter(Boolean)); }",
"function podSpecFor(item) {",
" if (!isObject(item) || !isObject(item.spec)) return null;",
" if (item.kind === 'Pod') return item.spec;",
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
" return null;",
"}",
"function envValue(container, name) {",
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
" const item = container.env.find((env) => env && env.name === name);",
" return item ? item.value : undefined;",
"}",
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
"function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }",
"function workloadChecks() {",
" const metricsRefs = [];",
" const missingStartupProbes = [];",
" const publicRuntimeImages = [];",
" const staleGitReadUrls = [];",
" const wrongPublicEndpoints = [];",
" const wrongDbSslModes = [];",
" const rewriteSources = new Set((overlay.runtimeImageRewrites || []).map((item) => item && item.source).filter(Boolean));",
" for (const file of yamlFiles(runtimePath)) {",
" if (path.basename(file) === 'kustomization.yaml') continue;",
" for (const doc of readYamlDocuments(file)) {",
" for (const item of listItems(doc).filter(Boolean)) {",
" const podSpec = podSpecFor(item);",
" if (!isObject(podSpec)) continue;",
" for (const container of Array.isArray(podSpec.containers) ? podSpec.containers : []) {",
" if (!isObject(container)) continue;",
" if (container.name === 'hwlab-metrics' || (Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === 'hwlab-metrics-sidecar'))) metricsRefs.push(workloadRef(item, file, container));",
" if (isEnvReuseContainer(container) && (container.readinessProbe || container.livenessProbe) && !container.startupProbe) missingStartupProbes.push(workloadRef(item, file, container));",
" if (typeof container.image === 'string' && rewriteSources.has(container.image)) publicRuntimeImages.push({ ...workloadRef(item, file, container), image: container.image });",
" if (Array.isArray(container.env) && container.env.some((env) => env && typeof env.value === 'string' && env.value.includes('git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git') && env.value !== overlay.gitReadUrl)) staleGitReadUrls.push(workloadRef(item, file, container));",
" const publicEndpoint = envValue(container, 'HWLAB_PUBLIC_ENDPOINT');",
" if (publicEndpoint !== undefined && publicEndpoint !== expectedPublicEndpoint(item)) wrongPublicEndpoints.push({ ...workloadRef(item, file, container), value: publicEndpoint, expected: expectedPublicEndpoint(item) });",
" const dbSslMode = envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE');",
" if (overlay.externalPostgres && overlay.externalPostgres.sslmode && dbSslMode !== undefined && dbSslMode !== overlay.externalPostgres.sslmode) wrongDbSslModes.push({ ...workloadRef(item, file, container), value: dbSslMode, expected: overlay.externalPostgres.sslmode });",
" }",
" if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));",
" }",
" }",
" }",
" return { metricsRefs, missingStartupProbes, publicRuntimeImages, staleGitReadUrls, wrongPublicEndpoints, wrongDbSslModes };",
"}",
"const checks = [];",
"const workloadCheck = workloadChecks();",
"const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');",
"if (overlay.observability && overlay.observability.prometheusOperator === false) {",
" if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');",
" const resources = readYaml(kustomizationPath).resources || [];",
" if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });",
" if (workloadCheck.metricsRefs.length > 0) fail('observability-sidecar-still-rendered', { refs: workloadCheck.metricsRefs.slice(0, 12), count: workloadCheck.metricsRefs.length });",
" checks.push('observability-disabled');",
"}",
"if (workloadCheck.missingStartupProbes.length > 0) fail('env-reuse-startup-probe-missing', { refs: workloadCheck.missingStartupProbes.slice(0, 12), count: workloadCheck.missingStartupProbes.length });",
"checks.push('env-reuse-startup-probes');",
"if (workloadCheck.publicRuntimeImages.length > 0) fail('runtime-image-rewrite-missing', { refs: workloadCheck.publicRuntimeImages.slice(0, 12), count: workloadCheck.publicRuntimeImages.length });",
"if ((overlay.runtimeImageRewrites || []).length > 0) checks.push('runtime-image-rewrites');",
"if (workloadCheck.staleGitReadUrls.length > 0) fail('runtime-git-read-url-stale', { refs: workloadCheck.staleGitReadUrls.slice(0, 12), count: workloadCheck.staleGitReadUrls.length, expected: overlay.gitReadUrl });",
"checks.push('runtime-git-read-url');",
"if (workloadCheck.wrongPublicEndpoints.length > 0) fail('runtime-public-endpoint-mismatch', { refs: workloadCheck.wrongPublicEndpoints.slice(0, 12), count: workloadCheck.wrongPublicEndpoints.length });",
"checks.push('runtime-public-endpoint');",
"if (workloadCheck.wrongDbSslModes.length > 0) fail('runtime-db-ssl-mode-mismatch', { refs: workloadCheck.wrongDbSslModes.slice(0, 12), count: workloadCheck.wrongDbSslModes.length });",
"if (overlay.externalPostgres && overlay.externalPostgres.sslmode) checks.push('runtime-db-ssl-mode');",
"const pg = overlay.externalPostgres;",
"if (pg && pg.serviceName) {",
" const file = path.join(runtimePath, 'external-postgres.yaml');",
" if (!fs.existsSync(file)) fail('external-postgres-missing');",
" const items = listItems(readYaml(file)).filter(Boolean);",
" const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);",
" const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');",
" if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });",
" if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });",
" const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;",
" const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;",
" const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;",
" if (Number(servicePort) !== Number(pg.port) || Number(endpointPort) !== Number(pg.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: pg.port });",
" if (String(endpointAddress) !== String(pg.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: pg.endpointAddress });",
" checks.push('external-postgres-bridge');",
"}",
"const exposure = overlay.publicExposure;",
"if (exposure && exposure.enabled) {",
" const file = path.join(runtimePath, 'node-frpc.yaml');",
" if (!fs.existsSync(file)) fail('public-exposure-frpc-missing');",
" const items = allItemsFromFile(file);",
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
" const configKey = String(exposure.secretKey || 'frpc.toml');",
" const tokenKey = String(exposure.tokenKey || 'token');",
" const configMap = items.find((item) => item && item.kind === 'ConfigMap' && item.metadata && item.metadata.name === configName);",
" const deployment = items.find((item) => item && item.kind === 'Deployment' && item.metadata && item.metadata.name === deploymentName);",
" if (!configMap) fail('public-exposure-frpc-configmap-missing', { expected: configName });",
" if (!deployment) fail('public-exposure-frpc-deployment-missing', { expected: deploymentName });",
" const toml = configMap.data && configMap.data[configKey];",
" if (typeof toml !== 'string') fail('public-exposure-frpc-config-missing', { expectedKey: configKey });",
" for (const expected of [String(exposure.serverAddr), String(exposure.serverPort), String(exposure.webProxy.name), String(exposure.webProxy.remotePort), String(exposure.apiProxy.name), String(exposure.apiProxy.remotePort), 'HWLAB_FRP_TOKEN']) {",
" if (!toml.includes(expected)) fail('public-exposure-frpc-config-mismatch', { expected });",
" }",
" const podSpec = podSpecFor(deployment);",
" const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];",
" const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;",
" if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });",
" const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));",
" const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;",
" const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;",
" if (!secretRef || secretRef.name !== exposure.secretName || secretRef.key !== tokenKey) fail('public-exposure-frpc-token-env-mismatch', { expectedSecret: exposure.secretName, expectedKey: tokenKey });",
" checks.push('public-exposure-frpc');",
"}",
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));",
"NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;",
"}",
"function patchScript(script) {",
" let result = String(script || '');",
" if (result.includes('npm run gitops:ts:check')) {",
" result = result.replace(/\\n[ \\t]*npm run gitops:ts:check\\n/g, '\\n echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-replaces-tsc-gate\"}\\' >&2\\n');",
" }",
" const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');",
" if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {",
" result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());",
" }",
" const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';",
" if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
" result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);",
" }",
" const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';",
" if (result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
" result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);",
" }",
" result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);",
" result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
" result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
" result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
" result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
" result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);",
" result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);",
" const legacyRuntimePath = (() => {",
" const runtimePath = String(overlay.runtimePath || '');",
" const parts = runtimePath.split('/').filter(Boolean);",
" if (parts.length < 2) return '';",
" const leaf = parts.at(-1);",
" const parent = parts.slice(0, -2).join('/');",
" return parent && leaf ? `${parent}/${leaf}` : '';",
" })();",
" if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {",
" result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);",
" result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');",
" result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');",
" }",
" const bootstrap = stepEnvBootstrapScript();",
" if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {",
" result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);",
" }",
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {",
" let next = match;",
" if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
" if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
" return next;",
" });",
" result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {",
" if (match.includes('--check')) return runtimeGitopsVerifyScript();",
" return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;",
" });",
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
" validatePrepareSourceDependencyScript(result);",
" return result;",
"}",
"function patchManifestObject(doc) {",
" if (!doc || typeof doc !== 'object') return false;",
" if (doc.kind !== 'Pipeline' || !doc.spec) return false;",
" const defaults = {",
" 'git-url': overlay.gitUrl,",
" 'git-read-url': overlay.gitReadUrl,",
" 'git-write-url': overlay.gitWriteUrl,",
" 'catalog-path': overlay.catalogPath,",
" 'runtime-path': overlay.runtimePath,",
" 'registry-prefix': overlay.registryPrefix,",
" };",
" for (const param of doc.spec?.params || []) {",
" if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];",
" }",
" doc.metadata = doc.metadata || {};",
" doc.metadata.annotations = doc.metadata.annotations || {};",
" doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;",
" doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;",
" for (const task of doc.spec?.tasks || []) {",
" for (const sidecar of task.taskSpec?.sidecars || []) {",
" if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;",
" }",
" for (const step of task.taskSpec?.steps || []) {",
" if (Array.isArray(step.env)) {",
" for (const env of step.env) {",
" if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];",
" }",
" }",
" step.env = Array.isArray(step.env) ? step.env : [];",
" const existingEnv = new Set(step.env.map((env) => env.name));",
" for (const [name, value] of Object.entries(stepEnv)) {",
" if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });",
" }",
" if (typeof step.script === 'string') step.script = patchScript(step.script);",
" }",
" }",
" return true;",
"}",
"function patchStructuredPipeline() {",
" try {",
" const doc = JSON.parse(text);",
" if (!patchManifestObject(doc)) return false;",
" text = JSON.stringify(doc, null, 2) + '\\n';",
" return true;",
" } catch {}",
" if (YAML) {",
" try {",
" const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);",
" const changed = docs.some((doc) => patchManifestObject(doc));",
" if (!changed) return false;",
" text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';",
" return true;",
" } catch {}",
" }",
" return false;",
"}",
"const structured = patchStructuredPipeline();",
"function replaceParamDefault(name, value) {",
" const namePattern = escapeRegExp(name);",
" text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);",
" text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);",
"}",
"replaceParamDefault('git-url', overlay.gitUrl);",
"replaceParamDefault('git-read-url', overlay.gitReadUrl);",
"replaceParamDefault('git-write-url', overlay.gitWriteUrl);",
"replaceParamDefault('catalog-path', overlay.catalogPath);",
"replaceParamDefault('runtime-path', overlay.runtimePath);",
"replaceParamDefault('registry-prefix', overlay.registryPrefix);",
"text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);",
"text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);",
"if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);",
"text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
"text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
"text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
"text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
"for (const [name, value] of Object.entries(stepEnv)) {",
" if (value === undefined) continue;",
" text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);",
" text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);",
"}",
"const bootstrap = stepEnvBootstrapScript();",
"if (bootstrap) {",
" text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {",
" const previous = fullText.slice(Math.max(0, offset - 500), offset);",
" if (previous.includes('unidesk-step-env-bootstrap')) return match;",
" return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;",
" });",
"}",
"if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {",
" text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);",
"}",
"const quotedRoot = JSON.stringify(overlay.gitopsRoot);",
"const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');",
"const replacements = [",
" ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],",
" ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],",
"];",
"let changed = false;",
"for (const [needle, replacement] of replacements) {",
" if (!text.includes(needle)) continue;",
" text = text.split(needle).join(replacement);",
" changed = true;",
"}",
"if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }",
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }",
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && !text.includes('NODE_UNIDESK_YAML_DEPENDENCY')) { throw new Error(`generated pipeline missing UniDesk yaml dependency install in ${pipelinePath}`); }",
"if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }",
"fs.writeFileSync(pipelinePath, text);",
"function patchArgoYaml(filePath) {",
" if (!YAML || !fs.existsSync(filePath)) return;",
" const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
" let changed = false;",
" for (const doc of docs) {",
" if (!doc || typeof doc !== 'object') continue;",
" if (doc.kind === 'AppProject') {",
" doc.spec = doc.spec || {};",
" doc.spec.sourceRepos = [overlay.argoRepoUrl];",
" changed = true;",
" }",
" if (doc.kind === 'Application') {",
" doc.spec = doc.spec || {};",
" doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };",
" changed = true;",
" }",
" }",
" if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
"}",
"patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));",
"patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));",
"NODE",
];
}
function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
return {
nodeId: spec.nodeId,
lane: spec.lane,
sourceBranch: spec.sourceBranch,
gitopsBranch: spec.gitopsBranch,
gitopsRoot: nodeRuntimeGitopsRoot(spec),
runtimePath: spec.runtimePath,
runtimeRenderDir: spec.runtimeRenderDir,
runtimeNamespace: spec.runtimeNamespace,
catalogPath: spec.catalogPath,
tektonDir: spec.tektonDir,
argoApplicationFile: spec.argoApplicationFile,
argoRepoUrl: spec.argoRepoUrl,
gitUrl: spec.gitUrl,
gitReadUrl: spec.gitReadUrl,
gitWriteUrl: spec.gitWriteUrl,
networkProfileId: spec.networkProfileId,
downloadProfileId: spec.downloadProfileId,
gitSshProxyHost: gitSshProxy?.host,
gitSshProxyPort: gitSshProxy?.port,
proxyHttp: spec.networkProfile.proxy.http,
proxyHttps: spec.networkProfile.proxy.https,
proxyAll: spec.networkProfile.proxy.all,
noProxy: spec.networkProfile.proxy.noProxy.join(","),
dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http,
dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https,
dockerProxyAll: spec.networkProfile.dockerBuildProxy.all,
dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy,
npmRegistry: spec.downloadProfile.npm.registry,
npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000,
npmRetries: spec.downloadProfile.npm.retries,
stepEnv: spec.stepEnv,
observability: spec.observability,
runtimeImageRewrites: spec.runtimeImageRewrites,
registryPrefix: spec.registryPrefix,
buildkitSidecarImage: spec.buildkit?.sidecarImage,
publicWebUrl: spec.publicWebUrl,
publicApiUrl: spec.publicApiUrl,
publicExposure: spec.publicExposure === null ? undefined : {
enabled: spec.publicExposure.enabled,
mode: spec.publicExposure.mode,
publicBaseUrl: spec.publicExposure.publicBaseUrl,
hostname: spec.publicExposure.hostname,
expectedA: spec.publicExposure.expectedA,
serverAddr: spec.publicExposure.serverAddr,
serverPort: spec.publicExposure.serverPort,
secretName: spec.publicExposure.secretName,
secretKey: spec.publicExposure.secretKey,
tokenKey: spec.publicExposure.tokenKey,
webProxy: spec.publicExposure.webProxy,
apiProxy: spec.publicExposure.apiProxy,
},
externalPostgres: spec.externalPostgres === undefined ? undefined : {
enabled: true,
serviceName: spec.externalPostgres.serviceName,
endpointAddress: spec.externalPostgres.endpointAddress,
port: spec.externalPostgres.port,
sslmode: spec.externalPostgres.sslmode,
},
};
}
function httpProxyEndpoint(value: string): { host: string; port: number } | null {
let parsed: URL;
try {
parsed = new URL(value);
} catch {
return null;
}
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10);
if (!Number.isInteger(port) || port <= 0) return null;
return { host: parsed.hostname, port };
}
function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
return [
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,
`${renderDir}/argocd/project.yaml`,
`${renderDir}/argocd/${spec.argoApplicationFile}`,
];
}
function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
const files = nodeRuntimeControlPlaneFiles(spec, renderDir);
const args = [
"kubectl",
"apply",
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
...files.flatMap((file) => ["-f", file]),
];
return runNodeK3sArgs(spec, args, timeoutSeconds);
}
function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir)
.map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`)
.join("");
const args = [
"kubectl",
"apply",
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
"-f",
"-",
];
return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest);
}
function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult {
const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
const script = [
"set -eu",
`render_dir=${shellQuote(renderDir)}`,
`prefix=${shellQuote(prefix)}`,
"case \"$render_dir\" in",
" \"$prefix\"*) rm -rf \"$render_dir\" ;;",
" *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;",
"esac",
].join("\n");
return runNodeHostScript(spec, script, 60);
}
function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult {
const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`;
if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) {
return {
command: ["rm", "-rf", "<refused>"],
cwd: repoRoot,
exitCode: 44,
stdout: "",
stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`,
signal: null,
timedOut: false,
};
}
return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 });
}
function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
return {
apiVersion: "tekton.dev/v1",
kind: "PipelineRun",
metadata: {
name: pipelineRun,
namespace: "hwlab-ci",
labels: {
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/gitops-target": spec.lane,
"hwlab.pikastech.local/source-commit": sourceCommit,
"hwlab.pikastech.local/trigger": "unidesk-node-cli",
},
annotations: {
"hwlab.pikastech.local/node": spec.nodeId,
"hwlab.pikastech.local/source-branch": spec.sourceBranch,
"hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
"hwlab.pikastech.local/runtime-path": spec.runtimePath,
"hwlab.pikastech.local/network-profile": spec.networkProfileId,
"hwlab.pikastech.local/download-profile": spec.downloadProfileId,
},
},
spec: {
pipelineRef: { name: spec.pipeline },
taskRunTemplate: {
serviceAccountName: spec.serviceAccountName,
podTemplate: {
hostNetwork: true,
dnsPolicy: "ClusterFirstWithHostNet",
securityContext: { fsGroup: 1000 },
},
},
params: [
{ name: "git-url", value: spec.gitUrl },
{ name: "git-read-url", value: spec.gitReadUrl },
{ name: "git-write-url", value: spec.gitWriteUrl },
{ name: "source-branch", value: spec.sourceBranch },
{ name: "gitops-branch", value: spec.gitopsBranch },
{ name: "lane", value: spec.lane },
{ name: "catalog-path", value: spec.catalogPath },
{ name: "image-tag-mode", value: "full" },
{ name: "runtime-path", value: spec.runtimePath },
{ name: "revision", value: sourceCommit },
{ name: "registry-prefix", value: spec.registryPrefix },
{ name: "services", value: spec.serviceIds.join(",") },
{ name: "base-image", value: spec.baseImage },
],
workspaces: [
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
],
},
};
}
function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult {
const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64");
const script = [
"set -eu",
`manifest_b64=${shellQuote(manifestB64)}`,
`pipeline_run=${shellQuote(pipelineRun)}`,
`rerun=${rerun ? "true" : "false"}`,
"manifest_path=\"/tmp/$pipeline_run.json\"",
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
"existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
"if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then",
" kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
"elif [ \"$existing_status\" = False ]; then",
" kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
"fi",
"if kubectl create -f \"$manifest_path\"; then",
" :",
"else",
" code=$?",
" if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then",
" printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2",
" else",
" exit \"$code\"",
" fi",
"fi",
"kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'",
].join("\n");
return runNodeK3sScript(spec, script, timeoutSeconds);
}
function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`);
}
function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60);
if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) };
const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u);
return {
exists: true,
name: pipelineRun,
status: status || null,
reason: reason || null,
message: message || null,
createdAt: createdAt || null,
result: compactRuntimeCommand(result),
};
}
function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
const pg = spec.externalPostgres;
if (pg === undefined) return null;
const secretRoot = externalPostgresSecretSourceRoot(spec);
const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey);
const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey);
const missing = [
...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]),
...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]),
];
if (missing.length > 0) {
if (dryRun) {
return {
ok: true,
mode: "dry-run",
mutation: false,
skipped: true,
reason: "external-postgres-secret-source-missing",
missing,
sourceRoot: secretRoot,
valuesPrinted: false,
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
};
}
return {
ok: false,
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
mutation: false,
degradedReason: "external-postgres-secret-source-missing",
missing,
sourceRoot: secretRoot,
valuesPrinted: false,
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
};
}
const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds);
if (!isCommandSuccess(setup)) {
return {
ok: false,
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
mutation: false,
phase: "namespace-authn-setup",
setup: compactRuntimeCommand(setup),
valuesPrinted: false,
};
}
const manifest = {
apiVersion: "v1",
kind: "List",
items: [
{
apiVersion: "v1",
kind: "Secret",
metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace },
type: "Opaque",
stringData: { [pg.cloudApi.secretKey]: cloudApi.value },
},
{
apiVersion: "v1",
kind: "Secret",
metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace },
type: "Opaque",
stringData: { [pg.openfga.secretKey]: openfga.value },
},
],
};
const applyScript = [
"set -eu",
[
"kubectl",
"apply",
"--server-side",
"--force-conflicts",
"--field-manager=unidesk-hwlab-node-external-postgres-secret",
...(dryRun ? ["--dry-run=server"] : []),
"-f",
"-",
].map(shellQuote).join(" "),
].join("\n");
const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest));
return {
ok: isCommandSuccess(apply),
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
mutation: !dryRun && isCommandSuccess(apply),
namespace: spec.runtimeNamespace,
sourceRoot: secretRoot,
secrets: [
{ name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey },
{ name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null },
],
setup: compactRuntimeCommand(setup),
apply: compactRuntimeCommand(apply),
valuesPrinted: false,
};
}
function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
if (spec.baseImageSource === undefined) return null;
const script = [
"set -eu",
`target=${shellQuote(spec.baseImage)}`,
`source=${shellQuote(spec.baseImageSource)}`,
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
"repo_tag=${target#*/}",
"repo=${repo_tag%:*}",
"tag=${repo_tag##*:}",
"if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
"registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
"present=false",
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi",
"action=none",
"if [ \"$present\" = false ]; then",
" action=seed",
" if [ \"$dry_run\" = false ]; then",
" docker image inspect \"$source\" >/dev/null",
" docker tag \"$source\" \"$target\"",
" docker push \"$target\" >/tmp/hwlab-node-base-image-push.out",
" fi",
"fi",
"after=false",
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi",
"printf 'target\\t%s\\n' \"$target\"",
"printf 'source\\t%s\\n' \"$source\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'presentBefore\\t%s\\n' \"$present\"",
"printf 'presentAfter\\t%s\\n' \"$after\"",
"printf 'action\\t%s\\n' \"$action\"",
"if [ \"$dry_run\" = true ] || [ \"$after\" = true ]; then exit 0; fi",
"exit 1",
].join("\n");
const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
const fields = keyValueLinesFromText(statusText(result));
return {
ok: isCommandSuccess(result),
dryRun,
target: fields.target ?? spec.baseImage,
source: fields.source ?? spec.baseImageSource,
presentBefore: fields.presentBefore === "true",
presentAfter: fields.presentAfter === "true",
action: fields.action ?? null,
result: compactRuntimeCommand(result),
};
}
function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
const pg = spec.externalPostgres;
if (pg === undefined) return "true";
const authnKey = pg.openfga.authnKey ?? "authn-preshared-key";
return [
"set -eu",
`namespace=${shellQuote(spec.runtimeNamespace)}`,
`secret=${shellQuote(pg.openfga.secretName)}`,
`authn_key=${shellQuote(authnKey)}`,
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
"namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"",
"if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi",
"kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -",
"authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')",
"if [ \"${authn_present:-0}\" -gt 0 ]; then",
" action=kept",
"elif [ \"$dry_run\" = true ]; then",
" action=would-generate-authn-key",
"else",
" if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi",
" kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null",
" authn_value=",
" action=generated-authn-key",
"fi",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$secret\"",
"printf 'authnKey\\t%s\\n' \"$authn_key\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'valuesPrinted\\tfalse\\n'",
].join("\n");
}
function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string {
const configRef = spec.externalPostgres?.configRef;
if (configRef === undefined) return join(repoRoot, ".state", "secrets");
const configPath = rootPath(configRef);
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown;
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`);
const secrets = (parsed as Record<string, unknown>).secrets;
if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`);
const root = (secrets as Record<string, unknown>).root;
if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`);
return root.startsWith("/") ? root : rootPath(root);
}
function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } {
const sourcePath = join(secretRoot, sourceRef);
if (!existsSync(sourcePath)) return { ok: false, sourcePath };
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
const value = values[key];
if (value === undefined || value.length === 0) return { ok: false, sourcePath };
return { ok: true, value, sourcePath };
}
function parseEnvFile(text: string): Record<string, string> {
const values: Record<string, string> = {};
for (const rawLine of text.split(/\r?\n/u)) {
const line = rawLine.trim();
if (line.length === 0 || line.startsWith("#")) continue;
const eq = line.indexOf("=");
if (eq <= 0) continue;
const key = line.slice(0, eq).trim();
const rawValue = line.slice(eq + 1).trim();
values[key] = rawValue.startsWith("'") && rawValue.endsWith("'")
? rawValue.slice(1, -1).replace(/'"'"'/gu, "'")
: rawValue.startsWith("\"") && rawValue.endsWith("\"")
? rawValue.slice(1, -1)
: rawValue;
}
return values;
}
function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean {
if (!/postgres|pgsql|pgdata/iu.test(name)) return false;
const externalServiceName = spec.externalPostgres?.serviceName;
if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false;
return true;
}
function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
const pg = spec.externalPostgres;
if (pg === undefined) return { required: false, ready: true };
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "name"], 60);
const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
const [address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u);
return {
required: true,
ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && address === pg.endpointAddress && Number(port) === pg.port,
serviceName: pg.serviceName,
endpointAddress: address || null,
expectedEndpointAddress: pg.endpointAddress,
port: numericField(port),
expectedPort: pg.port,
service: compactRuntimeCommand(service),
endpointSlice: compactRuntimeCommand(endpointSlice),
};
}
function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
const pg = spec.externalPostgres;
if (pg === undefined) return { required: false, ready: true };
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey);
const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey);
const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key");
return {
required: true,
ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present,
valuesPrinted: false,
cloudApi,
openfga: {
datastoreUri: openfgaUri,
authnPresharedKey: openfgaAuthn,
},
};
}
function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record<string, unknown> {
const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60);
return {
secret,
key,
present: result.exitCode === 0 && result.stdout.trim().length > 0,
valueBytesBase64: result.stdout.trim().length,
exitCode: result.exitCode,
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500),
valuesPrinted: false,
};
}
function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const commandArgs = [
"hwlab",
"nodes",
options.domain,
...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
"--node",
options.node,
"--lane",
options.lane,
"--confirm",
"--timeout-seconds",
String(options.timeoutSeconds),
"--wait",
];
const command = ["bun", "scripts/cli.ts", ...commandArgs];
const job = startJob(
`hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
command,
`Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
);
return {
ok: true,
command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
mode: "async-job",
reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
job,
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
waitCommand: command.join(" "),
};
}
function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const rewritten = rewriteDelegatedNodeValue(value, scoped);
const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
return {
...result,
node: scoped.node,
commandFamily: "hwlab nodes",
};
}
function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
if (typeof value !== "object" || value === null) return value;
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
}
function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
return text
.replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
.replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
};
return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
}
function parseSecretOptions(args: string[]): NodeSecretOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-vNN-bootstrap-admin|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]");
}
const node = requiredOption(args, "--node");
assertNodeId(node);
const lane = requiredOption(args, "--lane");
assertLane(lane);
const spec = runtimeSecretSpec({ node, lane });
const name = optionValue(args, "--name") ?? spec.openFgaSecret;
const key = optionValue(args, "--key");
const confirm = args.includes("--confirm");
const explicitDryRun = args.includes("--dry-run");
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
if (actionRaw === "cleanup-owned-postgres") {
if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`);
return {
action: actionRaw,
node,
lane,
name: spec.postgresSecret,
preset: "owned-postgres-cleanup",
confirm,
dryRun: explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
if (actionRaw === "cleanup-obsolete") {
if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup");
if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key");
if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`);
return {
action: actionRaw,
node,
lane,
name,
preset: "obsolete-secret-cleanup",
confirm,
dryRun: explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
if (name === spec.masterAdminApiKeySecret) {
if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
return {
action: actionRaw,
node,
lane,
name,
key: key ?? MASTER_ADMIN_API_KEY_KEY,
preset: "master-server-admin-api-key",
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
if (name === spec.bootstrapAdminSecret) {
if (key !== undefined && key !== BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY) throw new Error(`secret ${name} supports only key ${BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY}`);
return {
action: actionRaw,
node,
lane,
name,
key: key ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
preset: "bootstrap-admin",
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
if (name === spec.codeAgentProviderSecret) {
if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) {
throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`);
}
return {
action: actionRaw,
node,
lane,
name,
key,
preset: "code-agent-provider",
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
if (name === spec.cloudApiDbSecret) {
if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`);
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
}
return {
action: actionRaw,
node,
lane,
name,
key: key ?? spec.cloudApiDbKey,
preset: "cloud-api-db",
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
};
}
if (name !== spec.openFgaSecret) {
throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.bootstrapAdminSecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`);
}
if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
}
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
}
return {
action: actionRaw,
node,
lane,
name,
key,
preset: "openfga",
confirm,
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec {
const namespace = `hwlab-${input.lane}`;
const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined;
const externalPostgres = runtimeLaneSpec?.externalPostgres;
const platformDb = externalPostgres !== undefined || /^v0*[3-9]\d*$/.test(input.lane);
const platformPostgresService = externalPostgres?.serviceName ?? "g14-platform-postgres";
const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`;
const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`;
const platformPostgresEndpointSlice = `${platformPostgresService}-host`;
const openFgaDbName = externalPostgres?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga");
const openFgaDbUser = externalPostgres?.openfga.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga");
const cloudApiDbName = externalPostgres?.database ?? `hwlab_${input.lane}`;
const cloudApiDbUser = externalPostgres?.cloudApi.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`);
return {
node: input.node,
lane: input.lane,
namespace,
platformDb,
...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }),
...(externalPostgres === undefined ? {} : { externalPostgres }),
platformPostgresService,
platformPostgresEndpointAddress: externalPostgres?.endpointAddress,
platformPostgresEndpointSlice,
postgresSecret: `${namespace}-postgres`,
postgresStatefulSet: `${namespace}-postgres`,
postgresAdminUser: `hwlab_${input.lane}`,
openFgaSecret: externalPostgres?.openfga.secretName ?? `${namespace}-openfga`,
openFgaDbName,
openFgaDbUser,
openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`,
bootstrapAdminSecret: `${namespace}-bootstrap-admin`,
bootstrapAdminPasswordHashKey: BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE,
bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET,
cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? `hwlab-cloud-api-${input.lane}-db`,
cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? CLOUD_API_DB_KEY,
cloudApiDbName,
cloudApiDbUser,
cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
cloudApiDeployment: "hwlab-cloud-api",
obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`,
obsoleteHwpodDbName: `hwpod_${input.lane}`,
obsoleteHwpodDbUser: `hwpod_${input.lane}_app`,
codeAgentProviderSecret: `${namespace}-code-agent-provider`,
codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET,
fieldManager: `unidesk-hwlab-node-${input.lane}-secret`,
};
}
function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
const spec = runtimeSecretSpec(options);
if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec);
if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
return runExternalPostgresSecretEnsure(options, spec);
}
const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun
? readMasterAdminApiKey().key
: "";
const script = options.preset === "openfga"
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
: options.preset === "master-server-admin-api-key"
? masterAdminApiKeySecretScript(options, spec)
: options.preset === "bootstrap-admin"
? bootstrapAdminSecretScript(options, spec)
: options.preset === "cloud-api-db"
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
: options.preset === "owned-postgres-cleanup"
? ownedPostgresCleanupScript(options, spec)
: codeAgentProviderSecretScript(options, spec);
const result = runTransScript(options.node, script, input, options.timeoutSeconds);
const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true;
const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true;
return {
ok,
command: `hwlab nodes secret ${options.action}`,
node: options.node,
lane: options.lane,
namespace: spec.namespace,
secret: options.name,
key: options.key ?? null,
preset: options.preset,
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure",
status,
mutation: status.mutation === true,
result: compactCommandResult(result),
valuesRedacted: true,
next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec),
};
}
function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, string> {
if (options.action === "cleanup-owned-postgres") {
return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
}
if (options.action === "cleanup-obsolete") {
return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` };
}
if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
}
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
return {
status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
};
}
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
}
function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
const runtimeLaneSpec = spec.runtimeLaneSpec;
if (runtimeLaneSpec === undefined) {
return {
ok: false,
command: `hwlab nodes secret ${options.action}`,
node: options.node,
lane: options.lane,
namespace: spec.namespace,
secret: options.name,
key: options.key ?? null,
preset: options.preset,
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
mutation: false,
degradedReason: "external-postgres-runtime-lane-spec-missing",
valuesRedacted: true,
};
}
const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds);
const shouldReadStatus = sync !== null && sync.ok === true;
const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null;
const status = statusResult === null
? null
: secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec);
const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true);
return {
ok,
command: `hwlab nodes secret ${options.action}`,
node: options.node,
lane: options.lane,
namespace: spec.namespace,
secret: options.name,
key: options.key ?? null,
preset: options.preset,
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
status: status ?? sync,
externalPostgresSecretSync: sync,
mutation: sync?.mutation === true,
result: statusResult === null ? null : compactCommandResult(statusResult),
valuesRedacted: true,
next: ok ? undefined : nextSecretCommand(options, spec),
};
}
function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record<string, unknown> {
return {
mode: exposure.mode,
publicBaseUrl: exposure.publicBaseUrl,
hostname: exposure.hostname,
expectedA: exposure.expectedA,
frpc: {
serverAddr: exposure.serverAddr,
serverPort: exposure.serverPort,
tokenSourceRef: exposure.tokenSourceRef,
tokenSourceKey: exposure.tokenSourceKey,
secretName: exposure.secretName,
secretKey: exposure.secretKey,
tokenKey: exposure.tokenKey,
webProxy: exposure.webProxy,
apiProxy: exposure.apiProxy,
},
caddy: {
route: exposure.caddyRoute,
configPath: exposure.caddyConfigPath,
serviceName: exposure.caddyServiceName,
tls: exposure.caddyTls,
responseHeaderTimeoutSeconds: exposure.responseHeaderTimeoutSeconds,
},
};
}
function runNodePublicExposure(options: NodePublicExposureOptions): Record<string, unknown> {
const exposure = options.spec.publicExposure;
if (exposure === null || !exposure.enabled) {
return {
ok: false,
command: "hwlab nodes control-plane public-exposure",
node: options.node,
lane: options.lane,
configPath: hwlabRuntimeLaneConfigPath(),
error: "publicExposure is not configured for this node/lane target",
mutation: false,
};
}
const source = readPublicExposureTokenSource(exposure);
if (!source.ok) {
return {
ok: false,
command: "hwlab nodes control-plane public-exposure",
node: options.node,
lane: options.lane,
mode: options.dryRun ? "dry-run" : "confirmed",
configPath: hwlabRuntimeLaneConfigPath(),
publicExposure: publicExposureSummary(exposure),
source,
mutation: false,
valuesRedacted: true,
next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=<redacted>` },
};
}
const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds);
let secretFields = keyValueLinesFromText(statusText(secretApplyResult));
let secretResult = secretApplyResult;
if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") {
const restartResult = runPublicExposureFrpcRecreate(options);
secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) };
secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult);
}
const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds);
const caddyFields = keyValueLinesFromText(statusText(caddyResult));
const secretStatus = publicExposureSecretStatus(secretFields, secretResult);
const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult);
const ok = secretStatus.ok === true && caddyStatus.ok === true;
return {
ok,
command: "hwlab nodes control-plane public-exposure",
node: options.node,
lane: options.lane,
mode: options.dryRun ? "dry-run" : "confirmed",
mutation: !options.dryRun && (secretFields.mutation === "true" || caddyFields.mutation === "true"),
configPath: hwlabRuntimeLaneConfigPath(),
publicExposure: publicExposureSummary(exposure),
source: {
ok: source.ok,
path: source.path,
key: exposure.tokenSourceKey,
bytes: source.value?.length ?? 0,
fingerprint: source.fingerprint,
valueRedacted: true,
},
secret: secretStatus,
caddy: caddyStatus,
valuesRedacted: true,
next: ok
? {
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${options.node} --lane ${options.lane} --confirm`,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
}
: { retry: `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${options.node} --lane ${options.lane} --confirm` },
};
}
function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } {
const checkedPaths = publicExposureTokenSourcePaths(exposure);
const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef);
if (!existsSync(path)) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-source-missing" };
const values = parseEnvFile(readFileSync(path, "utf8"));
const value = values[exposure.tokenSourceKey];
if (value === undefined || value.length === 0) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-key-missing" };
return {
ok: true,
path,
checkedPaths,
key: exposure.tokenSourceKey,
value,
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
};
}
function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec): string[] {
const paths = [join(repoRoot, ".state", "secrets", exposure.tokenSourceRef)];
const marker = "/.worktree/";
const index = repoRoot.indexOf(marker);
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", exposure.tokenSourceRef));
return [...new Set(paths)];
}
function publicExposureSecretStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
const dryRun = fields.dryRun === "true";
return {
ok: result.exitCode === 0 && (dryRun || (
fields.afterSecretExists === "yes"
&& fields.strategyPatchExitCode === "0"
&& fields.rolloutRestartExitCode === "0"
&& fields.rolloutStatusExitCode === "0"
)),
dryRun,
mutation: fields.mutation === "true",
namespace: fields.namespace || null,
secret: fields.secret || null,
deployment: fields.deployment || null,
beforeExists: fields.beforeSecretExists === "yes",
afterExists: fields.afterSecretExists === "yes",
tokenBytes: numericField(fields.tokenBytes),
applyExitCode: numericField(fields.applyExitCode),
restartMode: fields.restartMode || null,
strategyPatchExitCode: numericField(fields.strategyPatchExitCode),
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
scaleDownExitCode: numericField(fields.scaleDownExitCode),
scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode),
scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount),
scaleUpExitCode: numericField(fields.scaleUpExitCode),
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
readyReplicas: numericField(fields.readyReplicas),
availableReplicas: numericField(fields.availableReplicas),
desiredReplicas: numericField(fields.desiredReplicas),
strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null,
restartErrorPreview: fields.restartErrorPreview || null,
scaleDownErrorPreview: fields.scaleDownErrorPreview || null,
scaleUpErrorPreview: fields.scaleUpErrorPreview || null,
readyErrorPreview: fields.readyErrorPreview || null,
exitCode: result.exitCode,
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
};
}
function publicExposureCaddyStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
const dryRun = fields.dryRun === "true";
return {
ok: result.exitCode === 0
&& fields.afterBlockPresent === "yes"
&& fields.validateExitCode === "0"
&& (dryRun || (
fields.active === "active"
&& fields.localWebStatus === "200"
&& fields.localApiStatus === "200"
)),
dryRun,
mutation: fields.mutation === "true",
hostname: fields.hostname || null,
configPath: fields.configPath || null,
beforeBlockPresent: fields.beforeBlockPresent === "yes",
afterBlockPresent: fields.afterBlockPresent === "yes",
staleBlocksRemoved: numericField(fields.staleBlocksRemoved),
pythonExitCode: numericField(fields.pythonExitCode),
validateMode: fields.validateMode || null,
validateExitCode: numericField(fields.validateExitCode),
reloadExitCode: numericField(fields.reloadExitCode),
active: fields.active || null,
localWebStatus: fields.localWebStatus || null,
localApiStatus: fields.localApiStatus || null,
validateErrorPreview: fields.validateErrorPreview || null,
localWebErrorPreview: fields.localWebErrorPreview || null,
localApiErrorPreview: fields.localApiErrorPreview || null,
exitCode: result.exitCode,
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
};
}
function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
const deployment = `${options.spec.runtimeNamespace}-frpc`;
return [
"set +e",
`namespace=${shellQuote(options.spec.runtimeNamespace)}`,
`secret=${shellQuote(exposure.secretName)}`,
`deployment=${shellQuote(deployment)}`,
`token_key=${shellQuote(exposure.tokenKey)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"token=$(cat)",
"before_secret_exists=no",
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes",
"token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')",
"apply_exit=",
"restart_mode=recreate-rollout-restart",
"strategy_patch_exit=",
"rollout_restart_exit=",
"scale_down_exit=",
"scale_down_wait_exit=",
"scale_down_wait_pod_count=",
"scale_up_exit=",
"rollout_status_exit=",
"ready_replicas=",
"available_replicas=",
"desired_replicas=",
"mutation=false",
"if [ \"$dry_run\" = false ]; then",
" tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)",
" token_b64=$(printf '%s' \"$token\" | base64 | tr -d '\\n')",
" cat >\"$tmp\" <<EOF",
"apiVersion: v1",
"kind: Secret",
"metadata:",
" name: $secret",
" namespace: $namespace",
" labels:",
" app.kubernetes.io/part-of: hwlab",
" app.kubernetes.io/managed-by: unidesk",
"type: Opaque",
"data:",
" $token_key: $token_b64",
"EOF",
" kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-public-exposure -f \"$tmp\" >/tmp/hwlab-public-frpc-secret.apply.out 2>/tmp/hwlab-public-frpc-secret.apply.err",
" apply_exit=$?",
" rm -f \"$tmp\"",
" if [ \"$apply_exit\" = 0 ]; then mutation=true; fi",
"fi",
"after_secret_exists=no",
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$secret\"",
"printf 'deployment\\t%s\\n' \"$deployment\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
"printf 'tokenBytes\\t%s\\n' \"$token_bytes\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"printf 'restartMode\\t%s\\n' \"$restart_mode\"",
"printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"",
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
"printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"",
"printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"",
"printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"",
"printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"",
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
"printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"",
"printf 'availableReplicas\\t%s\\n' \"$available_replicas\"",
"printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"",
"if [ \"$dry_run\" = true ]; then exit 0; fi",
"[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]",
].join("\n");
}
function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult {
const namespace = options.spec.runtimeNamespace;
const deployment = `${namespace}-frpc`;
const fields: Record<string, string> = {
deployment,
restartMode: "recreate-rollout-restart",
strategyPatchExitCode: "",
rolloutRestartExitCode: "",
scaleDownExitCode: "",
scaleDownWaitExitCode: "",
scaleDownWaitPodCount: "",
scaleUpExitCode: "",
rolloutStatusExitCode: "",
readyReplicas: "",
availableReplicas: "",
desiredReplicas: "",
strategyPatchErrorPreview: "",
restartErrorPreview: "",
scaleDownErrorPreview: "",
scaleUpErrorPreview: "",
readyErrorPreview: "",
};
const stdoutParts: string[] = [];
const stderrParts: string[] = [];
const addResult = (result: CommandResult): void => {
if (result.stdout.trim()) stdoutParts.push(result.stdout.trim());
if (result.stderr.trim()) stderrParts.push(result.stderr.trim());
};
const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55);
const strategyPatch = runTransScript(options.node, [
"set +e",
`namespace=${shellQuote(namespace)}`,
`deployment=${shellQuote(deployment)}`,
"kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err",
"code=$?",
"printf 'strategyPatchExitCode\\t%s\\n' \"$code\"",
"printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-strategy-patch.err | cut -c1-1000 || true)\"",
"exit \"$code\"",
].join("\n"), "", shortTimeout);
addResult(strategyPatch);
Object.assign(fields, keyValueLinesFromText(statusText(strategyPatch)));
if (strategyPatch.exitCode === 0) {
const restart = runTransScript(options.node, [
"set +e",
`namespace=${shellQuote(namespace)}`,
`deployment=${shellQuote(deployment)}`,
"kubectl -n \"$namespace\" rollout restart deploy/\"$deployment\" >/tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err",
"code=$?",
"printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"",
"printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-restart.err | cut -c1-1000 || true)\"",
"exit \"$code\"",
].join("\n"), "", shortTimeout);
addResult(restart);
Object.assign(fields, keyValueLinesFromText(statusText(restart)));
}
if (fields.rolloutRestartExitCode === "0") {
const deadline = Date.now() + 55_000;
while (Date.now() <= deadline) {
const probe = runTransScript(options.node, [
"set +e",
`namespace=${shellQuote(namespace)}`,
`deployment=${shellQuote(deployment)}`,
"kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.replicas}{\"\\t\"}{.status.readyReplicas}{\"\\t\"}{.status.availableReplicas}' >/tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err",
"code=$?",
"values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)",
"desired=$(printf '%s' \"$values\" | cut -f1)",
"ready=$(printf '%s' \"$values\" | cut -f2)",
"available=$(printf '%s' \"$values\" | cut -f3)",
"printf 'desiredReplicas\\t%s\\n' \"$desired\"",
"printf 'readyReplicas\\t%s\\n' \"$ready\"",
"printf 'availableReplicas\\t%s\\n' \"$available\"",
"printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-ready.err | cut -c1-1000 || true)\"",
"[ \"$code\" = 0 ] && [ \"$desired\" = 1 ] && [ \"$ready\" = 1 ] && [ \"$available\" = 1 ]",
].join("\n"), "", shortTimeout);
addResult(probe);
Object.assign(fields, keyValueLinesFromText(statusText(probe)));
if (probe.exitCode === 0) {
fields.rolloutStatusExitCode = "0";
break;
}
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
}
}
if (fields.rolloutStatusExitCode === "") fields.rolloutStatusExitCode = "1";
const ok = fields.strategyPatchExitCode === "0"
&& fields.rolloutRestartExitCode === "0"
&& fields.rolloutStatusExitCode === "0";
const stdout = Object.entries(fields).map(([key, value]) => `${key}\t${value}`).join("\n") + "\n";
return {
command: [transPath(), `${options.node}:k3s`, "script", "--", "<public-exposure-frpc-recreate>"],
cwd: repoRoot,
exitCode: ok ? 0 : 1,
stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n",
stderr: stderrParts.join("\n"),
signal: null,
timedOut: false,
};
}
function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult {
return {
command: [...first.command, "&&", ...second.command],
cwd: repoRoot,
exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode,
stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n",
stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"),
signal: second.signal ?? first.signal,
timedOut: first.timedOut || second.timedOut,
};
}
function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64");
const marker = `unidesk managed ${exposure.hostname}`;
return [
"set +e",
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`hostname=${shellQuote(exposure.hostname)}`,
`config_path=${shellQuote(exposure.caddyConfigPath)}`,
`service=${shellQuote(exposure.caddyServiceName)}`,
`marker=${shellQuote(marker)}`,
`block_b64=${shellQuote(blockB64)}`,
`expected_a=${shellQuote(exposure.expectedA)}`,
`web_port=${shellQuote(String(exposure.webProxy.remotePort))}`,
`api_port=${shellQuote(String(exposure.apiProxy.remotePort))}`,
"rm -f /tmp/hwlab-public-caddy-validate.out /tmp/hwlab-public-caddy-validate.err /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-web.err /tmp/hwlab-public-caddy-api.err",
"tmp=$(mktemp -d)",
"block=\"$tmp/block\"",
"next=\"$tmp/Caddyfile\"",
"printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
"before_present=no",
"if [ -f \"$config_path\" ] && grep -Fq \"# BEGIN $marker\" \"$config_path\"; then before_present=yes; fi",
"if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
"stale_blocks_removed=$(python3 - \"$next\" \"$block\" \"$marker\" \"$web_port\" \"$api_port\" 2>/tmp/hwlab-public-caddy-python.err <<'PY'",
"import pathlib, re, sys",
"config = pathlib.Path(sys.argv[1])",
"block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
"marker = sys.argv[3]",
"web_port = sys.argv[4]",
"api_port = sys.argv[5]",
"current_name = marker[len('unidesk managed '):] if marker.startswith('unidesk managed ') else marker",
"text = config.read_text(encoding='utf-8') if config.exists() else ''",
"begin = f'# BEGIN {marker}'",
"end = f'# END {marker}'",
"managed = f'{begin}\\n{block.rstrip()}\\n{end}\\n'",
"stale_removed = [0]",
"pattern = re.compile(r'(?ms)^# BEGIN unidesk managed (?P<name>[^\\n]+)\\n(?P<body>.*?)\\n# END unidesk managed (?P=name)\\n*')",
"def keep(match):",
" name = match.group('name')",
" body = match.group('body')",
" if name != current_name and (f'127.0.0.1:{web_port}' in body or f'127.0.0.1:{api_port}' in body):",
" stale_removed[0] += 1",
" return ''",
" return match.group(0)",
"text = pattern.sub(keep, text)",
"if begin in text and end in text:",
" start = text.index(begin)",
" stop = text.index(end, start) + len(end)",
" text = text[:start].rstrip() + '\\n\\n' + managed.rstrip() + '\\n' + text[stop:].lstrip('\\n')",
"else:",
" text = text.rstrip() + '\\n\\n' + managed",
"config.write_text(text, encoding='utf-8')",
"print(stale_removed[0])",
"PY",
")",
"python_exit=$?",
"validate_exit=1",
"validate_mode=validate",
"if [ \"$python_exit\" != 0 ]; then",
" validate_mode=python",
" validate_exit=$python_exit",
"else",
" if [ \"$dry_run\" = true ]; then",
" validate_mode=adapt",
" caddy adapt --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
" else",
" sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
" fi",
" validate_exit=$?",
"fi",
"reload_exit=",
"mutation=false",
"if [ \"$dry_run\" = false ] && [ \"$validate_exit\" = 0 ]; then",
" sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/hwlab-public-caddy-install.out 2>/tmp/hwlab-public-caddy-install.err",
" install_exit=$?",
" if [ \"$install_exit\" = 0 ]; then",
" mutation=true",
" sudo systemctl reload \"$service\" >/tmp/hwlab-public-caddy-reload.out 2>/tmp/hwlab-public-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/hwlab-public-caddy-reload.out 2>>/tmp/hwlab-public-caddy-reload.err",
" reload_exit=$?",
" else",
" reload_exit=$install_exit",
" fi",
"fi",
"active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
"after_present=no",
"if [ \"$dry_run\" = true ]; then grep -Fq \"# BEGIN $marker\" \"$next\" && after_present=yes; else grep -Fq \"# BEGIN $marker\" \"$config_path\" && after_present=yes; fi",
"local_web_status=",
"local_api_status=",
"if [ \"$dry_run\" = false ]; then",
" local_web_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-web.err || true)",
" local_api_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/health/live\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-api.err || true)",
"fi",
"validate_err_preview=$(cat /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-validate.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
"local_web_err_preview=$([ -f /tmp/hwlab-public-caddy-web.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-web.err 2>/dev/null | cut -c1-1000 || true)",
"local_api_err_preview=$([ -f /tmp/hwlab-public-caddy-api.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-api.err 2>/dev/null | cut -c1-1000 || true)",
"printf 'hostname\\t%s\\n' \"$hostname\"",
"printf 'expectedA\\t%s\\n' \"$expected_a\"",
"printf 'configPath\\t%s\\n' \"$config_path\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeBlockPresent\\t%s\\n' \"$before_present\"",
"printf 'afterBlockPresent\\t%s\\n' \"$after_present\"",
"printf 'staleBlocksRemoved\\t%s\\n' \"$stale_blocks_removed\"",
"printf 'pythonExitCode\\t%s\\n' \"$python_exit\"",
"printf 'validateMode\\t%s\\n' \"$validate_mode\"",
"printf 'validateExitCode\\t%s\\n' \"$validate_exit\"",
"printf 'reloadExitCode\\t%s\\n' \"$reload_exit\"",
"printf 'active\\t%s\\n' \"$active\"",
"printf 'localWebStatus\\t%s\\n' \"$local_web_status\"",
"printf 'localApiStatus\\t%s\\n' \"$local_api_status\"",
"printf 'validateErrorPreview\\t%s\\n' \"$validate_err_preview\"",
"printf 'localWebErrorPreview\\t%s\\n' \"$local_web_err_preview\"",
"printf 'localApiErrorPreview\\t%s\\n' \"$local_api_err_preview\"",
"rm -rf \"$tmp\"",
"[ \"$validate_exit\" = 0 ] && [ \"$after_present\" = yes ]",
].join("\n");
}
function publicExposureCaddyBlock(exposure: HwlabRuntimePublicExposureSpec): string {
const tlsLines = exposure.caddyTls === "internal" ? " tls internal\n" : "";
return `${exposure.hostname} {
${tlsLines} @api path /health* /api* /v1* /openapi* /docs* /swagger*
reverse_proxy @api 127.0.0.1:${exposure.apiProxy.remotePort} {
transport http {
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
}
}
reverse_proxy 127.0.0.1:${exposure.webProxy.remotePort} {
transport http {
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
}
}
}`;
}
function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
return runCommand([transPath(), `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}
function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
return runCommand([transPath(), node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}
function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds);
const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec);
const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true };
const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds);
const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec);
const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false);
const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true;
return {
ok,
command: `hwlab nodes secret ${options.action}`,
node: options.node,
lane: options.lane,
namespace: spec.namespace,
secret: options.name,
key: null,
preset: options.preset,
mode: options.dryRun ? "dry-run" : "confirmed-delete",
status: {
ok,
preset: "obsolete-hwpod-db-cleanup",
dryRun: options.dryRun,
mutation,
kubernetesSecret: kubernetesStatus,
platformDatabase: platformDbStatus,
hostMutationSkipped: !options.dryRun && hostOptions.dryRun,
summary: ok
? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove`
: `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`,
},
mutation,
result: {
kubernetesSecret: compactCommandResult(kubernetesResult),
platformDatabase: compactCommandResult(platformDbResult),
},
valuesRedacted: true,
next: ok ? undefined : nextSecretCommand(options, spec),
};
}
function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm");
const dryRun = options.dryRun || !options.confirm;
const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds);
const fields = keyValueLinesFromText(statusText(result));
const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes";
const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes";
const afterExcluded = fields.afterEndpointResourcesExcluded === "yes";
const afterIgnored = fields.afterEndpointsIgnoreUpdates === "yes" || fields.afterEndpointSliceIgnoreUpdates === "yes";
const beforeExtraEndpointSlices = splitWhitespaceField(fields.beforeExtraEndpointSliceNames);
const afterExtraEndpointSlices = splitWhitespaceField(fields.afterExtraEndpointSliceNames);
const beforeLegacyEndpoints = fields.beforeLegacyEndpointsExists === "yes";
const afterLegacyEndpoints = fields.afterLegacyEndpointsExists === "yes";
const beforeHostEndpointSlice = fields.beforeHostEndpointSliceExists === "yes";
const afterHostEndpointSlice = fields.afterHostEndpointSliceExists === "yes";
const bridgeReady = !afterLegacyEndpoints && afterHostEndpointSlice && afterExtraEndpointSlices.length === 0;
const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored && bridgeReady;
return {
ok: dryRun ? result.exitCode === 0 : ok,
command: "hwlab nodes control-plane allow-endpoint-bridge",
node: options.node,
lane: options.lane,
namespace: "argocd",
application: fields.application || `hwlab-node-${options.lane}`,
mode: dryRun ? "dry-run" : "confirmed-control-plane-update",
status: {
action: fields.action || null,
dryRun,
mutation: fields.mutation === "true",
before: {
endpointResourcesExcluded: beforeExcluded,
endpointsIgnoreUpdates: fields.beforeEndpointsIgnoreUpdates === "yes",
endpointSliceIgnoreUpdates: fields.beforeEndpointSliceIgnoreUpdates === "yes",
legacyEndpointsExist: beforeLegacyEndpoints,
hostEndpointSliceExists: beforeHostEndpointSlice,
extraEndpointSlices: beforeExtraEndpointSlices,
},
after: {
endpointResourcesExcluded: afterExcluded,
endpointsIgnoreUpdates: fields.afterEndpointsIgnoreUpdates === "yes",
endpointSliceIgnoreUpdates: fields.afterEndpointSliceIgnoreUpdates === "yes",
legacyEndpointsExist: afterLegacyEndpoints,
hostEndpointSliceExists: afterHostEndpointSlice,
extraEndpointSlices: afterExtraEndpointSlices,
},
runtimeNamespace: fields.runtimeNamespace || `hwlab-${options.lane}`,
platformService: fields.platformService || "g14-platform-postgres",
hostEndpointSlice: fields.hostEndpointSlice || "g14-platform-postgres-host",
patchExitCode: numericField(fields.patchExitCode),
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
deleteLegacyEndpointsExitCode: numericField(fields.deleteLegacyEndpointsExitCode),
deleteExtraEndpointSlicesExitCode: numericField(fields.deleteExtraEndpointSlicesExitCode),
refreshExitCode: numericField(fields.refreshExitCode),
exitCode: result.exitCode,
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
summary: !afterExcluded && !afterIgnored && bridgeReady
? "Argo tracks HWLAB external Postgres EndpointSlice and no legacy Endpoints remain"
: "Argo endpoint bridge is not in final Service plus EndpointSlice shape",
},
result: compactCommandResult(result),
};
}
function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string {
const application = `hwlab-node-${options.lane}`;
const runtimeNamespace = `hwlab-${options.lane}`;
return [
"set +e",
"namespace=argocd",
`runtime_namespace=${shellQuote(runtimeNamespace)}`,
"configmap=argocd-cm",
`application=${shellQuote(application)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"platform_service=g14-platform-postgres",
"host_endpointslice=g14-platform-postgres-host",
"preset=endpoint-bridge-resource-tracking",
"cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
"cm_has_key() { value=$(cm_data \"$1\"); [ -n \"$value\" ] && [ \"$value\" != \"<no value>\" ] && printf yes || printf no; }",
"endpoint_resources_excluded() { exclusions=$(cm_data resource.exclusions); printf '%s' \"$exclusions\" | grep -Eq '(^|[[:space:]])(Endpoints|EndpointSlice)([[:space:]]|$)' && printf yes || printf no; }",
"resource_exists() { kubectl -n \"$runtime_namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
"extra_endpoint_slices() { kubectl -n \"$runtime_namespace\" get endpointslice -l \"kubernetes.io/service-name=$platform_service\" -o name 2>/dev/null | sed \"/\\/$host_endpointslice$/d\" | tr '\\n' ' ' | sed 's/[[:space:]]*$//'; }",
"wait_runtime_bridge_clean() {",
" for _ in $(seq 1 30); do",
" current_legacy=$(resource_exists endpoints \"$platform_service\")",
" current_extra=$(extra_endpoint_slices)",
" current_host=$(resource_exists endpointslice \"$host_endpointslice\")",
" if [ \"$current_legacy\" != yes ] && [ -z \"$current_extra\" ] && [ \"$current_host\" = yes ]; then return 0; fi",
" sleep 2",
" done",
" return 1",
"}",
"before_endpoint_resources_excluded=$(endpoint_resources_excluded)",
"before_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')",
"before_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')",
"before_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")",
"before_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")",
"before_extra_endpoint_slice_names=$(extra_endpoint_slices)",
"needs_argo_update=false",
"if [ \"$before_endpoint_resources_excluded\" = yes ] || [ \"$before_endpoints_ignore_updates\" = yes ] || [ \"$before_endpoint_slice_ignore_updates\" = yes ]; then needs_argo_update=true; fi",
"needs_runtime_cleanup=false",
"if [ \"$before_legacy_endpoints_exists\" = yes ] || [ -n \"$before_extra_endpoint_slice_names\" ]; then needs_runtime_cleanup=true; fi",
"action=observed",
"mutation=false",
"patch_exit=",
"rollout_restart_exit=",
"rollout_status_exit=",
"delete_legacy_endpoints_exit=",
"delete_extra_endpointslices_exit=",
"refresh_exit=",
"if [ \"$dry_run\" = true ]; then",
" if [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-old-endpoint-exclusions-and-legacy-endpoints",
" elif [ \"$needs_argo_update\" = true ]; then action=would-remove-old-endpoint-exclusions",
" elif [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-legacy-endpoints",
" else action=kept; fi",
"else",
" if [ \"$needs_argo_update\" = true ]; then",
" patch_file=$(mktemp /tmp/hwlab-argocd-endpoint-bridge.XXXXXX.json)",
" python3 - <<'PY' >\"$patch_file\"",
"import json",
"desired = '''### Internal Kubernetes resources excluded to reduce watch volume",
"- apiGroups:",
" - coordination.k8s.io",
" kinds:",
" - Lease",
"### Internal Kubernetes Authz/Authn resources excluded to reduce watched events",
"- apiGroups:",
" - authentication.k8s.io",
" - authorization.k8s.io",
" kinds:",
" - SelfSubjectReview",
" - TokenReview",
" - LocalSubjectAccessReview",
" - SelfSubjectAccessReview",
" - SelfSubjectRulesReview",
" - SubjectAccessReview",
"### Intermediate Certificate Request excluded to reduce watched events",
"- apiGroups:",
" - certificates.k8s.io",
" kinds:",
" - CertificateSigningRequest",
"- apiGroups:",
" - cert-manager.io",
" kinds:",
" - CertificateRequest",
"### Cilium internal resources excluded to reduce UI clutter",
"- apiGroups:",
" - cilium.io",
" kinds:",
" - CiliumIdentity",
" - CiliumEndpoint",
" - CiliumEndpointSlice",
"### Kyverno intermediate and reporting resources excluded to reduce watched events",
"- apiGroups:",
" - kyverno.io",
" - reports.kyverno.io",
" - wgpolicyk8s.io",
" kinds:",
" - PolicyReport",
" - ClusterPolicyReport",
" - EphemeralReport",
" - ClusterEphemeralReport",
" - AdmissionReport",
" - ClusterAdmissionReport",
" - BackgroundScanReport",
" - ClusterBackgroundScanReport",
" - UpdateRequest",
"'''",
"print(json.dumps({",
" 'data': {",
" 'resource.exclusions': desired,",
" 'resource.customizations.ignoreResourceUpdates.Endpoints': None,",
" 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice': None,",
" }",
"}))",
"PY",
" kubectl -n \"$namespace\" patch configmap \"$configmap\" --type merge --patch-file \"$patch_file\" >/tmp/hwlab-argocd-endpoint-bridge-patch.out 2>/tmp/hwlab-argocd-endpoint-bridge-patch.err",
" patch_exit=$?",
" rm -f \"$patch_file\"",
" if [ \"$patch_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout restart statefulset/argocd-application-controller >/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.err",
" rollout_restart_exit=$?",
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout status statefulset/argocd-application-controller --timeout=180s >/tmp/hwlab-argocd-endpoint-bridge-rollout-status.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-status.err",
" rollout_status_exit=$?",
" fi",
" fi",
" fi",
" if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then action=patch-failed",
" elif [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
" else",
" if [ \"$needs_runtime_cleanup\" = true ]; then",
" kubectl -n \"$runtime_namespace\" delete endpoints \"$platform_service\" --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpoints-delete.out 2>/tmp/hwlab-platform-postgres-endpoints-delete.err",
" delete_legacy_endpoints_exit=$?",
" wait_runtime_bridge_clean",
" remaining_extra=$(extra_endpoint_slices)",
" if [ -n \"$remaining_extra\" ]; then",
" kubectl -n \"$runtime_namespace\" delete $remaining_extra --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpointslices-delete.out 2>/tmp/hwlab-platform-postgres-endpointslices-delete.err",
" delete_extra_endpointslices_exit=$?",
" wait_runtime_bridge_clean",
" fi",
" fi",
" if [ \"$needs_argo_update\" = true ] || [ \"$needs_runtime_cleanup\" = true ]; then",
" kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/hwlab-argocd-endpoint-bridge-refresh.out 2>/tmp/hwlab-argocd-endpoint-bridge-refresh.err",
" refresh_exit=$?",
" fi",
" if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then action=delete-legacy-endpoints-failed",
" elif [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then action=delete-extra-endpointslices-failed",
" elif [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then action=refresh-failed",
" elif [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=removed-old-endpoint-exclusions-and-legacy-endpoints; mutation=true",
" elif [ \"$needs_argo_update\" = true ]; then action=removed-old-endpoint-exclusions; mutation=true",
" elif [ \"$needs_runtime_cleanup\" = true ]; then action=removed-legacy-endpoints; mutation=true",
" else action=kept; fi",
" fi",
"fi",
"after_endpoint_resources_excluded=$(endpoint_resources_excluded)",
"after_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')",
"after_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')",
"after_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")",
"after_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")",
"after_extra_endpoint_slice_names=$(extra_endpoint_slices)",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"",
"printf 'configMap\\t%s\\n' \"$configmap\"",
"printf 'application\\t%s\\n' \"$application\"",
"printf 'platformService\\t%s\\n' \"$platform_service\"",
"printf 'hostEndpointSlice\\t%s\\n' \"$host_endpointslice\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeEndpointResourcesExcluded\\t%s\\n' \"$before_endpoint_resources_excluded\"",
"printf 'beforeEndpointsIgnoreUpdates\\t%s\\n' \"$before_endpoints_ignore_updates\"",
"printf 'beforeEndpointSliceIgnoreUpdates\\t%s\\n' \"$before_endpoint_slice_ignore_updates\"",
"printf 'beforeLegacyEndpointsExists\\t%s\\n' \"$before_legacy_endpoints_exists\"",
"printf 'beforeHostEndpointSliceExists\\t%s\\n' \"$before_host_endpointslice_exists\"",
"printf 'beforeExtraEndpointSliceNames\\t%s\\n' \"$before_extra_endpoint_slice_names\"",
"printf 'afterEndpointResourcesExcluded\\t%s\\n' \"$after_endpoint_resources_excluded\"",
"printf 'afterEndpointsIgnoreUpdates\\t%s\\n' \"$after_endpoints_ignore_updates\"",
"printf 'afterEndpointSliceIgnoreUpdates\\t%s\\n' \"$after_endpoint_slice_ignore_updates\"",
"printf 'afterLegacyEndpointsExists\\t%s\\n' \"$after_legacy_endpoints_exists\"",
"printf 'afterHostEndpointSliceExists\\t%s\\n' \"$after_host_endpointslice_exists\"",
"printf 'afterExtraEndpointSliceNames\\t%s\\n' \"$after_extra_endpoint_slice_names\"",
"printf 'patchExitCode\\t%s\\n' \"$patch_exit\"",
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
"printf 'deleteLegacyEndpointsExitCode\\t%s\\n' \"$delete_legacy_endpoints_exit\"",
"printf 'deleteExtraEndpointSlicesExitCode\\t%s\\n' \"$delete_extra_endpointslices_exit\"",
"printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"",
"if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi",
"if [ \"$dry_run\" != true ] && { [ \"$after_legacy_endpoints_exists\" = yes ] || [ -n \"$after_extra_endpoint_slice_names\" ] || [ \"$after_host_endpointslice_exists\" != yes ]; }; then exit 47; fi",
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
"if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then exit \"$delete_legacy_endpoints_exit\"; fi",
"if [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then exit \"$delete_extra_endpointslices_exit\"; fi",
"if [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then exit \"$refresh_exit\"; fi",
].join("\n");
}
function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
const pvc = `data-${spec.postgresSecret}-0`;
const platformService = spec.platformPostgresService;
const postgresService = spec.postgresSecret;
const postgresConfigMap = `${spec.postgresSecret}-init`;
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
`postgres_service=${shellQuote(postgresService)}`,
`postgres_configmap=${shellQuote(postgresConfigMap)}`,
`pvc=${shellQuote(pvc)}`,
`platform_service=${shellQuote(platformService)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"preset=owned-postgres-cleanup",
"exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }",
"pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }",
"phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }",
"before_secret_exists=$(exists_flag secret \"$postgres_secret\")",
"before_pvc_exists=$(exists_flag pvc \"$pvc\")",
"before_pvc_phase=$(phase_of_pvc)",
"before_pv=$(pv_name)",
"before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
"before_service_exists=$(exists_flag service \"$postgres_service\")",
"before_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
"platform_service_exists=$(exists_flag service \"$platform_service\")",
"action=observed",
"mutation=false",
"delete_statefulset_exit=",
"delete_service_exit=",
"delete_configmap_exit=",
"delete_secret_exit=",
"delete_pvc_exit=",
"before_any_owned=false",
"for flag in \"$before_statefulset_exists\" \"$before_service_exists\" \"$before_configmap_exists\" \"$before_secret_exists\" \"$before_pvc_exists\"; do",
" if [ \"$flag\" = yes ]; then before_any_owned=true; fi",
"done",
"if [ \"$dry_run\" = true ]; then",
" if [ \"$before_any_owned\" = true ]; then action=would-delete; else action=already-absent; fi",
"else",
" kubectl -n \"$namespace\" delete statefulset \"$postgres_statefulset\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-statefulset-delete.out 2>/tmp/hwlab-owned-postgres-statefulset-delete.err",
" delete_statefulset_exit=$?",
" kubectl -n \"$namespace\" delete service \"$postgres_service\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-service-delete.out 2>/tmp/hwlab-owned-postgres-service-delete.err",
" delete_service_exit=$?",
" kubectl -n \"$namespace\" delete configmap \"$postgres_configmap\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-configmap-delete.out 2>/tmp/hwlab-owned-postgres-configmap-delete.err",
" delete_configmap_exit=$?",
" kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err",
" delete_secret_exit=$?",
" kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err",
" delete_pvc_exit=$?",
" for _ in $(seq 1 30); do",
" current_statefulset=$(exists_flag statefulset \"$postgres_statefulset\")",
" current_service=$(exists_flag service \"$postgres_service\")",
" current_configmap=$(exists_flag configmap \"$postgres_configmap\")",
" current_secret=$(exists_flag secret \"$postgres_secret\")",
" current_pvc=$(exists_flag pvc \"$pvc\")",
" if [ \"$current_statefulset\" != yes ] && [ \"$current_service\" != yes ] && [ \"$current_configmap\" != yes ] && [ \"$current_secret\" != yes ] && [ \"$current_pvc\" != yes ]; then break; fi",
" sleep 2",
" done",
" if [ \"$delete_statefulset_exit\" -eq 0 ] && [ \"$delete_service_exit\" -eq 0 ] && [ \"$delete_configmap_exit\" -eq 0 ] && [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then",
" if [ \"$before_any_owned\" = true ]; then action=deleted; mutation=true; else action=already-absent; fi",
" else",
" action=delete-failed",
" fi",
"fi",
"after_secret_exists=$(exists_flag secret \"$postgres_secret\")",
"after_pvc_exists=$(exists_flag pvc \"$pvc\")",
"after_pvc_phase=$(phase_of_pvc)",
"after_pv=$(pv_name)",
"after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
"after_service_exists=$(exists_flag service \"$postgres_service\")",
"after_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$postgres_secret\"",
"printf 'statefulSet\\t%s\\n' \"$postgres_statefulset\"",
"printf 'service\\t%s\\n' \"$postgres_service\"",
"printf 'configMap\\t%s\\n' \"$postgres_configmap\"",
"printf 'pvc\\t%s\\n' \"$pvc\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
"printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"",
"printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"",
"printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"",
"printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"",
"printf 'beforeServiceExists\\t%s\\n' \"$before_service_exists\"",
"printf 'beforeConfigMapExists\\t%s\\n' \"$before_configmap_exists\"",
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
"printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"",
"printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"",
"printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"",
"printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"",
"printf 'afterServiceExists\\t%s\\n' \"$after_service_exists\"",
"printf 'afterConfigMapExists\\t%s\\n' \"$after_configmap_exists\"",
"printf 'deleteStatefulSetExitCode\\t%s\\n' \"$delete_statefulset_exit\"",
"printf 'deleteServiceExitCode\\t%s\\n' \"$delete_service_exit\"",
"printf 'deleteConfigMapExitCode\\t%s\\n' \"$delete_configmap_exit\"",
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
"printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"",
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
"if [ \"$after_statefulset_exists\" = yes ] || [ \"$after_service_exists\" = yes ] || [ \"$after_configmap_exists\" = yes ] || [ \"$after_secret_exists\" = yes ] || [ \"$after_pvc_exists\" = yes ]; then exit 45; fi",
"if [ -n \"$delete_statefulset_exit\" ] && [ \"$delete_statefulset_exit\" != 0 ]; then exit \"$delete_statefulset_exit\"; fi",
"if [ -n \"$delete_service_exit\" ] && [ \"$delete_service_exit\" != 0 ]; then exit \"$delete_service_exit\"; fi",
"if [ -n \"$delete_configmap_exit\" ] && [ \"$delete_configmap_exit\" != 0 ]; then exit \"$delete_configmap_exit\"; fi",
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
"if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi",
].join("\n");
}
function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`secret=${shellQuote(options.name)}`,
`expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"preset=obsolete-secret-cleanup",
"exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }",
"workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)",
"workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)",
"workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)",
"before_exists=$(exists_flag)",
"action=observed",
"mutation=false",
"delete_secret_exit=",
"if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi",
"if [ \"$action\" = observed ]; then",
" if [ \"$workload_refs_present\" = yes ]; then",
" action=blocked-referenced",
" elif [ \"$dry_run\" = true ]; then",
" if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
" else",
" kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err",
" delete_secret_exit=$?",
" for _ in $(seq 1 15); do",
" current_exists=$(exists_flag)",
" if [ \"$current_exists\" != yes ]; then break; fi",
" sleep 1",
" done",
" if [ \"$delete_secret_exit\" -eq 0 ]; then",
" if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
" else",
" action=delete-failed",
" fi",
" fi",
"fi",
"after_exists=$(exists_flag)",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$secret\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"",
"printf 'afterSecretExists\\t%s\\n' \"$after_exists\"",
"printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"",
"printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"",
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
"if [ \"$action\" = unsupported-secret ]; then exit 43; fi",
"if [ \"$workload_refs_present\" = yes ]; then exit 46; fi",
"if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi",
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
].join("\n");
}
function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`db_name=${shellQuote(spec.obsoleteHwpodDbName)}`,
`db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"preset=obsolete-platform-db-cleanup",
"database_exists_flag() {",
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)",
" code=$?",
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
" [ \"$output\" = t ] && printf yes || printf no",
"}",
"role_exists_flag() {",
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)",
" code=$?",
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
" [ \"$output\" = t ] && printf yes || printf no",
"}",
"before_database_exists=$(database_exists_flag)",
"before_database_probe_exit=$?",
"before_role_exists=$(role_exists_flag)",
"before_role_probe_exit=$?",
"action=observed",
"mutation=false",
"drop_database_exit=",
"drop_role_exit=",
"before_any=false",
"if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi",
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then",
" action=probe-failed",
"elif [ \"$dry_run\" = true ]; then",
" if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi",
"else",
" if [ \"$before_database_exists\" = yes ]; then",
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'",
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();",
"DROP DATABASE IF EXISTS :\"db_name\";",
"SQL",
" drop_database_exit=$?",
" else",
" drop_database_exit=0",
" fi",
" if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then",
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'",
"DROP ROLE IF EXISTS :\"db_user\";",
"SQL",
" drop_role_exit=$?",
" elif [ \"$drop_database_exit\" -eq 0 ]; then",
" drop_role_exit=0",
" else",
" drop_role_exit=",
" fi",
" if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then",
" if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi",
" else",
" action=drop-failed",
" fi",
"fi",
"after_database_exists=$(database_exists_flag)",
"after_database_probe_exit=$?",
"after_role_exists=$(role_exists_flag)",
"after_role_probe_exit=$?",
"printf 'database\\t%s\\n' \"$db_name\"",
"printf 'role\\t%s\\n' \"$db_user\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"",
"printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"",
"printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"",
"printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"",
"printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"",
"printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"",
"printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"",
"printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"",
"printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"",
"printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"",
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi",
"if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi",
"if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi",
"if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi",
].join("\n");
}
function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
const isOpenFga = options.preset === "openfga";
const platformEndpointSlice = spec.platformPostgresEndpointSlice;
const expectedUriHost = spec.platformPostgresEndpointAddress ?? (isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost);
const databaseUrlKey = isOpenFga ? spec.externalPostgres?.openfga.secretKey ?? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey;
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`,
`database_url_key=${shellQuote(databaseUrlKey)}`,
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
`legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`,
`platform_service=${shellQuote(spec.platformPostgresService)}`,
`platform_endpointslice=${shellQuote(platformEndpointSlice)}`,
`platform_host=${shellQuote(spec.platformPostgresService)}`,
`platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`,
`platform_endpoint_address=${shellQuote(spec.platformPostgresEndpointAddress ?? "")}`,
`db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`,
`db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`,
`db_host=${shellQuote(expectedUriHost)}`,
`selected_key=${shellQuote(options.key ?? "")}`,
`preset=${shellQuote(options.preset)}`,
"dry_run=true",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
"resource_exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
"endpointslice_exists_flag() { kubectl -n \"$namespace\" get endpointslice \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"uri_has_platform_host=no",
"uri_has_db_name=no",
"uri_has_db_user=no",
"uri_matches_expected() {",
" uri=$1",
" uri_has_platform_host=no",
" uri_has_db_name=no",
" uri_has_db_user=no",
" case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*|*\"@$db_host:\"*|*\"@$db_host/\"*) uri_has_platform_host=yes ;; esac",
" if [ -n \"$platform_endpoint_address\" ]; then",
" case \"$uri\" in *\"@$platform_endpoint_address:\"*|*\"@$platform_endpoint_address/\"*) uri_has_platform_host=yes ;; esac",
" fi",
" case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac",
" case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac",
"}",
"exists=$(secret_exists_flag \"$name\")",
"legacy_postgres_exists=$(secret_exists_flag \"$legacy_postgres_secret\")",
"uri_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
"uri_present=$([ -n \"$uri_b64\" ] && printf yes || printf no)",
"uri_bytes=$(decoded_length \"$uri_b64\")",
"uri_value=$(decoded_value \"$uri_b64\")",
"authn_b64=$(secret_b64_key \"$name\" \"$authn_key\")",
"authn_present=$([ -n \"$authn_b64\" ] && printf yes || printf no)",
"authn_bytes=$(decoded_length \"$authn_b64\")",
"pg_password_b64=$(secret_b64_key \"$name\" \"$postgres_password_key\")",
"pg_password_present=$([ -n \"$pg_password_b64\" ] && printf yes || printf no)",
"pg_password_bytes=$(decoded_length \"$pg_password_b64\")",
"platform_service_exists=$(resource_exists_flag service \"$platform_service\")",
"platform_endpoints_exists=$(resource_exists_flag endpoints \"$platform_service\")",
"platform_endpointslice_exists=$(endpointslice_exists_flag \"$platform_endpointslice\")",
"uri_matches_expected \"$uri_value\"",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'key\\t%s\\n' \"$database_url_key\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\tobserved\\n'",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\tfalse\\n'",
"printf 'platformDbMode\\ttrue\\n'",
"printf 'afterExists\\t%s\\n' \"$exists\"",
"printf 'afterDatabaseUrlPresent\\t%s\\n' \"$uri_present\"",
"printf 'afterDatabaseUrlBytes\\t%s\\n' \"$uri_bytes\"",
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$uri_present\"",
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$uri_bytes\"",
"printf 'afterAuthnPresent\\t%s\\n' \"$authn_present\"",
"printf 'afterAuthnBytes\\t%s\\n' \"$authn_bytes\"",
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$pg_password_present\"",
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$pg_password_bytes\"",
"printf 'legacyPostgresSecret\\t%s\\n' \"$legacy_postgres_secret\"",
"printf 'legacyPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
"printf 'afterPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
"printf 'platformService\\t%s\\n' \"$platform_service\"",
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
"printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"",
"printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"",
"printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"",
"printf 'platformEndpointAddress\\t%s\\n' \"$platform_endpoint_address\"",
"printf 'dbName\\t%s\\n' \"$db_name\"",
"printf 'dbUser\\t%s\\n' \"$db_user\"",
"printf 'dbHost\\t%s\\n' \"$db_host\"",
"printf 'dbHostMatchesPlatform\\t%s\\n' \"$uri_has_platform_host\"",
"printf 'dbNameMatchesExpected\\t%s\\n' \"$uri_has_db_name\"",
"printf 'dbUserMatchesExpected\\t%s\\n' \"$uri_has_db_user\"",
"uri_value=",
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
].join("\n");
}
function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`openfga_secret=${shellQuote(spec.openFgaSecret)}`,
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
`postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`,
`selected_key=${shellQuote(options.key ?? "")}`,
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
`datastore_uri_key=${shellQuote(OPENFGA_DATASTORE_URI_KEY)}`,
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
`db_name=${shellQuote(spec.openFgaDbName)}`,
`db_user=${shellQuote(spec.openFgaDbUser)}`,
`db_host=${shellQuote(spec.openFgaDbHost)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(spec.fieldManager)}`,
"preset=openfga",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
"probe_db() {",
" role_result=unknown",
" database_result=unknown",
" probe_exit=missing-postgres-admin-secret",
" if [ -n \"$postgres_admin_password\" ]; then",
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
" role_exit=$?",
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
" database_exit=$?",
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
" fi",
"}",
"before_exists=$(secret_exists_flag \"$openfga_secret\")",
"before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
"before_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")",
"before_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")",
"before_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")",
"postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)",
"before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)",
"before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)",
"before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)",
"before_authn_bytes=$(decoded_length \"$before_authn_b64\")",
"before_uri_bytes=$(decoded_length \"$before_uri_b64\")",
"before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")",
"authn_value=$(decoded_value \"$before_authn_b64\")",
"datastore_uri=$(decoded_value \"$before_uri_b64\")",
"pg_password=$(decoded_value \"$before_pg_password_b64\")",
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
"probe_db",
"db_role_exists_before=$role_result",
"db_database_exists_before=$database_result",
"db_probe_exit_before=$probe_exit",
"action=observed",
"mutation=false",
"postgres_secret_exit=",
"postgres_rollout_exit=",
"apply_exit=",
"db_ensure_exit=",
"if [ \"$action_request\" = ensure ]; then",
" missing_secret=false",
" [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true",
" [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true",
" [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true",
" missing_db=false",
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
" if [ \"$dry_run\" = true ]; then",
" if [ \"$before_postgres_exists\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
" elif ! command -v openssl >/dev/null 2>&1; then",
" action=openssl-missing; apply_exit=127",
" else",
" [ -n \"$postgres_admin_password\" ] || postgres_admin_password=$(openssl rand -hex 24)",
" kubectl -n \"$namespace\" create secret generic \"$postgres_secret\" --from-literal=\"POSTGRES_PASSWORD=$postgres_admin_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
" postgres_secret_exit=$?",
" if [ \"$postgres_secret_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout status \"statefulset/$postgres_statefulset\" --timeout=120s >/tmp/hwlab-postgres-rollout.out 2>/tmp/hwlab-postgres-rollout.err",
" postgres_rollout_exit=$?",
" fi",
" if [ \"$postgres_secret_exit\" -ne 0 ]; then action=postgres-secret-failed; apply_exit=$postgres_secret_exit",
" elif [ \"$postgres_rollout_exit\" -ne 0 ]; then action=postgres-rollout-failed; apply_exit=$postgres_rollout_exit",
" else",
" [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)",
" [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
" [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
" kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'",
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
"\\gexec",
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
"\\gexec",
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
"SQL",
" db_ensure_exit=$?",
" if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi",
" else action=apply-failed; fi",
" fi",
" fi",
"fi",
"after_exists=$(secret_exists_flag \"$openfga_secret\")",
"after_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
"after_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")",
"after_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")",
"after_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")",
"after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)",
"after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)",
"after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)",
"after_authn_bytes=$(decoded_length \"$after_authn_b64\")",
"after_uri_bytes=$(decoded_length \"$after_uri_b64\")",
"after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")",
"probe_db",
"db_role_exists_after=$role_result",
"db_database_exists_after=$database_result",
"db_probe_exit_after=$probe_exit",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$openfga_secret\"",
"printf 'key\\t%s\\n' \"$selected_key\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'afterPostgresSecretExists\\t%s\\n' \"$after_postgres_exists\"",
"printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"",
"printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"",
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"",
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"",
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"",
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"",
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
"printf 'postgresSecretExitCode\\t%s\\n' \"$postgres_secret_exit\"",
"printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
"authn_value= datastore_uri= pg_password= postgres_admin_password=",
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
].join("\n");
}
function masterAdminApiKeySecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`name=${shellQuote(spec.masterAdminApiKeySecret)}`,
`api_key_name=${shellQuote(MASTER_ADMIN_API_KEY_KEY)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(spec.fieldManager)}`,
"preset=master-server-admin-api-key",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-12; value=; fi; }",
"before_exists=$(secret_exists_flag)",
"before_api_key_b64=$(secret_b64_key \"$api_key_name\")",
"before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)",
"before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")",
"action=observed",
"mutation=false",
"apply_exit=",
"if [ \"$action_request\" = ensure ]; then",
" missing_secret=false",
" [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true",
" if [ \"$dry_run\" = true ]; then",
" if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi",
" else",
" api_key=$(cat)",
" case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac",
" if [ -z \"$apply_exit\" ]; then",
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
" fi",
" api_key=",
" fi",
"fi",
"after_exists=$(secret_exists_flag)",
"after_api_key_b64=$(secret_b64_key \"$api_key_name\")",
"after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)",
"after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")",
"after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'key\\t%s\\n' \"$api_key_name\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"",
"printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"",
"printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
].join("\n");
}
function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`name=${shellQuote(spec.bootstrapAdminSecret)}`,
`source_namespace=${shellQuote(spec.bootstrapAdminSourceNamespace)}`,
`source_name=${shellQuote(spec.bootstrapAdminSourceSecret)}`,
`password_hash_key=${shellQuote(spec.bootstrapAdminPasswordHashKey)}`,
`cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(spec.fieldManager)}`,
"preset=bootstrap-admin",
"secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"before_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
"before_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")",
"source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")",
"source_hash_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$password_hash_key\")",
"before_hash_present=$([ -n \"$before_hash_b64\" ] && printf yes || printf no)",
"source_hash_present=$([ -n \"$source_hash_b64\" ] && printf yes || printf no)",
"before_hash_bytes=$(decoded_length \"$before_hash_b64\")",
"source_hash_bytes=$(decoded_length \"$source_hash_b64\")",
"action=observed",
"mutation=false",
"apply_exit=",
"rollout_restart_exit=",
"rollout_status_exit=",
"if [ \"$action_request\" = ensure ]; then",
" missing_target=false",
" [ \"$before_exists\" = yes ] && [ \"$before_hash_bytes\" -gt 0 ] || missing_target=true",
" missing_source=false",
" [ \"$source_exists\" = yes ] && [ \"$source_hash_bytes\" -gt 0 ] || missing_source=true",
" if [ \"$missing_source\" = true ]; then",
" action=source-missing-password-hash",
" apply_exit=44",
" elif [ \"$dry_run\" = true ]; then",
" if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi",
" elif [ \"$missing_target\" = false ]; then",
" action=kept",
" else",
" cat <<EOF_SECRET | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
"apiVersion: v1",
"kind: Secret",
"metadata:",
" name: $name",
" namespace: $namespace",
" labels:",
" app.kubernetes.io/part-of: hwlab",
" hwlab.pikastech.local/secret-preset: bootstrap-admin",
"type: Opaque",
"data:",
" password-hash: $source_hash_b64",
"EOF_SECRET",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-bootstrap-admin-rollout-restart.out 2>/tmp/hwlab-bootstrap-admin-rollout-restart.err",
" rollout_restart_exit=$?",
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-bootstrap-admin-rollout-status.out 2>/tmp/hwlab-bootstrap-admin-rollout-status.err",
" rollout_status_exit=$?",
" fi",
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
" else action=copied-from-source; mutation=true; fi",
" else action=apply-failed; fi",
" fi",
"fi",
"after_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
"after_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")",
"after_hash_present=$([ -n \"$after_hash_b64\" ] && printf yes || printf no)",
"after_hash_bytes=$(decoded_length \"$after_hash_b64\")",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'key\\t%s\\n' \"$password_hash_key\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'sourceNamespace\\t%s\\n' \"$source_namespace\"",
"printf 'sourceSecret\\t%s\\n' \"$source_name\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'beforePasswordHashPresent\\t%s\\n' \"$before_hash_present\"",
"printf 'beforePasswordHashBytes\\t%s\\n' \"$before_hash_bytes\"",
"printf 'sourceExists\\t%s\\n' \"$source_exists\"",
"printf 'sourcePasswordHashPresent\\t%s\\n' \"$source_hash_present\"",
"printf 'sourcePasswordHashBytes\\t%s\\n' \"$source_hash_bytes\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'afterPasswordHashPresent\\t%s\\n' \"$after_hash_present\"",
"printf 'afterPasswordHashBytes\\t%s\\n' \"$after_hash_bytes\"",
"printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
].join("\n");
}
function codeAgentProviderSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`name=${shellQuote(spec.codeAgentProviderSecret)}`,
`source_namespace=${shellQuote(spec.codeAgentProviderSourceNamespace)}`,
`source_name=${shellQuote(spec.codeAgentProviderSourceSecret)}`,
`openai_key=${shellQuote(CODE_AGENT_PROVIDER_OPENAI_KEY)}`,
`opencode_key=${shellQuote(CODE_AGENT_PROVIDER_OPENCODE_KEY)}`,
`selected_key=${shellQuote(options.key ?? "")}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(spec.fieldManager)}`,
"preset=code-agent-provider",
"secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"before_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
"before_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")",
"before_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")",
"source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")",
"source_openai_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$openai_key\")",
"source_opencode_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$opencode_key\")",
"before_openai_present=$([ -n \"$before_openai_b64\" ] && printf yes || printf no)",
"before_opencode_present=$([ -n \"$before_opencode_b64\" ] && printf yes || printf no)",
"source_openai_present=$([ -n \"$source_openai_b64\" ] && printf yes || printf no)",
"source_opencode_present=$([ -n \"$source_opencode_b64\" ] && printf yes || printf no)",
"before_openai_bytes=$(decoded_length \"$before_openai_b64\")",
"before_opencode_bytes=$(decoded_length \"$before_opencode_b64\")",
"source_openai_bytes=$(decoded_length \"$source_openai_b64\")",
"source_opencode_bytes=$(decoded_length \"$source_opencode_b64\")",
"action=observed",
"mutation=false",
"apply_exit=",
"if [ \"$action_request\" = ensure ]; then",
" missing_target=false",
" [ \"$before_exists\" = yes ] && { [ \"$before_openai_bytes\" -gt 0 ] || [ \"$before_opencode_bytes\" -gt 0 ]; } || missing_target=true",
" missing_source=false",
" [ \"$source_exists\" = yes ] && { [ \"$source_openai_bytes\" -gt 0 ] || [ \"$source_opencode_bytes\" -gt 0 ]; } || missing_source=true",
" if [ \"$missing_source\" = true ]; then",
" action=source-missing-provider-key",
" apply_exit=44",
" elif [ \"$dry_run\" = true ]; then",
" if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi",
" elif [ \"$missing_target\" = false ]; then",
" action=kept",
" else",
" cat <<EOF_SECRET | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
"apiVersion: v1",
"kind: Secret",
"metadata:",
" name: $name",
" namespace: $namespace",
" labels:",
" app.kubernetes.io/part-of: hwlab",
" hwlab.pikastech.local/secret-preset: code-agent-provider",
"type: Opaque",
"data:",
" openai-api-key: $source_openai_b64",
" opencode-api-key: $source_opencode_b64",
"EOF_SECRET",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then action=copied-from-source; mutation=true; else action=apply-failed; fi",
" fi",
"fi",
"after_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
"after_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")",
"after_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")",
"after_openai_present=$([ -n \"$after_openai_b64\" ] && printf yes || printf no)",
"after_opencode_present=$([ -n \"$after_opencode_b64\" ] && printf yes || printf no)",
"after_openai_bytes=$(decoded_length \"$after_openai_b64\")",
"after_opencode_bytes=$(decoded_length \"$after_opencode_b64\")",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'sourceNamespace\\t%s\\n' \"$source_namespace\"",
"printf 'sourceSecret\\t%s\\n' \"$source_name\"",
"printf 'selectedKey\\t%s\\n' \"$selected_key\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'beforeOpenaiPresent\\t%s\\n' \"$before_openai_present\"",
"printf 'beforeOpenaiBytes\\t%s\\n' \"$before_openai_bytes\"",
"printf 'beforeOpencodePresent\\t%s\\n' \"$before_opencode_present\"",
"printf 'beforeOpencodeBytes\\t%s\\n' \"$before_opencode_bytes\"",
"printf 'sourceExists\\t%s\\n' \"$source_exists\"",
"printf 'sourceOpenaiPresent\\t%s\\n' \"$source_openai_present\"",
"printf 'sourceOpenaiBytes\\t%s\\n' \"$source_openai_bytes\"",
"printf 'sourceOpencodePresent\\t%s\\n' \"$source_opencode_present\"",
"printf 'sourceOpencodeBytes\\t%s\\n' \"$source_opencode_bytes\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'afterOpenaiPresent\\t%s\\n' \"$after_openai_present\"",
"printf 'afterOpenaiBytes\\t%s\\n' \"$after_openai_bytes\"",
"printf 'afterOpencodePresent\\t%s\\n' \"$after_opencode_present\"",
"printf 'afterOpencodeBytes\\t%s\\n' \"$after_opencode_bytes\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
].join("\n");
}
function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`name=${shellQuote(spec.cloudApiDbSecret)}`,
`database_url_key=${shellQuote(spec.cloudApiDbKey)}`,
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
`postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`,
`db_name=${shellQuote(spec.cloudApiDbName)}`,
`db_user=${shellQuote(spec.cloudApiDbUser)}`,
`db_host=${shellQuote(spec.cloudApiDbHost)}`,
`cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`,
`action_request=${shellQuote(options.action)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
`field_manager=${shellQuote(spec.fieldManager)}`,
"preset=cloud-api-db",
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
"probe_db() {",
" role_result=unknown",
" database_result=unknown",
" probe_exit=missing-postgres-admin-secret",
" if [ -n \"$postgres_admin_password\" ]; then",
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
" role_exit=$?",
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
" database_exit=$?",
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
" fi",
"}",
"before_exists=$(secret_exists_flag \"$name\")",
"before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
"before_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
"before_url_present=$([ -n \"$before_url_b64\" ] && printf yes || printf no)",
"before_url_bytes=$(decoded_length \"$before_url_b64\")",
"database_url=$(decoded_value \"$before_url_b64\")",
"postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)",
"postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)",
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
"probe_db",
"db_role_exists_before=$role_result",
"db_database_exists_before=$database_result",
"db_probe_exit_before=$probe_exit",
"action=observed",
"mutation=false",
"apply_exit=",
"db_ensure_exit=",
"rollout_restart_exit=",
"rollout_status_exit=",
"if [ \"$action_request\" = ensure ]; then",
" missing_secret=false",
" [ \"$before_url_present\" = yes ] && [ \"$before_url_bytes\" -gt 0 ] || missing_secret=true",
" missing_db=false",
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
" if [ \"$dry_run\" = true ]; then",
" if [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
" elif [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ -z \"$postgres_admin_password\" ]; then",
" action=postgres-admin-secret-missing",
" apply_exit=44",
" elif [ \"$missing_secret\" = false ] && [ \"$missing_db\" = false ]; then",
" action=kept",
" else",
" [ -n \"$database_url\" ] || database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"",
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$database_url_key=$database_url\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
" apply_exit=$?",
" if [ \"$apply_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$postgres_admin_password\" >/tmp/hwlab-cloud-api-db-psql.out 2>/tmp/hwlab-cloud-api-db-psql.err <<'SQL'",
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
"\\gexec",
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
"\\gexec",
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
"SQL",
" db_ensure_exit=$?",
" if [ \"$db_ensure_exit\" -eq 0 ]; then",
" if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then",
" kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-cloud-api-rollout-restart.out 2>/tmp/hwlab-cloud-api-rollout-restart.err",
" rollout_restart_exit=$?",
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
" kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-cloud-api-rollout-status.out 2>/tmp/hwlab-cloud-api-rollout-status.err",
" rollout_status_exit=$?",
" fi",
" fi",
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
" else action=ensured; mutation=true; fi",
" else action=db-ensure-failed; fi",
" else action=apply-failed; fi",
" fi",
"fi",
"after_exists=$(secret_exists_flag \"$name\")",
"after_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
"after_url_present=$([ -n \"$after_url_b64\" ] && printf yes || printf no)",
"after_url_bytes=$(decoded_length \"$after_url_b64\")",
"probe_db",
"db_role_exists_after=$role_result",
"db_database_exists_after=$database_result",
"db_probe_exit_after=$probe_exit",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$name\"",
"printf 'key\\t%s\\n' \"$database_url_key\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
"printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"",
"printf 'beforeDatabaseUrlPresent\\t%s\\n' \"$before_url_present\"",
"printf 'beforeDatabaseUrlBytes\\t%s\\n' \"$before_url_bytes\"",
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
"printf 'afterDatabaseUrlPresent\\t%s\\n' \"$after_url_present\"",
"printf 'afterDatabaseUrlBytes\\t%s\\n' \"$after_url_bytes\"",
"printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"",
"printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"",
"printf 'dbName\\t%s\\n' \"$db_name\"",
"printf 'dbUser\\t%s\\n' \"$db_user\"",
"printf 'dbHost\\t%s\\n' \"$db_host\"",
"printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"",
"printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"",
"printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"",
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
"printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"",
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
"database_url= postgres_admin_password=",
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
].join("\n");
}
function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
const fields = keyValueLinesFromText(text);
if (fields.preset === "owned-postgres-cleanup") {
const absent = fields.afterStatefulSetExists !== "yes" &&
fields.afterServiceExists !== "yes" &&
fields.afterConfigMapExists !== "yes" &&
fields.afterSecretExists !== "yes" &&
fields.afterPvcExists !== "yes";
const platformServiceReady = fields.platformServiceExists === "yes";
return {
ok: commandOk && absent && platformServiceReady,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.postgresSecret,
statefulSet: fields.statefulSet || spec.postgresStatefulSet,
service: fields.service || spec.postgresSecret,
configMap: fields.configMap || `${spec.postgresSecret}-init`,
pvc: fields.pvc || `data-${spec.postgresSecret}-0`,
preset: "owned-postgres-cleanup",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
before: {
statefulSetExists: fields.beforeStatefulSetExists === "yes",
serviceExists: fields.beforeServiceExists === "yes",
configMapExists: fields.beforeConfigMapExists === "yes",
secretExists: fields.beforeSecretExists === "yes",
pvcExists: fields.beforePvcExists === "yes",
pvcPhase: fields.beforePvcPhase || null,
persistentVolume: fields.beforePersistentVolume || null,
},
after: {
statefulSetExists: fields.afterStatefulSetExists === "yes",
serviceExists: fields.afterServiceExists === "yes",
configMapExists: fields.afterConfigMapExists === "yes",
secretExists: fields.afterSecretExists === "yes",
pvcExists: fields.afterPvcExists === "yes",
pvcPhase: fields.afterPvcPhase || null,
persistentVolume: fields.afterPersistentVolume || null,
},
platformService: {
name: "g14-platform-postgres",
exists: platformServiceReady,
},
deleteStatefulSetExitCode: numericField(fields.deleteStatefulSetExitCode),
deleteServiceExitCode: numericField(fields.deleteServiceExitCode),
deleteConfigMapExitCode: numericField(fields.deleteConfigMapExitCode),
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
deletePvcExitCode: numericField(fields.deletePvcExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: absent
? `${fields.statefulSet || spec.postgresStatefulSet}, ${fields.service || spec.postgresSecret}, ${fields.configMap || `${spec.postgresSecret}-init`}, ${fields.secret || spec.postgresSecret}, and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent`
: `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`,
};
}
if (fields.preset === "obsolete-secret-cleanup") {
const absent = fields.afterSecretExists !== "yes";
const refsAbsent = fields.workloadRefsPresent !== "yes";
const dryRun = fields.dryRun === "true";
return {
ok: commandOk && refsAbsent && (dryRun || absent),
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.obsoleteHwpodDbSecret,
preset: "obsolete-secret-cleanup",
action: fields.action || null,
dryRun,
mutation: fields.mutation === "true",
before: {
secretExists: fields.beforeSecretExists === "yes",
},
after: {
secretExists: fields.afterSecretExists === "yes",
},
workloadRefs: {
present: fields.workloadRefsPresent === "yes",
preview: fields.workloadRefsPreview || "",
},
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: refsAbsent && (dryRun || absent)
? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}`
: `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`,
};
}
if (fields.preset === "master-server-admin-api-key") {
const afterBytes = numericField(fields.afterApiKeyBytes);
const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.masterAdminApiKeySecret,
preset: "master-server-admin-api-key",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } },
applyExitCode: numericField(fields.applyExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy ? `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} exists` : `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} missing`,
};
}
if (fields.preset === "bootstrap-admin") {
const beforeHashBytes = numericField(fields.beforePasswordHashBytes);
const sourceHashBytes = numericField(fields.sourcePasswordHashBytes);
const afterHashBytes = numericField(fields.afterPasswordHashBytes);
const healthy = fields.afterExists === "yes" &&
fields.afterPasswordHashPresent === "yes" &&
typeof afterHashBytes === "number" && afterHashBytes > 0;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.bootstrapAdminSecret,
key: fields.key || spec.bootstrapAdminPasswordHashKey,
preset: "bootstrap-admin",
source: {
namespace: fields.sourceNamespace || spec.bootstrapAdminSourceNamespace,
secret: fields.sourceSecret || spec.bootstrapAdminSourceSecret,
exists: fields.sourceExists === "yes",
passwordHash: { keyPresent: fields.sourcePasswordHashPresent === "yes", valueBytes: sourceHashBytes },
},
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
before: {
exists: fields.beforeExists === "yes",
passwordHash: { keyPresent: fields.beforePasswordHashPresent === "yes", valueBytes: beforeHashBytes },
},
after: {
exists: fields.afterExists === "yes",
passwordHash: { keyPresent: fields.afterPasswordHashPresent === "yes", valueBytes: afterHashBytes },
},
cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment,
applyExitCode: numericField(fields.applyExitCode),
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy ? `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} exists` : `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} missing`,
};
}
if (fields.preset === "code-agent-provider") {
const beforeOpenaiBytes = numericField(fields.beforeOpenaiBytes);
const beforeOpencodeBytes = numericField(fields.beforeOpencodeBytes);
const sourceOpenaiBytes = numericField(fields.sourceOpenaiBytes);
const sourceOpencodeBytes = numericField(fields.sourceOpencodeBytes);
const afterOpenaiBytes = numericField(fields.afterOpenaiBytes);
const afterOpencodeBytes = numericField(fields.afterOpencodeBytes);
const openaiReady = fields.afterOpenaiPresent === "yes" && typeof afterOpenaiBytes === "number" && afterOpenaiBytes > 0;
const opencodeReady = fields.afterOpencodePresent === "yes" && typeof afterOpencodeBytes === "number" && afterOpencodeBytes > 0;
const healthy = fields.afterExists === "yes" && (openaiReady || opencodeReady);
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.codeAgentProviderSecret,
preset: "code-agent-provider",
source: {
namespace: fields.sourceNamespace || spec.codeAgentProviderSourceNamespace,
secret: fields.sourceSecret || spec.codeAgentProviderSourceSecret,
exists: fields.sourceExists === "yes",
openaiApiKey: { keyPresent: fields.sourceOpenaiPresent === "yes", valueBytes: sourceOpenaiBytes },
opencodeApiKey: { keyPresent: fields.sourceOpencodePresent === "yes", valueBytes: sourceOpencodeBytes },
},
selectedKey: fields.selectedKey || null,
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
before: {
exists: fields.beforeExists === "yes",
openaiApiKey: { keyPresent: fields.beforeOpenaiPresent === "yes", valueBytes: beforeOpenaiBytes },
opencodeApiKey: { keyPresent: fields.beforeOpencodePresent === "yes", valueBytes: beforeOpencodeBytes },
},
after: {
exists: fields.afterExists === "yes",
openaiApiKey: { keyPresent: fields.afterOpenaiPresent === "yes", valueBytes: afterOpenaiBytes },
opencodeApiKey: { keyPresent: fields.afterOpencodePresent === "yes", valueBytes: afterOpencodeBytes },
requiredAnyProviderKeyPresent: openaiReady || opencodeReady,
},
applyExitCode: numericField(fields.applyExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`,
};
}
if (fields.preset === "cloud-api-db") {
const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes);
const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes);
if (fields.platformDbMode === "true") {
const keysHealthy = fields.afterExists === "yes" &&
fields.afterDatabaseUrlPresent === "yes" &&
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
fields.platformEndpointsExists !== "yes" &&
fields.platformEndpointSliceExists === "yes";
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
fields.dbNameMatchesExpected === "yes" &&
fields.dbUserMatchesExpected === "yes";
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.cloudApiDbSecret,
key: fields.key || spec.cloudApiDbKey,
preset: "cloud-api-db",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
platformDbMode: true,
after: {
exists: fields.afterExists === "yes",
databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes },
},
legacyPostgresSecret: {
name: fields.legacyPostgresSecret || spec.postgresSecret,
exists: fields.legacyPostgresSecretExists === "yes",
},
platformService: {
name: fields.platformService || spec.platformPostgresService,
exists: fields.platformServiceExists === "yes",
endpointsExist: fields.platformEndpointsExists === "yes",
legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes",
endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`,
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
},
dbName: fields.dbName || spec.cloudApiDbName,
dbUser: fields.dbUser || spec.cloudApiDbUser,
dbHost: fields.dbHost || spec.cloudApiDbHost,
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy
? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}`
: `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`,
};
}
const keysHealthy = fields.afterExists === "yes" &&
fields.afterDatabaseUrlPresent === "yes" &&
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t";
const healthy = keysHealthy && databaseHealthy;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.cloudApiDbSecret,
key: fields.key || spec.cloudApiDbKey,
preset: "cloud-api-db",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
before: {
exists: fields.beforeExists === "yes",
postgresSecretExists: fields.beforePostgresSecretExists === "yes",
databaseUrl: { keyPresent: fields.beforeDatabaseUrlPresent === "yes", valueBytes: beforeUrlBytes },
database: {
roleExists: fields.dbRoleExistsBefore || "unknown",
databaseExists: fields.dbDatabaseExistsBefore || "unknown",
probeExitCode: fields.dbProbeExitCodeBefore || null,
},
},
after: {
exists: fields.afterExists === "yes",
databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes },
database: {
roleExists: fields.dbRoleExistsAfter || "unknown",
databaseExists: fields.dbDatabaseExistsAfter || "unknown",
probeExitCode: fields.dbProbeExitCodeAfter || null,
},
},
postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes",
postgresSecret: fields.postgresSecret || spec.postgresSecret,
dbName: fields.dbName || spec.cloudApiDbName,
dbUser: fields.dbUser || spec.cloudApiDbUser,
dbHost: fields.dbHost || spec.cloudApiDbHost,
cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment,
applyExitCode: numericField(fields.applyExitCode),
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`,
};
}
const afterAuthnBytes = numericField(fields.afterAuthnBytes);
const afterUriBytes = numericField(fields.afterDatastoreUriBytes);
const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes);
if (fields.platformDbMode === "true") {
const keysHealthy = fields.afterExists === "yes" &&
fields.afterAuthnPresent === "yes" &&
fields.afterDatastoreUriPresent === "yes" &&
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
typeof afterUriBytes === "number" && afterUriBytes > 0;
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
fields.platformEndpointsExists !== "yes" &&
fields.platformEndpointSliceExists === "yes";
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
fields.dbNameMatchesExpected === "yes" &&
fields.dbUserMatchesExpected === "yes";
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.openFgaSecret,
preset: fields.preset || "openfga",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
platformDbMode: true,
after: {
exists: fields.afterExists === "yes",
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
},
legacyPostgresSecret: {
name: fields.legacyPostgresSecret || spec.postgresSecret,
exists: fields.legacyPostgresSecretExists === "yes",
},
platformService: {
name: fields.platformService || spec.platformPostgresService,
exists: fields.platformServiceExists === "yes",
endpointsExist: fields.platformEndpointsExists === "yes",
legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes",
endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`,
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
},
dbName: fields.dbName || spec.openFgaDbName,
dbUser: fields.dbUser || spec.openFgaDbUser,
dbHost: fields.dbHost || spec.openFgaDbHost,
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy
? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.platformService || spec.platformPostgresService}`
: `${fields.secret || spec.openFgaSecret} datastore-uri is not aligned to platform DB`,
};
}
const keysHealthy = fields.afterExists === "yes" &&
fields.afterPostgresSecretExists === "yes" &&
fields.afterAuthnPresent === "yes" &&
fields.afterDatastoreUriPresent === "yes" &&
fields.afterPostgresPasswordPresent === "yes" &&
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
typeof afterUriBytes === "number" && afterUriBytes > 0 &&
typeof afterPasswordBytes === "number" && afterPasswordBytes > 0;
const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t";
const healthy = keysHealthy && databaseHealthy;
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.openFgaSecret,
preset: fields.preset || "openfga",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
after: {
exists: fields.afterExists === "yes",
postgresSecretExists: fields.afterPostgresSecretExists === "yes",
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null },
},
postgresSecretExitCode: numericField(fields.postgresSecretExitCode),
postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode),
applyExitCode: numericField(fields.applyExitCode),
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy ? `${fields.secret || spec.openFgaSecret} keys and Postgres database exist` : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`,
};
}
function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
const fields = keyValueLinesFromText(text);
const dryRun = fields.dryRun === "true";
const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown";
const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown";
const probesOk = fields.beforeDatabaseExists !== "unknown" &&
fields.beforeRoleExists !== "unknown" &&
fields.afterDatabaseExists !== "unknown" &&
fields.afterRoleExists !== "unknown";
return {
ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)),
database: fields.database || spec.obsoleteHwpodDbName,
role: fields.role || spec.obsoleteHwpodDbUser,
preset: "obsolete-platform-db-cleanup",
action: fields.action || null,
dryRun,
mutation: fields.mutation === "true",
before: {
databaseExists: fields.beforeDatabaseExists === "yes",
roleExists: fields.beforeRoleExists === "yes",
},
after: {
databaseExists: fields.afterDatabaseExists === "yes",
roleExists: fields.afterRoleExists === "yes",
},
beforeProbeExitCode: {
database: numericField(fields.beforeDatabaseProbeExitCode),
role: numericField(fields.beforeRoleProbeExitCode),
},
afterProbeExitCode: {
database: numericField(fields.afterDatabaseProbeExitCode),
role: numericField(fields.afterRoleProbeExitCode),
},
dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode),
dropRoleExitCode: numericField(fields.dropRoleExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: probesOk && (dryRun || (databaseAbsent && roleAbsent))
? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}`
: `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`,
};
}
export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record<string, unknown> {
return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane }));
}
function readMasterAdminApiKey(): { key: string; source: string } {
if (!existsSync(MASTER_ADMIN_API_KEY_ENV)) throw new Error(`HWLAB_API_KEY source missing: ${MASTER_ADMIN_API_KEY_ENV}`);
const content = readFileSync(MASTER_ADMIN_API_KEY_ENV, "utf8");
const match = content.match(/^HWLAB_API_KEY=(.+)$/m);
const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, "");
if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${MASTER_ADMIN_API_KEY_ENV}`);
return { key: raw, source: MASTER_ADMIN_API_KEY_ENV };
}
function optionValue(args: string[], name: string): string | undefined {
const index = args.indexOf(name);
if (index === -1) return undefined;
const value = args[index + 1];
if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`);
return value;
}
function requiredOption(args: string[], name: string): string {
const value = optionValue(args, name);
if (value === undefined) throw new Error(`${name} is required`);
return value;
}
function stripOption(args: string[], name: string): string[] {
return stripOptions(args, [name]);
}
function stripOptions(args: string[], names: readonly string[]): string[] {
const remove = new Set(names);
const without: string[] = [];
for (let index = 0; index < args.length; index += 1) {
const arg = args[index] ?? "";
if (remove.has(arg)) {
if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1;
continue;
}
without.push(arg);
}
return without;
}
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
const raw = optionValue(args, name);
if (raw === undefined) return defaultValue;
const value = Number(raw);
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
return Math.min(value, maxValue);
}
function assertLane(value: string): void {
if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`);
}
function assertNodeId(value: string): void {
if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`);
}
function shellQuote(value: string): string {
return `'${value.replace(/'/gu, `'"'"'`)}'`;
}
function statusText(result: CommandResult): string {
return result.stdout || result.stderr;
}
function keyValueLinesFromText(text: string): Record<string, string> {
const fields: Record<string, string> = {};
for (const line of text.split(/\r?\n/u)) {
const index = line.indexOf("\t");
if (index <= 0) continue;
fields[line.slice(0, index)] = line.slice(index + 1);
}
return fields;
}
function numericField(value: string | undefined): number | null {
if (value === undefined || value === "") return null;
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function commaListField(value: string | undefined): string[] {
if (!value) return [];
return value.split(",").map((item) => item.trim()).filter(Boolean);
}
function splitWhitespaceField(value: string | undefined): string[] {
if (!value) return [];
return value.split(/\s+/u).filter(Boolean);
}
function compactCommandResult(result: CommandResult): Record<string, unknown> {
return {
command: compactCommand(result.command),
exitCode: result.exitCode,
stdoutBytes: result.stdout.length,
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
timedOut: result.timedOut,
};
}
function compactCommand(command: string[]): string[] {
const scriptIndex = command.indexOf("--");
if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "<script omitted>"];
return command;
}
Reference in New Issue View Git Blame Copy Permalink