diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts new file mode 100644 index 00000000..5b72dc6f --- /dev/null +++ b/scripts/src/hwlab-node-help.ts @@ -0,0 +1,103 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Help payloads for the HWLAB node/lane CLI. +import { hwlabRuntimeLaneConfigPath } from "./hwlab-node-lanes"; + +export function hwlabNodeHelp(): Record { + return { + ok: true, + command: "hwlab nodes", + description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.", + configPath: hwlabRuntimeLaneConfigPath(), + examples: [ + "bun scripts/cli.ts hwlab nodes control-plane infra plan --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane plan --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 --full", + "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane runtime-image status --node G14 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03", + "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", + "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-bootstrap-admin", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-bootstrap-admin --confirm", + "bun scripts/cli.ts hwlab nodes secret ensure --node D601 --lane v03 --name hwlab-v03-bootstrap-admin --confirm --force", + "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db", + "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run", + "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm", + "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm", + "bun scripts/cli.ts hwlab nodes test-accounts status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes test-accounts sync --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000", + "bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03", + ], + }; +} + +export function hwlabNodeWebProbeHelp(): Record { + return { + ok: true, + command: "hwlab nodes web-probe", + description: "Run target node/lane HWLAB Cloud Web DOM probes with Web login credentials resolved from YAML-declared bootstrap admin sourceRef and injected only as one-shot stdin/env.", + examples: [ + "bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000", + "bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --url https://hwlab.pikapython.com --fresh-session --message 'ping'", + "bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 --script-file .state/probes/workbench.mjs", + "bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 <<'JS'\nexport default async ({ waitWorkbenchReady, fetchJson, fetchApiMatrix, recordStep, collectText, safeEvaluate, screenshot }) => {\n const ready = await waitWorkbenchReady();\n const workspace = await fetchJson('/v1/workbench/workspace?projectId=prj_hwpod_workbench');\n const apiMatrix = await fetchApiMatrix(['/v1/workbench/workspace?projectId=prj_hwpod_workbench', '/auth/session']);\n const workspaceText = await collectText('#workspace');\n const evaluated = await safeEvaluate(({ a, b }) => ({ sum: a + b }), { a: 1, b: 2 });\n await screenshot('workbench.png');\n recordStep('workbench-summary', { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, apiMatrixOk: apiMatrix.ok });\n return { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, workspaceText, evaluated };\n};\nJS", + ], + actions: { + run: "Run the repo-owned scripts/web-live-dom-probe.mjs helper.", + script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus fetchJson/safeFetchJson/fetchApiMatrix/recordStep/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.", + }, + notes: [ + "Prefer --script-file for reusable probes; stdin heredocs remain supported for one-off probes.", + "Issue-ready summary is available under probe.summary; full script report is persisted under probe.reportPath with a SHA-256 fingerprint.", + "Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.", + "Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).", + "Failures include failureKind, errorMessage, scriptSha256, runDir, lastUrl, and lastScreenshot when a screenshot can be captured.", + ], + }; +} + +export function hwlabNodeObservabilityHelp(): Record { + return { + ok: true, + command: "hwlab nodes observability", + description: "YAML-first node/lane observability control for HWLAB Workbench metrics. Runtime is queried only for presence, boundary, and summary evidence.", + configPath: hwlabRuntimeLaneConfigPath(), + examples: [ + "bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03", + ], + actions: { + plan: "Render the YAML-declared collection mode, scrape target, boundary, and required Workbench metric series.", + apply: "Apply the YAML-declared observability control plane. D601 pod-loopback mode validates the boundary and metrics without creating ad-hoc cluster objects.", + status: "Check Kubernetes service/pod presence, public raw metrics denial, and Workbench metric readiness.", + "workbench-summary": "Read the YAML-declared metrics endpoint via pod loopback and summarize required Workbench series.", + }, + }; +} diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts new file mode 100644 index 00000000..b4a68073 --- /dev/null +++ b/scripts/src/hwlab-node-impl.ts @@ -0,0 +1,8075 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: YAML-first node/lane operations, including Workbench observability control commands. +import { createHash, randomBytes } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { repoRoot, rootPath, type Config } from "./config"; +import { runCommand, type CommandResult } from "./command"; +import { startJob } from "./jobs"; +import { classifySshTcpPoolFailure } from "./ssh"; +import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; +import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes"; +import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source"; +import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "./hwlab-node-help"; +import { compactWebProbeResult, compactWebProbeScriptResult } from "./hwlab-node-web-probe-summary"; +import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "./hwlab-node-observability-promql"; +import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "./hwlab-node-transport"; + +type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; +type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; +type NodeRuntimeRenderLocation = "node-host" | "local"; + +interface NodeWebProbeRunOptions { + action: "run"; + node: string; + lane: string; + url: string; + timeoutMs: number; + waitAfterSubmitMs: number; + waitMessagesMs: number; + waitAgentTerminalMs: number; + traceSampleCount: number; + traceSampleIntervalMs: number; + message: string | null; + conversationId: string | null; + freshSession: boolean; + cancelRunning: boolean; + commandTimeoutSeconds: number; + commandTimeoutAutoSeconds: number; + commandTimeoutUserProvided: boolean; +} + +interface NodeWebProbeScriptOptions { + action: "script"; + node: string; + lane: string; + url: string; + timeoutMs: number; + viewport: string; + commandTimeoutSeconds: number; + scriptText: string; + scriptSource: { + kind: "stdin" | "file"; + path: string | null; + byteCount: number; + sha256: string; + }; +} + +type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions; + +type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary"; + +interface NodeObservabilityOptions { + action: NodeObservabilityAction; + node: string; + lane: HwlabRuntimeLane; + confirm: boolean; + dryRun: boolean; + full: boolean; + timeoutSeconds: number; + spec: HwlabRuntimeLaneSpec; +} + +interface NodeRuntimeRenderResult { + readonly result: CommandResult; + readonly renderDir: string; + readonly worktreeDir: string; + readonly location: NodeRuntimeRenderLocation; +} + +interface NodeSecretOptions { + action: SecretAction; + node: string; + lane: string; + name: string; + key?: string; + preset: SecretPreset; + dryRun: boolean; + confirm: boolean; + force?: boolean; + timeoutSeconds: number; +} + +interface BootstrapAdminSecretMaterial { + ok: boolean; + sourceRef: string | null; + sourceKey: string | null; + sourcePath: string | null; + sourcePresent: boolean; + sourceFingerprint: string | null; + passwordHash: string | null; + error: string | null; +} + +interface BootstrapAdminPasswordMaterial { + ok: boolean; + sourceRef: string | null; + sourceKey: string | null; + sourcePath: string | null; + sourcePresent: boolean; + sourceFingerprint: string | null; + password: string | null; + error: string | null; +} + +interface NodePublicExposureOptions { + action: "public-exposure"; + node: string; + lane: HwlabRuntimeLane; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; + spec: HwlabRuntimeLaneSpec; +} + +interface RuntimeSecretSpec { + node: string; + lane: string; + namespace: string; + platformDb: boolean; + runtimeLaneSpec?: HwlabRuntimeLaneSpec; + externalPostgres?: NonNullable; + platformPostgresService: string; + platformPostgresEndpointAddress?: string; + platformPostgresEndpointSlice: string; + postgresSecret: string; + postgresStatefulSet: string; + postgresAdminUser: string; + openFgaSecret: string; + openFgaDbName: string; + openFgaDbUser: string; + openFgaDbHost: string; + masterAdminApiKeySecret: string; + bootstrapAdminSecret: string; + bootstrapAdminPasswordHashKey: string; + bootstrapAdminUsername: string; + bootstrapAdminDisplayName: string; + bootstrapAdminPasswordSourceRef?: string; + bootstrapAdminPasswordSourceKey?: string; + bootstrapAdminPasswordHashTransform?: "hwlab-sha256"; + bootstrapAdminSourceNamespace: string; + bootstrapAdminSourceSecret: string; + cloudApiDbSecret: string; + cloudApiDbKey: string; + cloudApiDbName: string; + cloudApiDbUser: string; + cloudApiDbHost: string; + cloudApiDeployment: string; + obsoleteHwpodDbSecret: string; + obsoleteHwpodDbName: string; + obsoleteHwpodDbUser: string; + codeAgentProviderSecret: string; + codeAgentProviderSourceNamespace: string; + codeAgentProviderSourceSecret: string; + fieldManager: string; +} + +interface NodeRuntimeGitMirrorTargetSpec { + id: string; + node: string; + lane: string; + namespace: string; + serviceReadName: string; + serviceWriteName: string; + cachePvcName: string; + cacheHostPath: string | null; + secretName: string; + syncJobPrefix: string; + flushJobPrefix: string; + toolsImage: string; + sourceRepository: string; + sourceBranch: string; + gitopsBranch: string; +} + +const MASTER_ADMIN_API_KEY_KEY = "api-key"; +const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash"; +const BOOTSTRAP_ADMIN_SOURCE_NAMESPACE = "hwlab-v02"; +const BOOTSTRAP_ADMIN_SOURCE_SECRET = "hwlab-v02-bootstrap-admin"; +const OPENFGA_AUTHN_KEY = "authn-preshared-key"; +const OPENFGA_DATASTORE_URI_KEY = "datastore-uri"; +const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password"; +const CLOUD_API_DB_KEY = "database-url"; +const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key"; +const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key"; +const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02"; +const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider"; + +export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise> { + if (args.length === 0) return hwlabNodeHelp(); + const [domain] = args; + if (domain === "control-plane" && args[1] === "infra") { + if (args.length === 2 || args.includes("--help") || args.includes("-h") || args[2] === "help") return hwlabNodeControlPlaneInfraHelp(); + return runHwlabNodeControlPlaneInfra(args.slice(2)); + } + if (domain === "test-accounts") { + const { runHwlabTestAccountsCommand } = await import("./hwlab-test-accounts"); + return runHwlabTestAccountsCommand(args.slice(1)); + } + if (domain === "web-probe") { + if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeWebProbeHelp(); + return runNodeWebProbe(parseNodeWebProbeOptions(args.slice(1))); + } + if (domain === "observability") { + if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeObservabilityHelp(); + return runNodeObservability(parseNodeObservabilityOptions(args.slice(1))); + } + if (args.includes("--help") || args.includes("-h")) return hwlabNodeHelp(); + if (domain === "control-plane" || domain === "git-mirror") { + return runNodeDelegatedDomain(_config, domain, args.slice(1)); + } + if (domain !== "secret") { + return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes observability, hwlab nodes secret, hwlab nodes test-accounts, hwlab nodes web-probe" }; + } + const options = parseSecretOptions(args.slice(1)); + return runNodeSecret(options); +} + +export { hwlabNodeHelp, hwlabNodeWebProbeHelp, hwlabNodeObservabilityHelp } from "./hwlab-node-help"; + +function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions { + const [actionRaw] = args; + if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) { + throw new Error("observability usage: observability ACTION --node NODE --lane vNN [--dry-run|--confirm]"); + } + if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary") { + throw new Error(`observability action must be plan, apply, status, or workbench-summary; got ${actionRaw}`); + } + assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--full", "--raw"])); + const node = requiredOption(args, "--node"); + assertNodeId(node); + const laneRaw = requiredOption(args, "--lane"); + if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`); + const confirm = args.includes("--confirm"); + const dryRun = args.includes("--dry-run"); + if (confirm && dryRun) throw new Error("observability accepts only one of --confirm or --dry-run"); + if (actionRaw === "apply" && !confirm && !dryRun) throw new Error("observability apply requires --dry-run or --confirm"); + if (actionRaw !== "apply" && (confirm || dryRun)) throw new Error(`observability ${actionRaw} is read-only and does not accept --confirm or --dry-run`); + return { + action: actionRaw, + node, + lane: laneRaw, + confirm, + dryRun, + full: args.includes("--full") || args.includes("--raw"), + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), + spec: hwlabRuntimeLaneSpecForNode(laneRaw, node), + }; +} + +async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise> { + const scoped = parseNodeScopedDelegatedOptions(domain, args); + const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane); + if (domain === "control-plane" && scoped.action === "public-exposure") { + return runNodePublicExposure({ + action: "public-exposure", + node: scoped.node, + lane: scoped.lane, + dryRun: scoped.dryRun || !scoped.confirm, + confirm: scoped.confirm, + timeoutSeconds: scoped.timeoutSeconds, + spec: scoped.spec, + }); + } + if (domain === "control-plane" && scoped.action === "runtime-image") { + return nodeRuntimeBaseImageCommand(scoped); + } + if (domain === "control-plane" && scoped.action === "plan") { + return nodeRuntimeControlPlanePlan(scoped); + } + if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) { + if (scoped.action === "status") return nodeRuntimeControlPlaneStatus(scoped); + if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration") { + if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped); + return nodeRuntimeControlPlaneRun(scoped); + } + return nodeRuntimeUnsupportedAction(scoped); + } + if (domain === "git-mirror" && scoped.node !== defaultSpec.nodeId) { + if (scoped.action === "status") return nodeRuntimeGitMirrorStatus(scoped); + if (scoped.action === "sync" || scoped.action === "flush") { + if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped); + return nodeRuntimeGitMirrorRun(scoped); + } + return nodeRuntimeUnsupportedAction(scoped); + } + if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") { + return runNodeEndpointBridge(scoped); + } + if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) { + return startNodeDelegatedJob(scoped); + } + if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) { + return startNodeDelegatedJob(scoped); + } + const delegatedArgs = stripOption(args, "--node"); + const result = await runDelegatedHwlabNodeCommand(config, domain, delegatedArgs); + return rewriteDelegatedNodeResult(result, scoped); +} + +function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): { + domain: DelegatedNodeDomain; + action: string; + runtimeImageAction: string | null; + node: string; + lane: HwlabRuntimeLane; + confirm: boolean; + dryRun: boolean; + wait: boolean; + rerun: boolean; + allowLiveDbRead: boolean; + timeoutSeconds: number; + originalArgs: string[]; + spec: HwlabRuntimeLaneSpec; +} { + const [actionRaw] = args; + if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`); + const runtimeImageAction = actionRaw === "runtime-image" && typeof args[1] === "string" && !args[1].startsWith("--") ? args[1] : null; + const node = requiredOption(args, "--node"); + assertNodeId(node); + const laneRaw = requiredOption(args, "--lane"); + if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`); + const spec = hwlabRuntimeLaneSpecForNode(laneRaw, node); + const confirm = args.includes("--confirm"); + const dryRun = args.includes("--dry-run"); + if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`); + return { + domain, + action: actionRaw, + runtimeImageAction, + node, + lane: laneRaw, + confirm, + dryRun, + wait: args.includes("--wait"), + rerun: args.includes("--rerun"), + allowLiveDbRead: args.includes("--allow-live-db-read"), + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600), + originalArgs: [...args], + spec, + }; +} + +function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record { + return { + configPath: hwlabRuntimeLaneConfigPath(), + node: spec.nodeId, + nodeRoute: spec.nodeRoute, + nodeKubeRoute: spec.nodeKubeRoute, + lane: spec.lane, + sourceBranch: spec.sourceBranch, + workspace: spec.workspace, + cicdRepo: spec.cicdRepo, + git: { + sourceUrl: spec.gitUrl, + readUrl: spec.gitReadUrl, + writeUrl: spec.gitWriteUrl, + }, + argo: { + repoURL: spec.argoRepoUrl, + }, + gitopsBranch: spec.gitopsBranch, + catalogPath: spec.catalogPath, + runtimePath: spec.runtimePath, + runtimeNamespace: spec.runtimeNamespace, + renderDir: spec.runtimeRenderDir, + pipeline: spec.pipeline, + pipelineRunPrefix: spec.pipelineRunPrefix, + serviceAccount: spec.serviceAccountName, + argoApplication: spec.app, + registryPrefix: spec.registryPrefix, + baseImage: { + image: spec.baseImage, + sourceImage: spec.baseImageSource ?? null, + }, + buildkit: spec.buildkit === undefined ? null : { + sidecarImage: spec.buildkit.sidecarImage, + }, + dockerBuildProxy: { + http: spec.networkProfile.dockerBuildProxy.http, + https: spec.networkProfile.dockerBuildProxy.https, + all: spec.networkProfile.dockerBuildProxy.all, + noProxy: spec.networkProfile.dockerBuildProxy.noProxy, + }, + stepEnv: spec.stepEnv, + public: { + webUrl: spec.publicWebUrl, + apiUrl: spec.publicApiUrl, + }, + bootstrapAdmin: spec.bootstrapAdmin === undefined ? null : { + username: spec.bootstrapAdmin.username, + displayName: spec.bootstrapAdmin.displayName, + passwordSourceRef: spec.bootstrapAdmin.passwordSourceRef, + passwordSourceKey: spec.bootstrapAdmin.passwordSourceKey, + passwordHashTransform: spec.bootstrapAdmin.passwordHashTransform, + secretName: spec.bootstrapAdmin.secretName, + secretKey: spec.bootstrapAdmin.secretKey, + rolloutDeployment: spec.bootstrapAdmin.rolloutDeployment, + valuesPrinted: false, + }, + publicExposure: spec.publicExposure === null ? null : publicExposureSummary(spec.publicExposure), + downloadProfile: { + id: spec.downloadProfileId, + git: spec.downloadProfile.git, + npm: spec.downloadProfile.npm, + }, + observability: spec.observability, + runtimeImageRewrites: spec.runtimeImageRewrites, + externalPostgres: spec.externalPostgres === undefined ? null : { + provider: spec.externalPostgres.provider, + configRef: spec.externalPostgres.configRef, + serviceName: spec.externalPostgres.serviceName, + endpointAddress: spec.externalPostgres.endpointAddress, + port: spec.externalPostgres.port, + sslmode: spec.externalPostgres.sslmode, + database: spec.externalPostgres.database, + cloudApi: { + secretName: spec.externalPostgres.cloudApi.secretName, + secretKey: spec.externalPostgres.cloudApi.secretKey, + sourceRef: spec.externalPostgres.cloudApi.sourceRef, + envKey: spec.externalPostgres.cloudApi.envKey, + role: spec.externalPostgres.cloudApi.role, + }, + openfga: { + secretName: spec.externalPostgres.openfga.secretName, + secretKey: spec.externalPostgres.openfga.secretKey, + sourceRef: spec.externalPostgres.openfga.sourceRef, + envKey: spec.externalPostgres.openfga.envKey, + authnKey: spec.externalPostgres.openfga.authnKey ?? null, + role: spec.externalPostgres.openfga.role, + schema: spec.externalPostgres.openfga.schema ?? null, + }, + valuesPrinted: false, + }, + localPostgres: { + shouldRender: spec.externalPostgres === undefined, + expectedAbsent: spec.externalPostgres !== undefined, + }, + }; +} + +function nodeRuntimeControlPlanePlan(scoped: ReturnType): Record { + return { + ok: true, + command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, + mode: "plan", + mutation: false, + node: scoped.node, + lane: scoped.lane, + expected: nodeRuntimeExpected(scoped.spec), + checks: { + nodeScopedTargetConfigured: true, + externalPostgresDeclared: scoped.spec.externalPostgres !== undefined, + secretValuesPrinted: false, + runtimeNamespace: scoped.spec.runtimeNamespace, + localPostgresExpectedAbsent: scoped.spec.externalPostgres !== undefined, + publicExposureDeclared: scoped.spec.publicExposure !== null, + }, + next: { + infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`, + status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, + platformDbStatus: scoped.spec.externalPostgres === undefined + ? null + : `bun scripts/cli.ts platform-db postgres status --config ${scoped.spec.externalPostgres.configRef}`, + publicExposure: scoped.spec.publicExposure === null ? null : `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${scoped.node} --lane ${scoped.lane} --confirm`, + }, + }; +} + +function runNodeObservability(options: NodeObservabilityOptions): Record { + if (options.action === "plan") return nodeObservabilityPlan(options); + if (options.action === "apply") return nodeObservabilityApply(options); + if (options.action === "status") return nodeObservabilityStatus(options); + return nodeObservabilityWorkbenchSummary(options); +} + +function nodeObservabilityPlan(options: NodeObservabilityOptions): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + return { + ok: configProblem === null, + command: `hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`, + mode: "node-observability-plan", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + expected: nodeObservabilityExpected(options.spec), + renderPlan: nodeObservabilityRenderPlan(options.spec.observability), + degradedReason: configProblem, + next: { + dryRun: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`, + apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`, + status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`, + workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`, + }, + }; +} + +function nodeObservabilityApply(options: NodeObservabilityOptions): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + const applyPlan = nodeObservabilityApplyPlan(options.spec.observability); + if (options.dryRun) { + return { + ok: configProblem === null && applyPlan.supported === true, + command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`, + mode: "node-observability-apply-dry-run", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + expected: nodeObservabilityExpected(options.spec), + applyPlan, + degradedReason: configProblem ?? (applyPlan.supported === true ? undefined : "node-observability-apply-mode-unsupported"), + }; + } + if (configProblem !== null || applyPlan.supported !== true) { + return { + ok: false, + command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`, + mode: "node-observability-apply", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + applyPlan, + degradedReason: configProblem ?? "node-observability-apply-mode-unsupported", + }; + } + const status = nodeObservabilityStatus({ ...options, full: true }); + return { + ok: status.ok === true, + command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`, + mode: "node-observability-apply", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + applyPlan, + appliedResources: [], + applyResult: { + mode: applyPlan.mode, + reason: applyPlan.reason, + runtimeValidationOnly: true, + }, + status: options.full ? status : summarizeNodeObservabilityStatus(status), + degradedReason: status.ok === true ? undefined : "node-observability-status-not-ready", + }; +} + +function nodeObservabilityStatus(options: NodeObservabilityOptions): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + const serviceStatus = nodeObservabilityServiceStatus(options); + const publicRawMetrics = nodeObservabilityPublicRawMetricsProbe(options); + const workbenchSummary = nodeObservabilityWorkbenchSummary(options, serviceStatus); + const ok = configProblem === null && serviceStatus.ready === true && publicRawMetrics.ok === true && workbenchSummary.ok === true; + const status = { + ok, + command: `hwlab nodes observability status --node ${options.node} --lane ${options.lane}`, + mode: "node-observability-status", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + expected: nodeObservabilityExpected(options.spec), + service: serviceStatus, + publicRawMetrics, + workbenchSummary, + degradedReason: configProblem + ?? (serviceStatus.ready === true + ? publicRawMetrics.ok === true + ? workbenchSummary.ok === true ? undefined : "node-observability-workbench-metrics-not-ready" + : "node-observability-public-raw-metrics-boundary-failed" + : "node-observability-service-not-ready"), + next: { + plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`, + workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`, + }, + }; + return options.full ? status : summarizeNodeObservabilityStatus(status); +} + +function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, existingServiceStatus?: Record): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + const serviceStatus = existingServiceStatus ?? nodeObservabilityServiceStatus(options); + const podName = typeof serviceStatus.podName === "string" && serviceStatus.podName.length > 0 ? serviceStatus.podName : null; + const metricsProbe = podName === null ? null : nodeObservabilityMetricsProbe(options, podName); + const metrics = metricsProbe?.summary ?? summarizeNodeObservabilityMetrics(options.spec.observability, "", null); + const ok = configProblem === null && serviceStatus.ready === true && metricsProbe?.ok === true && metrics.ready === true; + return { + ok, + command: `hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`, + mode: "node-observability-workbench-summary", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + metricsEndpoint: nodeObservabilityEndpointSummary(options.spec.observability), + service: { + ready: serviceStatus.ready === true, + namespace: serviceStatus.namespace, + serviceName: serviceStatus.serviceName, + podName, + containerName: serviceStatus.containerName, + }, + recordingRules: nodeObservabilityRecordingRuleSummaries(options.spec.observability), + warningAlerts: nodeObservabilityWarningAlertSummaries(options.spec.observability), + metrics, + probe: metricsProbe === null ? null : { + ok: metricsProbe.ok, + httpStatus: metricsProbe.httpStatus, + bodyBytes: metricsProbe.bodyBytes, + result: compactRuntimeCommand(metricsProbe.result), + error: metricsProbe.error, + }, + degradedReason: configProblem + ?? (serviceStatus.ready === true + ? metricsProbe?.ok === true + ? metrics.ready === true ? undefined : "node-observability-required-series-missing" + : "node-observability-metrics-endpoint-not-readable" + : "node-observability-service-not-ready"), + }; +} + +function summarizeNodeObservabilityStatus(status: Record): Record { + const service = record(status.service); + const publicRawMetrics = record(status.publicRawMetrics); + const workbenchSummary = record(status.workbenchSummary); + const metrics = record(workbenchSummary.metrics); + const recordingRules = Array.isArray(workbenchSummary.recordingRules) ? workbenchSummary.recordingRules : []; + const warningAlerts = Array.isArray(workbenchSummary.warningAlerts) ? workbenchSummary.warningAlerts : []; + return { + ok: status.ok === true, + command: status.command, + mode: "node-observability-status-summary", + mutation: false, + node: status.node, + lane: status.lane, + degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null, + service: { + ready: service.ready === true, + namespace: service.namespace ?? null, + serviceName: service.serviceName ?? null, + podName: service.podName ?? null, + containerName: service.containerName ?? null, + }, + publicRawMetrics: { + ok: publicRawMetrics.ok === true, + expected: publicRawMetrics.expected ?? null, + httpStatus: publicRawMetrics.httpStatus ?? null, + exposesPrometheus: publicRawMetrics.exposesPrometheus === true, + }, + workbench: { + ok: workbenchSummary.ok === true, + httpStatus: metrics.httpStatus ?? null, + bodyBytes: metrics.bodyBytes ?? null, + recordingRuleCount: recordingRules.length, + warningAlertCount: warningAlerts.length, + seriesByPrefix: metrics.seriesByPrefix ?? {}, + missingSeries: metrics.missingSeries ?? [], + topSlowDimensions: metrics.topSlowDimensions ?? [], + deniedBackendEventLineCount: metrics.deniedBackendEventLineCount ?? null, + maxDeniedBackendEventLines: metrics.maxDeniedBackendEventLines ?? null, + }, + next: status.next, + }; +} + +function nodeObservabilityExpected(spec: HwlabRuntimeLaneSpec): Record { + return { + node: spec.nodeId, + lane: spec.lane, + namespace: spec.runtimeNamespace, + publicApiUrl: spec.publicApiUrl, + observability: spec.observability, + sourceOfTruth: hwlabRuntimeLaneConfigPath(), + runtimeAsSourceOfTruth: false, + }; +} + +function nodeObservabilityRenderPlan(observability: HwlabRuntimeObservabilitySpec): Record { + const endpoint = observability.metricsEndpoint; + return { + prometheusOperator: observability.prometheusOperator, + metricsEndpoint: nodeObservabilityEndpointSummary(observability), + workbench: observability.workbench ?? null, + recordingRules: nodeObservabilityRecordingRuleSummaries(observability), + warningAlerts: nodeObservabilityWarningAlertSummaries(observability), + clusterResources: observability.prometheusOperator + ? { + source: "HWLAB GitOps rendered manifests", + serviceMonitor: "YAML-rendered only when declared by target lane", + prometheusRule: "YAML-rendered only when declared by target lane", + } + : { + source: "none", + reason: endpoint?.scrapeMode === "pod-loopback" ? "target lane declares pod-loopback scrape mode" : "no YAML-declared scrape target", + }, + publicRawMetrics: endpoint?.publicRawMetrics ?? null, + }; +} + +function nodeObservabilityApplyPlan(observability: HwlabRuntimeObservabilitySpec): Record { + const endpoint = observability.metricsEndpoint; + if (endpoint === undefined) { + return { + supported: false, + mode: "unconfigured", + reason: "metricsEndpoint is missing from YAML", + }; + } + if (!observability.prometheusOperator && endpoint.scrapeMode === "pod-loopback") { + return { + supported: true, + mode: "validated-noop", + reason: "target lane declares pod-loopback metrics collection and no Prometheus Operator resources", + resources: [], + validates: ["kubernetes-service", "selected-pod", "pod-loopback-metrics", "public-raw-metrics-boundary", "workbench-required-series"], + }; + } + if (observability.prometheusOperator) { + return { + supported: false, + mode: "prometheus-operator", + reason: "Prometheus Operator object rendering must be declared in the target lane GitOps YAML before this node-scoped apply can own it", + }; + } + return { + supported: false, + mode: endpoint.scrapeMode, + reason: "unsupported metricsEndpoint.scrapeMode", + }; +} + +function nodeObservabilityConfigProblem(observability: HwlabRuntimeObservabilitySpec): string | null { + if (observability.metricsEndpoint === undefined) return "node-observability-metrics-endpoint-not-declared"; + if (observability.workbench === undefined) return "node-observability-workbench-not-declared"; + if (!observability.workbench.enabled) return "node-observability-workbench-disabled"; + return null; +} + +function nodeObservabilityEndpointSummary(observability: HwlabRuntimeObservabilitySpec): Record | null { + const endpoint = observability.metricsEndpoint; + if (endpoint === undefined) return null; + return { + serviceName: endpoint.serviceName, + containerName: endpoint.containerName, + port: endpoint.port, + scheme: endpoint.scheme, + path: endpoint.path, + scrapeMode: endpoint.scrapeMode, + publicRawMetrics: endpoint.publicRawMetrics, + }; +} + +function nodeObservabilityServiceStatus(options: NodeObservabilityOptions): Record { + const endpoint = options.spec.observability.metricsEndpoint; + if (endpoint === undefined) { + return { + ready: false, + namespace: options.spec.runtimeNamespace, + serviceName: null, + podName: null, + containerName: null, + degradedReason: "node-observability-metrics-endpoint-not-declared", + }; + } + const namespace = runNodeK3sArgs(options.spec, ["kubectl", "get", "ns", options.spec.runtimeNamespace, "-o", "name"], 60); + const namespaceExists = namespace.exitCode === 0; + const service = namespaceExists + ? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "svc", endpoint.serviceName, "-o", "json"], 60) + : null; + const serviceJson = service !== null && service.exitCode === 0 ? parseJsonRecordFromText(service.stdout) : {}; + const selector = k8sServiceSelector(serviceJson); + const selectorText = labelSelectorText(selector); + const pods = namespaceExists && service !== null && service.exitCode === 0 && selectorText.length > 0 + ? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "pods", "-l", selectorText, "-o", "json"], 60) + : null; + const pod = pods !== null && pods.exitCode === 0 ? selectK8sPod(parseJsonRecordFromText(pods.stdout), endpoint.containerName) : null; + const ready = namespaceExists && service !== null && service.exitCode === 0 && pod?.ready === true; + return { + ready, + namespace: options.spec.runtimeNamespace, + namespaceExists, + serviceName: endpoint.serviceName, + serviceExists: service !== null && service.exitCode === 0, + selector, + selectorText: selectorText.length > 0 ? selectorText : null, + podName: pod?.name ?? null, + podPhase: pod?.phase ?? null, + podReady: pod?.ready ?? false, + containerName: endpoint.containerName, + containerReady: pod?.containerReady ?? false, + probes: { + namespace: compactRuntimeCommand(namespace), + service: service === null ? null : compactRuntimeCommand(service), + pods: pods === null ? null : compactRuntimeCommand(pods), + }, + degradedReason: ready ? undefined : namespaceExists ? "node-observability-pod-or-service-not-ready" : "node-observability-namespace-missing", + }; +} + +interface NodeObservabilityMetricsProbeResult { + readonly ok: boolean; + readonly httpStatus: number | null; + readonly bodyBytes: number; + readonly summary: Record; + readonly result: CommandResult; + readonly error: string | null; +} + +function nodeObservabilityMetricsProbe(options: NodeObservabilityOptions, podName: string): NodeObservabilityMetricsProbeResult { + const endpoint = options.spec.observability.metricsEndpoint; + if (endpoint === undefined) throw new Error("metricsEndpoint is required before probing metrics"); + const workbench = options.spec.observability.workbench; + const summaryConfig = Buffer.from(JSON.stringify({ + metricPrefixes: workbench?.metricPrefixes ?? [], + requiredSeries: workbench?.requiredSeries ?? [], + backendLabelDenylist: workbench?.backendLabelDenylist ?? [], + maxDeniedBackendEventLines: workbench?.maxUnknownEventLines ?? 0, + lowSampleThreshold: workbench?.lowSampleThreshold ?? 0, + histogramMetrics: [ + { id: "workbench_journey", metric: "hwlab_workbench_journey_duration_seconds", kind: "workbench_journey", dimensionLabels: ["namespace", "gitops_target", "journey", "route", "backend", "transport", "target_state", "cache", "source", "entry", "outcome"] }, + { id: "workbench_event_phase", metric: "hwlab_workbench_event_phase_duration_seconds", kind: "workbench_event_phase", dimensionLabels: ["namespace", "gitops_target", "phase", "event_type", "backend", "transport", "outcome"] }, + { id: "workbench_backend_event_visible", metric: "hwlab_workbench_backend_event_visible_latency_seconds", kind: "workbench_backend_event_visible", dimensionLabels: ["namespace", "gitops_target", "event_type", "backend", "transport", "outcome"] }, + ], + }), "utf8").toString("base64"); + const source = [ + "const http = require('node:http');", + "const port = Number(process.argv[1]);", + "const path = process.argv[2];", + "const config = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));", + "function metricNameFromLine(line) { const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/.exec(line); return match ? match[1] : null; }", + "function lineMatchesMetric(line, name) { if (!line.startsWith(name)) return false; const next = line.charAt(name.length); return next === '' || next === '{' || /\\s/.test(next); }", + "function lineHasLabel(line, name, value) { return line.includes(`${name}=\\\"${String(value).replace(/\\\\/g, '\\\\\\\\').replace(/\\\"/g, '\\\\\\\"')}\\\"`); }", + "function isBackendEventMetric(name) { return name.startsWith('hwlab_workbench_event_') || name.startsWith('hwlab_workbench_backend_event_'); }", + "function compactLines(lines) { return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); }", + "function parseLabels(raw) { const labels = {}; for (const part of String(raw || '').split(',')) { const index = part.indexOf('='); if (index <= 0) continue; const key = part.slice(0, index); let value = part.slice(index + 1); if (value.startsWith('\\\"') && value.endsWith('\\\"')) value = value.slice(1, -1); labels[key] = value.replace(/\\\\\\\"/g, '\\\"').replace(/\\\\\\\\/g, '\\\\'); } return labels; }", + "function labelKey(metric, labels) { const copy = { ...labels }; delete copy.le; return `${metric}|${Object.keys(copy).sort().map((key) => `${key}=${copy[key]}`).join('|')}`; }", + "function pickLabels(labels, names) { return Object.fromEntries(names.map((name) => [name, labels[name] || 'unknown'])); }", + "function quantile(row, q) { const count = Number(row.count || 0); if (count <= 0) return 0; const rank = Math.max(1, Math.ceil(count * q)); const buckets = row.buckets.filter((item) => item.le !== '+Inf').sort((left, right) => Number(left.le) - Number(right.le)); for (const bucket of buckets) { if (bucket.value >= rank) return Number(bucket.le); } return buckets.length ? Number(buckets[buckets.length - 1].le) : 0; }", + "function summarizeHistograms(sampleLines) {", + " const metricConfig = new Map(config.histogramMetrics.map((item) => [item.metric, item]));", + " const series = new Map();", + " for (const line of sampleLines) {", + " const match = /^([A-Za-z_:][A-Za-z0-9_:]*?)_(bucket|sum|count)\\{([^}]*)\\}\\s+([0-9.eE+-]+)/.exec(line);", + " if (!match || !metricConfig.has(match[1])) continue;", + " const metric = match[1];", + " const suffix = match[2];", + " const labels = parseLabels(match[3]);", + " const value = Number(match[4]);", + " if (!Number.isFinite(value)) continue;", + " const key = labelKey(metric, labels);", + " const row = series.get(key) || { metric, labels: { ...labels }, buckets: [], sum: 0, count: 0 };", + " if (suffix === 'bucket') row.buckets.push({ le: labels.le || '+Inf', value });", + " if (suffix === 'sum') row.sum = value;", + " if (suffix === 'count') row.count = value;", + " series.set(key, row);", + " }", + " const lowSampleThreshold = Number(config.lowSampleThreshold || 0);", + " const groups = Object.fromEntries(config.histogramMetrics.map((item) => [item.id, []]));", + " const rows = [];", + " for (const row of series.values()) {", + " const cfg = metricConfig.get(row.metric);", + " const sampleCount = Number(row.count || 0);", + " const output = { kind: cfg.kind, metric: row.metric, sampleCount, average: sampleCount > 0 ? Number((row.sum / sampleCount).toFixed(4)) : 0, p50: quantile(row, 0.5), p75: quantile(row, 0.75), p95: quantile(row, 0.95), lowSample: sampleCount > 0 && sampleCount < lowSampleThreshold, sampleState: sampleCount <= 0 ? 'empty' : sampleCount < lowSampleThreshold ? 'low-sample' : 'ok', dimensions: pickLabels(row.labels, cfg.dimensionLabels) };", + " groups[cfg.id].push(output);", + " rows.push(output);", + " }", + " for (const key of Object.keys(groups)) groups[key].sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);", + " rows.sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);", + " return { groups: Object.fromEntries(Object.entries(groups).map(([key, value]) => [key, value.slice(0, 24)])), topSlowDimensions: rows.slice(0, 12) };", + "}", + "function summarize(text, statusCode) {", + " const lines = text.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean);", + " const sampleLines = lines.filter((line) => !line.startsWith('#'));", + " const seriesByPrefix = Object.fromEntries(config.metricPrefixes.map((prefix) => [prefix, sampleLines.filter((line) => (metricNameFromLine(line) || '').startsWith(prefix)).length]));", + " const requiredSeries = config.requiredSeries.map((name) => { const matching = sampleLines.filter((line) => lineMatchesMetric(line, name)); return { name, present: matching.length > 0, sampleCount: matching.length }; });", + " const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);", + " const deniedBackendEventLines = sampleLines.filter((line) => { const name = metricNameFromLine(line) || ''; if (!isBackendEventMetric(name)) return false; return config.backendLabelDenylist.some((label) => lineHasLabel(line, 'backend', label)); });", + " const histogramSummary = summarizeHistograms(sampleLines);", + " const ready = statusCode >= 200 && statusCode < 300 && missingSeries.length === 0 && deniedBackendEventLines.length <= config.maxDeniedBackendEventLines;", + " return {", + " ready,", + " httpStatus: statusCode,", + " bodyBytes: Buffer.byteLength(text),", + " lineCount: lines.length,", + " sampleLineCount: sampleLines.length,", + " metricPrefixes: config.metricPrefixes,", + " seriesByPrefix,", + " requiredSeries,", + " missingSeries,", + " backendLabelDenylist: config.backendLabelDenylist,", + " lowSampleThreshold: config.lowSampleThreshold,", + " workbenchHistograms: histogramSummary.groups,", + " topSlowDimensions: histogramSummary.topSlowDimensions,", + " deniedBackendEventLineCount: deniedBackendEventLines.length,", + " maxDeniedBackendEventLines: config.maxDeniedBackendEventLines,", + " deniedBackendEventLines: compactLines(deniedBackendEventLines),", + " backendVisibleCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_backend_event_visible_latency_seconds_count'))),", + " phaseCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_event_phase_duration_seconds_count'))),", + " degradedReason: ready ? undefined : missingSeries.length > 0 ? 'node-observability-required-series-missing' : 'node-observability-denied-backend-label-present',", + " };", + "}", + "let settled = false;", + "const finish = (payload, code = 0) => { if (settled) return; settled = true; console.log(JSON.stringify(payload)); process.exitCode = code; };", + "const req = http.get({ host: '127.0.0.1', port, path, timeout: 8000 }, (res) => {", + " let body = '';", + " res.setEncoding('utf8');", + " res.on('data', (chunk) => { body += chunk; if (body.length > 2000000) req.destroy(new Error('metrics body exceeded 2MB')); });", + " res.on('end', () => finish({ ok: res.statusCode >= 200 && res.statusCode < 300, statusCode: res.statusCode, bodyBytes: Buffer.byteLength(body), summary: summarize(body, res.statusCode || 0) }));", + "});", + "req.on('timeout', () => req.destroy(new Error('timeout')));", + "req.on('error', (error) => finish({ ok: false, statusCode: null, bodyBytes: 0, summary: null, error: String(error && error.message || error) }, 1));", + ].join("\n"); + const result = runNodeK3sArgs(options.spec, [ + "kubectl", + "-n", options.spec.runtimeNamespace, + "exec", podName, + "-c", endpoint.containerName, + "--", + "node", "-e", source, + String(endpoint.port), + endpoint.path, + summaryConfig, + ], options.timeoutSeconds); + const payload = parseJsonRecordFromText(result.stdout); + const httpStatus = typeof payload.statusCode === "number" ? payload.statusCode : null; + const error = typeof payload.error === "string" ? payload.error : null; + const summary = record(payload.summary); + return { + ok: result.exitCode === 0 && payload.ok === true && httpStatus !== null && httpStatus >= 200 && httpStatus < 300, + httpStatus, + bodyBytes: typeof payload.bodyBytes === "number" ? payload.bodyBytes : 0, + summary, + result, + error, + }; +} + +function nodeObservabilityPublicRawMetricsProbe(options: NodeObservabilityOptions): Record { + const endpoint = options.spec.observability.metricsEndpoint; + if (endpoint === undefined) { + return { + ok: false, + expected: null, + degradedReason: "node-observability-metrics-endpoint-not-declared", + }; + } + const url = joinUrlPath(options.spec.publicApiUrl, endpoint.path); + const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "-", "-w", "\nUNIDESK_HTTP_STATUS:%{http_code}", url], repoRoot, { timeoutMs: 15_000 }); + const parsed = splitCurlBodyStatus(result.stdout); + const exposesPrometheus = looksLikePrometheusMetrics(parsed.body, options.spec.observability.workbench?.metricPrefixes ?? []); + const denied = endpoint.publicRawMetrics === "denied" && (parsed.httpStatus === null || parsed.httpStatus >= 400 || !exposesPrometheus); + return { + ok: result.exitCode === 0 && denied, + expected: endpoint.publicRawMetrics, + url, + httpStatus: parsed.httpStatus, + exposesPrometheus, + bodyBytes: Buffer.byteLength(parsed.body), + result: compactRuntimeCommand(result), + degradedReason: denied ? undefined : "node-observability-public-raw-metrics-exposed", + }; +} + +function summarizeNodeObservabilityMetrics(observability: HwlabRuntimeObservabilitySpec, text: string, httpStatus: number | null): Record { + const workbench = observability.workbench; + if (workbench === undefined) { + return { + ready: false, + httpStatus, + degradedReason: "node-observability-workbench-not-declared", + }; + } + const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + const sampleLines = lines.filter((line) => !line.startsWith("#")); + const seriesByPrefix = Object.fromEntries(workbench.metricPrefixes.map((prefix) => [ + prefix, + sampleLines.filter((line) => (metricNameFromPrometheusLine(line) ?? "").startsWith(prefix)).length, + ])); + const requiredSeries = workbench.requiredSeries.map((name) => { + const matching = sampleLines.filter((line) => prometheusLineMatchesMetric(line, name)); + return { + name, + present: matching.length > 0, + sampleCount: matching.length, + }; + }); + const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name); + const deniedBackendEventLines = sampleLines.filter((line) => { + const name = metricNameFromPrometheusLine(line) ?? ""; + if (!isWorkbenchBackendEventMetric(name)) return false; + return workbench.backendLabelDenylist.some((label) => prometheusLineHasLabel(line, "backend", label)); + }); + const ready = httpStatus !== null + && httpStatus >= 200 + && httpStatus < 300 + && missingSeries.length === 0 + && deniedBackendEventLines.length <= workbench.maxUnknownEventLines; + return { + ready, + httpStatus, + bodyBytes: Buffer.byteLength(text), + lineCount: lines.length, + sampleLineCount: sampleLines.length, + metricPrefixes: workbench.metricPrefixes, + seriesByPrefix, + requiredSeries, + missingSeries, + backendLabelDenylist: workbench.backendLabelDenylist, + deniedBackendEventLineCount: deniedBackendEventLines.length, + maxDeniedBackendEventLines: workbench.maxUnknownEventLines, + deniedBackendEventLines: compactPrometheusLines(deniedBackendEventLines), + backendVisibleCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_backend_event_visible_latency_seconds_count"))), + phaseCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_event_phase_duration_seconds_count"))), + degradedReason: ready ? undefined : missingSeries.length > 0 ? "node-observability-required-series-missing" : "node-observability-denied-backend-label-present", + }; +} + +function splitCurlBodyStatus(stdout: string): { body: string; httpStatus: number | null } { + const marker = "\nUNIDESK_HTTP_STATUS:"; + const index = stdout.lastIndexOf(marker); + if (index < 0) return { body: stdout, httpStatus: null }; + const rawStatus = stdout.slice(index + marker.length).trim(); + const status = nullableInteger(rawStatus); + return { + body: stdout.slice(0, index), + httpStatus: status, + }; +} + +function looksLikePrometheusMetrics(text: string, prefixes: readonly string[]): boolean { + const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + return lines.some((line) => { + if (line.startsWith("# HELP ") || line.startsWith("# TYPE ")) return true; + const name = metricNameFromPrometheusLine(line); + return name !== null && (prefixes.length === 0 || prefixes.some((prefix) => name.startsWith(prefix))); + }); +} + +function metricNameFromPrometheusLine(line: string): string | null { + const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line); + return match?.[1] ?? null; +} + +function prometheusLineMatchesMetric(line: string, name: string): boolean { + if (!line.startsWith(name)) return false; + const next = line.charAt(name.length); + return next === "" || next === "{" || /\s/u.test(next); +} + +function prometheusLineHasLabel(line: string, name: string, value: string): boolean { + return line.includes(`${name}="${value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"')}"`); +} + +function isWorkbenchBackendEventMetric(name: string): boolean { + return name.startsWith("hwlab_workbench_event_") || name.startsWith("hwlab_workbench_backend_event_"); +} + +function compactPrometheusLines(lines: readonly string[]): string[] { + return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); +} + +function parseJsonRecordFromText(text: string): Record { + const trimmed = text.trim(); + if (trimmed.length === 0) return {}; + try { + const parsed = JSON.parse(trimmed) as unknown; + return record(parsed); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { + return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown); + } catch {} + } + } + return {}; +} + +function k8sServiceSelector(serviceJson: Record): Record { + const spec = record(serviceJson.spec); + const selector = record(spec.selector); + return Object.fromEntries(Object.entries(selector).filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].length > 0)); +} + +function labelSelectorText(selector: Record): string { + return Object.entries(selector).map(([key, value]) => `${key}=${value}`).join(","); +} + +function selectK8sPod(podsJson: Record, containerName: string): { name: string; phase: string | null; ready: boolean; containerReady: boolean } | null { + const items = Array.isArray(podsJson.items) ? podsJson.items.map(record) : []; + const pods = items.map((item) => { + const metadata = record(item.metadata); + const status = record(item.status); + const name = typeof metadata.name === "string" ? metadata.name : ""; + const phase = typeof status.phase === "string" ? status.phase : null; + const containerStatuses = Array.isArray(status.containerStatuses) ? status.containerStatuses.map(record) : []; + const targetContainer = containerStatuses.find((container) => container.name === containerName); + const containerReady = targetContainer?.ready === true; + return { + name, + phase, + ready: name.length > 0 && phase === "Running" && containerReady, + containerReady, + }; + }).filter((pod) => pod.name.length > 0); + return pods.find((pod) => pod.ready) ?? pods.find((pod) => pod.phase === "Running") ?? pods[0] ?? null; +} + +function compactRuntimeCommand(result: CommandResult): Record { + return { + exitCode: result.exitCode, + stdoutBytes: Buffer.byteLength(result.stdout), + stderrBytes: Buffer.byteLength(result.stderr), + stdoutTail: result.stdout.trim().slice(-2000), + stderrTail: result.stderr.trim().slice(-2000), + timedOut: result.timedOut, + }; +} + +function parseLastJsonLineObject(text: string): Record { + for (const line of text.split(/\r?\n/u).reverse()) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) continue; + try { + return record(JSON.parse(trimmed) as unknown); + } catch { + // Keep scanning: command tails can mix kubectl status, git output, and JSON payloads. + } + } + return {}; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); +} + +function nodeRuntimeGitMirrorRefSources(scoped: ReturnType, mirror: NodeRuntimeGitMirrorTargetSpec): Record { + return { + localSource: `refs/heads/${mirror.sourceBranch}`, + githubSource: `refs/mirror-stage/heads/${mirror.sourceBranch}`, + localGitops: `refs/heads/${mirror.gitopsBranch}`, + githubGitops: `refs/mirror-stage/heads/${mirror.gitopsBranch}`, + githubFieldsAreMirrorStageCache: true, + cacheRefresh: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`, + }; +} + +function nodeRuntimeGitMirrorFlushPartialSuccess(scoped: ReturnType, mirror: NodeRuntimeGitMirrorTargetSpec, result: CommandResult): Record | null { + if (scoped.action !== "flush") return null; + const text = `${result.stdout}\n${result.stderr}`; + const payload = parseLastJsonLineObject(text); + const partialSuccess = typeof payload.partialSuccess === "string" && payload.partialSuccess.length > 0 + ? payload.partialSuccess + : null; + const payloadStatus = typeof payload.status === "string" && payload.status.length > 0 ? payload.status : null; + const branch = escapeRegExp(mirror.gitopsBranch); + const pushSucceededInGitOutput = new RegExp(`\\b${branch}\\s*->\\s*${branch}\\b`, "u").test(text); + const postPushFetchFailed = /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|fatal:.*fetch|fetch-pack|early EOF/iu.test(text); + if (partialSuccess !== "push-succeeded-fetch-failed" && !(result.exitCode !== 0 && pushSucceededInGitOutput && postPushFetchFailed)) { + return null; + } + return { + status: payloadStatus ?? "partial-success", + partialSuccess: "push-succeeded-fetch-failed", + degradedReason: "node-runtime-git-mirror-flush-post-push-fetch-failed", + retryable: true, + message: "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Refresh mirror-stage cache before deciding another flush is needed.", + payload: Object.keys(payload).length > 0 ? payload : null, + refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), + next: { + sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`, + status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, + }, + }; +} + +function sshTcpPoolDiagnosticsFromCommand(spec: HwlabRuntimeLaneSpec, result: CommandResult): Record | null { + if (isCommandSuccess(result)) return null; + const failureKind = classifySshTcpPoolFailure(`${result.stderr}\n${result.stdout}`); + if (failureKind === null) return null; + return { + classification: "ssh-tcp-pool-transient", + failureKind, + providerId: spec.nodeId, + route: spec.nodeKubeRoute, + message: "SSH tcp data pool failed while running the controlled command; treat this as transport/data-pool transient until provider labels and retry say otherwise.", + next: { + poolStatus: `bun scripts/cli.ts debug ssh-pool ${spec.nodeId}`, + retrySmoke: `trans ${spec.nodeId} argv true`, + retryApply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + }, + }; +} + +function nodeRuntimeUnsupportedAction(scoped: ReturnType): Record { + return { + ok: false, + command: `hwlab nodes control-plane ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mutation: false, + degradedReason: "unsupported-node-scoped-runtime-action", + message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/runtime-image/runtime-migration", + expected: nodeRuntimeExpected(scoped.spec), + }; +} + +function transPath(): string { + return join(repoRoot, "scripts", "trans"); +} + +function runNodeHostScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { + return runCommand([transPath(), spec.nodeRoute, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function runNodeHostScriptAsync(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, label: string): CommandResult { + const token = nodeRuntimeRenderToken(); + const stateDir = `/tmp/unidesk-hwlab-node-runtime/${label}-${token}`; + const statusPath = `${stateDir}/status.json`; + const stdoutPath = `${stateDir}/stdout.log`; + const stderrPath = `${stateDir}/stderr.log`; + const runnerPath = `${stateDir}/run.sh`; + const startScript = [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + `status_path=${shellQuote(statusPath)}`, + `stdout_path=${shellQuote(stdoutPath)}`, + `stderr_path=${shellQuote(stderrPath)}`, + `runner_path=${shellQuote(runnerPath)}`, + "rm -rf \"$state_dir\"", + "mkdir -p \"$state_dir\"", + "cat > \"$runner_path\" <<'UNIDESK_REMOTE_RUNNER'", + script, + "UNIDESK_REMOTE_RUNNER", + "chmod 700 \"$runner_path\"", + "python3 - \"$status_path\" <<'PY'", + "import datetime, json, sys", + "json.dump({'state':'running','ok':None,'pid':None,'startedAt':datetime.datetime.utcnow().isoformat()+'Z','finishedAt':None,'exitCode':None}, open(sys.argv[1], 'w'), indent=2)", + "PY", + "(", + " set +e", + " sh \"$runner_path\" >\"$stdout_path\" 2>\"$stderr_path\"", + " code=$?", + " python3 - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'PY'", + "import datetime, json, pathlib, sys", + "status_path, stdout_path, stderr_path, code = sys.argv[1:5]", + "def tail(path):", + " try:", + " text = pathlib.Path(path).read_text(errors='replace')", + " except FileNotFoundError:", + " return ''", + " return text[-12000:]", + "payload = {'state':'finished','ok':code == '0','pid':None,'startedAt':None,'finishedAt':datetime.datetime.utcnow().isoformat()+'Z','exitCode':int(code),'stdout':tail(stdout_path),'stderr':tail(stderr_path)}", + "try:", + " current = json.load(open(status_path))", + " payload['startedAt'] = current.get('startedAt')", + "except Exception:", + " pass", + "json.dump(payload, open(status_path, 'w'), indent=2)", + "PY", + ") >/dev/null 2>&1 &", + "pid=$!", + "python3 - \"$status_path\" \"$pid\" <<'PY'", + "import json, sys", + "path, pid = sys.argv[1:3]", + "payload = json.load(open(path))", + "payload['pid'] = int(pid)", + "json.dump(payload, open(path, 'w'), indent=2)", + "PY", + "printf '{\"ok\":true,\"state\":\"started\",\"pid\":%s,\"statusPath\":\"%s\"}\\n' \"$pid\" \"$status_path\"", + ].join("\n"); + const start = runNodeHostScript(spec, startScript, 55); + if (!isCommandSuccess(start)) return start; + const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000; + let lastStatus: CommandResult | null = null; + let payload: Record = {}; + while (Date.now() <= deadline) { + runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); + const status = runNodeHostScript(spec, `cat ${shellQuote(statusPath)} 2>/dev/null || printf '{"state":"missing","ok":false,"exitCode":127,"stdout":"","stderr":"missing async status"}\\n'`, 55); + lastStatus = status; + payload = parseJsonObject(status.stdout.trim()); + if (payload.state === "finished" || payload.ok === true || payload.ok === false) { + return commandResultFromAsync(spec, payload, statusPath, false); + } + } + const kill = runNodeHostScript(spec, [ + "set +e", + `status_path=${shellQuote(statusPath)}`, + "pid=$(python3 - \"$status_path\" <<'PY' 2>/dev/null", + "import json, sys", + "try: print(json.load(open(sys.argv[1])).get('pid') or '')", + "except Exception: print('')", + "PY", + ")", + "if [ -n \"$pid\" ]; then kill \"$pid\" 2>/dev/null || true; fi", + "cat \"$status_path\" 2>/dev/null || true", + ].join("\n"), 55); + return { + command: [transPath(), spec.nodeRoute, "sh", "--", ""], + cwd: repoRoot, + exitCode: 124, + stdout: typeof payload.stdout === "string" ? payload.stdout : "", + stderr: [ + `remote async command timed out after ${timeoutSeconds}s; statusPath=${statusPath}`, + typeof payload.stderr === "string" ? payload.stderr : "", + lastStatus?.stderr.trim() ?? "", + kill.stderr.trim(), + ].filter(Boolean).join("\n"), + signal: null, + timedOut: true, + }; +} + +function parseJsonObject(text: string): Record { + try { + const parsed = JSON.parse(text) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; + } catch { + return {}; + } +} + +function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record, statusPath: string, timedOut: boolean): CommandResult { + return { + command: [transPath(), spec.nodeRoute, "sh", "--", ""], + cwd: repoRoot, + exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1, + stdout: typeof payload.stdout === "string" ? payload.stdout : "", + stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`, + signal: null, + timedOut, + }; +} + +function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult { + return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 }); +} + +function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { + return runCommand([transPath(), spec.nodeKubeRoute, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function isCommandSuccess(result: CommandResult): boolean { + return result.exitCode === 0 && !result.timedOut; +} + +function shortSha(value: string): string { + return value.slice(0, 12).toLowerCase(); +} + +function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { + return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`; +} + +function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string { + const suffix = `/${spec.runtimeRenderDir}`; + if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length); + return spec.gitopsRoot; +} + +function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { + const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 }); + if (!isCommandSuccess(result)) return { sourceCommit: null, result }; + const match = /[0-9a-f]{40}/iu.exec(statusText(result)); + return { sourceCommit: match?.[0].toLowerCase() ?? null, result }; +} + +function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { + const gitRetries = Math.max(1, spec.downloadProfile.git.retries); + const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); + return [ + `cicd_repo=${shellQuote(spec.cicdRepo)}`, + `cicd_url=${shellQuote(spec.gitUrl)}`, + `cicd_branch=${shellQuote(spec.sourceBranch)}`, + `cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`, + `git_retries=${shellQuote(String(gitRetries))}`, + `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, + "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", + "retry_git() {", + " op=$1", + " shift", + " attempt=1", + " while [ \"$attempt\" -le \"$git_retries\" ]; do", + " echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2", + " run_git \"$@\"", + " code=$?", + " if [ \"$code\" -eq 0 ]; then return 0; fi", + " echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2", + " attempt=$((attempt + 1))", + " done", + " return 1", + "}", + "mkdir -p \"$(dirname \"$cicd_repo\")\"", + "if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", + " :", + "elif [ -e \"$cicd_repo\" ]; then", + " echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", + " exit 41", + "else", + " retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", + "fi", + "git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"", + "git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", + "if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then", + " rm -rf \"$cicd_repo\"", + " retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", + " git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", + " retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune", + "fi", + ].join("\n"); +} + +function nodeRuntimeControlPlaneRun(scoped: ReturnType): Record { + if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped); + if (scoped.action === "sync") return nodeRuntimeSync(scoped); + if (scoped.action === "apply") return nodeRuntimeApply(scoped); + if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped); + if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped); + return nodeRuntimeUnsupportedAction(scoped); +} + +function nodeRuntimeBaseImageCommand(scoped: ReturnType): Record { + const action = scoped.runtimeImageAction; + if (action === null) { + return { + ok: false, + command: `hwlab nodes control-plane runtime-image --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mutation: false, + degradedReason: "node-runtime-image-action-missing", + message: "runtime-image requires one of: status, preload, build", + expected: nodeRuntimeExpected(scoped.spec), + }; + } + if (action !== "status" && action !== "preload" && action !== "build") { + return { + ok: false, + command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mutation: false, + degradedReason: "unsupported-node-runtime-image-action", + message: "runtime-image currently supports status/preload/build", + expected: nodeRuntimeExpected(scoped.spec), + }; + } + const statusBefore = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds); + if (action === "status") { + return { + ok: statusBefore.ok, + command: `hwlab nodes control-plane runtime-image status --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "status", + mutation: false, + status: statusBefore, + degradedReason: statusBefore.ok ? undefined : "node-runtime-base-image-not-ready", + next: statusBefore.ok ? undefined : { + preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm`, + }, + }; + } + if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-image preload/build requires --dry-run or --confirm"); + const preload = ensureNodeBaseImage(scoped.spec, scoped.dryRun, scoped.timeoutSeconds); + const statusAfter = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds); + return { + ok: preload !== null && preload.ok === true && (scoped.dryRun || statusAfter.ok === true), + command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-preload", + requestedAction: action, + effectiveAction: "preload", + mutation: !scoped.dryRun && preload !== null && preload.ok === true, + statusBefore, + preload, + statusAfter, + degradedReason: preload === null + ? "node-runtime-base-image-source-missing" + : preload.ok === true && (scoped.dryRun || statusAfter.ok === true) + ? undefined + : "node-runtime-base-image-seed-failed", + next: scoped.dryRun + ? { preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm` } + : { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun` }, + }; +} + +function nodeRuntimeMigration(scoped: ReturnType): Record { + const spec = scoped.spec; + if (scoped.allowLiveDbRead && scoped.confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm"); + if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-migration requires --dry-run or --confirm"); + const head = resolveNodeRuntimeLaneHead(spec); + const sourceCommit = head.sourceCommit; + if (sourceCommit === null) { + return { + ok: false, + command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "source-head", + mutation: false, + degradedReason: "node-runtime-source-head-unresolved", + headProbe: compactRuntimeCommand(head.result), + }; + } + const reportPath = `/tmp/hwlab-${scoped.node.toLowerCase()}-${scoped.lane}-runtime-migration-${shortSha(sourceCommit)}.json`; + const migrationArgs = scoped.dryRun + ? [ + ...(scoped.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]), + "--report", + reportPath, + ] + : [ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--report", + reportPath, + ]; + const result = runNodeK3sArgs(spec, [ + "kubectl", + "exec", + "-n", + spec.runtimeNamespace, + "deployment/hwlab-cloud-api", + "-c", + "hwlab-cloud-api", + "--", + "sh", + "-lc", + `cd /workspace/hwlab-boot/repo && exec bun cmd/hwlab-cloud-api/migrate.ts ${migrationArgs.map(shellQuote).join(" ")}`, + ], scoped.timeoutSeconds); + const ok = isCommandSuccess(result); + return { + ok, + command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? scoped.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply", + sourceCommit, + runtimeNamespace: spec.runtimeNamespace, + target: "deployment/hwlab-cloud-api -c hwlab-cloud-api", + workingDirectory: "/workspace/hwlab-boot/repo", + migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs], + reportPath, + mutation: !scoped.dryRun && ok, + result: compactRuntimeCommand(result), + degradedReason: ok ? undefined : "node-runtime-migration-failed", + next: scoped.dryRun + ? { + liveReadDryRun: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --allow-live-db-read --dry-run`, + apply: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --confirm`, + } + : { + health: `curl -fsS --max-time 20 ${spec.publicApiUrl}/health/live`, + status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, + }, + }; +} + +function nodeRuntimeApply(scoped: ReturnType): Record { + const spec = scoped.spec; + const head = resolveNodeRuntimeLaneHead(spec); + const sourceCommit = head.sourceCommit; + if (sourceCommit === null) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "source-head", + degradedReason: "node-runtime-source-head-unresolved", + headProbe: compactRuntimeCommand(head.result), + }; + } + const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds); + if (secrets !== null && secrets.ok !== true) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + phase: "external-postgres-secret-sync", + sourceCommit, + secrets, + degradedReason: "external-postgres-secret-sync-failed", + }; + } + const baseImage = ensureNodeBaseImage(spec, scoped.dryRun, scoped.timeoutSeconds); + if (baseImage !== null && baseImage.ok !== true) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + phase: "base-image-seed", + sourceCommit, + secrets, + baseImage, + degradedReason: "node-runtime-base-image-seed-failed", + }; + } + const render = renderNodeRuntimeControlPlane(spec, sourceCommit, scoped.timeoutSeconds); + if (!isCommandSuccess(render.result)) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + phase: "source-render", + sourceCommit, + secrets, + baseImage, + renderDir: render.renderDir, + renderLocation: render.location, + render: compactRuntimeCommand(render.result), + degradedReason: "node-runtime-control-plane-render-failed", + }; + } + const apply = render.location === "local" + ? applyLocalNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds) + : applyNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds); + const cleanup = render.location === "local" + ? cleanupLocalNodeRuntimeRenderDir(spec, render) + : cleanupNodeRuntimeRenderDir(spec, render.renderDir); + const sshTcpPoolDiagnostics = sshTcpPoolDiagnosticsFromCommand(spec, apply); + return { + ok: isCommandSuccess(apply), + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + mutation: !scoped.dryRun && isCommandSuccess(apply), + sourceCommit, + expected: nodeRuntimeExpected(spec), + secrets, + baseImage, + renderDir: render.renderDir, + renderLocation: render.location, + render: compactRuntimeCommand(render.result), + apply: compactRuntimeCommand(apply), + cleanupRenderDir: compactRuntimeCommand(cleanup), + diagnostics: sshTcpPoolDiagnostics, + degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed", + next: scoped.dryRun + ? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` } + : { + triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, + ...(sshTcpPoolDiagnostics === null ? {} : { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` }), + }, + }; +} + +function nodeRuntimeRefresh(scoped: ReturnType): Record { + const spec = scoped.spec; + const script = [ + "set +e", + `app=${shellQuote(spec.app)}`, + "argo_namespace=argocd", + `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`, + "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)", + "if [ \"$dry_run\" = true ]; then", + " output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)", + "else", + " output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)", + "fi", + "code=$?", + "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)", + "printf 'app\\t%s\\n' \"$app\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'before\\t%s\\n' \"$before\"", + "printf 'after\\t%s\\n' \"$after\"", + "printf 'annotateExitCode\\t%s\\n' \"$code\"", + "printf 'annotateOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", + "exit \"$code\"", + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const fields = keyValueLinesFromText(statusText(result)); + return { + ok: isCommandSuccess(result), + command: `hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-refresh", + mutation: !scoped.dryRun && isCommandSuccess(result), + argoApplication: fields.app || spec.app, + before: fields.before || null, + after: fields.after || null, + annotateExitCode: numericField(fields.annotateExitCode), + result: compactRuntimeCommand(result), + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, + }; +} + +function nodeRuntimeSync(scoped: ReturnType): Record { + const spec = scoped.spec; + const operation = { + operation: { + initiatedBy: { username: "unidesk-hwlab-node-cli" }, + retry: { limit: 1 }, + sync: { + prune: true, + revision: spec.gitopsBranch, + source: { + repoURL: spec.argoRepoUrl, + targetRevision: spec.gitopsBranch, + path: spec.runtimePath, + }, + syncOptions: ["CreateNamespace=true"], + syncStrategy: { hook: {} }, + }, + }, + }; + const patchB64 = Buffer.from(JSON.stringify(operation), "utf8").toString("base64"); + const script = [ + "set +e", + `app=${shellQuote(spec.app)}`, + "argo_namespace=argocd", + `runtime_namespace=${shellQuote(spec.runtimeNamespace)}`, + `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`, + `patch_b64=${shellQuote(patchB64)}`, + "terminate_patch_file=$(mktemp /tmp/hwlab-node-argocd-terminate.XXXXXX.json)", + "patch_file=$(mktemp /tmp/hwlab-node-argocd-sync.XXXXXX.json)", + "printf '{\"operation\":null}' > \"$terminate_patch_file\"", + "printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"", + "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)", + "operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)", + "terminate_code=0", + "terminate_output=", + "terminated_operation=false", + "if [ \"$operation_phase\" = Running ]; then", + " if [ \"$dry_run\" = true ]; then", + " terminated_operation=would-terminate", + " else", + " terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)", + " terminate_code=$?", + " if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi", + " sleep 2", + " fi", + "fi", + "failed_hook_count=0", + "deleted_hook_count=0", + "failed_hooks=", + "deleted_hooks=", + "hook_delete_errors=", + "for job in $(kubectl -n \"$runtime_namespace\" get job -o name 2>/dev/null || true); do", + " tracking=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ index .metadata.annotations \"argocd.argoproj.io/tracking-id\" }}' 2>/dev/null || true)", + " failed=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ range .status.conditions }}{{ if or (eq .type \"Failed\") (eq .type \"FailureTarget\") }}{{ .status }} {{ end }}{{ end }}' 2>/dev/null || true)", + " case \"$tracking\" in \"$app:\"*) ;; *) continue ;; esac", + " case \"$failed\" in *True*) ;; *) continue ;; esac", + " hook_name=${job#job.batch/}", + " failed_hook_count=$((failed_hook_count + 1))", + " if [ -z \"$failed_hooks\" ]; then failed_hooks=$hook_name; else failed_hooks=$failed_hooks,$hook_name; fi", + " if [ \"$dry_run\" = false ]; then", + " delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$job\" --wait=false 2>&1)", + " delete_code=$?", + " if [ \"$delete_code\" -eq 0 ]; then", + " deleted_hook_count=$((deleted_hook_count + 1))", + " if [ -z \"$deleted_hooks\" ]; then deleted_hooks=$hook_name; else deleted_hooks=$deleted_hooks,$hook_name; fi", + " else", + " if [ -z \"$hook_delete_errors\" ]; then hook_delete_errors=\"$hook_name:$delete_code\"; else hook_delete_errors=\"$hook_delete_errors,$hook_name:$delete_code\"; fi", + " fi", + " fi", + "done", + "if [ \"$dry_run\" = true ]; then", + " output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)", + "else", + " output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" -o name 2>&1)", + "fi", + "code=$?", + "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.operation.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)", + "printf 'app\\t%s\\n' \"$app\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'before\\t%s\\n' \"$before\"", + "printf 'after\\t%s\\n' \"$after\"", + "printf 'operationPhase\\t%s\\n' \"$operation_phase\"", + "printf 'terminatedOperation\\t%s\\n' \"$terminated_operation\"", + "printf 'terminateExitCode\\t%s\\n' \"$terminate_code\"", + "printf 'terminateOutput\\t%s\\n' \"$(printf '%s' \"$terminate_output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", + "printf 'failedHookCount\\t%s\\n' \"$failed_hook_count\"", + "printf 'failedHooks\\t%s\\n' \"$failed_hooks\"", + "printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"", + "printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"", + "printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"", + "printf 'patchExitCode\\t%s\\n' \"$code\"", + "printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", + "rm -f \"$patch_file\" \"$terminate_patch_file\"", + "exit \"$code\"", + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const fields = keyValueLinesFromText(statusText(result)); + return { + ok: isCommandSuccess(result), + command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-sync", + mutation: !scoped.dryRun && isCommandSuccess(result), + argoApplication: fields.app || spec.app, + syncSource: { + repoURL: spec.argoRepoUrl, + targetRevision: spec.gitopsBranch, + path: spec.runtimePath, + }, + before: fields.before || null, + after: fields.after || null, + operationRecovery: { + operationPhase: fields.operationPhase || null, + terminatedOperation: fields.terminatedOperation || "false", + terminateExitCode: numericField(fields.terminateExitCode), + terminateOutput: fields.terminateOutput || null, + failedHookCount: numericField(fields.failedHookCount), + failedHooks: commaListField(fields.failedHooks), + deletedHookCount: numericField(fields.deletedHookCount), + deletedHooks: commaListField(fields.deletedHooks), + hookDeleteErrors: fields.hookDeleteErrors || null, + }, + patchExitCode: numericField(fields.patchExitCode), + patchOutput: fields.patchOutput || null, + result: compactRuntimeCommand(result), + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, + }; +} + +function nodeRuntimeTriggerCurrent(scoped: ReturnType): Record { + const spec = scoped.spec; + printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" }); + const head = resolveNodeRuntimeLaneHead(spec); + const sourceCommit = head.sourceCommit; + if (sourceCommit === null) { + printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "failed" }); + return { + ok: false, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "source-head", + degradedReason: "node-runtime-source-head-unresolved", + headProbe: compactRuntimeCommand(head.result), + }; + } + const pipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit); + printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun }); + printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun }); + const before = getNodeRuntimePipelineRun(spec, pipelineRun); + printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null }); + if (scoped.dryRun) { + const gitMirror = nodeRuntimeGitMirrorStatus(scoped); + return { + ok: true, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "dry-run", + sourceCommit, + pipelineRun, + rerun: scoped.rerun, + before, + gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror), + manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun), + next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; + } + if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) { + const pipelineWait = before.status === "Unknown" + ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds) + : { ok: true, status: "already-succeeded", pipelineRun: before, polls: 0, elapsedMs: 0 }; + const waitedPipelineRun = record(pipelineWait.pipelineRun); + const postFlush = waitedPipelineRun.status === "True" + ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun) + : null; + const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true); + return { + ok, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "confirmed-trigger", + sourceCommit, + pipelineRun, + before, + pipelineWait, + postFlush, + skipped: true, + reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running", + rerunAvailable: true, + degradedReason: ok ? undefined : postFlush !== null && postFlush.ok !== true ? "node-runtime-git-mirror-post-flush-failed" : "node-runtime-existing-pipelinerun-not-succeeded", + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` }, + }; + } + printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun }); + const gitMirror = nodeRuntimeEnsureGitMirrorSourceCurrent(scoped, sourceCommit, pipelineRun); + if (gitMirror.ok !== true) { + printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "failed", sourceCommit, pipelineRun, reason: gitMirror.degradedReason ?? null }); + return { + ok: false, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "git-mirror-pre-sync", + sourceCommit, + pipelineRun, + before, + gitMirror, + degradedReason: "node-runtime-git-mirror-pre-sync-failed", + }; + } + printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "succeeded", sourceCommit, pipelineRun }); + printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun }); + const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false }); + if (refresh.ok !== true) { + const diagnostics = record(refresh.diagnostics); + printNodeRuntimeTriggerProgress(spec, { + stage: "control-plane-refresh", + status: "failed", + sourceCommit, + pipelineRun, + ...(diagnostics.classification === "ssh-tcp-pool-transient" + ? { reason: "ssh-tcp-pool-transient", failureKind: diagnostics.failureKind ?? null } + : {}), + }); + return { + ok: false, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "control-plane-apply", + sourceCommit, + pipelineRun, + refresh, + diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null, + degradedReason: "node-runtime-control-plane-apply-before-trigger-failed", + next: { + retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, + ...(diagnostics.classification === "ssh-tcp-pool-transient" ? { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` } : {}), + }, + }; + } + printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun }); + printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun }); + const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun); + const after = getNodeRuntimePipelineRun(spec, pipelineRun); + const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True"); + const createOk = isCommandSuccess(create) || createObserved; + printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined }); + const pipelineWait = createOk ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds) : null; + const waitedPipelineRun = record(pipelineWait?.pipelineRun); + const postFlush = waitedPipelineRun.status === "True" + ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun) + : null; + const pipelineReady = pipelineWait !== null && pipelineWait.ok === true; + const postFlushOk = postFlush === null || postFlush.ok === true; + const ok = createOk && pipelineReady && postFlushOk; + return { + ok, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "confirmed-trigger", + mutation: createOk, + sourceCommit, + pipelineRun, + rerun: scoped.rerun, + before, + gitMirror, + refresh, + create: compactRuntimeCommand(create), + after, + pipelineWait, + postFlush, + createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined, + degradedReason: ok + ? undefined + : !createOk + ? "node-runtime-pipelinerun-create-failed" + : !pipelineReady + ? "node-runtime-pipelinerun-not-succeeded" + : "node-runtime-git-mirror-post-flush-failed", + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` }, + }; +} + +function nodeRuntimeGitMirrorStatus(scoped: ReturnType): Record { + const spec = scoped.spec; + const mirror = nodeRuntimeGitMirrorTarget(spec); + const script = [ + "set +e", + `namespace=${shellQuote(mirror.namespace)}`, + `read_deploy=${shellQuote(mirror.serviceReadName)}`, + `write_deploy=${shellQuote(mirror.serviceWriteName)}`, + `read_svc=${shellQuote(mirror.serviceReadName)}`, + `write_svc=${shellQuote(mirror.serviceWriteName)}`, + `cache_pvc=${shellQuote(mirror.cachePvcName)}`, + `cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`, + "deploy_ready() { desired=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n \"$desired\" ] && [ \"$desired\" -gt 0 ] 2>/dev/null && [ \"${ready:-0}\" = \"$desired\" ] && printf true || printf false; }", + "exists_res() { kubectl -n \"$1\" get \"$2\" \"$3\" >/dev/null 2>&1 && printf true || printf false; }", + "endpoint_ready() { endpoints=$(kubectl -n \"$1\" get endpoints \"$2\" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n \"$endpoints\" ] && printf true || printf false; }", + "summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc '/etc/git-mirror/status.sh' 2>/tmp/hwlab-node-gitmirror-status.err || true)", + "if [ -z \"$summary_json\" ]; then summary_json='{}'; fi", + "read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")", + "write_deployment_ready=$(deploy_ready \"$namespace\" \"$write_deploy\")", + "read_service_exists=$(exists_res \"$namespace\" service \"$read_svc\")", + "write_service_exists=$(exists_res \"$namespace\" service \"$write_svc\")", + "read_endpoints_ready=$(endpoint_ready \"$namespace\" \"$read_svc\")", + "write_endpoints_ready=$(endpoint_ready \"$namespace\" \"$write_svc\")", + "cache_pvc_exists=$(exists_res \"$namespace\" pvc \"$cache_pvc\")", + "cache_host_path_exists=false", + "if [ -n \"$cache_host_path\" ] && kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_exists=true; fi", + "SUMMARY_JSON=\"$summary_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" node <<'NODE'", + "const summary = (() => { try { return JSON.parse(process.env.SUMMARY_JSON || '{}'); } catch { return {}; } })();", + "const env = process.env;", + "const ok = env.read_deployment_ready === 'true' && env.write_deployment_ready === 'true' && env.read_service_exists === 'true' && env.write_service_exists === 'true' && env.read_endpoints_ready === 'true' && env.write_endpoints_ready === 'true' && (env.cache_pvc_exists === 'true' || env.cache_host_path_exists === 'true') && summary.localSource;", + "console.log(JSON.stringify({", + " ok: Boolean(ok),", + " resources: {", + " readDeploymentReady: env.read_deployment_ready === 'true',", + " writeDeploymentReady: env.write_deployment_ready === 'true',", + " readServiceExists: env.read_service_exists === 'true',", + " writeServiceExists: env.write_service_exists === 'true',", + " readEndpointsReady: env.read_endpoints_ready === 'true',", + " writeEndpointsReady: env.write_endpoints_ready === 'true',", + " cachePvcExists: env.cache_pvc_exists === 'true',", + " cacheHostPathConfigured: Boolean(env.cache_host_path),", + " cacheHostPathExists: env.cache_host_path_exists === 'true'", + " },", + " summary,", + " valuesPrinted: false", + "}));", + "NODE", + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const parsed = parseJsonObject(statusText(result)); + const summary = record(parsed.summary); + const resources = record(parsed.resources); + const ok = result.exitCode === 0 && parsed.ok === true; + return { + ok, + command: `hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "status", + mutation: false, + namespace: mirror.namespace, + readUrl: spec.gitReadUrl, + writeUrl: spec.gitWriteUrl, + sourceBranch: mirror.sourceBranch, + gitopsBranch: mirror.gitopsBranch, + resources, + summary, + refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), + result: compactRuntimeCommand(result), + degradedReason: ok ? undefined : "node-runtime-git-mirror-not-ready", + valuesPrinted: false, + next: { + sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm`, + flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm`, + }, + }; +} + +function nodeRuntimeGitMirrorRun(scoped: ReturnType): Record { + if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped); + if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`); + const spec = scoped.spec; + const mirror = nodeRuntimeGitMirrorTarget(spec); + const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action); + const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName); + const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + const script = [ + "set -eu", + `namespace=${shellQuote(mirror.namespace)}`, + `job=${shellQuote(jobName)}`, + `manifest_b64=${shellQuote(manifestB64)}`, + "manifest_path=\"/tmp/$job.json\"", + "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", + scoped.dryRun + ? "kubectl create --dry-run=server -f \"$manifest_path\" -o name" + : [ + "kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null", + "kubectl create -f \"$manifest_path\"", + `deadline=$(( $(date +%s) + ${scoped.timeoutSeconds} ))`, + "while :; do", + " status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)", + " succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')", + " failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')", + " if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi", + " if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi", + " if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi", + " sleep 2", + "done", + "kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true", + ].join("\n"), + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result); + const syncFallback = scoped.action === "sync" && !scoped.dryRun && !isCommandSuccess(result) && nodeRuntimeGitMirrorSyncFallbackAllowed(result) + ? nodeRuntimeGitMirrorSyncFallback(scoped.spec, mirror, scoped.timeoutSeconds) + : null; + const flushFallback = scoped.action === "flush" && !scoped.dryRun && !isCommandSuccess(result) && nodeRuntimeGitMirrorSyncFallbackAllowed(result) + ? nodeRuntimeGitMirrorFlushFallback(scoped.spec, mirror, scoped.timeoutSeconds) + : null; + const fallback = syncFallback ?? flushFallback; + const partialSuccessRecovery = partialSuccess !== null && !scoped.dryRun + ? nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true }) + : null; + const partialSuccessRecoveryStatus = record(partialSuccessRecovery?.status); + const partialSuccessRecovered = partialSuccess !== null + && partialSuccessRecovery?.ok === true + && Object.keys(partialSuccessRecoveryStatus).length > 0 + && !nodeRuntimeGitMirrorNeedsFlush(partialSuccessRecoveryStatus); + const actionSucceeded = isCommandSuccess(result) || fallback?.ok === true || partialSuccessRecovered; + const status = scoped.dryRun || !actionSucceeded + ? undefined + : partialSuccessRecovered + ? partialSuccessRecovery?.status + : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); + return { + ok: actionSucceeded && (status === undefined || status.ok === true), + command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`, + mutation: !scoped.dryRun && actionSucceeded, + namespace: mirror.namespace, + jobName, + manifest: scoped.dryRun ? manifest : undefined, + result: compactRuntimeCommand(result), + fallback: fallback ?? undefined, + status, + partialSuccess: partialSuccess?.partialSuccess ?? undefined, + partialSuccessRecovered: partialSuccessRecovered ? true : undefined, + recovery: partialSuccess ?? undefined, + postPushRecovery: partialSuccessRecovery === null ? undefined : compactNodeRuntimeGitMirrorRun(partialSuccessRecovery), + degradedReason: partialSuccess !== null + ? partialSuccessRecovered + ? undefined + : "node-runtime-git-mirror-flush-post-push-fetch-failed" + : actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : fallback === null ? `node-runtime-git-mirror-${scoped.action}-failed` : `node-runtime-git-mirror-${scoped.action}-fallback-failed`, + next: partialSuccess !== null + ? partialSuccessRecovered + ? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` } + : partialSuccess.next + : scoped.dryRun + ? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` } + : { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` }, + }; +} + +function nodeRuntimeGitMirrorSyncFallbackAllowed(result: CommandResult): boolean { + const text = `${result.stdout}\n${result.stderr}`; + return /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com/iu.test(text); +} + +function nodeRuntimeGitMirrorSyncFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record { + const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror); + if (writeUrl === null) { + return { + ok: false, + mode: "host-workspace-fallback", + degradedReason: "node-runtime-git-mirror-write-service-unresolved", + next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` }, + }; + } + const script = [ + "set -eu", + `repository=${shellQuote(mirror.sourceRepository)}`, + `source_branch=${shellQuote(mirror.sourceBranch)}`, + `gitops_branch=${shellQuote(mirror.gitopsBranch)}`, + `mirror_write=${shellQuote(writeUrl)}`, + "github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", + "tmp_ns=\"refs/unidesk/git-mirror-sync/${source_branch}/$(date +%s)-$$\"", + "source_tmp=\"$tmp_ns/source\"", + "gitops_tmp=\"$tmp_ns/gitops\"", + "cleanup() { git update-ref -d \"$source_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$gitops_tmp\" >/dev/null 2>&1 || true; }", + "trap cleanup EXIT", + "export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", + "git fetch --no-tags \"$github_remote\" \"+refs/heads/${source_branch}:${source_tmp}\"", + "source_sha=$(git rev-parse --verify \"${source_tmp}^{commit}\")", + "git push \"$mirror_write\" \"$source_sha:refs/heads/${source_branch}\" \"$source_sha:refs/mirror-stage/heads/${source_branch}\"", + "gitops_sha=", + "if git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${gitops_tmp}\"; then", + " gitops_sha=$(git rev-parse --verify \"${gitops_tmp}^{commit}\")", + " git push \"$mirror_write\" \"$gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"", + "fi", + "printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"sourceCommit\":\"%s\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$source_sha\" \"$gitops_sha\" \"$mirror_write\"", + ].join("\n"); + const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300))); + return { + ok: isCommandSuccess(result), + mode: "host-workspace-fallback", + reason: "k3s git-mirror sync job could not reach GitHub SSH; fallback updates the node-local mirror from the target workspace through ssh.github.com:443", + mirrorWrite: writeUrl, + result: compactRuntimeCommand(result), + degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed", + valuesPrinted: false, + }; +} + +function nodeRuntimeGitMirrorFlushFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record { + const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror); + if (writeUrl === null) { + return { + ok: false, + mode: "host-workspace-fallback", + degradedReason: "node-runtime-git-mirror-write-service-unresolved", + next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` }, + }; + } + const script = [ + "set -eu", + `repository=${shellQuote(mirror.sourceRepository)}`, + `gitops_branch=${shellQuote(mirror.gitopsBranch)}`, + `mirror_write=${shellQuote(writeUrl)}`, + "github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", + "tmp_ns=\"refs/unidesk/git-mirror-flush/${gitops_branch}/$(date +%s)-$$\"", + "local_gitops_tmp=\"$tmp_ns/local-gitops\"", + "github_gitops_tmp=\"$tmp_ns/github-gitops\"", + "cleanup() { git update-ref -d \"$local_gitops_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$github_gitops_tmp\" >/dev/null 2>&1 || true; }", + "trap cleanup EXIT", + "export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", + "git fetch --no-tags \"$mirror_write\" \"+refs/heads/${gitops_branch}:${local_gitops_tmp}\"", + "local_gitops_sha=$(git rev-parse --verify \"${local_gitops_tmp}^{commit}\")", + "git push \"$github_remote\" \"$local_gitops_sha:refs/heads/${gitops_branch}\"", + "git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${github_gitops_tmp}\"", + "github_gitops_sha=$(git rev-parse --verify \"${github_gitops_tmp}^{commit}\")", + "if [ \"$github_gitops_sha\" != \"$local_gitops_sha\" ]; then", + " printf 'github gitops mismatch after fallback flush: local=%s github=%s\\n' \"$local_gitops_sha\" \"$github_gitops_sha\" >&2", + " exit 46", + "fi", + "git push \"$mirror_write\" \"$github_gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"", + "printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$github_gitops_sha\" \"$mirror_write\"", + ].join("\n"); + const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300))); + return { + ok: isCommandSuccess(result), + mode: "host-workspace-fallback", + reason: "k3s git-mirror flush job could not reach GitHub SSH; fallback pushes GitOps from the target workspace through ssh.github.com:443 and refreshes mirror-stage cache", + mirrorWrite: writeUrl, + result: compactRuntimeCommand(result), + degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed", + valuesPrinted: false, + }; +} + +function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null { + const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60); + const value = result.stdout.trim(); + if (result.exitCode !== 0 || value.length === 0 || value.startsWith(":")) return null; + return `http://${value}/${mirror.sourceRepository}.git`; +} + +function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType, sourceCommit: string, pipelineRun: string): Record { + const full = nodeScopedFullOutput(scoped); + const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); + const beforeSummary = compactNodeRuntimeGitMirrorStatus(before); + if (before.ok === true && beforeSummary.localSource === sourceCommit && beforeSummary.githubSource === sourceCommit) { + const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before); + return { + ok: flush.ok === true, + mode: "already-current", + sourceCommit, + beforeSummary, + before: full ? before : undefined, + flush, + degradedReason: flush.ok === true ? undefined : "node-runtime-git-mirror-pre-flush-failed", + }; + } + const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true }); + const after = record(sync.status); + const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {}; + const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit; + const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after) : null; + const ok = sourceOk && (flush === null || flush.ok === true); + return { + ok, + mode: "synced-before-trigger", + sourceCommit, + beforeSummary, + before: full ? before : undefined, + sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync), + afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null, + after: full ? sync.status ?? null : undefined, + flush, + degradedReason: ok + ? undefined + : sourceOk + ? "node-runtime-git-mirror-pre-flush-failed" + : "node-runtime-git-mirror-local-source-not-current-after-sync", + }; +} + +function nodeRuntimeEnsureGitMirrorFlushed( + scoped: ReturnType, + phase: "pre" | "post", + sourceCommit: string, + pipelineRun: string | null, + statusInput: Record | null = null, +): Record { + const stage = `git-mirror-${phase}-flush`; + const full = nodeScopedFullOutput(scoped); + const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); + const beforeSummary = compactNodeRuntimeGitMirrorStatus(before); + if (before.ok !== true) { + printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "failed", sourceCommit, pipelineRun, reason: "git-mirror-status-failed", ...beforeSummary }); + return { + ok: false, + phase, + mode: "status-failed", + executed: false, + before: full ? before : undefined, + beforeSummary, + degradedReason: `node-runtime-git-mirror-${phase}-status-failed`, + next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` }, + }; + } + const flushNeeded = nodeRuntimeGitMirrorNeedsFlush(before); + if (!flushNeeded) { + printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "skipped", sourceCommit, pipelineRun, flushNeeded: false, ...beforeSummary }); + return { + ok: true, + phase, + mode: "already-flushed", + executed: false, + before: full ? before : undefined, + beforeSummary, + after: full ? before : undefined, + afterSummary: beforeSummary, + }; + } + printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun, flushNeeded: true, ...beforeSummary }); + const flush = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true }); + const after = record(flush.status); + const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {}; + const ok = flush.ok === true && Object.keys(after).length > 0 && !nodeRuntimeGitMirrorNeedsFlush(after); + printNodeRuntimeTriggerProgress(scoped.spec, { + stage, + status: ok ? "succeeded" : "failed", + sourceCommit, + pipelineRun, + flushNeeded: true, + jobName: flush.jobName ?? null, + ...afterSummary, + }); + return { + ok, + phase, + mode: "flushed", + executed: true, + before: full ? before : undefined, + beforeSummary, + flush: full ? flush : compactNodeRuntimeGitMirrorRun(flush), + jobName: flush.jobName ?? null, + after: full ? (Object.keys(after).length > 0 ? after : null) : undefined, + afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null, + degradedReason: ok ? undefined : `node-runtime-git-mirror-${phase}-flush-failed`, + next: ok ? undefined : flush.next ?? { flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` }, + }; +} + +function nodeRuntimeGitMirrorNeedsFlush(status: Record): boolean { + const summary = record(status.summary); + const localGitops = typeof summary.localGitops === "string" ? summary.localGitops : null; + const githubGitops = typeof summary.githubGitops === "string" ? summary.githubGitops : null; + return summary.pendingFlush === true + || summary.flushNeeded === true + || summary.githubInSync === false + || (localGitops !== null && githubGitops !== null && localGitops !== githubGitops); +} + +function nodeScopedFullOutput(scoped: ReturnType): boolean { + return scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw"); +} + +function compactNodeRuntimeGitMirrorObservation(status: Record): Record { + return { + ok: status.ok === true, + mode: status.mode ?? null, + mutation: status.mutation === true, + summary: compactNodeRuntimeGitMirrorStatus(status), + degradedReason: status.degradedReason ?? null, + next: status.next ?? null, + }; +} + +function compactNodeRuntimeGitMirrorRun(result: Record): Record { + const status = record(result.status); + return { + ok: result.ok === true, + action: result.action ?? null, + mode: result.mode ?? null, + mutation: result.mutation === true, + jobName: result.jobName ?? null, + partialSuccess: result.partialSuccess ?? null, + fallback: result.fallback ?? null, + degradedReason: result.degradedReason ?? null, + statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null, + next: result.next ?? null, + }; +} + +function compactNodeRuntimeGitMirrorStatus(status: Record): Record { + const summary = record(status.summary); + return { + ok: status.ok === true, + localSource: summary.localSource ?? null, + githubSource: summary.githubSource ?? null, + localGitops: summary.localGitops ?? null, + githubGitops: summary.githubGitops ?? null, + pendingFlush: summary.pendingFlush === true, + flushNeeded: summary.flushNeeded === true, + githubInSync: summary.githubInSync === true, + }; +} + +function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string { + const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix; + return `${prefix}-${Date.now().toString(36)}`.slice(0, 63); +} + +function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record { + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName, + namespace: mirror.namespace, + labels: { + "app.kubernetes.io/name": "git-mirror", + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "app.kubernetes.io/component": `${action}-controller`, + "hwlab.pikastech.local/node": mirror.node, + "hwlab.pikastech.local/lane": mirror.lane, + "hwlab.pikastech.local/trigger": "manual-cli", + }, + }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 600, + ttlSecondsAfterFinished: 3600, + template: { + metadata: { + labels: { + "app.kubernetes.io/name": "git-mirror", + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "app.kubernetes.io/component": `${action}-controller`, + "hwlab.pikastech.local/node": mirror.node, + "hwlab.pikastech.local/lane": mirror.lane, + }, + }, + spec: { + restartPolicy: "Never", + volumes: [ + { name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, + { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } }, + { name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } }, + ], + containers: [{ + name: action, + image: mirror.toolsImage, + imagePullPolicy: "IfNotPresent", + command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"], + volumeMounts: [ + { name: "cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + { name: "script", mountPath: "/script", readOnly: true }, + ], + }], + }, + }, + }, + }; +} + +function nodeRuntimeControlPlaneStatus(scoped: ReturnType): Record { + const spec = scoped.spec; + const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit"); + const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run"); + const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null; + const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null; + const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit)); + const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60); + const namespaceExists = namespace.exitCode === 0; + const postgresObjects = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60) + : null; + const localPostgresObjects = postgresObjects === null + ? [] + : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec)); + const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60); + const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60); + const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60); + const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u); + const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun); + const workloads = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60) + : null; + const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + const workloadReadinessProbe = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], 60) + : null; + const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? ""); + const bridge = externalPostgresBridgeStatus(spec, namespaceExists); + const secrets = externalPostgresSecretStatus(spec, namespaceExists); + const publicProbes = nodeRuntimePublicProbeStatus(spec); + const gitMirror = nodeRuntimeGitMirrorStatus(scoped); + const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror); + const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0; + const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready); + const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && workloadsReady && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready)); + const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy"; + const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True"; + const publicReady = publicProbes.ready === true; + const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true; + const fullStatus = { + ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady, + command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, + mode: "node-scoped-runtime-status", + mutation: false, + node: scoped.node, + lane: scoped.lane, + sourceCommit, + pipelineRun, + expected: nodeRuntimeExpected(spec), + sourceHead: head === null + ? { ok: sourceCommit !== null, value: sourceCommit, source: "option" } + : { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) }, + controlPlane: { + ready: controlPlaneReady, + serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) }, + pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) }, + }, + argo: { + ready: argoReady, + application: spec.app, + repoURL, + expectedRepoURL: spec.argoRepoUrl, + targetRevision, + path, + syncRevision, + syncStatus, + health, + result: compactRuntimeCommand(argo), + }, + pipelineRun: pipelineRunProbe, + runtime: { + ready: runtimeReady, + namespace: spec.runtimeNamespace, + namespaceExists, + localPostgresObjects, + localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0, + workloadNames, + workloadCount: workloadNames.length, + workloadReadiness, + workloadsReady, + workloadReadinessResult: workloadReadinessProbe === null ? null : compactRuntimeCommand(workloadReadinessProbe), + workloadResult: workloads === null ? null : compactRuntimeCommand(workloads), + externalPostgresBridge: bridge, + externalPostgresSecrets: secrets, + }, + publicProbes, + gitMirror: { + ...gitMirror, + compact: gitMirrorCompact, + ready: gitMirrorReady, + }, + probes: { + namespace: compactRuntimeCommand(namespace), + postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects), + }, + degradedReason: controlPlaneReady + ? runtimeReady + ? argoReady + ? pipelineRunReady + ? publicReady + ? gitMirrorReady ? undefined : "git-mirror-pending-flush" + : "public-probe-not-ready" + : "pipelinerun-not-succeeded" + : "argo-not-synced-healthy" + : namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing" + : "control-plane-not-ready", + next: { + plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, + apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`, + triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, + }, + }; + const summary = summarizeNodeRuntimeControlPlaneStatus(fullStatus, scoped); + if (scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw")) return { ...fullStatus, summary }; + return summary; +} + +function parseNodeRuntimeWorkloadReadiness(text: string): Array> { + return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => { + const [ref = "", counts = ""] = line.split(/\t/u); + const [readyRaw = "", currentRaw = "", desiredRaw = ""] = counts.split("/"); + const readyReplicas = nullableInteger(readyRaw); + const currentReplicas = nullableInteger(currentRaw); + const desiredReplicas = nullableInteger(desiredRaw); + const ready = desiredReplicas !== null + ? (readyReplicas ?? 0) >= desiredReplicas + : readyReplicas !== null && currentReplicas !== null && readyReplicas >= currentReplicas; + return { + ref, + readyReplicas, + currentReplicas, + desiredReplicas, + ready, + }; + }); +} + +function nullableInteger(value: string): number | null { + if (!/^[0-9]+$/u.test(value)) return null; + return Number(value); +} + +function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record { + const web = publicHttpProbe("web", spec.publicWebUrl); + const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live")); + return { + ready: web.ok === true && apiHealth.ok === true, + web, + apiHealth, + }; +} + +function publicHttpProbe(kind: string, url: string): Record { + const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "/dev/null", "-w", "%{http_code}", url], repoRoot, { timeoutMs: 15_000 }); + const httpStatus = nullableInteger(result.stdout.trim().slice(-3)); + return { + kind, + url, + ok: result.exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400, + httpStatus, + result: compactRuntimeCommand(result), + }; +} + +function joinUrlPath(baseUrl: string, suffix: string): string { + return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`; +} + +function summarizeNodeRuntimeControlPlaneStatus(status: Record, scoped: ReturnType): Record { + const pipelineRun = record(status.pipelineRun); + const argo = record(status.argo); + const runtime = record(status.runtime); + const publicProbes = record(status.publicProbes); + const gitMirror = record(status.gitMirror); + const gitMirrorCompact = record(gitMirror.compact); + const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : []; + const readyWorkloads = workloadReadiness.filter((item) => item.ready === true).length; + const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length; + const webProbe = record(publicProbes.web); + const apiProbe = record(publicProbes.apiHealth); + return { + ok: status.ok === true, + command: status.command, + mode: "node-scoped-runtime-status-summary", + mutation: false, + node: status.node, + lane: status.lane, + sourceCommit: status.sourceCommit, + degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null, + pipelineRun: { + name: pipelineRun.name ?? null, + exists: pipelineRun.exists === true, + status: pipelineRun.status ?? null, + reason: pipelineRun.reason ?? null, + message: pipelineRun.message ?? null, + createdAt: pipelineRun.createdAt ?? null, + ready: pipelineRun.status === "True", + }, + argo: { + application: argo.application ?? null, + ready: argo.ready === true, + syncRevision: argo.syncRevision ?? null, + syncStatus: argo.syncStatus ?? null, + health: argo.health ?? null, + }, + runtime: { + namespace: runtime.namespace ?? null, + namespaceExists: runtime.namespaceExists === true, + ready: runtime.ready === true, + workloadCount, + workloadReady: `${readyWorkloads}/${workloadReadiness.length}`, + localPostgresAbsent: runtime.localPostgresAbsent === true, + externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined + ? null + : record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true, + }, + publicProbe: { + ready: publicProbes.ready === true, + web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null }, + apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null }, + }, + gitMirror: { + ready: gitMirror.ready === true, + localSource: gitMirrorCompact.localSource ?? null, + githubSource: gitMirrorCompact.githubSource ?? null, + localGitops: gitMirrorCompact.localGitops ?? null, + githubGitops: gitMirrorCompact.githubGitops ?? null, + pendingFlush: gitMirrorCompact.pendingFlush === true, + flushNeeded: gitMirrorCompact.flushNeeded === true, + githubInSync: gitMirrorCompact.githubInSync === true, + }, + nextAction: nodeRuntimeStatusNextAction(status, scoped), + next: { + full: `${nodeRuntimeStatusCommand(scoped)} --full`, + plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, + triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, + gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`, + webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`, + }, + }; +} + +function nodeRuntimeStatusNextAction(status: Record, scoped: ReturnType): string { + const reason = typeof status.degradedReason === "string" ? status.degradedReason : null; + if (reason === null) return `${nodeRuntimeStatusCommand(scoped)} --full`; + if (reason === "control-plane-not-ready" || reason === "runtime-namespace-missing") { + return `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`; + } + if (reason === "runtime-not-ready") return `${nodeRuntimeStatusCommand(scoped)} --full`; + if (reason === "argo-not-synced-healthy") { + return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`; + } + if (reason === "pipelinerun-not-succeeded") { + return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`; + } + if (reason === "public-probe-not-ready") { + return `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`; + } + if (reason === "git-mirror-pending-flush") { + return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`; + } + return `${nodeRuntimeStatusCommand(scoped)} --full`; +} + +function nodeRuntimeStatusCommand(scoped: ReturnType): string { + const sourceCommit = optionValue(scoped.originalArgs, "--source-commit"); + const pipelineRun = optionValue(scoped.originalArgs, "--pipeline-run"); + return [ + `bun scripts/cli.ts hwlab nodes control-plane status --node ${shellQuote(scoped.node)} --lane ${shellQuote(scoped.lane)}`, + sourceCommit === undefined ? "" : `--source-commit ${shellQuote(sourceCommit)}`, + pipelineRun === undefined ? "" : `--pipeline-run ${shellQuote(pipelineRun)}`, + ].filter(Boolean).join(" "); +} + +function nodeRuntimeRenderToken(): string { + return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-"); +} + +function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { + if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds); + return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds); +} + +function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean { + return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId; +} + +function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] { + const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds)); + const retryCount = Math.max(0, Math.floor(retries)); + const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-"); + return [ + `yaml_registry=${shellQuote(registry)}`, + `yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`, + `yaml_fetch_retries=${shellQuote(String(retryCount))}`, + `yaml_dependency_context=${shellQuote(safeContext)}`, + "yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }", + "yaml_npm_debug_log_tail() {", + " yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"", + " if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi", + " yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"", + " if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then", + " echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2", + " tail -n 80 \"$yaml_npm_log\" >&2 || true", + " fi", + "}", + "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then", + " mkdir -p node_modules/yaml", + " yaml_tarball=\"${yaml_registry%/}/yaml/-/yaml-2.8.3.tgz\"", + " yaml_tgz=\"$(mktemp)\"", + " if command -v curl >/dev/null 2>&1 && command -v tar >/dev/null 2>&1; then", + " if timeout \"$yaml_fetch_timeout\" curl -fsSL --retry \"$yaml_fetch_retries\" --connect-timeout 10 -o \"$yaml_tgz\" \"$yaml_tarball\"; then", + " tar -xzf \"$yaml_tgz\" -C node_modules/yaml --strip-components=1", + " yaml_dependency_log installed tarball", + " else", + " yaml_dependency_log tarball-failed tarball", + " fi", + " fi", + " rm -f \"$yaml_tgz\"", + "fi", + "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then", + " rm -rf node_modules/yaml", + " if timeout \"$yaml_fetch_timeout\" bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then", + " yaml_dependency_log installed bun", + " else", + " yaml_dependency_log bun-failed bun", + " fi", + "fi", + "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then", + " rm -rf node_modules/yaml", + " command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }", + " if npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then", + " yaml_dependency_log installed npm", + " else", + " yaml_dependency_log npm-failed npm", + " yaml_npm_debug_log_tail", + " exit 34", + " fi", + "fi", + "node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }", + ]; +} + +function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { + const token = nodeRuntimeRenderToken(); + const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`; + const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`; + const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); + const script = [ + "set -eu", + runtimeLaneCicdRepoEnsureScript(spec), + `source_commit=${shellQuote(sourceCommit)}`, + `render_dir=${shellQuote(renderDir)}`, + `worktree_dir=${shellQuote(worktreeDir)}`, + `overlay_b64=${shellQuote(overlay)}`, + "cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }", + "trap cleanup_render_worktree EXIT", + `test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`, + "rm -rf \"$render_dir\" \"$worktree_dir\"", + "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", + "git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"", + "git -C \"$worktree_dir\" checkout --detach \"$source_commit\"", + "cd \"$worktree_dir\"", + ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"), + "node - \"$overlay_b64\" <<'NODE'", + "const fs = require('fs');", + "const YAML = require('yaml');", + "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", + "const path = 'deploy/deploy.yaml';", + "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", + "doc.nodes = doc.nodes || {};", + "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", + "doc.lanes = doc.lanes || {};", + "const lane = doc.lanes[overlay.lane] || {};", + "const downloadStack = {", + " ...(lane.envRecipe?.downloadStack || {}),", + " httpProxy: overlay.dockerProxyHttp,", + " httpsProxy: overlay.dockerProxyHttps,", + " noProxy: overlay.dockerNoProxyList,", + "};", + "doc.lanes[overlay.lane] = {", + " ...lane,", + " node: overlay.nodeId,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " namespace: overlay.runtimeNamespace,", + " endpoint: overlay.publicApiUrl,", + " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", + " artifactCatalog: overlay.catalogPath,", + " runtimePath: overlay.runtimePath,", + " imageTagMode: 'full',", + " sourceRepo: overlay.gitUrl,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", + "};", + "fs.writeFileSync(path, YAML.stringify(doc));", + "NODE", + "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", + [ + "node scripts/run-bun.mjs \"$render_script\"", + `--lane ${shellQuote(spec.lane)}`, + `--node ${shellQuote(spec.nodeId)}`, + `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, + `--catalog-path ${shellQuote(spec.catalogPath)}`, + "--image-tag-mode full", + `--source-revision ${shellQuote(sourceCommit)}`, + `--source-repo ${shellQuote(spec.gitUrl)}`, + `--source-branch ${shellQuote(spec.sourceBranch)}`, + `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, + `--git-read-url ${shellQuote(spec.gitReadUrl)}`, + `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, + `--registry-prefix ${shellQuote(spec.registryPrefix)}`, + `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, + `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, + `--out ${shellQuote(renderDir)}`, + ].join(" "), + ...nodeRuntimePipelinePostprocessScript(), + ].join("\n"); + return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" }; +} + +function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { + const token = nodeRuntimeRenderToken(); + const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`; + const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`; + const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); + const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); + const script = [ + "set -eu", + `source_url=${shellQuote(spec.gitUrl)}`, + `source_branch=${shellQuote(spec.sourceBranch)}`, + `source_commit=${shellQuote(sourceCommit)}`, + `render_dir=${shellQuote(renderDir)}`, + `worktree_dir=${shellQuote(worktreeDir)}`, + `overlay_b64=${shellQuote(overlay)}`, + `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, + "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", + "rm -rf \"$render_dir\" \"$worktree_dir\"", + "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", + "echo \"phase=local-git-clone-worktree\" >&2", + "run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"", + "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"", + "cd \"$worktree_dir\"", + "echo \"phase=local-install-yaml\" >&2", + ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"), + "node - \"$overlay_b64\" <<'NODE'", + "const fs = require('fs');", + "const YAML = require('yaml');", + "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", + "const path = 'deploy/deploy.yaml';", + "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", + "doc.nodes = doc.nodes || {};", + "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", + "doc.lanes = doc.lanes || {};", + "const lane = doc.lanes[overlay.lane] || {};", + "const downloadStack = {", + " ...(lane.envRecipe?.downloadStack || {}),", + " httpProxy: overlay.dockerProxyHttp,", + " httpsProxy: overlay.dockerProxyHttps,", + " noProxy: overlay.dockerNoProxyList,", + "};", + "doc.lanes[overlay.lane] = {", + " ...lane,", + " node: overlay.nodeId,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " namespace: overlay.runtimeNamespace,", + " endpoint: overlay.publicApiUrl,", + " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", + " artifactCatalog: overlay.catalogPath,", + " runtimePath: overlay.runtimePath,", + " imageTagMode: 'full',", + " sourceRepo: overlay.gitUrl,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", + "};", + "fs.writeFileSync(path, YAML.stringify(doc));", + "NODE", + "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", + "echo \"phase=local-gitops-render\" >&2", + [ + "node scripts/run-bun.mjs \"$render_script\"", + `--lane ${shellQuote(spec.lane)}`, + `--node ${shellQuote(spec.nodeId)}`, + `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, + `--catalog-path ${shellQuote(spec.catalogPath)}`, + "--image-tag-mode full", + `--source-revision ${shellQuote(sourceCommit)}`, + `--source-repo ${shellQuote(spec.gitUrl)}`, + `--source-branch ${shellQuote(spec.sourceBranch)}`, + `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, + `--git-read-url ${shellQuote(spec.gitReadUrl)}`, + `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, + `--registry-prefix ${shellQuote(spec.registryPrefix)}`, + `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, + `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, + `--out ${shellQuote(renderDir)}`, + ].join(" "), + ...nodeRuntimePipelinePostprocessScript(), + ].join("\n"); + return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" }; +} + +function nodeRuntimePipelinePostprocessScript(): string[] { + return [ + "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'", + "const fs = require('fs');", + "const path = require('path');", + "const vm = require('node:vm');", + "const renderDir = process.argv[2];", + "const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));", + "const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');", + "let text = fs.readFileSync(pipelinePath, 'utf8');", + "let YAML = null;", + "try { YAML = require('yaml'); } catch {}", + "const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');", + "const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;", + "const yamlString = (value) => JSON.stringify(String(value));", + "const proxyEnv = {", + " HTTP_PROXY: overlay.proxyHttp,", + " HTTPS_PROXY: overlay.proxyHttps,", + " ALL_PROXY: overlay.proxyAll,", + " NO_PROXY: overlay.noProxy,", + " http_proxy: overlay.proxyHttp,", + " https_proxy: overlay.proxyHttps,", + " all_proxy: overlay.proxyAll,", + " no_proxy: overlay.noProxy,", + "};", + "const dockerProxyEnv = {", + " HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,", + " HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,", + " HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,", + "};", + "const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };", + "function prepareSourceDependencyScript() {", + " const registry = String(overlay.npmRegistry || 'https://registry.npmjs.org/');", + " const timeoutSeconds = Math.max(15, Math.ceil(Number(overlay.npmFetchTimeoutMs || 120000) / 1000));", + " const retryCount = Math.max(0, Math.floor(Number(overlay.npmRetries || 3)));", + " return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"", + "node <<'NODE_UNIDESK_YAML_DEPENDENCY'", + "const { spawnSync } = require('node:child_process');", + "const fs = require('node:fs');", + "const os = require('node:os');", + "const path = require('node:path');", + "const registry = ${JSON.stringify(registry)};", + "const timeoutMs = ${JSON.stringify(timeoutSeconds * 1000)};", + "const timeoutSeconds = ${JSON.stringify(timeoutSeconds)};", + "const retryCount = ${JSON.stringify(retryCount)};", + "const dependency = 'yaml';", + "const version = '2.8.3';", + "function emit(status, extra = {}) { console.error(JSON.stringify({ event: 'prepare-source-dependencies', status, dependency, ...extra })); }", + "function hasYaml() { try { require.resolve('yaml'); return true; } catch { return false; } }", + "function run(command, args) {", + " const result = spawnSync(command, args, { stdio: 'inherit', env: process.env, timeout: timeoutMs });", + " if (result.error) console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'command-error', command, error: result.error.message }));", + " return result.status === 0;", + "}", + "function tailNpmLog() {", + " const dir = path.join(process.env.HOME || '/tmp', '.npm', '_logs');", + " if (!fs.existsSync(dir)) return;", + " const files = fs.readdirSync(dir).filter((name) => name.includes('debug') && name.endsWith('.log')).sort();", + " const file = files[files.length - 1];", + " if (!file) return;", + " const full = path.join(dir, file);", + " console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'npm-debug-log', path: full }));", + " const lines = fs.readFileSync(full, 'utf8').split(String.fromCharCode(10));", + " console.error(lines.slice(-80).join(String.fromCharCode(10)));", + "}", + "if (hasYaml()) { emit('cached'); process.exit(0); }", + "fs.mkdirSync('node_modules/yaml', { recursive: true });", + "let registryBase = registry;", + "while (registryBase.endsWith('/')) registryBase = registryBase.slice(0, -1);", + "const tarball = registryBase + '/yaml/-/yaml-' + version + '.tgz';", + "const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hwlab-yaml-'));", + "const tgz = path.join(tmpDir, 'yaml.tgz');", + "if (run('curl', ['-fsSL', '--retry', String(retryCount), '--connect-timeout', '10', '--max-time', String(timeoutSeconds), '-o', tgz, tarball]) && run('tar', ['-xzf', tgz, '-C', 'node_modules/yaml', '--strip-components=1'])) emit('installed', { manager: 'tarball' });", + "else emit('tarball-failed', { manager: 'tarball' });", + "fs.rmSync(tmpDir, { recursive: true, force: true });", + "if (!hasYaml()) {", + " fs.rmSync('node_modules/yaml', { recursive: true, force: true });", + " if (run('bun', ['add', '--no-save', '--ignore-scripts', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'bun' });", + " else emit('bun-failed', { manager: 'bun' });", + "}", + "if (!hasYaml()) {", + " fs.rmSync('node_modules/yaml', { recursive: true, force: true });", + " if (run('npm', ['install', '--package-lock=false', '--no-save', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'npm' });", + " else { emit('npm-failed', { manager: 'npm' }); tailNpmLog(); process.exit(34); }", + "}", + "if (!hasYaml()) { emit('failed', { reason: 'unresolved' }); process.exit(34); }", + "NODE_UNIDESK_YAML_DEPENDENCY", + "ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;", + "}", + "function validatePrepareSourceDependencyScript(script) {", + " const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';", + " let offset = 0;", + " while (true) {", + " const markerStart = script.indexOf(\"node <<'\" + marker + \"'\", offset);", + " if (markerStart === -1) return;", + " const bodyStart = script.indexOf('\\n', markerStart);", + " if (bodyStart === -1) throw new Error('prepare-source dependency heredoc body missing newline');", + " const bodyEnd = script.indexOf('\\n' + marker, bodyStart + 1);", + " if (bodyEnd === -1) throw new Error('prepare-source dependency heredoc terminator missing');", + " const body = script.slice(bodyStart + 1, bodyEnd);", + " try { new vm.Script(body, { filename: 'NODE_UNIDESK_YAML_DEPENDENCY.js' }); } catch (error) { throw new Error(`generated prepare-source yaml dependency script is invalid: ${error.message}`); }", + " offset = bodyEnd + marker.length + 1;", + " }", + "}", + "function deployYamlOverlayScript() {", + " const runtimeOverlay = JSON.stringify({", + " nodeId: overlay.nodeId,", + " lane: overlay.lane,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " gitopsRoot: overlay.gitopsRoot,", + " runtimePath: overlay.runtimePath,", + " runtimeRenderDir: overlay.runtimeRenderDir,", + " runtimeNamespace: overlay.runtimeNamespace,", + " catalogPath: overlay.catalogPath,", + " gitUrl: overlay.gitUrl,", + " publicWebUrl: overlay.publicWebUrl,", + " publicApiUrl: overlay.publicApiUrl,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " runtimeImageRewrites: overlay.runtimeImageRewrites,", + " dockerProxyHttp: overlay.dockerProxyHttp,", + " dockerProxyHttps: overlay.dockerProxyHttps,", + " dockerNoProxyList: overlay.dockerNoProxyList,", + " npmRegistry: overlay.npmRegistry,", + " npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,", + " npmRetries: overlay.npmRetries,", + " });", + " return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'", + "const fs = require('fs');", + "const YAML = require('yaml');", + "const overlay = ${runtimeOverlay};", + "const file = 'deploy/deploy.yaml';", + "if (!fs.existsSync(file)) {", + " console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));", + " process.exit(45);", + "}", + "const doc = YAML.parse(fs.readFileSync(file, 'utf8'));", + "doc.nodes = doc.nodes || {};", + "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", + "doc.lanes = doc.lanes || {};", + "const lane = doc.lanes[overlay.lane] || {};", + "const envRecipe = lane.envRecipe || {};", + "const downloadStack = {", + " ...(envRecipe.downloadStack || {}),", + " httpProxy: overlay.dockerProxyHttp,", + " httpsProxy: overlay.dockerProxyHttps,", + " noProxy: overlay.dockerNoProxyList,", + "};", + "if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;", + "if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;", + "doc.lanes[overlay.lane] = {", + " ...lane,", + " node: overlay.nodeId,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " namespace: overlay.runtimeNamespace,", + " endpoint: overlay.publicApiUrl,", + " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", + " artifactCatalog: overlay.catalogPath,", + " runtimePath: overlay.runtimePath,", + " imageTagMode: 'full',", + " sourceRepo: overlay.gitUrl,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " envRecipe: { ...envRecipe, downloadStack },", + "};", + "fs.writeFileSync(file, YAML.stringify(doc));", + "console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));", + "NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;", + "}", + "function runtimePathOverlayScript() {", + " const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;", + " const targetPath = String(overlay.runtimePath || '');", + " if (!targetPath || sourcePath === targetPath) return '';", + " return [", + " `if [ ! -d ${shellSingle(targetPath)} ]; then`,", + " ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,", + " ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,", + " ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,", + " ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,", + " `fi`,", + " ].join('\\n');", + "}", + "function stepEnvBootstrapScript() {", + " const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);", + " if (entries.length === 0) return '';", + " const lines = ['# unidesk-step-env-bootstrap'];", + " for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);", + " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');", + " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');", + " return lines.join('\\n');", + "}", + "function runtimeGitopsPostprocessScript() {", + " const runtimeOverlay = JSON.stringify({", + " gitopsRoot: overlay.gitopsRoot,", + " runtimePath: overlay.runtimePath,", + " runtimeRenderDir: overlay.runtimeRenderDir,", + " runtimeNamespace: overlay.runtimeNamespace,", + " externalPostgres: overlay.externalPostgres,", + " publicExposure: overlay.publicExposure,", + " observability: overlay.observability,", + " runtimeImageRewrites: overlay.runtimeImageRewrites,", + " gitReadUrl: overlay.gitReadUrl,", + " publicWebUrl: overlay.publicWebUrl,", + " publicApiUrl: overlay.publicApiUrl,", + " });", + " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'", + "const fs = require('fs');", + "const path = require('path');", + "const crypto = require('crypto');", + "const YAML = require('yaml');", + "const overlay = ${runtimeOverlay};", + "const runtimePath = String(overlay.runtimePath || '');", + "const renderDir = String(overlay.runtimeRenderDir || '');", + "const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';", + "const candidates = [...new Set([", + " runtimePath,", + " renderDir,", + " overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',", + " legacyRuntimePath,", + "].filter(Boolean))];", + "const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;", + "if (!runtimePath || !fs.existsSync(sourcePath)) {", + " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));", + " process.exit(47);", + "}", + "if (sourcePath !== runtimePath) {", + " fs.rmSync(runtimePath, { recursive: true, force: true });", + " fs.mkdirSync(path.dirname(runtimePath), { recursive: true });", + " fs.cpSync(sourcePath, runtimePath, { recursive: true });", + " fs.rmSync(sourcePath, { recursive: true, force: true });", + "}", + "for (const candidate of candidates) {", + " if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });", + "}", + "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", + "function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }", + "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", + "function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }", + "function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }", + "function yamlFiles(dir) {", + " const files = [];", + " for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {", + " const file = path.join(dir, entry.name);", + " if (entry.isDirectory()) files.push(...yamlFiles(file));", + " else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);", + " }", + " return files;", + "}", + "function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", + "function writeYamlDocuments(file, docs) { fs.writeFileSync(file, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\\\n---\\\\n') + '\\\\n'); }", + "function podSpecFor(item) {", + " if (!isObject(item) || !isObject(item.spec)) return null;", + " if (item.kind === 'Pod') return item.spec;", + " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", + " if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", + " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;", + " return null;", + "}", + "function templateMetadataFor(item) {", + " if (!isObject(item) || !isObject(item.spec)) return null;", + " if (item.kind === 'Pod') return item.metadata || null;", + " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;", + " if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;", + " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;", + " return null;", + "}", + "function stripMonitoringMetadata(metadata) {", + " if (!isObject(metadata)) return false;", + " let changed = false;", + " if (isObject(metadata.labels) && metadata.labels['hwlab.pikastech.local/monitoring'] !== undefined && metadata.labels['hwlab.pikastech.local/monitoring'] !== 'disabled') {", + " metadata.labels['hwlab.pikastech.local/monitoring'] = 'disabled';", + " changed = true;", + " }", + " if (isObject(metadata.annotations) && metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'] !== undefined) {", + " delete metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'];", + " changed = true;", + " }", + " return changed;", + "}", + "function containerHasVolumeMount(container, name) { return isObject(container) && Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === name); }", + "function removeMetricsSidecar(podSpec) {", + " if (!isObject(podSpec)) return false;", + " let changed = false;", + " if (Array.isArray(podSpec.containers)) {", + " const next = podSpec.containers.filter((container) => !(isObject(container) && (container.name === 'hwlab-metrics' || (Array.isArray(container.command) && container.command.includes('/metrics/metrics-sidecar.mjs')))));", + " if (next.length !== podSpec.containers.length) { podSpec.containers = next; changed = true; }", + " }", + " for (const group of ['containers', 'initContainers']) {", + " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", + " if (!isObject(container) || !Array.isArray(container.volumeMounts)) continue;", + " const nextMounts = container.volumeMounts.filter((mount) => !(mount && mount.name === 'hwlab-metrics-sidecar'));", + " if (nextMounts.length !== container.volumeMounts.length) { container.volumeMounts = nextMounts; changed = true; }", + " }", + " }", + " if (Array.isArray(podSpec.volumes)) {", + " const nextVolumes = podSpec.volumes.filter((volume) => !(volume && volume.name === 'hwlab-metrics-sidecar'));", + " if (nextVolumes.length !== podSpec.volumes.length) { podSpec.volumes = nextVolumes; changed = true; }", + " }", + " return changed;", + "}", + "function envValue(container, name) {", + " if (!isObject(container) || !Array.isArray(container.env)) return undefined;", + " const item = container.env.find((env) => env && env.name === name);", + " return item ? item.value : undefined;", + "}", + "function setEnvValue(container, name, value) {", + " if (!isObject(container) || typeof value !== 'string') return false;", + " container.env = Array.isArray(container.env) ? container.env : [];", + " let item = container.env.find((env) => env && env.name === name);", + " if (!item) { item = { name }; container.env.push(item); }", + " const changed = item.value !== value || item.valueFrom !== undefined;", + " item.value = value;", + " delete item.valueFrom;", + " return changed;", + "}", + "function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }", + "function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }", + "function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }", + "function startupProbeFrom(probe) {", + " const next = JSON.parse(JSON.stringify(probe));", + " next.periodSeconds = 10;", + " next.timeoutSeconds = Math.max(Number(next.timeoutSeconds || 0), 2);", + " next.failureThreshold = 30;", + " next.successThreshold = 1;", + " delete next.initialDelaySeconds;", + " return next;", + "}", + "function addEnvReuseStartupProbe(podSpec) {", + " if (!isObject(podSpec) || !Array.isArray(podSpec.containers)) return false;", + " let changed = false;", + " for (const container of podSpec.containers) {", + " if (!isObject(container) || !isEnvReuseContainer(container) || container.startupProbe) continue;", + " const sourceProbe = container.readinessProbe || container.livenessProbe;", + " if (!sourceProbe) continue;", + " container.startupProbe = startupProbeFrom(sourceProbe);", + " changed = true;", + " }", + " return changed;", + "}", + "function rewriteRuntimeImage(image) {", + " if (typeof image !== 'string') return image;", + " const match = (overlay.runtimeImageRewrites || []).find((item) => item && item.source === image);", + " return match ? match.target : image;", + "}", + "function patchRuntimeImages(podSpec) {", + " if (!isObject(podSpec)) return false;", + " let changed = false;", + " for (const group of ['containers', 'initContainers']) {", + " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", + " if (!isObject(container) || typeof container.image !== 'string') continue;", + " const nextImage = rewriteRuntimeImage(container.image);", + " if (nextImage !== container.image) { container.image = nextImage; changed = true; }", + " }", + " }", + " return changed;", + "}", + "function rewriteEnvValue(value) {", + " if (typeof value !== 'string') return value;", + " return value.replaceAll('http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git', overlay.gitReadUrl);", + "}", + "function patchGitReadUrlEnv(podSpec) {", + " if (!isObject(podSpec)) return false;", + " let changed = false;", + " for (const group of ['containers', 'initContainers']) {", + " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", + " if (!isObject(container) || !Array.isArray(container.env)) continue;", + " for (const env of container.env) {", + " if (!isObject(env) || typeof env.value !== 'string') continue;", + " const nextValue = rewriteEnvValue(env.value);", + " if (nextValue !== env.value) { env.value = nextValue; changed = true; }", + " }", + " }", + " }", + " return changed;", + "}", + "function patchRuntimeEnv(item, podSpec) {", + " if (!isObject(podSpec)) return { publicEndpointChanged: false, dbSslModeChanged: false };", + " let publicEndpointChanged = false;", + " let dbSslModeChanged = false;", + " const pg = overlay.externalPostgres;", + " for (const group of ['containers', 'initContainers']) {", + " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", + " if (!isObject(container)) continue;", + " if (envValue(container, 'HWLAB_PUBLIC_ENDPOINT') !== undefined) publicEndpointChanged = setEnvValue(container, 'HWLAB_PUBLIC_ENDPOINT', expectedPublicEndpoint(item)) || publicEndpointChanged;", + " if (pg && pg.sslmode && envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE') !== undefined) dbSslModeChanged = setEnvValue(container, 'HWLAB_CLOUD_DB_SSL_MODE', pg.sslmode) || dbSslModeChanged;", + " }", + " }", + " return { publicEndpointChanged, dbSslModeChanged };", + "}", + "function patchRuntimeWorkloads() {", + " let observabilityChanged = false;", + " let startupProbeChanged = false;", + " let imageRewriteChanged = false;", + " let gitReadUrlChanged = false;", + " let publicEndpointChanged = false;", + " let dbSslModeChanged = false;", + " for (const file of yamlFiles(runtimePath)) {", + " if (path.basename(file) === 'kustomization.yaml') continue;", + " const docs = readYamlDocuments(file);", + " let changed = false;", + " for (const doc of docs) {", + " for (const item of listItems(doc).filter(Boolean)) {", + " if (!isObject(item)) continue;", + " if (overlay.observability && overlay.observability.prometheusOperator === false) {", + " const metadataChanged = stripMonitoringMetadata(item.metadata);", + " const templateChanged = stripMonitoringMetadata(templateMetadataFor(item));", + " const sidecarChanged = removeMetricsSidecar(podSpecFor(item));", + " const monitoringChanged = metadataChanged || templateChanged || sidecarChanged;", + " changed = monitoringChanged || changed;", + " observabilityChanged = observabilityChanged || monitoringChanged;", + " }", + " const probeChanged = addEnvReuseStartupProbe(podSpecFor(item));", + " changed = probeChanged || changed;", + " startupProbeChanged = startupProbeChanged || probeChanged;", + " const imageChanged = patchRuntimeImages(podSpecFor(item));", + " changed = imageChanged || changed;", + " imageRewriteChanged = imageRewriteChanged || imageChanged;", + " const gitUrlChanged = patchGitReadUrlEnv(podSpecFor(item));", + " changed = gitUrlChanged || changed;", + " gitReadUrlChanged = gitReadUrlChanged || gitUrlChanged;", + " const envChanged = patchRuntimeEnv(item, podSpecFor(item));", + " changed = envChanged.publicEndpointChanged || envChanged.dbSslModeChanged || changed;", + " publicEndpointChanged = publicEndpointChanged || envChanged.publicEndpointChanged;", + " dbSslModeChanged = dbSslModeChanged || envChanged.dbSslModeChanged;", + " }", + " }", + " if (changed) writeYamlDocuments(file, docs);", + " }", + " return { observabilityChanged, startupProbeChanged, imageRewriteChanged, gitReadUrlChanged, publicEndpointChanged, dbSslModeChanged };", + "}", + "function patchKustomization() {", + " const file = path.join(runtimePath, 'kustomization.yaml');", + " if (!fs.existsSync(file)) return false;", + " const doc = readYaml(file) || {};", + " const resources = Array.isArray(doc.resources) ? doc.resources : [];", + " const next = resources.filter((item) => !(overlay.observability && overlay.observability.prometheusOperator === false && item === 'observability.yaml'));", + " let changed = false;", + " if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); changed = true; }", + " const observabilityFile = path.join(runtimePath, 'observability.yaml');", + " if (overlay.observability && overlay.observability.prometheusOperator === false && fs.existsSync(observabilityFile)) { fs.rmSync(observabilityFile, { force: true }); changed = true; }", + " return changed;", + "}", + "function patchExternalPostgres() {", + " const pg = overlay.externalPostgres;", + " if (!pg || !pg.serviceName) return false;", + " const file = path.join(runtimePath, 'external-postgres.yaml');", + " if (!fs.existsSync(file)) return false;", + " const doc = readYaml(file);", + " const items = listItems(doc).filter(Boolean);", + " let changed = false;", + " const endpointSliceName = String(pg.serviceName) + '-host';", + " for (const item of items) {", + " if (!item || typeof item !== 'object') continue;", + " item.metadata = item.metadata || {};", + " item.metadata.namespace = overlay.runtimeNamespace;", + " item.metadata.labels = item.metadata.labels || {};", + " item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;", + " if (item.kind === 'Service') {", + " item.metadata.name = pg.serviceName;", + " item.spec = item.spec || {};", + " item.spec.ports = [{ name: 'postgres', port: pg.port, targetPort: pg.port, protocol: 'TCP' }];", + " delete item.spec.selector;", + " changed = true;", + " }", + " if (item.kind === 'EndpointSlice') {", + " item.metadata.name = endpointSliceName;", + " item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;", + " item.addressType = 'IPv4';", + " item.ports = [{ name: 'postgres', port: pg.port, protocol: 'TCP' }];", + " item.endpoints = [{ addresses: [pg.endpointAddress], conditions: { ready: true } }];", + " changed = true;", + " }", + " }", + " if (changed) writeYaml(file, normalizeList(items));", + " return changed;", + "}", + "function patchHealthContract() {", + " const file = path.join(runtimePath, 'health-contract.yaml');", + " if (!fs.existsSync(file)) return false;", + " const doc = readYaml(file);", + " const items = listItems(doc).filter(Boolean);", + " let changed = false;", + " const pg = overlay.externalPostgres;", + " for (const item of items) {", + " if (!item || item.kind !== 'ConfigMap') continue;", + " item.data = item.data || {};", + " if (item.data.endpoint !== overlay.publicWebUrl) { item.data.endpoint = overlay.publicWebUrl; changed = true; }", + " const cloudApi = 'GET /health/live through ' + overlay.publicApiUrl;", + " if (item.data['cloud-api'] !== cloudApi) { item.data['cloud-api'] = cloudApi; changed = true; }", + " const cloudWeb = 'GET /health/live on ' + overlay.publicWebUrl + '; consumes cloud-api only';", + " if (item.data['cloud-web'] !== cloudWeb) { item.data['cloud-web'] = cloudWeb; changed = true; }", + " if (pg && pg.sslmode && typeof item.data['cloud-api-db'] === 'string') {", + " const next = item.data['cloud-api-db'].replace(/HWLAB_CLOUD_DB_SSL_MODE=[A-Za-z0-9_-]+/g, 'HWLAB_CLOUD_DB_SSL_MODE=' + pg.sslmode);", + " if (next !== item.data['cloud-api-db']) { item.data['cloud-api-db'] = next; changed = true; }", + " }", + " }", + " if (changed) writeYaml(file, normalizeList(items));", + " return changed;", + "}", + "function renderPublicExposureFrpcToml(exposure) {", + " return [", + " 'serverAddr = ' + JSON.stringify(String(exposure.serverAddr)),", + " 'serverPort = ' + Number(exposure.serverPort),", + " 'loginFailExit = true',", + " 'auth.token = \"{{ .Envs.HWLAB_FRP_TOKEN }}\"',", + " '',", + " '[[proxies]]',", + " 'name = ' + JSON.stringify(String(exposure.webProxy.name)),", + " 'type = \"tcp\"',", + " 'localIP = ' + JSON.stringify(String(exposure.webProxy.localIP)),", + " 'localPort = ' + Number(exposure.webProxy.localPort),", + " 'remotePort = ' + Number(exposure.webProxy.remotePort),", + " '',", + " '[[proxies]]',", + " 'name = ' + JSON.stringify(String(exposure.apiProxy.name)),", + " 'type = \"tcp\"',", + " 'localIP = ' + JSON.stringify(String(exposure.apiProxy.localIP)),", + " 'localPort = ' + Number(exposure.apiProxy.localPort),", + " 'remotePort = ' + Number(exposure.apiProxy.remotePort),", + " '',", + " ].join('\\\\n');", + "}", + "function setEnvFromSecret(container, name, secretName, secretKey) {", + " if (!isObject(container)) return false;", + " container.env = Array.isArray(container.env) ? container.env : [];", + " let item = container.env.find((env) => env && env.name === name);", + " if (!item) { item = { name }; container.env.push(item); }", + " const nextValueFrom = { secretKeyRef: { name: secretName, key: secretKey } };", + " const changed = item.value !== undefined || JSON.stringify(item.valueFrom || {}) !== JSON.stringify(nextValueFrom);", + " delete item.value;", + " item.valueFrom = nextValueFrom;", + " return changed;", + "}", + "function patchPublicExposure() {", + " const exposure = overlay.publicExposure;", + " if (!exposure || !exposure.enabled) return { configured: false, changed: false };", + " const file = path.join(runtimePath, 'node-frpc.yaml');", + " if (!fs.existsSync(file)) {", + " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));", + " process.exit(49);", + " }", + " const docs = readYamlDocuments(file);", + " const configName = String(overlay.runtimeNamespace) + '-frpc-config';", + " const deploymentName = String(overlay.runtimeNamespace) + '-frpc';", + " const configKey = String(exposure.secretKey || 'frpc.toml');", + " const tokenKey = String(exposure.tokenKey || 'token');", + " const toml = renderPublicExposureFrpcToml(exposure);", + " let changed = false;", + " let foundConfigMap = false;", + " let foundDeployment = false;", + " for (const doc of docs) {", + " for (const item of listItems(doc).filter(Boolean)) {", + " if (!isObject(item)) continue;", + " item.metadata = item.metadata || {};", + " if (item.kind === 'ConfigMap' && item.metadata.name === configName) {", + " foundConfigMap = true;", + " item.data = item.data || {};", + " if (item.data[configKey] !== toml) { item.data[configKey] = toml; changed = true; }", + " }", + " if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {", + " foundDeployment = true;", + " item.spec = item.spec || {};", + " const nextStrategy = { type: 'Recreate' };", + " if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }", + " const podSpec = podSpecFor(item);", + " for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {", + " if (!isObject(container)) continue;", + " if (container.name === 'frpc' || String(container.image || '').includes('frpc')) changed = setEnvFromSecret(container, 'HWLAB_FRP_TOKEN', exposure.secretName, tokenKey) || changed;", + " }", + " }", + " }", + " }", + " if (!foundConfigMap || !foundDeployment) {", + " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'frpc-resource-missing', filePath: file, configName, deploymentName, foundConfigMap, foundDeployment }));", + " process.exit(50);", + " }", + " if (changed) writeYamlDocuments(file, docs);", + " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: true, applied: true, changed, filePath: file, hostname: exposure.hostname, serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, webProxy: exposure.webProxy.name, apiProxy: exposure.apiProxy.name, configSha256: crypto.createHash('sha256').update(toml).digest('hex') }));", + " return { configured: true, changed, foundConfigMap, foundDeployment };", + "}", + "const kustomizationChanged = patchKustomization();", + "const runtimeWorkloadsChanged = patchRuntimeWorkloads();", + "const externalPostgresChanged = patchExternalPostgres();", + "const healthContractChanged = patchHealthContract();", + "const publicExposureChanged = patchPublicExposure();", + "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, runtimeImageRewriteCount: (overlay.runtimeImageRewrites || []).length, kustomizationChanged, observabilityWorkloadsChanged: runtimeWorkloadsChanged.observabilityChanged, startupProbeChanged: runtimeWorkloadsChanged.startupProbeChanged, runtimeImageRewriteChanged: runtimeWorkloadsChanged.imageRewriteChanged, gitReadUrlChanged: runtimeWorkloadsChanged.gitReadUrlChanged, publicEndpointChanged: runtimeWorkloadsChanged.publicEndpointChanged, dbSslModeChanged: runtimeWorkloadsChanged.dbSslModeChanged, externalPostgresChanged, healthContractChanged, publicExposureChanged }));", + "NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;", + "}", + "function runtimeGitopsVerifyScript() {", + " const runtimeOverlay = JSON.stringify({", + " runtimePath: overlay.runtimePath,", + " runtimeNamespace: overlay.runtimeNamespace,", + " externalPostgres: overlay.externalPostgres,", + " publicExposure: overlay.publicExposure,", + " observability: overlay.observability,", + " runtimeImageRewrites: overlay.runtimeImageRewrites,", + " gitReadUrl: overlay.gitReadUrl,", + " publicWebUrl: overlay.publicWebUrl,", + " publicApiUrl: overlay.publicApiUrl,", + " });", + " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'", + "const fs = require('fs');", + "const path = require('path');", + "const YAML = require('yaml');", + "const overlay = ${runtimeOverlay};", + "const runtimePath = String(overlay.runtimePath || '');", + "function fail(reason, extra = {}) {", + " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));", + " process.exit(48);", + "}", + "if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');", + "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", + "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", + "function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }", + "function yamlFiles(dir) {", + " const files = [];", + " for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {", + " const file = path.join(dir, entry.name);", + " if (entry.isDirectory()) files.push(...yamlFiles(file));", + " else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);", + " }", + " return files;", + "}", + "function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", + "function allItemsFromFile(file) { return readYamlDocuments(file).flatMap((doc) => listItems(doc).filter(Boolean)); }", + "function podSpecFor(item) {", + " if (!isObject(item) || !isObject(item.spec)) return null;", + " if (item.kind === 'Pod') return item.spec;", + " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", + " if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", + " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;", + " return null;", + "}", + "function envValue(container, name) {", + " if (!isObject(container) || !Array.isArray(container.env)) return undefined;", + " const item = container.env.find((env) => env && env.name === name);", + " return item ? item.value : undefined;", + "}", + "function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }", + "function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }", + "function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }", + "function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }", + "function workloadChecks() {", + " const metricsRefs = [];", + " const missingStartupProbes = [];", + " const publicRuntimeImages = [];", + " const staleGitReadUrls = [];", + " const wrongPublicEndpoints = [];", + " const wrongDbSslModes = [];", + " const rewriteSources = new Set((overlay.runtimeImageRewrites || []).map((item) => item && item.source).filter(Boolean));", + " for (const file of yamlFiles(runtimePath)) {", + " if (path.basename(file) === 'kustomization.yaml') continue;", + " for (const doc of readYamlDocuments(file)) {", + " for (const item of listItems(doc).filter(Boolean)) {", + " const podSpec = podSpecFor(item);", + " if (!isObject(podSpec)) continue;", + " for (const container of Array.isArray(podSpec.containers) ? podSpec.containers : []) {", + " if (!isObject(container)) continue;", + " if (container.name === 'hwlab-metrics' || (Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === 'hwlab-metrics-sidecar'))) metricsRefs.push(workloadRef(item, file, container));", + " if (isEnvReuseContainer(container) && (container.readinessProbe || container.livenessProbe) && !container.startupProbe) missingStartupProbes.push(workloadRef(item, file, container));", + " if (typeof container.image === 'string' && rewriteSources.has(container.image)) publicRuntimeImages.push({ ...workloadRef(item, file, container), image: container.image });", + " if (Array.isArray(container.env) && container.env.some((env) => env && typeof env.value === 'string' && env.value.includes('git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git') && env.value !== overlay.gitReadUrl)) staleGitReadUrls.push(workloadRef(item, file, container));", + " const publicEndpoint = envValue(container, 'HWLAB_PUBLIC_ENDPOINT');", + " if (publicEndpoint !== undefined && publicEndpoint !== expectedPublicEndpoint(item)) wrongPublicEndpoints.push({ ...workloadRef(item, file, container), value: publicEndpoint, expected: expectedPublicEndpoint(item) });", + " const dbSslMode = envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE');", + " if (overlay.externalPostgres && overlay.externalPostgres.sslmode && dbSslMode !== undefined && dbSslMode !== overlay.externalPostgres.sslmode) wrongDbSslModes.push({ ...workloadRef(item, file, container), value: dbSslMode, expected: overlay.externalPostgres.sslmode });", + " }", + " if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));", + " }", + " }", + " }", + " return { metricsRefs, missingStartupProbes, publicRuntimeImages, staleGitReadUrls, wrongPublicEndpoints, wrongDbSslModes };", + "}", + "const checks = [];", + "const workloadCheck = workloadChecks();", + "const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');", + "if (overlay.observability && overlay.observability.prometheusOperator === false) {", + " if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');", + " const resources = readYaml(kustomizationPath).resources || [];", + " if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });", + " if (workloadCheck.metricsRefs.length > 0) fail('observability-sidecar-still-rendered', { refs: workloadCheck.metricsRefs.slice(0, 12), count: workloadCheck.metricsRefs.length });", + " checks.push('observability-disabled');", + "}", + "if (workloadCheck.missingStartupProbes.length > 0) fail('env-reuse-startup-probe-missing', { refs: workloadCheck.missingStartupProbes.slice(0, 12), count: workloadCheck.missingStartupProbes.length });", + "checks.push('env-reuse-startup-probes');", + "if (workloadCheck.publicRuntimeImages.length > 0) fail('runtime-image-rewrite-missing', { refs: workloadCheck.publicRuntimeImages.slice(0, 12), count: workloadCheck.publicRuntimeImages.length });", + "if ((overlay.runtimeImageRewrites || []).length > 0) checks.push('runtime-image-rewrites');", + "if (workloadCheck.staleGitReadUrls.length > 0) fail('runtime-git-read-url-stale', { refs: workloadCheck.staleGitReadUrls.slice(0, 12), count: workloadCheck.staleGitReadUrls.length, expected: overlay.gitReadUrl });", + "checks.push('runtime-git-read-url');", + "if (workloadCheck.wrongPublicEndpoints.length > 0) fail('runtime-public-endpoint-mismatch', { refs: workloadCheck.wrongPublicEndpoints.slice(0, 12), count: workloadCheck.wrongPublicEndpoints.length });", + "checks.push('runtime-public-endpoint');", + "if (workloadCheck.wrongDbSslModes.length > 0) fail('runtime-db-ssl-mode-mismatch', { refs: workloadCheck.wrongDbSslModes.slice(0, 12), count: workloadCheck.wrongDbSslModes.length });", + "if (overlay.externalPostgres && overlay.externalPostgres.sslmode) checks.push('runtime-db-ssl-mode');", + "const pg = overlay.externalPostgres;", + "if (pg && pg.serviceName) {", + " const file = path.join(runtimePath, 'external-postgres.yaml');", + " if (!fs.existsSync(file)) fail('external-postgres-missing');", + " const items = listItems(readYaml(file)).filter(Boolean);", + " const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);", + " const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');", + " if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });", + " if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });", + " const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;", + " const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;", + " const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;", + " if (Number(servicePort) !== Number(pg.port) || Number(endpointPort) !== Number(pg.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: pg.port });", + " if (String(endpointAddress) !== String(pg.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: pg.endpointAddress });", + " checks.push('external-postgres-bridge');", + "}", + "const exposure = overlay.publicExposure;", + "if (exposure && exposure.enabled) {", + " const file = path.join(runtimePath, 'node-frpc.yaml');", + " if (!fs.existsSync(file)) fail('public-exposure-frpc-missing');", + " const items = allItemsFromFile(file);", + " const configName = String(overlay.runtimeNamespace) + '-frpc-config';", + " const deploymentName = String(overlay.runtimeNamespace) + '-frpc';", + " const configKey = String(exposure.secretKey || 'frpc.toml');", + " const tokenKey = String(exposure.tokenKey || 'token');", + " const configMap = items.find((item) => item && item.kind === 'ConfigMap' && item.metadata && item.metadata.name === configName);", + " const deployment = items.find((item) => item && item.kind === 'Deployment' && item.metadata && item.metadata.name === deploymentName);", + " if (!configMap) fail('public-exposure-frpc-configmap-missing', { expected: configName });", + " if (!deployment) fail('public-exposure-frpc-deployment-missing', { expected: deploymentName });", + " const toml = configMap.data && configMap.data[configKey];", + " if (typeof toml !== 'string') fail('public-exposure-frpc-config-missing', { expectedKey: configKey });", + " for (const expected of [String(exposure.serverAddr), String(exposure.serverPort), String(exposure.webProxy.name), String(exposure.webProxy.remotePort), String(exposure.apiProxy.name), String(exposure.apiProxy.remotePort), 'HWLAB_FRP_TOKEN']) {", + " if (!toml.includes(expected)) fail('public-exposure-frpc-config-mismatch', { expected });", + " }", + " const podSpec = podSpecFor(deployment);", + " const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];", + " const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;", + " if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });", + " const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));", + " const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;", + " const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;", + " if (!secretRef || secretRef.name !== exposure.secretName || secretRef.key !== tokenKey) fail('public-exposure-frpc-token-env-mismatch', { expectedSecret: exposure.secretName, expectedKey: tokenKey });", + " checks.push('public-exposure-frpc');", + "}", + "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));", + "NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;", + "}", + "function patchScript(script) {", + " let result = String(script || '');", + " if (result.includes('npm run gitops:ts:check')) {", + " result = result.replace(/\\n[ \\t]*npm run gitops:ts:check\\n/g, '\\n echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-replaces-tsc-gate\"}\\' >&2\\n');", + " }", + " const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');", + " if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {", + " result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());", + " }", + " const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';", + " if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", + " result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);", + " }", + " const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';", + " if (result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", + " result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);", + " }", + " result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);", + " result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", + " result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", + " result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", + " result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", + " result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);", + " result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);", + " const legacyRuntimePath = (() => {", + " const runtimePath = String(overlay.runtimePath || '');", + " const parts = runtimePath.split('/').filter(Boolean);", + " if (parts.length < 2) return '';", + " const leaf = parts.at(-1);", + " const parent = parts.slice(0, -2).join('/');", + " return parent && leaf ? `${parent}/${leaf}` : '';", + " })();", + " if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {", + " result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);", + " result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');", + " result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');", + " }", + " const bootstrap = stepEnvBootstrapScript();", + " if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {", + " result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);", + " }", + " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {", + " let next = match;", + " if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", + " if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", + " return next;", + " });", + " result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {", + " if (match.includes('--check')) return runtimeGitopsVerifyScript();", + " return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;", + " });", + " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", + " validatePrepareSourceDependencyScript(result);", + " return result;", + "}", + "function patchManifestObject(doc) {", + " if (!doc || typeof doc !== 'object') return false;", + " if (doc.kind !== 'Pipeline' || !doc.spec) return false;", + " const defaults = {", + " 'git-url': overlay.gitUrl,", + " 'git-read-url': overlay.gitReadUrl,", + " 'git-write-url': overlay.gitWriteUrl,", + " 'catalog-path': overlay.catalogPath,", + " 'runtime-path': overlay.runtimePath,", + " 'registry-prefix': overlay.registryPrefix,", + " };", + " for (const param of doc.spec?.params || []) {", + " if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];", + " }", + " doc.metadata = doc.metadata || {};", + " doc.metadata.annotations = doc.metadata.annotations || {};", + " doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;", + " doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;", + " for (const task of doc.spec?.tasks || []) {", + " for (const sidecar of task.taskSpec?.sidecars || []) {", + " if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;", + " }", + " for (const step of task.taskSpec?.steps || []) {", + " if (Array.isArray(step.env)) {", + " for (const env of step.env) {", + " if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];", + " }", + " }", + " step.env = Array.isArray(step.env) ? step.env : [];", + " const existingEnv = new Set(step.env.map((env) => env.name));", + " for (const [name, value] of Object.entries(stepEnv)) {", + " if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });", + " }", + " if (typeof step.script === 'string') step.script = patchScript(step.script);", + " }", + " }", + " return true;", + "}", + "function patchStructuredPipeline() {", + " try {", + " const doc = JSON.parse(text);", + " if (!patchManifestObject(doc)) return false;", + " text = JSON.stringify(doc, null, 2) + '\\n';", + " return true;", + " } catch {}", + " if (YAML) {", + " try {", + " const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);", + " const changed = docs.some((doc) => patchManifestObject(doc));", + " if (!changed) return false;", + " text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';", + " return true;", + " } catch {}", + " }", + " return false;", + "}", + "const structured = patchStructuredPipeline();", + "function replaceParamDefault(name, value) {", + " const namePattern = escapeRegExp(name);", + " text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);", + " text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);", + "}", + "replaceParamDefault('git-url', overlay.gitUrl);", + "replaceParamDefault('git-read-url', overlay.gitReadUrl);", + "replaceParamDefault('git-write-url', overlay.gitWriteUrl);", + "replaceParamDefault('catalog-path', overlay.catalogPath);", + "replaceParamDefault('runtime-path', overlay.runtimePath);", + "replaceParamDefault('registry-prefix', overlay.registryPrefix);", + "text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);", + "text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);", + "if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);", + "text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", + "text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", + "text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", + "text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", + "for (const [name, value] of Object.entries(stepEnv)) {", + " if (value === undefined) continue;", + " text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);", + " text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);", + "}", + "const bootstrap = stepEnvBootstrapScript();", + "if (bootstrap) {", + " text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {", + " const previous = fullText.slice(Math.max(0, offset - 500), offset);", + " if (previous.includes('unidesk-step-env-bootstrap')) return match;", + " return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;", + " });", + "}", + "if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {", + " text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);", + "}", + "const quotedRoot = JSON.stringify(overlay.gitopsRoot);", + "const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');", + "const replacements = [", + " ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],", + " ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],", + "];", + "let changed = false;", + "for (const [needle, replacement] of replacements) {", + " if (!text.includes(needle)) continue;", + " text = text.split(needle).join(replacement);", + " changed = true;", + "}", + "if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }", + "if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }", + "if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && !text.includes('NODE_UNIDESK_YAML_DEPENDENCY')) { throw new Error(`generated pipeline missing UniDesk yaml dependency install in ${pipelinePath}`); }", + "if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }", + "fs.writeFileSync(pipelinePath, text);", + "function patchArgoYaml(filePath) {", + " if (!YAML || !fs.existsSync(filePath)) return;", + " const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);", + " let changed = false;", + " for (const doc of docs) {", + " if (!doc || typeof doc !== 'object') continue;", + " if (doc.kind === 'AppProject') {", + " doc.spec = doc.spec || {};", + " doc.spec.sourceRepos = [overlay.argoRepoUrl];", + " changed = true;", + " }", + " if (doc.kind === 'Application') {", + " doc.spec = doc.spec || {};", + " doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };", + " changed = true;", + " }", + " }", + " if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", + "}", + "patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));", + "patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));", + "NODE", + ]; +} + +function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record { + const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http); + return { + nodeId: spec.nodeId, + lane: spec.lane, + sourceBranch: spec.sourceBranch, + gitopsBranch: spec.gitopsBranch, + gitopsRoot: nodeRuntimeGitopsRoot(spec), + runtimePath: spec.runtimePath, + runtimeRenderDir: spec.runtimeRenderDir, + runtimeNamespace: spec.runtimeNamespace, + catalogPath: spec.catalogPath, + tektonDir: spec.tektonDir, + argoApplicationFile: spec.argoApplicationFile, + argoRepoUrl: spec.argoRepoUrl, + gitUrl: spec.gitUrl, + gitReadUrl: spec.gitReadUrl, + gitWriteUrl: spec.gitWriteUrl, + networkProfileId: spec.networkProfileId, + downloadProfileId: spec.downloadProfileId, + gitSshProxyHost: gitSshProxy?.host, + gitSshProxyPort: gitSshProxy?.port, + proxyHttp: spec.networkProfile.proxy.http, + proxyHttps: spec.networkProfile.proxy.https, + proxyAll: spec.networkProfile.proxy.all, + noProxy: spec.networkProfile.proxy.noProxy.join(","), + dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http, + dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https, + dockerProxyAll: spec.networkProfile.dockerBuildProxy.all, + dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","), + dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy, + npmRegistry: spec.downloadProfile.npm.registry, + npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000, + npmRetries: spec.downloadProfile.npm.retries, + stepEnv: spec.stepEnv, + observability: spec.observability, + runtimeImageRewrites: spec.runtimeImageRewrites, + registryPrefix: spec.registryPrefix, + buildkitSidecarImage: spec.buildkit?.sidecarImage, + publicWebUrl: spec.publicWebUrl, + publicApiUrl: spec.publicApiUrl, + publicExposure: spec.publicExposure === null ? undefined : { + enabled: spec.publicExposure.enabled, + mode: spec.publicExposure.mode, + publicBaseUrl: spec.publicExposure.publicBaseUrl, + hostname: spec.publicExposure.hostname, + expectedA: spec.publicExposure.expectedA, + serverAddr: spec.publicExposure.serverAddr, + serverPort: spec.publicExposure.serverPort, + secretName: spec.publicExposure.secretName, + secretKey: spec.publicExposure.secretKey, + tokenKey: spec.publicExposure.tokenKey, + webProxy: spec.publicExposure.webProxy, + apiProxy: spec.publicExposure.apiProxy, + }, + externalPostgres: spec.externalPostgres === undefined ? undefined : { + enabled: true, + serviceName: spec.externalPostgres.serviceName, + endpointAddress: spec.externalPostgres.endpointAddress, + port: spec.externalPostgres.port, + sslmode: spec.externalPostgres.sslmode, + }, + }; +} + +function httpProxyEndpoint(value: string): { host: string; port: number } | null { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return null; + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; + const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10); + if (!Number.isInteger(port) || port <= 0) return null; + return { host: parsed.hostname, port }; +} + + +function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] { + return [ + `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`, + `${renderDir}/${spec.tektonDir}/rbac.yaml`, + `${renderDir}/${spec.tektonDir}/pipeline.yaml`, + `${renderDir}/argocd/project.yaml`, + `${renderDir}/argocd/${spec.argoApplicationFile}`, + ]; +} + +function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { + const files = nodeRuntimeControlPlaneFiles(spec, renderDir); + const args = [ + "kubectl", + "apply", + ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), + ...files.flatMap((file) => ["-f", file]), + ]; + return runNodeK3sArgs(spec, args, timeoutSeconds); +} + +function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { + const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir) + .map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`) + .join(""); + const args = [ + "kubectl", + "apply", + ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), + "-f", + "-", + ]; + return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest); +} + +function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult { + const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; + const script = [ + "set -eu", + `render_dir=${shellQuote(renderDir)}`, + `prefix=${shellQuote(prefix)}`, + "case \"$render_dir\" in", + " \"$prefix\"*) rm -rf \"$render_dir\" ;;", + " *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;", + "esac", + ].join("\n"); + return runNodeHostScript(spec, script, 60); +} + +function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult { + const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; + const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`; + if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) { + return { + command: ["rm", "-rf", ""], + cwd: repoRoot, + exitCode: 44, + stdout: "", + stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`, + signal: null, + timedOut: false, + }; + } + return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 }); +} + +function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record { + return { + apiVersion: "tekton.dev/v1", + kind: "PipelineRun", + metadata: { + name: pipelineRun, + namespace: "hwlab-ci", + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": spec.lane, + "hwlab.pikastech.local/source-commit": sourceCommit, + "hwlab.pikastech.local/trigger": "unidesk-node-cli", + }, + annotations: { + "hwlab.pikastech.local/node": spec.nodeId, + "hwlab.pikastech.local/source-branch": spec.sourceBranch, + "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch, + "hwlab.pikastech.local/runtime-path": spec.runtimePath, + "hwlab.pikastech.local/network-profile": spec.networkProfileId, + "hwlab.pikastech.local/download-profile": spec.downloadProfileId, + }, + }, + spec: { + pipelineRef: { name: spec.pipeline }, + taskRunTemplate: { + serviceAccountName: spec.serviceAccountName, + podTemplate: { + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + securityContext: { fsGroup: 1000 }, + }, + }, + params: [ + { name: "git-url", value: spec.gitUrl }, + { name: "git-read-url", value: spec.gitReadUrl }, + { name: "git-write-url", value: spec.gitWriteUrl }, + { name: "source-branch", value: spec.sourceBranch }, + { name: "gitops-branch", value: spec.gitopsBranch }, + { name: "lane", value: spec.lane }, + { name: "catalog-path", value: spec.catalogPath }, + { name: "image-tag-mode", value: "full" }, + { name: "runtime-path", value: spec.runtimePath }, + { name: "revision", value: sourceCommit }, + { name: "registry-prefix", value: spec.registryPrefix }, + { name: "services", value: spec.serviceIds.join(",") }, + { name: "base-image", value: spec.baseImage }, + ], + workspaces: [ + { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, + { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, + ], + }, + }; +} + +function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult { + const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64"); + const script = [ + "set -eu", + `manifest_b64=${shellQuote(manifestB64)}`, + `pipeline_run=${shellQuote(pipelineRun)}`, + `rerun=${rerun ? "true" : "false"}`, + "manifest_path=\"/tmp/$pipeline_run.json\"", + "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", + "existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", + "if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then", + " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + "elif [ \"$existing_status\" = False ]; then", + " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + "fi", + "if kubectl create -f \"$manifest_path\"; then", + " :", + "else", + " code=$?", + " if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then", + " printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2", + " else", + " exit \"$code\"", + " fi", + "fi", + "kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'", + ].join("\n"); + return runNodeK3sScript(spec, script, timeoutSeconds); +} + +function waitForNodeRuntimePipelineRunTerminal(spec: HwlabRuntimeLaneSpec, pipelineRun: string, timeoutSeconds: number): Record { + const startedAt = Date.now(); + const deadline = startedAt + timeoutSeconds * 1000; + let polls = 0; + let last: Record = { exists: false, name: pipelineRun }; + printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds }); + while (Date.now() <= deadline) { + polls += 1; + last = getNodeRuntimePipelineRun(spec, pipelineRun); + const status = typeof last.status === "string" ? last.status : null; + const reason = typeof last.reason === "string" ? last.reason : null; + printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); + if (status === "True" || status === "False") { + const ok = status === "True"; + printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); + return { + ok, + status: ok ? "succeeded" : "failed", + pipelineRun: last, + polls, + elapsedMs: Date.now() - startedAt, + degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed", + }; + } + sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now()))); + } + printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "timeout", pipelineRun, polls, elapsedMs: Date.now() - startedAt }); + return { + ok: false, + status: "timeout", + pipelineRun: last, + polls, + elapsedMs: Date.now() - startedAt, + degradedReason: "node-runtime-pipelinerun-wait-timeout", + }; +} + +function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { + process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`); +} + +function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record { + const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60); + if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) }; + const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u); + return { + exists: true, + name: pipelineRun, + status: status || null, + reason: reason || null, + message: message || null, + createdAt: createdAt || null, + result: compactRuntimeCommand(result), + }; +} + +function sleepSync(ms: number): void { + const buffer = new SharedArrayBuffer(4); + Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms)); +} + +function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { + const pg = spec.externalPostgres; + if (pg === undefined) return null; + const secretRoot = externalPostgresSecretSourceRoot(spec); + const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey); + const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey); + const missing = [ + ...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]), + ...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]), + ]; + if (missing.length > 0) { + if (dryRun) { + return { + ok: true, + mode: "dry-run", + mutation: false, + skipped: true, + reason: "external-postgres-secret-source-missing", + missing, + sourceRoot: secretRoot, + valuesPrinted: false, + next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, + }; + } + return { + ok: false, + mode: dryRun ? "dry-run" : "confirmed-secret-sync", + mutation: false, + degradedReason: "external-postgres-secret-source-missing", + missing, + sourceRoot: secretRoot, + valuesPrinted: false, + next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, + }; + } + const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds); + if (!isCommandSuccess(setup)) { + return { + ok: false, + mode: dryRun ? "dry-run" : "confirmed-secret-sync", + mutation: false, + phase: "namespace-authn-setup", + setup: compactRuntimeCommand(setup), + valuesPrinted: false, + }; + } + const manifest = { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "Secret", + metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace }, + type: "Opaque", + stringData: { [pg.cloudApi.secretKey]: cloudApi.value }, + }, + { + apiVersion: "v1", + kind: "Secret", + metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace }, + type: "Opaque", + stringData: { [pg.openfga.secretKey]: openfga.value }, + }, + ], + }; + const applyScript = [ + "set -eu", + [ + "kubectl", + "apply", + "--server-side", + "--force-conflicts", + "--field-manager=unidesk-hwlab-node-external-postgres-secret", + ...(dryRun ? ["--dry-run=server"] : []), + "-f", + "-", + ].map(shellQuote).join(" "), + ].join("\n"); + const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest)); + return { + ok: isCommandSuccess(apply), + mode: dryRun ? "dry-run" : "confirmed-secret-sync", + mutation: !dryRun && isCommandSuccess(apply), + namespace: spec.runtimeNamespace, + sourceRoot: secretRoot, + secrets: [ + { name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey }, + { name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null }, + ], + setup: compactRuntimeCommand(setup), + apply: compactRuntimeCommand(apply), + valuesPrinted: false, + }; +} + +function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record { + const source = spec.baseImageSource ?? ""; + const script = [ + "set -eu", + `target=${shellQuote(spec.baseImage)}`, + `source=${shellQuote(source)}`, + "repo_tag=${target#*/}", + "repo=${repo_tag%:*}", + "tag=${repo_tag##*:}", + "if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", + "registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", + "registry_present=false", + "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi", + "target_present=false", + "if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi", + "source_present=false", + "if [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi", + "printf 'target\\t%s\\n' \"$target\"", + "printf 'source\\t%s\\n' \"$source\"", + "printf 'sourceConfigured\\t%s\\n' \"$([ -n \"$source\" ] && printf true || printf false)\"", + "printf 'registryUrl\\t%s\\n' \"$registry_url\"", + "printf 'registryTagPresent\\t%s\\n' \"$registry_present\"", + "printf 'targetImagePresent\\t%s\\n' \"$target_present\"", + "printf 'sourceImagePresent\\t%s\\n' \"$source_present\"", + ].join("\n"); + const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); + const fields = keyValueLinesFromText(statusText(result)); + const sourceConfigured = fields.sourceConfigured === "true"; + const registryTagPresent = fields.registryTagPresent === "true"; + return { + ok: isCommandSuccess(result) && sourceConfigured && registryTagPresent, + target: fields.target ?? spec.baseImage, + source: fields.source || null, + sourceConfigured, + registryUrl: fields.registryUrl ?? null, + registryTagPresent, + targetImagePresent: fields.targetImagePresent === "true", + sourceImagePresent: fields.sourceImagePresent === "true", + result: compactRuntimeCommand(result), + }; +} + +function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { + if (spec.baseImageSource === undefined) return null; + const script = [ + "set -eu", + `target=${shellQuote(spec.baseImage)}`, + `source=${shellQuote(spec.baseImageSource)}`, + `dry_run=${shellQuote(dryRun ? "true" : "false")}`, + `pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`, + "repo_tag=${target#*/}", + "repo=${repo_tag%:*}", + "tag=${repo_tag##*:}", + "if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", + "registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", + "present=false", + "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi", + "action=none", + "if [ \"$present\" = false ]; then", + " action=seed", + " if [ \"$dry_run\" = false ]; then", + " source_present_before=true", + " if ! docker image inspect \"$source\" >/dev/null 2>&1; then", + " source_present_before=false", + " attempt=1", + " pulled_source=false", + " while [ \"$attempt\" -le \"$pull_retries\" ]; do", + " pull_attempts=$attempt", + " if docker pull \"$source\" >/tmp/hwlab-node-base-image-pull.out 2>&1; then pulled_source=true; break; fi", + " attempt=$((attempt + 1))", + " sleep 2", + " done", + " if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-base-image-pull.out >&2 2>/dev/null || true; fi", + " fi", + " docker image inspect \"$source\" >/dev/null", + " docker tag \"$source\" \"$target\"", + " docker push \"$target\" >/tmp/hwlab-node-base-image-push.out", + " fi", + "fi", + "after=false", + "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi", + "printf 'target\\t%s\\n' \"$target\"", + "printf 'source\\t%s\\n' \"$source\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'presentBefore\\t%s\\n' \"$present\"", + "printf 'presentAfter\\t%s\\n' \"$after\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'pullRetries\\t%s\\n' \"$pull_retries\"", + "printf 'sourcePresentBefore\\t%s\\n' \"${source_present_before:-unknown}\"", + "printf 'pulledSource\\t%s\\n' \"${pulled_source:-false}\"", + "printf 'pullAttempts\\t%s\\n' \"${pull_attempts:-0}\"", + "if [ \"$dry_run\" = true ] || [ \"$after\" = true ]; then exit 0; fi", + "exit 1", + ].join("\n"); + const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); + const fields = keyValueLinesFromText(statusText(result)); + return { + ok: isCommandSuccess(result), + dryRun, + target: fields.target ?? spec.baseImage, + source: fields.source ?? spec.baseImageSource, + presentBefore: fields.presentBefore === "true", + presentAfter: fields.presentAfter === "true", + action: fields.action ?? null, + pullRetries: numericField(fields.pullRetries), + sourcePresentBefore: fields.sourcePresentBefore ?? null, + pulledSource: fields.pulledSource === "true", + pullAttempts: numericField(fields.pullAttempts), + result: compactRuntimeCommand(result), + }; +} + +function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string { + const pg = spec.externalPostgres; + if (pg === undefined) return "true"; + const authnKey = pg.openfga.authnKey ?? "authn-preshared-key"; + return [ + "set -eu", + `namespace=${shellQuote(spec.runtimeNamespace)}`, + `secret=${shellQuote(pg.openfga.secretName)}`, + `authn_key=${shellQuote(authnKey)}`, + `dry_run=${shellQuote(dryRun ? "true" : "false")}`, + "namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"", + "if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi", + "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -", + "authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')", + "if [ \"${authn_present:-0}\" -gt 0 ]; then", + " action=kept", + "elif [ \"$dry_run\" = true ]; then", + " action=would-generate-authn-key", + "else", + " if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi", + " kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null", + " authn_value=", + " action=generated-authn-key", + "fi", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$secret\"", + "printf 'authnKey\\t%s\\n' \"$authn_key\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'valuesPrinted\\tfalse\\n'", + ].join("\n"); +} + +function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string { + const configRef = spec.externalPostgres?.configRef; + if (configRef === undefined) return join(repoRoot, ".state", "secrets"); + const configPath = rootPath(configRef); + const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`); + const secrets = (parsed as Record).secrets; + if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`); + const root = (secrets as Record).root; + if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`); + return root.startsWith("/") ? root : rootPath(root); +} + +function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } { + const sourcePath = join(secretRoot, sourceRef); + if (!existsSync(sourcePath)) return { ok: false, sourcePath }; + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const value = values[key]; + if (value === undefined || value.length === 0) return { ok: false, sourcePath }; + return { ok: true, value, sourcePath }; +} + +function secretSourcePaths(sourceRef: string): string[] { + const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef)); + return [...new Set(paths)]; +} + +function displayRepoPath(path: string): string { + const normalizedRoot = repoRoot.replace(/\/+$/u, ""); + if (path === normalizedRoot) return "."; + if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1); + const marker = "/.worktree/"; + const index = normalizedRoot.indexOf(marker); + if (index >= 0) { + const mainRoot = normalizedRoot.slice(0, index); + if (path === mainRoot) return "."; + if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1); + } + return path; +} + +function hwlabPasswordHash(password: string): string { + const salt = randomBytes(16).toString("hex"); + return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`; +} + +function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial { + const sourceRef = spec.bootstrapAdminPasswordSourceRef; + const sourceKey = spec.bootstrapAdminPasswordSourceKey; + if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { + return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "bootstrap-admin-yaml-source-missing" }; + } + const paths = secretSourcePaths(sourceRef); + const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); + if (!existsSync(sourcePath)) { + return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "secret-source-missing" }; + } + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const password = values[sourceKey]; + if (password === undefined || password.length === 0) { + return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, passwordHash: null, error: "secret-key-missing" }; + } + return { + ok: true, + sourceRef, + sourceKey, + sourcePath, + sourcePresent: true, + sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`, + passwordHash: hwlabPasswordHash(password), + error: null, + }; +} + +function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial { + const sourceRef = spec.bootstrapAdminPasswordSourceRef; + const sourceKey = spec.bootstrapAdminPasswordSourceKey; + if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { + return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, password: null, error: "bootstrap-admin-yaml-source-missing" }; + } + const paths = secretSourcePaths(sourceRef); + const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); + if (!existsSync(sourcePath)) { + return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" }; + } + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const password = values[sourceKey]; + if (password === undefined || password.length === 0) { + return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" }; + } + return { + ok: true, + sourceRef, + sourceKey, + sourcePath, + sourcePresent: true, + sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`, + password, + error: null, + }; +} + +function parseEnvFile(text: string): Record { + const values: Record = {}; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + const rawValue = line.slice(eq + 1).trim(); + values[key] = rawValue.startsWith("'") && rawValue.endsWith("'") + ? rawValue.slice(1, -1).replace(/'"'"'/gu, "'") + : rawValue.startsWith("\"") && rawValue.endsWith("\"") + ? rawValue.slice(1, -1) + : rawValue; + } + return values; +} + +function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean { + if (!/postgres|pgsql|pgdata/iu.test(name)) return false; + const externalServiceName = spec.externalPostgres?.serviceName; + if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false; + return true; +} + +function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { + const pg = spec.externalPostgres; + if (pg === undefined) return { required: false, ready: true }; + if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; + const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "name"], 60); + const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60); + const [address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u); + return { + required: true, + ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && address === pg.endpointAddress && Number(port) === pg.port, + serviceName: pg.serviceName, + endpointAddress: address || null, + expectedEndpointAddress: pg.endpointAddress, + port: numericField(port), + expectedPort: pg.port, + service: compactRuntimeCommand(service), + endpointSlice: compactRuntimeCommand(endpointSlice), + }; +} + +function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { + const pg = spec.externalPostgres; + if (pg === undefined) return { required: false, ready: true }; + if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; + const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey); + const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey); + const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key"); + return { + required: true, + ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present, + valuesPrinted: false, + cloudApi, + openfga: { + datastoreUri: openfgaUri, + authnPresharedKey: openfgaAuthn, + }, + }; +} + +function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec { + const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); + const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown); + const targets = Array.isArray(parsed.targets) ? parsed.targets : []; + const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane); + if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); + const gitMirror = record(target.gitMirror); + const source = record(target.source); + const gitops = record(target.gitops); + const tekton = record(target.tekton); + const toolsImage = record(tekton.toolsImage); + return { + id: stringValue(target.id, "target.id"), + node: stringValue(target.node, "target.node"), + lane: stringValue(target.lane, "target.lane"), + namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"), + serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"), + serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"), + cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"), + cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"), + secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"), + syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"), + flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"), + toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"), + sourceRepository: stringValue(source.repository, "source.repository"), + sourceBranch: stringValue(source.branch, "source.branch"), + gitopsBranch: stringValue(gitops.branch, "gitops.branch"), + }; +} + +function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record { + const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60); + return { + secret, + key, + present: result.exitCode === 0 && result.stdout.trim().length > 0, + valueBytesBase64: result.stdout.trim().length, + exitCode: result.exitCode, + stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500), + valuesPrinted: false, + }; +} + +function startNodeDelegatedJob(options: ReturnType): Record { + const commandArgs = [ + "hwlab", + "nodes", + options.domain, + ...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]), + "--node", + options.node, + "--lane", + options.lane, + "--confirm", + "--timeout-seconds", + String(options.timeoutSeconds), + "--wait", + ]; + const command = ["bun", "scripts/cli.ts", ...commandArgs]; + const job = startJob( + `hwlab_nodes_${options.lane}_${options.domain}_${options.action}`, + command, + `Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`, + ); + return { + ok: true, + command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + mode: "async-job", + reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking", + job, + statusCommand: `bun scripts/cli.ts job status ${job.id}`, + waitCommand: command.join(" "), + }; +} + +function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType): Record { + const rewritten = rewriteDelegatedNodeValue(value, scoped); + const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record : { value: rewritten }; + return { + ...result, + node: scoped.node, + commandFamily: "hwlab nodes", + }; +} + +function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType): unknown { + if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped); + if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped)); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)])); +} + +function rewriteDelegatedNodeString(value: string, scoped: ReturnType): string { + const replaceCommand = (text: string, domain: DelegatedNodeDomain) => { + const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + return text + .replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`) + .replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`); + }; + return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror"); +} + +function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions { + const [actionRaw] = args; + if (actionRaw !== "run" && actionRaw !== "script") throw new Error("web-probe usage: run|script --node NODE --lane vNN [--url URL]"); + const node = requiredOption(args, "--node"); + assertNodeId(node); + const lane = requiredOption(args, "--lane"); + assertLane(lane); + if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`); + const spec = hwlabRuntimeLaneSpecForNode(lane, node); + if (actionRaw === "script") { + assertKnownOptions(args.slice(1), new Set([ + "--node", + "--lane", + "--url", + "--timeout-ms", + "--viewport", + "--script-file", + "--command-timeout-seconds", + ]), new Set([])); + const scriptFile = optionValue(args, "--script-file"); + if (scriptFile === undefined && process.stdin.isTTY) { + throw new Error("web-probe script requires a stdin heredoc or --script-file "); + } + const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8"); + if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script"); + const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000); + const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600); + return { + action: "script", + node, + lane, + url: optionValue(args, "--url") ?? spec.publicWebUrl, + timeoutMs, + viewport: optionValue(args, "--viewport") ?? "1440x900", + commandTimeoutSeconds, + scriptText, + scriptSource: { + kind: scriptFile === undefined ? "stdin" : "file", + path: scriptFile ?? null, + byteCount: Buffer.byteLength(scriptText), + sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`, + }, + }; + } + assertKnownOptions(args.slice(1), new Set([ + "--node", + "--lane", + "--url", + "--timeout-ms", + "--wait-after-submit-ms", + "--wait-messages-ms", + "--wait-agent-terminal-ms", + "--trace-sample-count", + "--trace-sample-interval-ms", + "--message", + "--conversation-id", + "--command-timeout-seconds", + ]), new Set([ + "--fresh-session", + "--no-cancel-running", + ])); + const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000); + const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000); + const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000); + const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000); + const message = optionValue(args, "--message") ?? null; + const hasMessage = message !== null; + const traceSampleCountDefault = hasMessage ? 2 : 0; + const traceSampleIntervalDefault = hasMessage ? 1500 : 0; + const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200); + const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000); + const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({ + timeoutMs, + waitAfterSubmitMs, + waitMessagesMs, + waitAgentTerminalMs, + traceSampleCount, + traceSampleIntervalMs, + freshSession: args.includes("--fresh-session"), + hasMessage, + }); + const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds"); + const commandTimeoutUserProvided = commandTimeoutRaw !== undefined; + const commandTimeoutSeconds = commandTimeoutUserProvided + ? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds) + : commandTimeoutAutoSeconds; + return { + action: "run", + node, + lane, + url: optionValue(args, "--url") ?? spec.publicWebUrl, + timeoutMs, + waitAfterSubmitMs, + waitMessagesMs, + waitAgentTerminalMs, + traceSampleCount, + traceSampleIntervalMs, + message, + conversationId: optionValue(args, "--conversation-id") ?? null, + freshSession: args.includes("--fresh-session"), + cancelRunning: !args.includes("--no-cancel-running"), + commandTimeoutSeconds, + commandTimeoutAutoSeconds, + commandTimeoutUserProvided, + }; +} + +function nodeWebProbeAutoCommandTimeoutSeconds(input: { + timeoutMs: number; + waitAfterSubmitMs: number; + waitMessagesMs: number; + waitAgentTerminalMs: number; + traceSampleCount: number; + traceSampleIntervalMs: number; + freshSession: boolean; + hasMessage: boolean; +}): number { + const traceWindowMs = input.traceSampleCount > 0 + ? Math.max(0, input.traceSampleCount - 1) * input.traceSampleIntervalMs + : 0; + const startupBudgetMs = input.timeoutMs + 30_000; + const freshnessBudgetMs = input.freshSession ? Math.min(input.timeoutMs, 30_000) : 0; + const submitBudgetMs = input.hasMessage ? input.waitAfterSubmitMs + input.waitMessagesMs + 15_000 : 0; + const terminalBudgetMs = input.waitAgentTerminalMs > 0 ? input.waitAgentTerminalMs : 0; + const totalMs = startupBudgetMs + freshnessBudgetMs + submitBudgetMs + terminalBudgetMs + traceWindowMs + 15_000; + return Math.min(3600, Math.max(60, Math.ceil(totalMs / 1000))); +} + +function assertKnownOptions(args: string[], valueOptions: Set, flagOptions: Set): void { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + if (!arg.startsWith("--")) continue; + if (flagOptions.has(arg)) continue; + if (valueOptions.has(arg)) { + index += 1; + continue; + } + throw new Error(`unknown option: ${arg}`); + } +} + +function runNodeWebProbe(options: NodeWebProbeOptions): Record { + const lane = options.lane; + if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`); + const spec = hwlabRuntimeLaneSpecForNode(lane, options.node); + const secretSpec = runtimeSecretSpec({ node: options.node, lane }); + const material = readBootstrapAdminPasswordMaterial(secretSpec); + const credential = webProbeCredential(secretSpec, material); + if (!material.ok || material.password === null) { + return { + ok: false, + status: "blocked", + command: `hwlab nodes web-probe ${options.action} --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + degradedReason: "web_login_secret_missing", + credential, + next: { secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${secretSpec.bootstrapAdminSecret}` }, + }; + } + if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential); + if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential); + const probeArgs = nodeWebProbeRunArgs(options, "run"); + const script = [ + "set -eu", + `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password)} ${probeArgs.map(shellQuote).join(" ")}`, + ].join("\n"); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); + const probe = compactWebProbeResult(parseJsonObject(result.stdout)); + const passed = result.exitCode === 0 && probe?.status === "pass"; + const summary = nullableRecord(probe?.summary); + const degradedReason = result.timedOut + ? "web-probe-command-timeout" + : typeof probe?.degradedReason === "string" + ? probe.degradedReason + : null; + return { + ok: passed, + status: passed ? "pass" : "blocked", + command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + credential, + commandTimeout: { + seconds: options.commandTimeoutSeconds, + autoSeconds: options.commandTimeoutAutoSeconds, + userProvided: options.commandTimeoutUserProvided, + timedOut: result.timedOut, + }, + degradedReason, + failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : null, + summary, + probe, + result: compactCommandResult(result), + valuesRedacted: true, + }; +} + +function nodeWebProbeRunArgs(options: NodeWebProbeRunOptions, command: "run" | "start"): string[] { + const probeArgs = [ + "node", + "scripts/web-live-dom-probe.mjs", + command, + "--url", options.url, + "--timeout-ms", String(options.timeoutMs), + "--wait-after-submit-ms", String(options.waitAfterSubmitMs), + "--wait-messages-ms", String(options.waitMessagesMs), + ]; + if (options.waitAgentTerminalMs > 0) probeArgs.push("--wait-agent-terminal-ms", String(options.waitAgentTerminalMs)); + if (options.traceSampleCount > 0) probeArgs.push("--trace-sample-count", String(options.traceSampleCount), "--trace-sample-interval-ms", String(options.traceSampleIntervalMs)); + if (options.freshSession) probeArgs.push("--fresh-session"); + if (!options.cancelRunning) probeArgs.push("--no-cancel-running"); + if (options.conversationId !== null) probeArgs.push("--conversation-id", options.conversationId); + if (options.message !== null) probeArgs.push("--message", options.message); + return probeArgs; +} + +function runNodeWebProbeAsync( + options: NodeWebProbeRunOptions, + spec: HwlabRuntimeLaneSpec, + secretSpec: RuntimeSecretSpec, + material: BootstrapAdminPasswordMaterial, + credential: Record, +): Record { + const startArgs = nodeWebProbeRunArgs(options, "start"); + const startScript = [ + "set -eu", + `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password ?? "")} ${startArgs.map(shellQuote).join(" ")}`, + ].join("\n"); + const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55); + const start = parseJsonObject(startResult.stdout); + const jobId = typeof start?.jobId === "string" ? start.jobId : null; + if (startResult.exitCode !== 0 || jobId === null) { + return { + ok: false, + status: "blocked", + command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + credential, + mode: "async-start", + commandTimeout: webProbeCommandTimeoutSummary(options, false), + degradedReason: startResult.timedOut ? "web-probe-command-timeout" : "web-probe-async-start-failed", + start: start ?? null, + result: compactCommandResultRedacted(startResult, [material.password ?? ""]), + valuesRedacted: true, + }; + } + const poll = pollNodeWebProbeJob(options, spec, jobId); + const statusReport = record(record(poll.status).report); + const reportPath = typeof start.reportPath === "string" ? start.reportPath : null; + const reportLoad = Object.keys(statusReport).length > 0 + ? { source: "status", report: statusReport, result: null as CommandResult | null, degradedReason: null as string | null, path: null as string | null } + : readNodeWebProbeReport(options, spec, reportPath); + const report = reportLoad.report ?? {}; + const probe = compactWebProbeResult(Object.keys(report).length > 0 ? report : null); + const passed = probe?.status === "pass"; + const summary = nullableRecord(probe?.summary); + const degradedReason = poll.timedOut + ? "web-probe-command-timeout" + : typeof probe?.degradedReason === "string" + ? probe.degradedReason + : poll.status === null + ? reportLoad.degradedReason ?? "web-probe-async-status-failed" + : reportLoad.degradedReason; + return { + ok: passed, + status: passed ? "pass" : "blocked", + command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + credential, + mode: "async", + commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut), + degradedReason, + failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : null, + summary, + job: { + jobId, + startedAt: start.startedAt ?? null, + polls: poll.polls, + elapsedMs: poll.elapsedMs, + statusCommand: start.statusCommand ?? `node scripts/web-live-dom-probe.mjs status ${jobId}`, + }, + probe, + start: { + ok: start.ok === true, + status: start.status ?? null, + jobId, + traceSampling: start.traceSampling ?? null, + reportPath: start.reportPath ?? null, + screenshotPath: start.screenshotPath ?? null, + }, + statusResult: poll.result === null ? null : compactCommandResult(poll.result), + reportLoad: { + source: reportLoad.source, + path: reportLoad.path, + result: reportLoad.result === null ? null : compactCommandResult(reportLoad.result), + degradedReason: reportLoad.degradedReason, + }, + valuesRedacted: true, + }; +} + +function readNodeWebProbeReport( + options: NodeWebProbeRunOptions, + spec: HwlabRuntimeLaneSpec, + reportPath: string | null, +): { source: string; report: Record | null; result: CommandResult | null; degradedReason: string | null; path: string | null } { + if (!reportPath) return { source: "missing", report: null, result: null, degradedReason: "web-probe-report-path-missing", path: null }; + if (!isSafeWebProbeReportPath(reportPath)) return { source: "unsafe-path", report: null, result: null, degradedReason: "web-probe-report-path-invalid", path: reportPath }; + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, `cat ${shellQuote(reportPath)}`, 55); + if (result.exitCode !== 0 || result.timedOut) return { source: "report-file", report: null, result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-report-read-failed", path: reportPath }; + const report = parseJsonObject(result.stdout); + return { + source: "report-file", + report, + result, + degradedReason: report === null ? "web-probe-report-parse-failed" : null, + path: reportPath, + }; +} + +function isSafeWebProbeReportPath(reportPath: string): boolean { + return reportPath.includes("/.state/web-live-dom-probe/") && reportPath.endsWith(".result.json") && !reportPath.includes("\0"); +} + +function pollNodeWebProbeJob(options: NodeWebProbeRunOptions, spec: HwlabRuntimeLaneSpec, jobId: string): { + status: Record | null; + result: CommandResult | null; + polls: number; + elapsedMs: number; + timedOut: boolean; +} { + const startedAt = Date.now(); + const deadline = startedAt + options.commandTimeoutSeconds * 1000; + let lastStatus: Record | null = null; + let lastResult: CommandResult | null = null; + let polls = 0; + while (Date.now() < deadline) { + polls += 1; + const script = `node scripts/web-live-dom-probe.mjs status ${shellQuote(jobId)}`; + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55); + lastResult = result; + lastStatus = parseJsonObject(result.stdout); + const status = typeof lastStatus?.status === "string" ? lastStatus.status : null; + if (result.exitCode === 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false }; + if (result.exitCode !== 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false }; + sleepSync(Math.min(5000, Math.max(500, deadline - Date.now()))); + } + return { status: lastStatus, result: lastResult, polls, elapsedMs: Date.now() - startedAt, timedOut: true }; +} + +function webProbeCommandTimeoutSummary(options: NodeWebProbeRunOptions, timedOut: boolean): Record { + return { + seconds: options.commandTimeoutSeconds, + autoSeconds: options.commandTimeoutAutoSeconds, + userProvided: options.commandTimeoutUserProvided, + transportMode: options.commandTimeoutSeconds > 55 ? "async-start-status" : "direct", + timedOut, + }; +} + +function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAdminPasswordMaterial): Record { + return { + username: secretSpec.bootstrapAdminUsername, + sourceRef: material.sourceRef, + sourceKey: material.sourceKey, + sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath), + sourcePresent: material.sourcePresent, + sourceFingerprint: material.sourceFingerprint, + injectedVia: material.ok ? "stdin-env" : null, + valuesRedacted: true, + error: material.error, + }; +} + +function runNodeWebProbeScript( + options: NodeWebProbeScriptOptions, + spec: HwlabRuntimeLaneSpec, + secretSpec: RuntimeSecretSpec, + material: BootstrapAdminPasswordMaterial, + credential: Record, +): Record { + const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? ""); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); + const report = compactWebProbeScriptResult(parseJsonObject(result.stdout)); + const passed = result.exitCode === 0 && report?.ok === true; + const summary = nullableRecord(report?.summary); + const degradedReason = typeof summary?.degradedReason === "string" + ? summary.degradedReason + : typeof report?.failureKind === "string" + ? report.failureKind + : result.timedOut + ? "web-probe-command-timeout" + : null; + return { + ok: passed, + status: passed ? "pass" : "blocked", + command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + credential, + scriptSource: options.scriptSource, + degradedReason, + failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : typeof report?.failureKind === "string" ? report.failureKind : null, + summary, + probe: report, + result: compactCommandResultRedacted(result, [material.password ?? ""]), + valuesRedacted: true, + }; +} + +function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string { + const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64"); + const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64"); + return [ + "set -eu", + "run_root=.state/web-probe-script", + "mkdir -p \"$run_root\"", + "run_dir=$(mktemp -d \"$run_root/run.XXXXXX\")", + "chmod 700 \"$run_dir\"", + "user_script=\"$run_dir/user-script.mjs\"", + "runner=\"$run_dir/runner.mjs\"", + `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`, + `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`, + [ + `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`, + `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, + `HWLAB_WEB_PASS=${shellQuote(password)}`, + "UNIDESK_WEB_PROBE_RUN_DIR=\"$run_dir\"", + "UNIDESK_WEB_PROBE_USER_SCRIPT=\"$user_script\"", + `UNIDESK_WEB_PROBE_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, + `UNIDESK_WEB_PROBE_VIEWPORT=${shellQuote(options.viewport)}`, + "node \"$runner\"", + ].join(" "), + ].join("\n"); +} + + + +function parseSecretOptions(args: string[]): NodeSecretOptions { + const [actionRaw] = args; + if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") { + throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-vNN-bootstrap-admin|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]"); + } + const node = requiredOption(args, "--node"); + assertNodeId(node); + const lane = requiredOption(args, "--lane"); + assertLane(lane); + const spec = runtimeSecretSpec({ node, lane }); + const name = optionValue(args, "--name") ?? spec.openFgaSecret; + const key = optionValue(args, "--key"); + const confirm = args.includes("--confirm"); + const explicitDryRun = args.includes("--dry-run"); + if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run"); + if (actionRaw === "cleanup-owned-postgres") { + if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres"); + if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key"); + if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`); + return { + action: actionRaw, + node, + lane, + name: spec.postgresSecret, + preset: "owned-postgres-cleanup", + confirm, + dryRun: explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; + } + if (actionRaw === "cleanup-obsolete") { + if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup"); + if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key"); + if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`); + return { + action: actionRaw, + node, + lane, + name, + preset: "obsolete-secret-cleanup", + confirm, + dryRun: explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; + } + if (name === spec.masterAdminApiKeySecret) { + if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`); + return { + action: actionRaw, + node, + lane, + name, + key: key ?? MASTER_ADMIN_API_KEY_KEY, + preset: "master-server-admin-api-key", + confirm, + dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; + } + if (name === spec.bootstrapAdminSecret) { + if (key !== undefined && key !== BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY) throw new Error(`secret ${name} supports only key ${BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY}`); + const force = args.includes("--force"); + if (force && actionRaw !== "ensure") throw new Error("secret --force is only supported with ensure"); + return { + action: actionRaw, + node, + lane, + name, + key: key ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, + preset: "bootstrap-admin", + confirm, + force, + dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; + } + if (name === spec.codeAgentProviderSecret) { + if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) { + throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`); + } + return { + action: actionRaw, + node, + lane, + name, + key, + preset: "code-agent-provider", + confirm, + dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; + } + if (name === spec.cloudApiDbSecret) { + if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`); + if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { + throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); + } + return { + action: actionRaw, + node, + lane, + name, + key: key ?? spec.cloudApiDbKey, + preset: "cloud-api-db", + confirm, + dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900), + }; + } + if (name !== spec.openFgaSecret) { + throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.bootstrapAdminSecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`); + } + if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) { + throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`); + } + if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { + throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); + } + return { + action: actionRaw, + node, + lane, + name, + key, + preset: "openfga", + confirm, + dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900), + }; +} + +function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec { + const namespace = `hwlab-${input.lane}`; + const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined; + const externalPostgres = runtimeLaneSpec?.externalPostgres; + const bootstrapAdmin = runtimeLaneSpec?.bootstrapAdmin; + const platformDb = externalPostgres !== undefined || /^v0*[3-9]\d*$/.test(input.lane); + const platformPostgresService = externalPostgres?.serviceName ?? "g14-platform-postgres"; + const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`; + const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`; + const platformPostgresEndpointSlice = `${platformPostgresService}-host`; + const openFgaDbName = externalPostgres?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga"); + const openFgaDbUser = externalPostgres?.openfga.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga"); + const cloudApiDbName = externalPostgres?.database ?? `hwlab_${input.lane}`; + const cloudApiDbUser = externalPostgres?.cloudApi.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`); + return { + node: input.node, + lane: input.lane, + namespace, + platformDb, + ...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }), + ...(externalPostgres === undefined ? {} : { externalPostgres }), + platformPostgresService, + platformPostgresEndpointAddress: externalPostgres?.endpointAddress, + platformPostgresEndpointSlice, + postgresSecret: `${namespace}-postgres`, + postgresStatefulSet: `${namespace}-postgres`, + postgresAdminUser: `hwlab_${input.lane}`, + openFgaSecret: externalPostgres?.openfga.secretName ?? `${namespace}-openfga`, + openFgaDbName, + openFgaDbUser, + openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, + masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`, + bootstrapAdminSecret: bootstrapAdmin?.secretName ?? `${namespace}-bootstrap-admin`, + bootstrapAdminPasswordHashKey: bootstrapAdmin?.secretKey ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, + bootstrapAdminUsername: bootstrapAdmin?.username ?? "admin", + bootstrapAdminDisplayName: bootstrapAdmin?.displayName ?? `HWLAB ${input.lane} Admin`, + ...(bootstrapAdmin?.passwordSourceRef === undefined ? {} : { bootstrapAdminPasswordSourceRef: bootstrapAdmin.passwordSourceRef }), + ...(bootstrapAdmin?.passwordSourceKey === undefined ? {} : { bootstrapAdminPasswordSourceKey: bootstrapAdmin.passwordSourceKey }), + ...(bootstrapAdmin?.passwordHashTransform === undefined ? {} : { bootstrapAdminPasswordHashTransform: bootstrapAdmin.passwordHashTransform }), + bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE, + bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET, + cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? `hwlab-cloud-api-${input.lane}-db`, + cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? CLOUD_API_DB_KEY, + cloudApiDbName, + cloudApiDbUser, + cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, + cloudApiDeployment: "hwlab-cloud-api", + obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`, + obsoleteHwpodDbName: `hwpod_${input.lane}`, + obsoleteHwpodDbUser: `hwpod_${input.lane}_app`, + codeAgentProviderSecret: `${namespace}-code-agent-provider`, + codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE, + codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET, + fieldManager: `unidesk-hwlab-node-${input.lane}-secret`, + }; +} + +function runNodeSecret(options: NodeSecretOptions): Record { + const spec = runtimeSecretSpec(options); + if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec); + if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) { + return runExternalPostgresSecretEnsure(options, spec); + } + const bootstrapAdminMaterial = options.preset === "bootstrap-admin" ? readBootstrapAdminSecretMaterial(spec) : null; + const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun + ? readMasterAdminApiKey(spec).key + : options.preset === "bootstrap-admin" && options.action === "ensure" && !options.dryRun && bootstrapAdminMaterial?.ok === true + ? bootstrapAdminMaterial.passwordHash ?? "" + : ""; + const script = options.preset === "openfga" + ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec) + : options.preset === "master-server-admin-api-key" + ? masterAdminApiKeySecretScript(options, spec) + : options.preset === "bootstrap-admin" + ? bootstrapAdminSecretScript(options, spec, bootstrapAdminMaterial) + : options.preset === "cloud-api-db" + ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec) + : options.preset === "owned-postgres-cleanup" + ? ownedPostgresCleanupScript(options, spec) + : codeAgentProviderSecretScript(options, spec); + const result = runTransScript(options.node, script, input, options.timeoutSeconds); + const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec); + const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0; + const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0; + const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true; + const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true; + return { + ok, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure", + status, + mutation: status.mutation === true, + result: compactCommandResult(result), + valuesRedacted: true, + next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec), + }; +} + +function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { + if (options.action === "cleanup-owned-postgres") { + return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` }; + } + if (options.action === "cleanup-obsolete") { + return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` }; + } + if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) { + return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; + } + if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) { + return { + status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`, + controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`, + }; + } + return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; +} + +function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { + const runtimeLaneSpec = spec.runtimeLaneSpec; + if (runtimeLaneSpec === undefined) { + return { + ok: false, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-ensure", + mutation: false, + degradedReason: "external-postgres-runtime-lane-spec-missing", + valuesRedacted: true, + }; + } + const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds); + const shouldReadStatus = sync !== null && sync.ok === true; + const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null; + const status = statusResult === null + ? null + : secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec); + const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true); + return { + ok, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-ensure", + status: status ?? sync, + externalPostgresSecretSync: sync, + mutation: sync?.mutation === true, + result: statusResult === null ? null : compactCommandResult(statusResult), + valuesRedacted: true, + next: ok ? undefined : nextSecretCommand(options, spec), + }; +} + +function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record { + return { + mode: exposure.mode, + publicBaseUrl: exposure.publicBaseUrl, + hostname: exposure.hostname, + expectedA: exposure.expectedA, + frpc: { + serverAddr: exposure.serverAddr, + serverPort: exposure.serverPort, + tokenSourceRef: exposure.tokenSourceRef, + tokenSourceKey: exposure.tokenSourceKey, + secretName: exposure.secretName, + secretKey: exposure.secretKey, + tokenKey: exposure.tokenKey, + webProxy: exposure.webProxy, + apiProxy: exposure.apiProxy, + }, + caddy: { + route: exposure.caddyRoute, + configPath: exposure.caddyConfigPath, + serviceName: exposure.caddyServiceName, + tls: exposure.caddyTls, + responseHeaderTimeoutSeconds: exposure.responseHeaderTimeoutSeconds, + }, + }; +} + +function runNodePublicExposure(options: NodePublicExposureOptions): Record { + const exposure = options.spec.publicExposure; + if (exposure === null || !exposure.enabled) { + return { + ok: false, + command: "hwlab nodes control-plane public-exposure", + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + error: "publicExposure is not configured for this node/lane target", + mutation: false, + }; + } + const source = readPublicExposureTokenSource(exposure); + if (!source.ok) { + return { + ok: false, + command: "hwlab nodes control-plane public-exposure", + node: options.node, + lane: options.lane, + mode: options.dryRun ? "dry-run" : "confirmed", + configPath: hwlabRuntimeLaneConfigPath(), + publicExposure: publicExposureSummary(exposure), + source, + mutation: false, + valuesRedacted: true, + next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=` }, + }; + } + const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds); + let secretFields = keyValueLinesFromText(statusText(secretApplyResult)); + let secretResult = secretApplyResult; + if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") { + const restartResult = runPublicExposureFrpcRecreate(options); + secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) }; + secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult); + } + const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds); + const caddyFields = keyValueLinesFromText(statusText(caddyResult)); + const secretStatus = publicExposureSecretStatus(secretFields, secretResult); + const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult); + const ok = secretStatus.ok === true && caddyStatus.ok === true; + return { + ok, + command: "hwlab nodes control-plane public-exposure", + node: options.node, + lane: options.lane, + mode: options.dryRun ? "dry-run" : "confirmed", + mutation: !options.dryRun && (secretFields.mutation === "true" || caddyFields.mutation === "true"), + configPath: hwlabRuntimeLaneConfigPath(), + publicExposure: publicExposureSummary(exposure), + source: { + ok: source.ok, + path: source.path, + key: exposure.tokenSourceKey, + bytes: source.value?.length ?? 0, + fingerprint: source.fingerprint, + valueRedacted: true, + }, + secret: secretStatus, + caddy: caddyStatus, + valuesRedacted: true, + next: ok + ? { + triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${options.node} --lane ${options.lane} --confirm`, + status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`, + } + : { retry: `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${options.node} --lane ${options.lane} --confirm` }, + }; +} + +function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } { + const checkedPaths = publicExposureTokenSourcePaths(exposure); + const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef); + if (!existsSync(path)) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-source-missing" }; + const values = parseEnvFile(readFileSync(path, "utf8")); + const value = values[exposure.tokenSourceKey]; + if (value === undefined || value.length === 0) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-key-missing" }; + return { + ok: true, + path, + checkedPaths, + key: exposure.tokenSourceKey, + value, + fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`, + }; +} + +function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec): string[] { + const paths = [join(repoRoot, ".state", "secrets", exposure.tokenSourceRef)]; + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", exposure.tokenSourceRef)); + return [...new Set(paths)]; +} + +function publicExposureSecretStatus(fields: Record, result: CommandResult): Record { + const dryRun = fields.dryRun === "true"; + return { + ok: result.exitCode === 0 && (dryRun || ( + fields.afterSecretExists === "yes" + && fields.strategyPatchExitCode === "0" + && fields.rolloutRestartExitCode === "0" + && fields.rolloutStatusExitCode === "0" + )), + dryRun, + mutation: fields.mutation === "true", + namespace: fields.namespace || null, + secret: fields.secret || null, + deployment: fields.deployment || null, + beforeExists: fields.beforeSecretExists === "yes", + afterExists: fields.afterSecretExists === "yes", + tokenBytes: numericField(fields.tokenBytes), + applyExitCode: numericField(fields.applyExitCode), + restartMode: fields.restartMode || null, + strategyPatchExitCode: numericField(fields.strategyPatchExitCode), + rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), + scaleDownExitCode: numericField(fields.scaleDownExitCode), + scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode), + scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount), + scaleUpExitCode: numericField(fields.scaleUpExitCode), + rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), + readyReplicas: numericField(fields.readyReplicas), + availableReplicas: numericField(fields.availableReplicas), + desiredReplicas: numericField(fields.desiredReplicas), + strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null, + restartErrorPreview: fields.restartErrorPreview || null, + scaleDownErrorPreview: fields.scaleDownErrorPreview || null, + scaleUpErrorPreview: fields.scaleUpErrorPreview || null, + readyErrorPreview: fields.readyErrorPreview || null, + exitCode: result.exitCode, + stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), + }; +} + +function publicExposureCaddyStatus(fields: Record, result: CommandResult): Record { + const dryRun = fields.dryRun === "true"; + return { + ok: result.exitCode === 0 + && fields.afterBlockPresent === "yes" + && fields.validateExitCode === "0" + && (dryRun || ( + fields.active === "active" + && fields.localWebStatus === "200" + && fields.localApiStatus === "200" + )), + dryRun, + mutation: fields.mutation === "true", + hostname: fields.hostname || null, + configPath: fields.configPath || null, + beforeBlockPresent: fields.beforeBlockPresent === "yes", + afterBlockPresent: fields.afterBlockPresent === "yes", + staleBlocksRemoved: numericField(fields.staleBlocksRemoved), + pythonExitCode: numericField(fields.pythonExitCode), + validateMode: fields.validateMode || null, + validateExitCode: numericField(fields.validateExitCode), + reloadExitCode: numericField(fields.reloadExitCode), + active: fields.active || null, + localWebStatus: fields.localWebStatus || null, + localApiStatus: fields.localApiStatus || null, + validateErrorPreview: fields.validateErrorPreview || null, + localWebErrorPreview: fields.localWebErrorPreview || null, + localApiErrorPreview: fields.localApiErrorPreview || null, + exitCode: result.exitCode, + stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), + }; +} + +function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string { + const deployment = `${options.spec.runtimeNamespace}-frpc`; + return [ + "set +e", + `namespace=${shellQuote(options.spec.runtimeNamespace)}`, + `secret=${shellQuote(exposure.secretName)}`, + `deployment=${shellQuote(deployment)}`, + `token_key=${shellQuote(exposure.tokenKey)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + "token=$(cat)", + "before_secret_exists=no", + "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes", + "token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')", + "apply_exit=", + "restart_mode=recreate-rollout-restart", + "strategy_patch_exit=", + "rollout_restart_exit=", + "scale_down_exit=", + "scale_down_wait_exit=", + "scale_down_wait_pod_count=", + "scale_up_exit=", + "rollout_status_exit=", + "ready_replicas=", + "available_replicas=", + "desired_replicas=", + "mutation=false", + "if [ \"$dry_run\" = false ]; then", + " tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)", + " token_b64=$(printf '%s' \"$token\" | base64 | tr -d '\\n')", + " cat >\"$tmp\" </tmp/hwlab-public-frpc-secret.apply.out 2>/tmp/hwlab-public-frpc-secret.apply.err", + " apply_exit=$?", + " rm -f \"$tmp\"", + " if [ \"$apply_exit\" = 0 ]; then mutation=true; fi", + "fi", + "after_secret_exists=no", + "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$secret\"", + "printf 'deployment\\t%s\\n' \"$deployment\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", + "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", + "printf 'tokenBytes\\t%s\\n' \"$token_bytes\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'restartMode\\t%s\\n' \"$restart_mode\"", + "printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", + "printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"", + "printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"", + "printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"", + "printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"", + "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", + "printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"", + "printf 'availableReplicas\\t%s\\n' \"$available_replicas\"", + "printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"", + "if [ \"$dry_run\" = true ]; then exit 0; fi", + "[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]", + ].join("\n"); +} + +function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult { + const namespace = options.spec.runtimeNamespace; + const deployment = `${namespace}-frpc`; + const fields: Record = { + deployment, + restartMode: "recreate-rollout-restart", + strategyPatchExitCode: "", + rolloutRestartExitCode: "", + scaleDownExitCode: "", + scaleDownWaitExitCode: "", + scaleDownWaitPodCount: "", + scaleUpExitCode: "", + rolloutStatusExitCode: "", + readyReplicas: "", + availableReplicas: "", + desiredReplicas: "", + strategyPatchErrorPreview: "", + restartErrorPreview: "", + scaleDownErrorPreview: "", + scaleUpErrorPreview: "", + readyErrorPreview: "", + }; + const stdoutParts: string[] = []; + const stderrParts: string[] = []; + const addResult = (result: CommandResult): void => { + if (result.stdout.trim()) stdoutParts.push(result.stdout.trim()); + if (result.stderr.trim()) stderrParts.push(result.stderr.trim()); + }; + const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55); + const strategyPatch = runTransScript(options.node, [ + "set +e", + `namespace=${shellQuote(namespace)}`, + `deployment=${shellQuote(deployment)}`, + "kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err", + "code=$?", + "printf 'strategyPatchExitCode\\t%s\\n' \"$code\"", + "printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' /tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err", + "code=$?", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"", + "printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' /tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err", + "code=$?", + "values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)", + "desired=$(printf '%s' \"$values\" | cut -f1)", + "ready=$(printf '%s' \"$values\" | cut -f2)", + "available=$(printf '%s' \"$values\" | cut -f3)", + "printf 'desiredReplicas\\t%s\\n' \"$desired\"", + "printf 'readyReplicas\\t%s\\n' \"$ready\"", + "printf 'availableReplicas\\t%s\\n' \"$available\"", + "printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' `${key}\t${value}`).join("\n") + "\n"; + return { + command: [transPath(), `${options.node}:k3s`, "sh", "--", ""], + cwd: repoRoot, + exitCode: ok ? 0 : 1, + stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n", + stderr: stderrParts.join("\n"), + signal: null, + timedOut: false, + }; +} + +function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult { + return { + command: [...first.command, "&&", ...second.command], + cwd: repoRoot, + exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode, + stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n", + stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"), + signal: second.signal ?? first.signal, + timedOut: first.timedOut || second.timedOut, + }; +} + +function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string { + const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64"); + const marker = `unidesk managed ${exposure.hostname}`; + return [ + "set +e", + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `hostname=${shellQuote(exposure.hostname)}`, + `config_path=${shellQuote(exposure.caddyConfigPath)}`, + `service=${shellQuote(exposure.caddyServiceName)}`, + `marker=${shellQuote(marker)}`, + `block_b64=${shellQuote(blockB64)}`, + `expected_a=${shellQuote(exposure.expectedA)}`, + `web_port=${shellQuote(String(exposure.webProxy.remotePort))}`, + `api_port=${shellQuote(String(exposure.apiProxy.remotePort))}`, + "rm -f /tmp/hwlab-public-caddy-validate.out /tmp/hwlab-public-caddy-validate.err /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-web.err /tmp/hwlab-public-caddy-api.err", + "tmp=$(mktemp -d)", + "block=\"$tmp/block\"", + "next=\"$tmp/Caddyfile\"", + "printf '%s' \"$block_b64\" | base64 -d >\"$block\"", + "before_present=no", + "if [ -f \"$config_path\" ] && grep -Fq \"# BEGIN $marker\" \"$config_path\"; then before_present=yes; fi", + "if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi", + "stale_blocks_removed=$(python3 - \"$next\" \"$block\" \"$marker\" \"$web_port\" \"$api_port\" 2>/tmp/hwlab-public-caddy-python.err <<'PY'", + "import pathlib, re, sys", + "config = pathlib.Path(sys.argv[1])", + "block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')", + "marker = sys.argv[3]", + "web_port = sys.argv[4]", + "api_port = sys.argv[5]", + "current_name = marker[len('unidesk managed '):] if marker.startswith('unidesk managed ') else marker", + "text = config.read_text(encoding='utf-8') if config.exists() else ''", + "begin = f'# BEGIN {marker}'", + "end = f'# END {marker}'", + "managed = f'{begin}\\n{block.rstrip()}\\n{end}\\n'", + "stale_removed = [0]", + "pattern = re.compile(r'(?ms)^# BEGIN unidesk managed (?P[^\\n]+)\\n(?P.*?)\\n# END unidesk managed (?P=name)\\n*')", + "def keep(match):", + " name = match.group('name')", + " body = match.group('body')", + " if name != current_name and (f'127.0.0.1:{web_port}' in body or f'127.0.0.1:{api_port}' in body):", + " stale_removed[0] += 1", + " return ''", + " return match.group(0)", + "text = pattern.sub(keep, text)", + "if begin in text and end in text:", + " start = text.index(begin)", + " stop = text.index(end, start) + len(end)", + " text = text[:start].rstrip() + '\\n\\n' + managed.rstrip() + '\\n' + text[stop:].lstrip('\\n')", + "else:", + " text = text.rstrip() + '\\n\\n' + managed", + "config.write_text(text, encoding='utf-8')", + "print(stale_removed[0])", + "PY", + ")", + "python_exit=$?", + "validate_exit=1", + "validate_mode=validate", + "if [ \"$python_exit\" != 0 ]; then", + " validate_mode=python", + " validate_exit=$python_exit", + "else", + " if [ \"$dry_run\" = true ]; then", + " validate_mode=adapt", + " caddy adapt --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err", + " else", + " sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err", + " fi", + " validate_exit=$?", + "fi", + "reload_exit=", + "mutation=false", + "if [ \"$dry_run\" = false ] && [ \"$validate_exit\" = 0 ]; then", + " sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/hwlab-public-caddy-install.out 2>/tmp/hwlab-public-caddy-install.err", + " install_exit=$?", + " if [ \"$install_exit\" = 0 ]; then", + " mutation=true", + " sudo systemctl reload \"$service\" >/tmp/hwlab-public-caddy-reload.out 2>/tmp/hwlab-public-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/hwlab-public-caddy-reload.out 2>>/tmp/hwlab-public-caddy-reload.err", + " reload_exit=$?", + " else", + " reload_exit=$install_exit", + " fi", + "fi", + "active=$(systemctl is-active \"$service\" 2>/dev/null || true)", + "after_present=no", + "if [ \"$dry_run\" = true ]; then grep -Fq \"# BEGIN $marker\" \"$next\" && after_present=yes; else grep -Fq \"# BEGIN $marker\" \"$config_path\" && after_present=yes; fi", + "local_web_status=", + "local_api_status=", + "if [ \"$dry_run\" = false ]; then", + " local_web_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-web.err || true)", + " local_api_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/health/live\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-api.err || true)", + "fi", + "validate_err_preview=$(cat /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-validate.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)", + "local_web_err_preview=$([ -f /tmp/hwlab-public-caddy-web.err ] && tr '\\n' ';' /dev/null | cut -c1-1000 || true)", + "local_api_err_preview=$([ -f /tmp/hwlab-public-caddy-api.err ] && tr '\\n' ';' /dev/null | cut -c1-1000 || true)", + "printf 'hostname\\t%s\\n' \"$hostname\"", + "printf 'expectedA\\t%s\\n' \"$expected_a\"", + "printf 'configPath\\t%s\\n' \"$config_path\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeBlockPresent\\t%s\\n' \"$before_present\"", + "printf 'afterBlockPresent\\t%s\\n' \"$after_present\"", + "printf 'staleBlocksRemoved\\t%s\\n' \"$stale_blocks_removed\"", + "printf 'pythonExitCode\\t%s\\n' \"$python_exit\"", + "printf 'validateMode\\t%s\\n' \"$validate_mode\"", + "printf 'validateExitCode\\t%s\\n' \"$validate_exit\"", + "printf 'reloadExitCode\\t%s\\n' \"$reload_exit\"", + "printf 'active\\t%s\\n' \"$active\"", + "printf 'localWebStatus\\t%s\\n' \"$local_web_status\"", + "printf 'localApiStatus\\t%s\\n' \"$local_api_status\"", + "printf 'validateErrorPreview\\t%s\\n' \"$validate_err_preview\"", + "printf 'localWebErrorPreview\\t%s\\n' \"$local_web_err_preview\"", + "printf 'localApiErrorPreview\\t%s\\n' \"$local_api_err_preview\"", + "rm -rf \"$tmp\"", + "[ \"$validate_exit\" = 0 ] && [ \"$after_present\" = yes ]", + ].join("\n"); +} + +function publicExposureCaddyBlock(exposure: HwlabRuntimePublicExposureSpec): string { + const tlsLines = exposure.caddyTls === "internal" ? " tls internal\n" : ""; + return `${exposure.hostname} { +${tlsLines} @api path /health* /auth* /v1* /json-rpc* /openapi* /docs* /swagger* + reverse_proxy @api 127.0.0.1:${exposure.apiProxy.remotePort} { + transport http { + response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s + } + } + reverse_proxy 127.0.0.1:${exposure.webProxy.remotePort} { + transport http { + response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s + } + } +}`; +} + +function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { + return runCommand([transPath(), `${node}:k3s`, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { + return runCommand([transPath(), node, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function runTransWorkspaceStdinScript(node: string, workspace: string, script: string, timeoutSeconds: number): CommandResult { + return runCommand([transPath(), `${node}:${workspace}`, "sh"], repoRoot, { input: script, timeoutMs: timeoutSeconds * 1000 }); +} + +function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { + const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds); + const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec); + const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true }; + const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds); + const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec); + const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false); + const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true; + return { + ok, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-delete", + status: { + ok, + preset: "obsolete-hwpod-db-cleanup", + dryRun: options.dryRun, + mutation, + kubernetesSecret: kubernetesStatus, + platformDatabase: platformDbStatus, + hostMutationSkipped: !options.dryRun && hostOptions.dryRun, + summary: ok + ? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove` + : `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`, + }, + mutation, + result: { + kubernetesSecret: compactCommandResult(kubernetesResult), + platformDatabase: compactCommandResult(platformDbResult), + }, + valuesRedacted: true, + next: ok ? undefined : nextSecretCommand(options, spec), + }; +} + +function runNodeEndpointBridge(options: ReturnType): Record { + if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm"); + const dryRun = options.dryRun || !options.confirm; + const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds); + const fields = keyValueLinesFromText(statusText(result)); + const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes"; + const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes"; + const afterExcluded = fields.afterEndpointResourcesExcluded === "yes"; + const afterIgnored = fields.afterEndpointsIgnoreUpdates === "yes" || fields.afterEndpointSliceIgnoreUpdates === "yes"; + const beforeExtraEndpointSlices = splitWhitespaceField(fields.beforeExtraEndpointSliceNames); + const afterExtraEndpointSlices = splitWhitespaceField(fields.afterExtraEndpointSliceNames); + const beforeLegacyEndpoints = fields.beforeLegacyEndpointsExists === "yes"; + const afterLegacyEndpoints = fields.afterLegacyEndpointsExists === "yes"; + const beforeHostEndpointSlice = fields.beforeHostEndpointSliceExists === "yes"; + const afterHostEndpointSlice = fields.afterHostEndpointSliceExists === "yes"; + const bridgeReady = !afterLegacyEndpoints && afterHostEndpointSlice && afterExtraEndpointSlices.length === 0; + const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored && bridgeReady; + return { + ok: dryRun ? result.exitCode === 0 : ok, + command: "hwlab nodes control-plane allow-endpoint-bridge", + node: options.node, + lane: options.lane, + namespace: "argocd", + application: fields.application || `hwlab-node-${options.lane}`, + mode: dryRun ? "dry-run" : "confirmed-control-plane-update", + status: { + action: fields.action || null, + dryRun, + mutation: fields.mutation === "true", + before: { + endpointResourcesExcluded: beforeExcluded, + endpointsIgnoreUpdates: fields.beforeEndpointsIgnoreUpdates === "yes", + endpointSliceIgnoreUpdates: fields.beforeEndpointSliceIgnoreUpdates === "yes", + legacyEndpointsExist: beforeLegacyEndpoints, + hostEndpointSliceExists: beforeHostEndpointSlice, + extraEndpointSlices: beforeExtraEndpointSlices, + }, + after: { + endpointResourcesExcluded: afterExcluded, + endpointsIgnoreUpdates: fields.afterEndpointsIgnoreUpdates === "yes", + endpointSliceIgnoreUpdates: fields.afterEndpointSliceIgnoreUpdates === "yes", + legacyEndpointsExist: afterLegacyEndpoints, + hostEndpointSliceExists: afterHostEndpointSlice, + extraEndpointSlices: afterExtraEndpointSlices, + }, + runtimeNamespace: fields.runtimeNamespace || `hwlab-${options.lane}`, + platformService: fields.platformService || "g14-platform-postgres", + hostEndpointSlice: fields.hostEndpointSlice || "g14-platform-postgres-host", + patchExitCode: numericField(fields.patchExitCode), + rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), + rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), + deleteLegacyEndpointsExitCode: numericField(fields.deleteLegacyEndpointsExitCode), + deleteExtraEndpointSlicesExitCode: numericField(fields.deleteExtraEndpointSlicesExitCode), + refreshExitCode: numericField(fields.refreshExitCode), + exitCode: result.exitCode, + stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), + summary: !afterExcluded && !afterIgnored && bridgeReady + ? "Argo tracks HWLAB external Postgres EndpointSlice and no legacy Endpoints remain" + : "Argo endpoint bridge is not in final Service plus EndpointSlice shape", + }, + result: compactCommandResult(result), + }; +} + +function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string { + const application = `hwlab-node-${options.lane}`; + const runtimeNamespace = `hwlab-${options.lane}`; + return [ + "set +e", + "namespace=argocd", + `runtime_namespace=${shellQuote(runtimeNamespace)}`, + "configmap=argocd-cm", + `application=${shellQuote(application)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + "platform_service=g14-platform-postgres", + "host_endpointslice=g14-platform-postgres-host", + "preset=endpoint-bridge-resource-tracking", + "cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", + "cm_has_key() { value=$(cm_data \"$1\"); [ -n \"$value\" ] && [ \"$value\" != \"\" ] && printf yes || printf no; }", + "endpoint_resources_excluded() { exclusions=$(cm_data resource.exclusions); printf '%s' \"$exclusions\" | grep -Eq '(^|[[:space:]])(Endpoints|EndpointSlice)([[:space:]]|$)' && printf yes || printf no; }", + "resource_exists() { kubectl -n \"$runtime_namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", + "extra_endpoint_slices() { kubectl -n \"$runtime_namespace\" get endpointslice -l \"kubernetes.io/service-name=$platform_service\" -o name 2>/dev/null | sed \"/\\/$host_endpointslice$/d\" | tr '\\n' ' ' | sed 's/[[:space:]]*$//'; }", + "wait_runtime_bridge_clean() {", + " for _ in $(seq 1 30); do", + " current_legacy=$(resource_exists endpoints \"$platform_service\")", + " current_extra=$(extra_endpoint_slices)", + " current_host=$(resource_exists endpointslice \"$host_endpointslice\")", + " if [ \"$current_legacy\" != yes ] && [ -z \"$current_extra\" ] && [ \"$current_host\" = yes ]; then return 0; fi", + " sleep 2", + " done", + " return 1", + "}", + "before_endpoint_resources_excluded=$(endpoint_resources_excluded)", + "before_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')", + "before_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')", + "before_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")", + "before_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")", + "before_extra_endpoint_slice_names=$(extra_endpoint_slices)", + "needs_argo_update=false", + "if [ \"$before_endpoint_resources_excluded\" = yes ] || [ \"$before_endpoints_ignore_updates\" = yes ] || [ \"$before_endpoint_slice_ignore_updates\" = yes ]; then needs_argo_update=true; fi", + "needs_runtime_cleanup=false", + "if [ \"$before_legacy_endpoints_exists\" = yes ] || [ -n \"$before_extra_endpoint_slice_names\" ]; then needs_runtime_cleanup=true; fi", + "action=observed", + "mutation=false", + "patch_exit=", + "rollout_restart_exit=", + "rollout_status_exit=", + "delete_legacy_endpoints_exit=", + "delete_extra_endpointslices_exit=", + "refresh_exit=", + "if [ \"$dry_run\" = true ]; then", + " if [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-old-endpoint-exclusions-and-legacy-endpoints", + " elif [ \"$needs_argo_update\" = true ]; then action=would-remove-old-endpoint-exclusions", + " elif [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-legacy-endpoints", + " else action=kept; fi", + "else", + " if [ \"$needs_argo_update\" = true ]; then", + " patch_file=$(mktemp /tmp/hwlab-argocd-endpoint-bridge.XXXXXX.json)", + " python3 - <<'PY' >\"$patch_file\"", + "import json", + "desired = '''### Internal Kubernetes resources excluded to reduce watch volume", + "- apiGroups:", + " - coordination.k8s.io", + " kinds:", + " - Lease", + "### Internal Kubernetes Authz/Authn resources excluded to reduce watched events", + "- apiGroups:", + " - authentication.k8s.io", + " - authorization.k8s.io", + " kinds:", + " - SelfSubjectReview", + " - TokenReview", + " - LocalSubjectAccessReview", + " - SelfSubjectAccessReview", + " - SelfSubjectRulesReview", + " - SubjectAccessReview", + "### Intermediate Certificate Request excluded to reduce watched events", + "- apiGroups:", + " - certificates.k8s.io", + " kinds:", + " - CertificateSigningRequest", + "- apiGroups:", + " - cert-manager.io", + " kinds:", + " - CertificateRequest", + "### Cilium internal resources excluded to reduce UI clutter", + "- apiGroups:", + " - cilium.io", + " kinds:", + " - CiliumIdentity", + " - CiliumEndpoint", + " - CiliumEndpointSlice", + "### Kyverno intermediate and reporting resources excluded to reduce watched events", + "- apiGroups:", + " - kyverno.io", + " - reports.kyverno.io", + " - wgpolicyk8s.io", + " kinds:", + " - PolicyReport", + " - ClusterPolicyReport", + " - EphemeralReport", + " - ClusterEphemeralReport", + " - AdmissionReport", + " - ClusterAdmissionReport", + " - BackgroundScanReport", + " - ClusterBackgroundScanReport", + " - UpdateRequest", + "'''", + "print(json.dumps({", + " 'data': {", + " 'resource.exclusions': desired,", + " 'resource.customizations.ignoreResourceUpdates.Endpoints': None,", + " 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice': None,", + " }", + "}))", + "PY", + " kubectl -n \"$namespace\" patch configmap \"$configmap\" --type merge --patch-file \"$patch_file\" >/tmp/hwlab-argocd-endpoint-bridge-patch.out 2>/tmp/hwlab-argocd-endpoint-bridge-patch.err", + " patch_exit=$?", + " rm -f \"$patch_file\"", + " if [ \"$patch_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout restart statefulset/argocd-application-controller >/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.err", + " rollout_restart_exit=$?", + " if [ \"$rollout_restart_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout status statefulset/argocd-application-controller --timeout=180s >/tmp/hwlab-argocd-endpoint-bridge-rollout-status.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-status.err", + " rollout_status_exit=$?", + " fi", + " fi", + " fi", + " if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then action=patch-failed", + " elif [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", + " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", + " else", + " if [ \"$needs_runtime_cleanup\" = true ]; then", + " kubectl -n \"$runtime_namespace\" delete endpoints \"$platform_service\" --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpoints-delete.out 2>/tmp/hwlab-platform-postgres-endpoints-delete.err", + " delete_legacy_endpoints_exit=$?", + " wait_runtime_bridge_clean", + " remaining_extra=$(extra_endpoint_slices)", + " if [ -n \"$remaining_extra\" ]; then", + " kubectl -n \"$runtime_namespace\" delete $remaining_extra --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpointslices-delete.out 2>/tmp/hwlab-platform-postgres-endpointslices-delete.err", + " delete_extra_endpointslices_exit=$?", + " wait_runtime_bridge_clean", + " fi", + " fi", + " if [ \"$needs_argo_update\" = true ] || [ \"$needs_runtime_cleanup\" = true ]; then", + " kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/hwlab-argocd-endpoint-bridge-refresh.out 2>/tmp/hwlab-argocd-endpoint-bridge-refresh.err", + " refresh_exit=$?", + " fi", + " if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then action=delete-legacy-endpoints-failed", + " elif [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then action=delete-extra-endpointslices-failed", + " elif [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then action=refresh-failed", + " elif [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=removed-old-endpoint-exclusions-and-legacy-endpoints; mutation=true", + " elif [ \"$needs_argo_update\" = true ]; then action=removed-old-endpoint-exclusions; mutation=true", + " elif [ \"$needs_runtime_cleanup\" = true ]; then action=removed-legacy-endpoints; mutation=true", + " else action=kept; fi", + " fi", + "fi", + "after_endpoint_resources_excluded=$(endpoint_resources_excluded)", + "after_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')", + "after_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')", + "after_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")", + "after_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")", + "after_extra_endpoint_slice_names=$(extra_endpoint_slices)", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"", + "printf 'configMap\\t%s\\n' \"$configmap\"", + "printf 'application\\t%s\\n' \"$application\"", + "printf 'platformService\\t%s\\n' \"$platform_service\"", + "printf 'hostEndpointSlice\\t%s\\n' \"$host_endpointslice\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeEndpointResourcesExcluded\\t%s\\n' \"$before_endpoint_resources_excluded\"", + "printf 'beforeEndpointsIgnoreUpdates\\t%s\\n' \"$before_endpoints_ignore_updates\"", + "printf 'beforeEndpointSliceIgnoreUpdates\\t%s\\n' \"$before_endpoint_slice_ignore_updates\"", + "printf 'beforeLegacyEndpointsExists\\t%s\\n' \"$before_legacy_endpoints_exists\"", + "printf 'beforeHostEndpointSliceExists\\t%s\\n' \"$before_host_endpointslice_exists\"", + "printf 'beforeExtraEndpointSliceNames\\t%s\\n' \"$before_extra_endpoint_slice_names\"", + "printf 'afterEndpointResourcesExcluded\\t%s\\n' \"$after_endpoint_resources_excluded\"", + "printf 'afterEndpointsIgnoreUpdates\\t%s\\n' \"$after_endpoints_ignore_updates\"", + "printf 'afterEndpointSliceIgnoreUpdates\\t%s\\n' \"$after_endpoint_slice_ignore_updates\"", + "printf 'afterLegacyEndpointsExists\\t%s\\n' \"$after_legacy_endpoints_exists\"", + "printf 'afterHostEndpointSliceExists\\t%s\\n' \"$after_host_endpointslice_exists\"", + "printf 'afterExtraEndpointSliceNames\\t%s\\n' \"$after_extra_endpoint_slice_names\"", + "printf 'patchExitCode\\t%s\\n' \"$patch_exit\"", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", + "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", + "printf 'deleteLegacyEndpointsExitCode\\t%s\\n' \"$delete_legacy_endpoints_exit\"", + "printf 'deleteExtraEndpointSlicesExitCode\\t%s\\n' \"$delete_extra_endpointslices_exit\"", + "printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"", + "if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi", + "if [ \"$dry_run\" != true ] && { [ \"$after_legacy_endpoints_exists\" = yes ] || [ -n \"$after_extra_endpoint_slice_names\" ] || [ \"$after_host_endpointslice_exists\" != yes ]; }; then exit 47; fi", + "if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi", + "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", + "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", + "if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then exit \"$delete_legacy_endpoints_exit\"; fi", + "if [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then exit \"$delete_extra_endpointslices_exit\"; fi", + "if [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then exit \"$refresh_exit\"; fi", + ].join("\n"); +} + +function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + const pvc = `data-${spec.postgresSecret}-0`; + const platformService = spec.platformPostgresService; + const postgresService = spec.postgresSecret; + const postgresConfigMap = `${spec.postgresSecret}-init`; + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `postgres_secret=${shellQuote(spec.postgresSecret)}`, + `postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`, + `postgres_service=${shellQuote(postgresService)}`, + `postgres_configmap=${shellQuote(postgresConfigMap)}`, + `pvc=${shellQuote(pvc)}`, + `platform_service=${shellQuote(platformService)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + "preset=owned-postgres-cleanup", + "exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }", + "pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }", + "phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }", + "before_secret_exists=$(exists_flag secret \"$postgres_secret\")", + "before_pvc_exists=$(exists_flag pvc \"$pvc\")", + "before_pvc_phase=$(phase_of_pvc)", + "before_pv=$(pv_name)", + "before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")", + "before_service_exists=$(exists_flag service \"$postgres_service\")", + "before_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")", + "platform_service_exists=$(exists_flag service \"$platform_service\")", + "action=observed", + "mutation=false", + "delete_statefulset_exit=", + "delete_service_exit=", + "delete_configmap_exit=", + "delete_secret_exit=", + "delete_pvc_exit=", + "before_any_owned=false", + "for flag in \"$before_statefulset_exists\" \"$before_service_exists\" \"$before_configmap_exists\" \"$before_secret_exists\" \"$before_pvc_exists\"; do", + " if [ \"$flag\" = yes ]; then before_any_owned=true; fi", + "done", + "if [ \"$dry_run\" = true ]; then", + " if [ \"$before_any_owned\" = true ]; then action=would-delete; else action=already-absent; fi", + "else", + " kubectl -n \"$namespace\" delete statefulset \"$postgres_statefulset\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-statefulset-delete.out 2>/tmp/hwlab-owned-postgres-statefulset-delete.err", + " delete_statefulset_exit=$?", + " kubectl -n \"$namespace\" delete service \"$postgres_service\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-service-delete.out 2>/tmp/hwlab-owned-postgres-service-delete.err", + " delete_service_exit=$?", + " kubectl -n \"$namespace\" delete configmap \"$postgres_configmap\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-configmap-delete.out 2>/tmp/hwlab-owned-postgres-configmap-delete.err", + " delete_configmap_exit=$?", + " kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err", + " delete_secret_exit=$?", + " kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err", + " delete_pvc_exit=$?", + " for _ in $(seq 1 30); do", + " current_statefulset=$(exists_flag statefulset \"$postgres_statefulset\")", + " current_service=$(exists_flag service \"$postgres_service\")", + " current_configmap=$(exists_flag configmap \"$postgres_configmap\")", + " current_secret=$(exists_flag secret \"$postgres_secret\")", + " current_pvc=$(exists_flag pvc \"$pvc\")", + " if [ \"$current_statefulset\" != yes ] && [ \"$current_service\" != yes ] && [ \"$current_configmap\" != yes ] && [ \"$current_secret\" != yes ] && [ \"$current_pvc\" != yes ]; then break; fi", + " sleep 2", + " done", + " if [ \"$delete_statefulset_exit\" -eq 0 ] && [ \"$delete_service_exit\" -eq 0 ] && [ \"$delete_configmap_exit\" -eq 0 ] && [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then", + " if [ \"$before_any_owned\" = true ]; then action=deleted; mutation=true; else action=already-absent; fi", + " else", + " action=delete-failed", + " fi", + "fi", + "after_secret_exists=$(exists_flag secret \"$postgres_secret\")", + "after_pvc_exists=$(exists_flag pvc \"$pvc\")", + "after_pvc_phase=$(phase_of_pvc)", + "after_pv=$(pv_name)", + "after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")", + "after_service_exists=$(exists_flag service \"$postgres_service\")", + "after_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$postgres_secret\"", + "printf 'statefulSet\\t%s\\n' \"$postgres_statefulset\"", + "printf 'service\\t%s\\n' \"$postgres_service\"", + "printf 'configMap\\t%s\\n' \"$postgres_configmap\"", + "printf 'pvc\\t%s\\n' \"$pvc\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", + "printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"", + "printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"", + "printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"", + "printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"", + "printf 'beforeServiceExists\\t%s\\n' \"$before_service_exists\"", + "printf 'beforeConfigMapExists\\t%s\\n' \"$before_configmap_exists\"", + "printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"", + "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", + "printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"", + "printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"", + "printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"", + "printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"", + "printf 'afterServiceExists\\t%s\\n' \"$after_service_exists\"", + "printf 'afterConfigMapExists\\t%s\\n' \"$after_configmap_exists\"", + "printf 'deleteStatefulSetExitCode\\t%s\\n' \"$delete_statefulset_exit\"", + "printf 'deleteServiceExitCode\\t%s\\n' \"$delete_service_exit\"", + "printf 'deleteConfigMapExitCode\\t%s\\n' \"$delete_configmap_exit\"", + "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", + "printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"", + "if [ \"$platform_service_exists\" != yes ]; then exit 44; fi", + "if [ \"$after_statefulset_exists\" = yes ] || [ \"$after_service_exists\" = yes ] || [ \"$after_configmap_exists\" = yes ] || [ \"$after_secret_exists\" = yes ] || [ \"$after_pvc_exists\" = yes ]; then exit 45; fi", + "if [ -n \"$delete_statefulset_exit\" ] && [ \"$delete_statefulset_exit\" != 0 ]; then exit \"$delete_statefulset_exit\"; fi", + "if [ -n \"$delete_service_exit\" ] && [ \"$delete_service_exit\" != 0 ]; then exit \"$delete_service_exit\"; fi", + "if [ -n \"$delete_configmap_exit\" ] && [ \"$delete_configmap_exit\" != 0 ]; then exit \"$delete_configmap_exit\"; fi", + "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", + "if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi", + ].join("\n"); +} + +function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `secret=${shellQuote(options.name)}`, + `expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + "preset=obsolete-secret-cleanup", + "exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }", + "workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)", + "workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)", + "workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)", + "before_exists=$(exists_flag)", + "action=observed", + "mutation=false", + "delete_secret_exit=", + "if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi", + "if [ \"$action\" = observed ]; then", + " if [ \"$workload_refs_present\" = yes ]; then", + " action=blocked-referenced", + " elif [ \"$dry_run\" = true ]; then", + " if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi", + " else", + " kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err", + " delete_secret_exit=$?", + " for _ in $(seq 1 15); do", + " current_exists=$(exists_flag)", + " if [ \"$current_exists\" != yes ]; then break; fi", + " sleep 1", + " done", + " if [ \"$delete_secret_exit\" -eq 0 ]; then", + " if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi", + " else", + " action=delete-failed", + " fi", + " fi", + "fi", + "after_exists=$(exists_flag)", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$secret\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"", + "printf 'afterSecretExists\\t%s\\n' \"$after_exists\"", + "printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"", + "printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"", + "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", + "if [ \"$action\" = unsupported-secret ]; then exit 43; fi", + "if [ \"$workload_refs_present\" = yes ]; then exit 46; fi", + "if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi", + "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", + ].join("\n"); +} + +function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `db_name=${shellQuote(spec.obsoleteHwpodDbName)}`, + `db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + "preset=obsolete-platform-db-cleanup", + "database_exists_flag() {", + " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)", + " code=$?", + " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", + " [ \"$output\" = t ] && printf yes || printf no", + "}", + "role_exists_flag() {", + " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)", + " code=$?", + " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", + " [ \"$output\" = t ] && printf yes || printf no", + "}", + "before_database_exists=$(database_exists_flag)", + "before_database_probe_exit=$?", + "before_role_exists=$(role_exists_flag)", + "before_role_probe_exit=$?", + "action=observed", + "mutation=false", + "drop_database_exit=", + "drop_role_exit=", + "before_any=false", + "if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi", + "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then", + " action=probe-failed", + "elif [ \"$dry_run\" = true ]; then", + " if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi", + "else", + " if [ \"$before_database_exists\" = yes ]; then", + " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'", + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();", + "DROP DATABASE IF EXISTS :\"db_name\";", + "SQL", + " drop_database_exit=$?", + " else", + " drop_database_exit=0", + " fi", + " if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then", + " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'", + "DROP ROLE IF EXISTS :\"db_user\";", + "SQL", + " drop_role_exit=$?", + " elif [ \"$drop_database_exit\" -eq 0 ]; then", + " drop_role_exit=0", + " else", + " drop_role_exit=", + " fi", + " if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then", + " if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi", + " else", + " action=drop-failed", + " fi", + "fi", + "after_database_exists=$(database_exists_flag)", + "after_database_probe_exit=$?", + "after_role_exists=$(role_exists_flag)", + "after_role_probe_exit=$?", + "printf 'database\\t%s\\n' \"$db_name\"", + "printf 'role\\t%s\\n' \"$db_user\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"", + "printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"", + "printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"", + "printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"", + "printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"", + "printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"", + "printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"", + "printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"", + "printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"", + "printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"", + "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi", + "if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi", + "if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi", + "if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi", + ].join("\n"); +} + +function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + const isOpenFga = options.preset === "openfga"; + const platformEndpointSlice = spec.platformPostgresEndpointSlice; + const expectedUriHost = spec.platformPostgresEndpointAddress ?? (isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost); + const databaseUrlKey = isOpenFga ? spec.externalPostgres?.openfga.secretKey ?? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey; + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`, + `database_url_key=${shellQuote(databaseUrlKey)}`, + `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, + `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, + `legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`, + `platform_service=${shellQuote(spec.platformPostgresService)}`, + `platform_endpointslice=${shellQuote(platformEndpointSlice)}`, + `platform_host=${shellQuote(spec.platformPostgresService)}`, + `platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`, + `platform_endpoint_address=${shellQuote(spec.platformPostgresEndpointAddress ?? "")}`, + `db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`, + `db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`, + `db_host=${shellQuote(expectedUriHost)}`, + `selected_key=${shellQuote(options.key ?? "")}`, + `preset=${shellQuote(options.preset)}`, + "dry_run=true", + "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", + "resource_exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", + "endpointslice_exists_flag() { kubectl -n \"$namespace\" get endpointslice \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", + "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "uri_has_platform_host=no", + "uri_has_db_name=no", + "uri_has_db_user=no", + "uri_matches_expected() {", + " uri=$1", + " uri_has_platform_host=no", + " uri_has_db_name=no", + " uri_has_db_user=no", + " case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*|*\"@$db_host:\"*|*\"@$db_host/\"*) uri_has_platform_host=yes ;; esac", + " if [ -n \"$platform_endpoint_address\" ]; then", + " case \"$uri\" in *\"@$platform_endpoint_address:\"*|*\"@$platform_endpoint_address/\"*) uri_has_platform_host=yes ;; esac", + " fi", + " case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac", + " case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac", + "}", + "exists=$(secret_exists_flag \"$name\")", + "legacy_postgres_exists=$(secret_exists_flag \"$legacy_postgres_secret\")", + "uri_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", + "uri_present=$([ -n \"$uri_b64\" ] && printf yes || printf no)", + "uri_bytes=$(decoded_length \"$uri_b64\")", + "uri_value=$(decoded_value \"$uri_b64\")", + "authn_b64=$(secret_b64_key \"$name\" \"$authn_key\")", + "authn_present=$([ -n \"$authn_b64\" ] && printf yes || printf no)", + "authn_bytes=$(decoded_length \"$authn_b64\")", + "pg_password_b64=$(secret_b64_key \"$name\" \"$postgres_password_key\")", + "pg_password_present=$([ -n \"$pg_password_b64\" ] && printf yes || printf no)", + "pg_password_bytes=$(decoded_length \"$pg_password_b64\")", + "platform_service_exists=$(resource_exists_flag service \"$platform_service\")", + "platform_endpoints_exists=$(resource_exists_flag endpoints \"$platform_service\")", + "platform_endpointslice_exists=$(endpointslice_exists_flag \"$platform_endpointslice\")", + "uri_matches_expected \"$uri_value\"", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$name\"", + "printf 'key\\t%s\\n' \"$database_url_key\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\tobserved\\n'", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\tfalse\\n'", + "printf 'platformDbMode\\ttrue\\n'", + "printf 'afterExists\\t%s\\n' \"$exists\"", + "printf 'afterDatabaseUrlPresent\\t%s\\n' \"$uri_present\"", + "printf 'afterDatabaseUrlBytes\\t%s\\n' \"$uri_bytes\"", + "printf 'afterDatastoreUriPresent\\t%s\\n' \"$uri_present\"", + "printf 'afterDatastoreUriBytes\\t%s\\n' \"$uri_bytes\"", + "printf 'afterAuthnPresent\\t%s\\n' \"$authn_present\"", + "printf 'afterAuthnBytes\\t%s\\n' \"$authn_bytes\"", + "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$pg_password_present\"", + "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$pg_password_bytes\"", + "printf 'legacyPostgresSecret\\t%s\\n' \"$legacy_postgres_secret\"", + "printf 'legacyPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"", + "printf 'afterPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"", + "printf 'platformService\\t%s\\n' \"$platform_service\"", + "printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"", + "printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"", + "printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"", + "printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"", + "printf 'platformEndpointAddress\\t%s\\n' \"$platform_endpoint_address\"", + "printf 'dbName\\t%s\\n' \"$db_name\"", + "printf 'dbUser\\t%s\\n' \"$db_user\"", + "printf 'dbHost\\t%s\\n' \"$db_host\"", + "printf 'dbHostMatchesPlatform\\t%s\\n' \"$uri_has_platform_host\"", + "printf 'dbNameMatchesExpected\\t%s\\n' \"$uri_has_db_name\"", + "printf 'dbUserMatchesExpected\\t%s\\n' \"$uri_has_db_user\"", + "uri_value=", + "if [ \"$platform_service_exists\" != yes ]; then exit 44; fi", + ].join("\n"); +} + +function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `openfga_secret=${shellQuote(spec.openFgaSecret)}`, + `postgres_secret=${shellQuote(spec.postgresSecret)}`, + `postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`, + `postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`, + `selected_key=${shellQuote(options.key ?? "")}`, + `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, + `datastore_uri_key=${shellQuote(OPENFGA_DATASTORE_URI_KEY)}`, + `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, + `db_name=${shellQuote(spec.openFgaDbName)}`, + `db_user=${shellQuote(spec.openFgaDbUser)}`, + `db_host=${shellQuote(spec.openFgaDbHost)}`, + `action_request=${shellQuote(options.action)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `field_manager=${shellQuote(spec.fieldManager)}`, + "preset=openfga", + "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", + "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", + "probe_db() {", + " role_result=unknown", + " database_result=unknown", + " probe_exit=missing-postgres-admin-secret", + " if [ -n \"$postgres_admin_password\" ]; then", + " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", + " role_exit=$?", + " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", + " database_exit=$?", + " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", + " fi", + "}", + "before_exists=$(secret_exists_flag \"$openfga_secret\")", + "before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", + "before_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", + "before_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", + "before_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", + "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", + "before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)", + "before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)", + "before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)", + "before_authn_bytes=$(decoded_length \"$before_authn_b64\")", + "before_uri_bytes=$(decoded_length \"$before_uri_b64\")", + "before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")", + "authn_value=$(decoded_value \"$before_authn_b64\")", + "datastore_uri=$(decoded_value \"$before_uri_b64\")", + "pg_password=$(decoded_value \"$before_pg_password_b64\")", + "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", + "probe_db", + "db_role_exists_before=$role_result", + "db_database_exists_before=$database_result", + "db_probe_exit_before=$probe_exit", + "action=observed", + "mutation=false", + "postgres_secret_exit=", + "postgres_rollout_exit=", + "apply_exit=", + "db_ensure_exit=", + "if [ \"$action_request\" = ensure ]; then", + " missing_secret=false", + " [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true", + " [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true", + " [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true", + " missing_db=false", + " [ \"$db_role_exists_before\" = t ] || missing_db=true", + " [ \"$db_database_exists_before\" = t ] || missing_db=true", + " if [ \"$dry_run\" = true ]; then", + " if [ \"$before_postgres_exists\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", + " elif ! command -v openssl >/dev/null 2>&1; then", + " action=openssl-missing; apply_exit=127", + " else", + " [ -n \"$postgres_admin_password\" ] || postgres_admin_password=$(openssl rand -hex 24)", + " kubectl -n \"$namespace\" create secret generic \"$postgres_secret\" --from-literal=\"POSTGRES_PASSWORD=$postgres_admin_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", + " postgres_secret_exit=$?", + " if [ \"$postgres_secret_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout status \"statefulset/$postgres_statefulset\" --timeout=120s >/tmp/hwlab-postgres-rollout.out 2>/tmp/hwlab-postgres-rollout.err", + " postgres_rollout_exit=$?", + " fi", + " if [ \"$postgres_secret_exit\" -ne 0 ]; then action=postgres-secret-failed; apply_exit=$postgres_secret_exit", + " elif [ \"$postgres_rollout_exit\" -ne 0 ]; then action=postgres-rollout-failed; apply_exit=$postgres_rollout_exit", + " else", + " [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)", + " [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)", + " [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"", + " kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", + " apply_exit=$?", + " if [ \"$apply_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'", + "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", + "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", + "\\gexec", + "ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';", + "SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')", + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')", + "\\gexec", + "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", + "SQL", + " db_ensure_exit=$?", + " if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi", + " else action=apply-failed; fi", + " fi", + " fi", + "fi", + "after_exists=$(secret_exists_flag \"$openfga_secret\")", + "after_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", + "after_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")", + "after_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")", + "after_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")", + "after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)", + "after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)", + "after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)", + "after_authn_bytes=$(decoded_length \"$after_authn_b64\")", + "after_uri_bytes=$(decoded_length \"$after_uri_b64\")", + "after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")", + "probe_db", + "db_role_exists_after=$role_result", + "db_database_exists_after=$database_result", + "db_probe_exit_after=$probe_exit", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$openfga_secret\"", + "printf 'key\\t%s\\n' \"$selected_key\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeExists\\t%s\\n' \"$before_exists\"", + "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", + "printf 'afterExists\\t%s\\n' \"$after_exists\"", + "printf 'afterPostgresSecretExists\\t%s\\n' \"$after_postgres_exists\"", + "printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"", + "printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"", + "printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"", + "printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"", + "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"", + "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"", + "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", + "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", + "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", + "printf 'postgresSecretExitCode\\t%s\\n' \"$postgres_secret_exit\"", + "printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", + "authn_value= datastore_uri= pg_password= postgres_admin_password=", + "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", + "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", + ].join("\n"); +} + +function masterAdminApiKeySecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `name=${shellQuote(spec.masterAdminApiKeySecret)}`, + `api_key_name=${shellQuote(MASTER_ADMIN_API_KEY_KEY)}`, + `action_request=${shellQuote(options.action)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `field_manager=${shellQuote(spec.fieldManager)}`, + `cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`, + "preset=master-server-admin-api-key", + "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-12; value=; fi; }", + "before_exists=$(secret_exists_flag)", + "before_api_key_b64=$(secret_b64_key \"$api_key_name\")", + "before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)", + "before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")", + "action=observed", + "mutation=false", + "apply_exit=", + "rollout_restart_exit=", + "rollout_status_exit=", + "if [ \"$action_request\" = ensure ]; then", + " missing_secret=false", + " [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true", + " if [ \"$dry_run\" = true ]; then", + " if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi", + " else", + " api_key=$(cat)", + " case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac", + " if [ -z \"$apply_exit\" ]; then", + " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", + " apply_exit=$?", + " if [ \"$apply_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-master-admin-api-key-rollout-restart.out 2>/tmp/hwlab-master-admin-api-key-rollout-restart.err", + " rollout_restart_exit=$?", + " if [ \"$rollout_restart_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-master-admin-api-key-rollout-status.out 2>/tmp/hwlab-master-admin-api-key-rollout-status.err", + " rollout_status_exit=$?", + " fi", + " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", + " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", + " else action=ensured; mutation=true; fi", + " else action=apply-failed; fi", + " fi", + " api_key=", + " fi", + "fi", + "after_exists=$(secret_exists_flag)", + "after_api_key_b64=$(secret_b64_key \"$api_key_name\")", + "after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)", + "after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")", + "after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$name\"", + "printf 'key\\t%s\\n' \"$api_key_name\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'afterExists\\t%s\\n' \"$after_exists\"", + "printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"", + "printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"", + "printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"", + "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", + "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", + "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", + "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", + "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", + ].join("\n"); +} + +function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec, material: BootstrapAdminSecretMaterial | null): string { + const yamlSourceEnabled = spec.bootstrapAdminPasswordSourceRef !== undefined && spec.bootstrapAdminPasswordSourceKey !== undefined && spec.bootstrapAdminPasswordHashTransform !== undefined; + if (!yamlSourceEnabled) return legacyBootstrapAdminSecretScript(options, spec); + const materialOk = material?.ok === true; + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `name=${shellQuote(spec.bootstrapAdminSecret)}`, + `password_hash_key=${shellQuote(spec.bootstrapAdminPasswordHashKey)}`, + `username=${shellQuote(spec.bootstrapAdminUsername)}`, + `display_name=${shellQuote(spec.bootstrapAdminDisplayName)}`, + `source_ref=${shellQuote(spec.bootstrapAdminPasswordSourceRef ?? "")}`, + `source_key=${shellQuote(spec.bootstrapAdminPasswordSourceKey ?? "")}`, + `source_path=${shellQuote(material?.sourcePath === null || material?.sourcePath === undefined ? "" : displayRepoPath(material.sourcePath))}`, + `source_present=${shellQuote(material?.sourcePresent === true ? "yes" : "no")}`, + `source_fingerprint=${shellQuote(material?.sourceFingerprint ?? "")}`, + `source_error=${shellQuote(material?.error ?? "")}`, + `transform=${shellQuote(spec.bootstrapAdminPasswordHashTransform ?? "")}`, + `material_ok=${shellQuote(materialOk ? "true" : "false")}`, + `force_sync=${shellQuote(options.force === true ? "true" : "false")}`, + `cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`, + `action_request=${shellQuote(options.action)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `field_manager=${shellQuote(spec.fieldManager)}`, + "preset=bootstrap-admin", + "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", + "secret_annotation() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ with .metadata.annotations }}{{ index . \\\"$1\\\" }}{{ end }}\" 2>/dev/null || true; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "before_exists=$(secret_exists_flag)", + "before_hash_b64=$(secret_b64_key \"$password_hash_key\")", + "before_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-ref)", + "before_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-key)", + "before_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-fingerprint)", + "before_username=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username)", + "before_hash_present=$([ -n \"$before_hash_b64\" ] && printf yes || printf no)", + "before_hash_bytes=$(decoded_length \"$before_hash_b64\")", + "action=observed", + "mutation=false", + "apply_exit=", + "rollout_restart_exit=", + "rollout_status_exit=", + "if [ \"$action_request\" = ensure ]; then", + " needs_sync=false", + " [ \"$before_exists\" = yes ] && [ \"$before_hash_bytes\" -gt 0 ] || needs_sync=true", + " [ \"$before_source_ref\" = \"$source_ref\" ] && [ \"$before_source_key\" = \"$source_key\" ] && [ \"$before_source_fingerprint\" = \"$source_fingerprint\" ] && [ \"$before_username\" = \"$username\" ] || needs_sync=true", + " [ \"$force_sync\" = true ] && needs_sync=true", + " if [ \"$material_ok\" != true ]; then", + " action=${source_error:-secret-source-invalid}", + " apply_exit=44", + " elif [ \"$dry_run\" = true ]; then", + " if [ \"$needs_sync\" = true ]; then action=would-sync-from-yaml-source; else action=kept; fi", + " elif [ \"$needs_sync\" = false ]; then", + " action=kept", + " else", + " password_hash=$(cat)", + " case \"$password_hash\" in sha256:*:*) ;; *) action=password-hash-invalid; apply_exit=45 ;; esac", + " if [ -z \"$apply_exit\" ]; then", + " cat </tmp/hwlab-bootstrap-admin-rollout-restart.out 2>/tmp/hwlab-bootstrap-admin-rollout-restart.err", + " rollout_restart_exit=$?", + " if [ \"$rollout_restart_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-bootstrap-admin-rollout-status.out 2>/tmp/hwlab-bootstrap-admin-rollout-status.err", + " rollout_status_exit=$?", + " fi", + " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", + " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", + " else action=synced-from-yaml-source; mutation=true; fi", + " else action=apply-failed; fi", + " password_hash=", + " fi", + "fi", + "after_exists=$(secret_exists_flag)", + "after_hash_b64=$(secret_b64_key \"$password_hash_key\")", + "after_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-ref)", + "after_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-key)", + "after_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-fingerprint)", + "after_username=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username)", + "after_hash_present=$([ -n \"$after_hash_b64\" ] && printf yes || printf no)", + "after_hash_bytes=$(decoded_length \"$after_hash_b64\")", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$name\"", + "printf 'key\\t%s\\n' \"$password_hash_key\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'username\\t%s\\n' \"$username\"", + "printf 'displayName\\t%s\\n' \"$display_name\"", + "printf 'sourceRef\\t%s\\n' \"$source_ref\"", + "printf 'sourceKey\\t%s\\n' \"$source_key\"", + "printf 'sourcePath\\t%s\\n' \"$source_path\"", + "printf 'sourceExists\\t%s\\n' \"$source_present\"", + "printf 'sourceFingerprint\\t%s\\n' \"$source_fingerprint\"", + "printf 'passwordHashTransform\\t%s\\n' \"$transform\"", + "printf 'forceSync\\t%s\\n' \"$force_sync\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeExists\\t%s\\n' \"$before_exists\"", + "printf 'beforePasswordHashPresent\\t%s\\n' \"$before_hash_present\"", + "printf 'beforePasswordHashBytes\\t%s\\n' \"$before_hash_bytes\"", + "printf 'beforeSourceRef\\t%s\\n' \"$before_source_ref\"", + "printf 'beforeSourceKey\\t%s\\n' \"$before_source_key\"", + "printf 'beforeSourceFingerprint\\t%s\\n' \"$before_source_fingerprint\"", + "printf 'beforeUsername\\t%s\\n' \"$before_username\"", + "printf 'afterExists\\t%s\\n' \"$after_exists\"", + "printf 'afterPasswordHashPresent\\t%s\\n' \"$after_hash_present\"", + "printf 'afterPasswordHashBytes\\t%s\\n' \"$after_hash_bytes\"", + "printf 'afterSourceRef\\t%s\\n' \"$after_source_ref\"", + "printf 'afterSourceKey\\t%s\\n' \"$after_source_key\"", + "printf 'afterSourceFingerprint\\t%s\\n' \"$after_source_fingerprint\"", + "printf 'afterUsername\\t%s\\n' \"$after_username\"", + "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", + "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", + "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", + "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", + "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", + ].join("\n"); +} + +function legacyBootstrapAdminSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `name=${shellQuote(spec.bootstrapAdminSecret)}`, + `source_namespace=${shellQuote(spec.bootstrapAdminSourceNamespace)}`, + `source_name=${shellQuote(spec.bootstrapAdminSourceSecret)}`, + `password_hash_key=${shellQuote(spec.bootstrapAdminPasswordHashKey)}`, + `cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`, + `action_request=${shellQuote(options.action)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `field_manager=${shellQuote(spec.fieldManager)}`, + "preset=bootstrap-admin", + "secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "before_exists=$(secret_exists_flag \"$namespace\" \"$name\")", + "before_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")", + "source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")", + "source_hash_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$password_hash_key\")", + "before_hash_present=$([ -n \"$before_hash_b64\" ] && printf yes || printf no)", + "source_hash_present=$([ -n \"$source_hash_b64\" ] && printf yes || printf no)", + "before_hash_bytes=$(decoded_length \"$before_hash_b64\")", + "source_hash_bytes=$(decoded_length \"$source_hash_b64\")", + "action=observed", + "mutation=false", + "apply_exit=", + "rollout_restart_exit=", + "rollout_status_exit=", + "if [ \"$action_request\" = ensure ]; then", + " missing_target=false", + " [ \"$before_exists\" = yes ] && [ \"$before_hash_bytes\" -gt 0 ] || missing_target=true", + " missing_source=false", + " [ \"$source_exists\" = yes ] && [ \"$source_hash_bytes\" -gt 0 ] || missing_source=true", + " if [ \"$missing_source\" = true ]; then", + " action=source-missing-password-hash", + " apply_exit=44", + " elif [ \"$dry_run\" = true ]; then", + " if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi", + " elif [ \"$missing_target\" = false ]; then", + " action=kept", + " else", + " cat </tmp/hwlab-bootstrap-admin-rollout-restart.out 2>/tmp/hwlab-bootstrap-admin-rollout-restart.err", + " rollout_restart_exit=$?", + " if [ \"$rollout_restart_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-bootstrap-admin-rollout-status.out 2>/tmp/hwlab-bootstrap-admin-rollout-status.err", + " rollout_status_exit=$?", + " fi", + " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", + " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", + " else action=copied-from-source; mutation=true; fi", + " else action=apply-failed; fi", + " fi", + "fi", + "after_exists=$(secret_exists_flag \"$namespace\" \"$name\")", + "after_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")", + "after_hash_present=$([ -n \"$after_hash_b64\" ] && printf yes || printf no)", + "after_hash_bytes=$(decoded_length \"$after_hash_b64\")", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$name\"", + "printf 'key\\t%s\\n' \"$password_hash_key\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'sourceNamespace\\t%s\\n' \"$source_namespace\"", + "printf 'sourceSecret\\t%s\\n' \"$source_name\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeExists\\t%s\\n' \"$before_exists\"", + "printf 'beforePasswordHashPresent\\t%s\\n' \"$before_hash_present\"", + "printf 'beforePasswordHashBytes\\t%s\\n' \"$before_hash_bytes\"", + "printf 'sourceExists\\t%s\\n' \"$source_exists\"", + "printf 'sourcePasswordHashPresent\\t%s\\n' \"$source_hash_present\"", + "printf 'sourcePasswordHashBytes\\t%s\\n' \"$source_hash_bytes\"", + "printf 'afterExists\\t%s\\n' \"$after_exists\"", + "printf 'afterPasswordHashPresent\\t%s\\n' \"$after_hash_present\"", + "printf 'afterPasswordHashBytes\\t%s\\n' \"$after_hash_bytes\"", + "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", + "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", + "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", + "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", + "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", + ].join("\n"); +} + +function codeAgentProviderSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `name=${shellQuote(spec.codeAgentProviderSecret)}`, + `source_namespace=${shellQuote(spec.codeAgentProviderSourceNamespace)}`, + `source_name=${shellQuote(spec.codeAgentProviderSourceSecret)}`, + `openai_key=${shellQuote(CODE_AGENT_PROVIDER_OPENAI_KEY)}`, + `opencode_key=${shellQuote(CODE_AGENT_PROVIDER_OPENCODE_KEY)}`, + `selected_key=${shellQuote(options.key ?? "")}`, + `action_request=${shellQuote(options.action)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + `field_manager=${shellQuote(spec.fieldManager)}`, + "preset=code-agent-provider", + "secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "before_exists=$(secret_exists_flag \"$namespace\" \"$name\")", + "before_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")", + "before_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")", + "source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")", + "source_openai_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$openai_key\")", + "source_opencode_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$opencode_key\")", + "before_openai_present=$([ -n \"$before_openai_b64\" ] && printf yes || printf no)", + "before_opencode_present=$([ -n \"$before_opencode_b64\" ] && printf yes || printf no)", + "source_openai_present=$([ -n \"$source_openai_b64\" ] && printf yes || printf no)", + "source_opencode_present=$([ -n \"$source_opencode_b64\" ] && printf yes || printf no)", + "before_openai_bytes=$(decoded_length \"$before_openai_b64\")", + "before_opencode_bytes=$(decoded_length \"$before_opencode_b64\")", + "source_openai_bytes=$(decoded_length \"$source_openai_b64\")", + "source_opencode_bytes=$(decoded_length \"$source_opencode_b64\")", + "action=observed", + "mutation=false", + "apply_exit=", + "if [ \"$action_request\" = ensure ]; then", + " missing_target=false", + " [ \"$before_exists\" = yes ] && { [ \"$before_openai_bytes\" -gt 0 ] || [ \"$before_opencode_bytes\" -gt 0 ]; } || missing_target=true", + " missing_source=false", + " [ \"$source_exists\" = yes ] && { [ \"$source_openai_bytes\" -gt 0 ] || [ \"$source_opencode_bytes\" -gt 0 ]; } || missing_source=true", + " if [ \"$missing_source\" = true ]; then", + " action=source-missing-provider-key", + " apply_exit=44", + " elif [ \"$dry_run\" = true ]; then", + " if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi", + " elif [ \"$missing_target\" = false ]; then", + " action=kept", + " else", + " cat </dev/null 2>&1 && printf yes || printf no; }", + "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }", + "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", + "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", + "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", + "probe_db() {", + " role_result=unknown", + " database_result=unknown", + " probe_exit=missing-postgres-admin-secret", + " if [ -n \"$postgres_admin_password\" ]; then", + " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", + " role_exit=$?", + " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", + " database_exit=$?", + " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", + " fi", + "}", + "before_exists=$(secret_exists_flag \"$name\")", + "before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")", + "before_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", + "before_url_present=$([ -n \"$before_url_b64\" ] && printf yes || printf no)", + "before_url_bytes=$(decoded_length \"$before_url_b64\")", + "database_url=$(decoded_value \"$before_url_b64\")", + "postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)", + "postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)", + "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", + "probe_db", + "db_role_exists_before=$role_result", + "db_database_exists_before=$database_result", + "db_probe_exit_before=$probe_exit", + "action=observed", + "mutation=false", + "apply_exit=", + "db_ensure_exit=", + "rollout_restart_exit=", + "rollout_status_exit=", + "if [ \"$action_request\" = ensure ]; then", + " missing_secret=false", + " [ \"$before_url_present\" = yes ] && [ \"$before_url_bytes\" -gt 0 ] || missing_secret=true", + " missing_db=false", + " [ \"$db_role_exists_before\" = t ] || missing_db=true", + " [ \"$db_database_exists_before\" = t ] || missing_db=true", + " if [ \"$dry_run\" = true ]; then", + " if [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", + " elif [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ -z \"$postgres_admin_password\" ]; then", + " action=postgres-admin-secret-missing", + " apply_exit=44", + " elif [ \"$missing_secret\" = false ] && [ \"$missing_db\" = false ]; then", + " action=kept", + " else", + " [ -n \"$database_url\" ] || database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"", + " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$database_url_key=$database_url\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", + " apply_exit=$?", + " if [ \"$apply_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$postgres_admin_password\" >/tmp/hwlab-cloud-api-db-psql.out 2>/tmp/hwlab-cloud-api-db-psql.err <<'SQL'", + "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", + "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", + "\\gexec", + "ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';", + "SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')", + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')", + "\\gexec", + "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", + "SQL", + " db_ensure_exit=$?", + " if [ \"$db_ensure_exit\" -eq 0 ]; then", + " if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then", + " kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-cloud-api-rollout-restart.out 2>/tmp/hwlab-cloud-api-rollout-restart.err", + " rollout_restart_exit=$?", + " if [ \"$rollout_restart_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-cloud-api-rollout-status.out 2>/tmp/hwlab-cloud-api-rollout-status.err", + " rollout_status_exit=$?", + " fi", + " fi", + " if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed", + " elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed", + " else action=ensured; mutation=true; fi", + " else action=db-ensure-failed; fi", + " else action=apply-failed; fi", + " fi", + "fi", + "after_exists=$(secret_exists_flag \"$name\")", + "after_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")", + "after_url_present=$([ -n \"$after_url_b64\" ] && printf yes || printf no)", + "after_url_bytes=$(decoded_length \"$after_url_b64\")", + "probe_db", + "db_role_exists_after=$role_result", + "db_database_exists_after=$database_result", + "db_probe_exit_after=$probe_exit", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$name\"", + "printf 'key\\t%s\\n' \"$database_url_key\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeExists\\t%s\\n' \"$before_exists\"", + "printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"", + "printf 'beforeDatabaseUrlPresent\\t%s\\n' \"$before_url_present\"", + "printf 'beforeDatabaseUrlBytes\\t%s\\n' \"$before_url_bytes\"", + "printf 'afterExists\\t%s\\n' \"$after_exists\"", + "printf 'afterDatabaseUrlPresent\\t%s\\n' \"$after_url_present\"", + "printf 'afterDatabaseUrlBytes\\t%s\\n' \"$after_url_bytes\"", + "printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"", + "printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"", + "printf 'dbName\\t%s\\n' \"$db_name\"", + "printf 'dbUser\\t%s\\n' \"$db_user\"", + "printf 'dbHost\\t%s\\n' \"$db_host\"", + "printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"", + "printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"", + "printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"", + "printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"", + "printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"", + "printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"", + "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", + "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", + "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", + "database_url= postgres_admin_password=", + "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", + "if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi", + "if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi", + "if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi", + ].join("\n"); +} + +function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { + const fields = keyValueLinesFromText(text); + if (fields.preset === "owned-postgres-cleanup") { + const absent = fields.afterStatefulSetExists !== "yes" && + fields.afterServiceExists !== "yes" && + fields.afterConfigMapExists !== "yes" && + fields.afterSecretExists !== "yes" && + fields.afterPvcExists !== "yes"; + const platformServiceReady = fields.platformServiceExists === "yes"; + return { + ok: commandOk && absent && platformServiceReady, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.postgresSecret, + statefulSet: fields.statefulSet || spec.postgresStatefulSet, + service: fields.service || spec.postgresSecret, + configMap: fields.configMap || `${spec.postgresSecret}-init`, + pvc: fields.pvc || `data-${spec.postgresSecret}-0`, + preset: "owned-postgres-cleanup", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + before: { + statefulSetExists: fields.beforeStatefulSetExists === "yes", + serviceExists: fields.beforeServiceExists === "yes", + configMapExists: fields.beforeConfigMapExists === "yes", + secretExists: fields.beforeSecretExists === "yes", + pvcExists: fields.beforePvcExists === "yes", + pvcPhase: fields.beforePvcPhase || null, + persistentVolume: fields.beforePersistentVolume || null, + }, + after: { + statefulSetExists: fields.afterStatefulSetExists === "yes", + serviceExists: fields.afterServiceExists === "yes", + configMapExists: fields.afterConfigMapExists === "yes", + secretExists: fields.afterSecretExists === "yes", + pvcExists: fields.afterPvcExists === "yes", + pvcPhase: fields.afterPvcPhase || null, + persistentVolume: fields.afterPersistentVolume || null, + }, + platformService: { + name: "g14-platform-postgres", + exists: platformServiceReady, + }, + deleteStatefulSetExitCode: numericField(fields.deleteStatefulSetExitCode), + deleteServiceExitCode: numericField(fields.deleteServiceExitCode), + deleteConfigMapExitCode: numericField(fields.deleteConfigMapExitCode), + deleteSecretExitCode: numericField(fields.deleteSecretExitCode), + deletePvcExitCode: numericField(fields.deletePvcExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: absent + ? `${fields.statefulSet || spec.postgresStatefulSet}, ${fields.service || spec.postgresSecret}, ${fields.configMap || `${spec.postgresSecret}-init`}, ${fields.secret || spec.postgresSecret}, and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent` + : `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`, + }; + } + if (fields.preset === "obsolete-secret-cleanup") { + const absent = fields.afterSecretExists !== "yes"; + const refsAbsent = fields.workloadRefsPresent !== "yes"; + const dryRun = fields.dryRun === "true"; + return { + ok: commandOk && refsAbsent && (dryRun || absent), + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.obsoleteHwpodDbSecret, + preset: "obsolete-secret-cleanup", + action: fields.action || null, + dryRun, + mutation: fields.mutation === "true", + before: { + secretExists: fields.beforeSecretExists === "yes", + }, + after: { + secretExists: fields.afterSecretExists === "yes", + }, + workloadRefs: { + present: fields.workloadRefsPresent === "yes", + preview: fields.workloadRefsPreview || "", + }, + deleteSecretExitCode: numericField(fields.deleteSecretExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: refsAbsent && (dryRun || absent) + ? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}` + : `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`, + }; + } + if (fields.preset === "master-server-admin-api-key") { + const afterBytes = numericField(fields.afterApiKeyBytes); + const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.masterAdminApiKeySecret, + preset: "master-server-admin-api-key", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } }, + cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, + applyExitCode: numericField(fields.applyExitCode), + rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), + rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy ? `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} exists` : `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} missing`, + }; + } + if (fields.preset === "bootstrap-admin") { + const beforeHashBytes = numericField(fields.beforePasswordHashBytes); + const sourceHashBytes = numericField(fields.sourcePasswordHashBytes); + const afterHashBytes = numericField(fields.afterPasswordHashBytes); + const yamlSourceMode = typeof fields.sourceRef === "string" && fields.sourceRef.length > 0; + const targetHashReady = fields.afterExists === "yes" && + fields.afterPasswordHashPresent === "yes" && + typeof afterHashBytes === "number" && afterHashBytes > 0; + const yamlSourceReady = !yamlSourceMode || ( + fields.sourceExists === "yes" && + typeof fields.sourceFingerprint === "string" && + fields.sourceFingerprint.length > 0 && + fields.afterSourceRef === fields.sourceRef && + fields.afterSourceKey === fields.sourceKey && + fields.afterSourceFingerprint === fields.sourceFingerprint && + fields.afterUsername === fields.username + ); + const healthy = targetHashReady && yamlSourceReady; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.bootstrapAdminSecret, + key: fields.key || spec.bootstrapAdminPasswordHashKey, + preset: "bootstrap-admin", + account: { + username: fields.username || spec.bootstrapAdminUsername, + displayName: fields.displayName || spec.bootstrapAdminDisplayName, + }, + source: yamlSourceMode + ? { + sourceRef: fields.sourceRef, + sourceKey: fields.sourceKey || null, + sourcePath: fields.sourcePath || null, + exists: fields.sourceExists === "yes", + fingerprint: fields.sourceFingerprint || null, + passwordHashTransform: fields.passwordHashTransform || spec.bootstrapAdminPasswordHashTransform || null, + valuesRedacted: true, + } + : { + namespace: fields.sourceNamespace || spec.bootstrapAdminSourceNamespace, + secret: fields.sourceSecret || spec.bootstrapAdminSourceSecret, + exists: fields.sourceExists === "yes", + passwordHash: { keyPresent: fields.sourcePasswordHashPresent === "yes", valueBytes: sourceHashBytes }, + }, + action: fields.action || null, + dryRun: fields.dryRun === "true", + forceSync: fields.forceSync === "true", + mutation: fields.mutation === "true", + before: { + exists: fields.beforeExists === "yes", + passwordHash: { keyPresent: fields.beforePasswordHashPresent === "yes", valueBytes: beforeHashBytes }, + ...(yamlSourceMode ? { + sourceRef: fields.beforeSourceRef || null, + sourceKey: fields.beforeSourceKey || null, + sourceFingerprint: fields.beforeSourceFingerprint || null, + username: fields.beforeUsername || null, + } : {}), + }, + after: { + exists: fields.afterExists === "yes", + passwordHash: { keyPresent: fields.afterPasswordHashPresent === "yes", valueBytes: afterHashBytes }, + ...(yamlSourceMode ? { + sourceRef: fields.afterSourceRef || null, + sourceKey: fields.afterSourceKey || null, + sourceFingerprint: fields.afterSourceFingerprint || null, + username: fields.afterUsername || null, + } : {}), + }, + cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, + applyExitCode: numericField(fields.applyExitCode), + rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), + rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy ? `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} exists` : `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} missing`, + }; + } + if (fields.preset === "code-agent-provider") { + const beforeOpenaiBytes = numericField(fields.beforeOpenaiBytes); + const beforeOpencodeBytes = numericField(fields.beforeOpencodeBytes); + const sourceOpenaiBytes = numericField(fields.sourceOpenaiBytes); + const sourceOpencodeBytes = numericField(fields.sourceOpencodeBytes); + const afterOpenaiBytes = numericField(fields.afterOpenaiBytes); + const afterOpencodeBytes = numericField(fields.afterOpencodeBytes); + const openaiReady = fields.afterOpenaiPresent === "yes" && typeof afterOpenaiBytes === "number" && afterOpenaiBytes > 0; + const opencodeReady = fields.afterOpencodePresent === "yes" && typeof afterOpencodeBytes === "number" && afterOpencodeBytes > 0; + const healthy = fields.afterExists === "yes" && (openaiReady || opencodeReady); + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.codeAgentProviderSecret, + preset: "code-agent-provider", + source: { + namespace: fields.sourceNamespace || spec.codeAgentProviderSourceNamespace, + secret: fields.sourceSecret || spec.codeAgentProviderSourceSecret, + exists: fields.sourceExists === "yes", + openaiApiKey: { keyPresent: fields.sourceOpenaiPresent === "yes", valueBytes: sourceOpenaiBytes }, + opencodeApiKey: { keyPresent: fields.sourceOpencodePresent === "yes", valueBytes: sourceOpencodeBytes }, + }, + selectedKey: fields.selectedKey || null, + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + before: { + exists: fields.beforeExists === "yes", + openaiApiKey: { keyPresent: fields.beforeOpenaiPresent === "yes", valueBytes: beforeOpenaiBytes }, + opencodeApiKey: { keyPresent: fields.beforeOpencodePresent === "yes", valueBytes: beforeOpencodeBytes }, + }, + after: { + exists: fields.afterExists === "yes", + openaiApiKey: { keyPresent: fields.afterOpenaiPresent === "yes", valueBytes: afterOpenaiBytes }, + opencodeApiKey: { keyPresent: fields.afterOpencodePresent === "yes", valueBytes: afterOpencodeBytes }, + requiredAnyProviderKeyPresent: openaiReady || opencodeReady, + }, + applyExitCode: numericField(fields.applyExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`, + }; + } + if (fields.preset === "cloud-api-db") { + const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes); + const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes); + if (fields.platformDbMode === "true") { + const keysHealthy = fields.afterExists === "yes" && + fields.afterDatabaseUrlPresent === "yes" && + typeof afterUrlBytes === "number" && afterUrlBytes > 0; + const platformBridgeHealthy = fields.platformServiceExists === "yes" && + fields.platformEndpointsExists !== "yes" && + fields.platformEndpointSliceExists === "yes"; + const uriHealthy = fields.dbHostMatchesPlatform === "yes" && + fields.dbNameMatchesExpected === "yes" && + fields.dbUserMatchesExpected === "yes"; + const healthy = keysHealthy && platformBridgeHealthy && uriHealthy; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.cloudApiDbSecret, + key: fields.key || spec.cloudApiDbKey, + preset: "cloud-api-db", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + platformDbMode: true, + after: { + exists: fields.afterExists === "yes", + databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes }, + }, + legacyPostgresSecret: { + name: fields.legacyPostgresSecret || spec.postgresSecret, + exists: fields.legacyPostgresSecretExists === "yes", + }, + platformService: { + name: fields.platformService || spec.platformPostgresService, + exists: fields.platformServiceExists === "yes", + endpointsExist: fields.platformEndpointsExists === "yes", + legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes", + endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`, + endpointSliceExists: fields.platformEndpointSliceExists === "yes", + }, + dbName: fields.dbName || spec.cloudApiDbName, + dbUser: fields.dbUser || spec.cloudApiDbUser, + dbHost: fields.dbHost || spec.cloudApiDbHost, + dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes", + dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes", + dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes", + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy + ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}` + : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`, + }; + } + const keysHealthy = fields.afterExists === "yes" && + fields.afterDatabaseUrlPresent === "yes" && + typeof afterUrlBytes === "number" && afterUrlBytes > 0; + const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; + const healthy = keysHealthy && databaseHealthy; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.cloudApiDbSecret, + key: fields.key || spec.cloudApiDbKey, + preset: "cloud-api-db", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + before: { + exists: fields.beforeExists === "yes", + postgresSecretExists: fields.beforePostgresSecretExists === "yes", + databaseUrl: { keyPresent: fields.beforeDatabaseUrlPresent === "yes", valueBytes: beforeUrlBytes }, + database: { + roleExists: fields.dbRoleExistsBefore || "unknown", + databaseExists: fields.dbDatabaseExistsBefore || "unknown", + probeExitCode: fields.dbProbeExitCodeBefore || null, + }, + }, + after: { + exists: fields.afterExists === "yes", + databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes }, + database: { + roleExists: fields.dbRoleExistsAfter || "unknown", + databaseExists: fields.dbDatabaseExistsAfter || "unknown", + probeExitCode: fields.dbProbeExitCodeAfter || null, + }, + }, + postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes", + postgresSecret: fields.postgresSecret || spec.postgresSecret, + dbName: fields.dbName || spec.cloudApiDbName, + dbUser: fields.dbUser || spec.cloudApiDbUser, + dbHost: fields.dbHost || spec.cloudApiDbHost, + cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, + applyExitCode: numericField(fields.applyExitCode), + dbEnsureExitCode: numericField(fields.dbEnsureExitCode), + rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), + rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`, + }; + } + const afterAuthnBytes = numericField(fields.afterAuthnBytes); + const afterUriBytes = numericField(fields.afterDatastoreUriBytes); + const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes); + if (fields.platformDbMode === "true") { + const keysHealthy = fields.afterExists === "yes" && + fields.afterAuthnPresent === "yes" && + fields.afterDatastoreUriPresent === "yes" && + typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && + typeof afterUriBytes === "number" && afterUriBytes > 0; + const platformBridgeHealthy = fields.platformServiceExists === "yes" && + fields.platformEndpointsExists !== "yes" && + fields.platformEndpointSliceExists === "yes"; + const uriHealthy = fields.dbHostMatchesPlatform === "yes" && + fields.dbNameMatchesExpected === "yes" && + fields.dbUserMatchesExpected === "yes"; + const healthy = keysHealthy && platformBridgeHealthy && uriHealthy; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.openFgaSecret, + preset: fields.preset || "openfga", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + platformDbMode: true, + after: { + exists: fields.afterExists === "yes", + authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, + datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, + postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, + }, + legacyPostgresSecret: { + name: fields.legacyPostgresSecret || spec.postgresSecret, + exists: fields.legacyPostgresSecretExists === "yes", + }, + platformService: { + name: fields.platformService || spec.platformPostgresService, + exists: fields.platformServiceExists === "yes", + endpointsExist: fields.platformEndpointsExists === "yes", + legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes", + endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`, + endpointSliceExists: fields.platformEndpointSliceExists === "yes", + }, + dbName: fields.dbName || spec.openFgaDbName, + dbUser: fields.dbUser || spec.openFgaDbUser, + dbHost: fields.dbHost || spec.openFgaDbHost, + dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes", + dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes", + dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes", + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy + ? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.platformService || spec.platformPostgresService}` + : `${fields.secret || spec.openFgaSecret} datastore-uri is not aligned to platform DB`, + }; + } + const keysHealthy = fields.afterExists === "yes" && + fields.afterPostgresSecretExists === "yes" && + fields.afterAuthnPresent === "yes" && + fields.afterDatastoreUriPresent === "yes" && + fields.afterPostgresPasswordPresent === "yes" && + typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && + typeof afterUriBytes === "number" && afterUriBytes > 0 && + typeof afterPasswordBytes === "number" && afterPasswordBytes > 0; + const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; + const healthy = keysHealthy && databaseHealthy; + return { + ok: commandOk && healthy, + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.openFgaSecret, + preset: fields.preset || "openfga", + action: fields.action || null, + dryRun: fields.dryRun === "true", + mutation: fields.mutation === "true", + after: { + exists: fields.afterExists === "yes", + postgresSecretExists: fields.afterPostgresSecretExists === "yes", + authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes }, + datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes }, + postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes }, + database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null }, + }, + postgresSecretExitCode: numericField(fields.postgresSecretExitCode), + postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode), + applyExitCode: numericField(fields.applyExitCode), + dbEnsureExitCode: numericField(fields.dbEnsureExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: healthy ? `${fields.secret || spec.openFgaSecret} keys and Postgres database exist` : `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`, + }; +} + +function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { + const fields = keyValueLinesFromText(text); + const dryRun = fields.dryRun === "true"; + const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown"; + const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown"; + const probesOk = fields.beforeDatabaseExists !== "unknown" && + fields.beforeRoleExists !== "unknown" && + fields.afterDatabaseExists !== "unknown" && + fields.afterRoleExists !== "unknown"; + return { + ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)), + database: fields.database || spec.obsoleteHwpodDbName, + role: fields.role || spec.obsoleteHwpodDbUser, + preset: "obsolete-platform-db-cleanup", + action: fields.action || null, + dryRun, + mutation: fields.mutation === "true", + before: { + databaseExists: fields.beforeDatabaseExists === "yes", + roleExists: fields.beforeRoleExists === "yes", + }, + after: { + databaseExists: fields.afterDatabaseExists === "yes", + roleExists: fields.afterRoleExists === "yes", + }, + beforeProbeExitCode: { + database: numericField(fields.beforeDatabaseProbeExitCode), + role: numericField(fields.beforeRoleProbeExitCode), + }, + afterProbeExitCode: { + database: numericField(fields.afterDatabaseProbeExitCode), + role: numericField(fields.afterRoleProbeExitCode), + }, + dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode), + dropRoleExitCode: numericField(fields.dropRoleExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: probesOk && (dryRun || (databaseAbsent && roleAbsent)) + ? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}` + : `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`, + }; +} + +export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record { + return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane })); +} + +function masterAdminApiKeyEnvPath(spec: RuntimeSecretSpec): string { + return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`; +} + +function readMasterAdminApiKey(spec: RuntimeSecretSpec): { key: string; source: string } { + const source = masterAdminApiKeyEnvPath(spec); + if (!existsSync(source)) throw new Error(`HWLAB_API_KEY source missing: ${source}`); + const content = readFileSync(source, "utf8"); + const match = content.match(/^HWLAB_API_KEY=(.+)$/m); + const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, ""); + if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${source}`); + return { key: raw, source }; +} + +function optionValue(args: string[], name: string): string | undefined { + const index = args.indexOf(name); + if (index === -1) return undefined; + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); + return value; +} + +function requiredOption(args: string[], name: string): string { + const value = optionValue(args, name); + if (value === undefined) throw new Error(`${name} is required`); + return value; +} + +function stripOption(args: string[], name: string): string[] { + return stripOptions(args, [name]); +} + +function stripOptions(args: string[], names: readonly string[]): string[] { + const remove = new Set(names); + const without: string[] = []; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + if (remove.has(arg)) { + if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1; + continue; + } + without.push(arg); + } + return without; +} + +function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { + const raw = optionValue(args, name); + if (raw === undefined) return defaultValue; + const value = Number(raw); + if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); + return Math.min(value, maxValue); +} + +function assertLane(value: string): void { + if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`); +} + +function assertNodeId(value: string): void { + if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`); +} + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function nullableRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; +} + +function stringValue(value: unknown, path: string): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`); + return value; +} + +function optionalStringValue(value: unknown, path: string): string | null { + if (value === undefined || value === null) return null; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string when set`); + return value; +} + +function parseJsonObject(text: string): Record { + const trimmed = text.trim(); + if (trimmed.length === 0) return {}; + try { + return record(JSON.parse(trimmed) as unknown); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { + return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown); + } catch {} + } + } + return {}; +} + +function shellQuote(value: string): string { + return `'${value.replace(/'/gu, `'"'"'`)}'`; +} + +function statusText(result: CommandResult): string { + return result.stdout || result.stderr; +} + +function keyValueLinesFromText(text: string): Record { + const fields: Record = {}; + for (const line of text.split(/\r?\n/u)) { + const index = line.indexOf("\t"); + if (index <= 0) continue; + fields[line.slice(0, index)] = line.slice(index + 1); + } + return fields; +} + +function numericField(value: string | undefined): number | null { + if (value === undefined || value === "") return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function commaListField(value: string | undefined): string[] { + if (!value) return []; + return value.split(",").map((item) => item.trim()).filter(Boolean); +} + +function splitWhitespaceField(value: string | undefined): string[] { + if (!value) return []; + return value.split(/\s+/u).filter(Boolean); +} + +function compactCommandResult(result: CommandResult): Record { + return { + command: compactCommand(result.command), + exitCode: result.exitCode, + stdoutBytes: result.stdout.length, + stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), + timedOut: result.timedOut, + }; +} + +function compactCommandResultRedacted(result: CommandResult, secrets: string[]): Record { + const compact = compactCommandResult(result); + if (typeof compact.stderr === "string" && compact.stderr.length > 0) { + compact.stderr = redactKnownSecrets(compact.stderr, secrets); + } + return compact; +} + +function redactKnownSecrets(text: string, secrets: string[]): string { + let next = text; + for (const secret of secrets.filter((item) => item.length > 0)) { + next = next.split(secret).join(""); + } + return next; +} + +function parseJsonObject(text: string): Record | null { + const trimmed = text.trim(); + if (trimmed.length === 0) return null; + try { + const parsed = JSON.parse(trimmed) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; + } catch { + const objectText = firstJsonObjectText(trimmed); + if (objectText) { + try { + const parsed = JSON.parse(objectText) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; + } catch {} + } + return null; + } +} + +function firstJsonObjectText(text: string): string | null { + const start = text.indexOf("{"); + if (start < 0) return null; + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === "\\") { + escaped = true; + } else if (char === "\"") { + inString = false; + } + continue; + } + if (char === "\"") { + inString = true; + continue; + } + if (char === "{") depth += 1; + else if (char === "}") { + depth -= 1; + if (depth === 0) return text.slice(start, index + 1); + } + } + return null; +} + +function compactCommand(command: string[]): string[] { + const scriptIndex = command.indexOf("--"); + if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "