// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-probe module for scripts/src/hwlab-node-impl.ts.

// Moved mechanically from scripts/src/hwlab-node-impl.ts:6032-7426 for #903.

// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
import { createHash, randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneCiGitWorkspaceSecret, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import type { RenderedCliResult } from "../output";

import type { BootstrapAdminPasswordMaterial, BootstrapAdminSecretMaterial, NodeRuntimeGitMirrorEgressProxySpec, NodeRuntimeGitMirrorGithubTransportSpec, NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult, NodeWebProbeOptions, RuntimeSecretSpec } from "./entry";
import { isCommandSuccess, nodeRuntimeGitopsRoot, runNodeK3sArgs, runNodeK3sScript } from "./cleanup";
import { nodeRuntimeCicdWaitWarning } from "./git-mirror";
import { parseNodeScopedDelegatedOptions } from "./plan";
import { nodeRuntimePipelineFailureSummary, nodeRuntimePipelineRunDiagnostics } from "./render";
import { compactRuntimeCommand, runNodeHostScript, runNodeHostScriptAsync } from "./runtime-common";
import { assertLane, assertNodeId, keyValueLinesFromText, numericField, optionValue, optionalStringValue, positiveIntegerOption, positiveIntegerValue, record, requiredOption, shellQuote, statusText, stringValue, stripOptions } from "./utils";
import { discoverWebObserveIndexEntry, readWebObserveIndexEntry } from "./web-observe-render";
import { assertKnownOptions, nodeWebProbeAutoCommandTimeoutSeconds, nodeWebProbeDefaultUrl, normalizeNodeWebProbeObserveArgs, parseNodeWebProbeObserveOptions, parseNodeWebProbeSentinelOptions, parseWebProbeBrowserProxyMode } from "./web-probe-observe";
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
import { resolveEgressProxySourceRef } from "../egress-proxy-sources";

export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
  const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
  const gitMirror = nodeRuntimeGitMirrorTarget(spec);
  const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
  const renderGitMirror = {
    ...gitMirror,
    egressProxy: gitMirror.egressProxy.mode === "direct" ? {
      mode: "direct",
      required: false,
    } : {
      ...gitMirror.egressProxy,
      mode: "node-global",
      required: true,
    },
  };
  return {
    nodeId: spec.nodeId,
    lane: spec.lane,
    sourceBranch: spec.sourceBranch,
    gitopsBranch: spec.gitopsBranch,
    gitopsRoot: nodeRuntimeGitopsRoot(spec),
    runtimePath: spec.runtimePath,
    runtimeRenderDir: spec.runtimeRenderDir,
    runtimeNamespace: spec.runtimeNamespace,
    catalogPath: spec.catalogPath,
    tektonDir: spec.tektonDir,
    pipelineName: spec.pipeline,
    argoApplicationFile: spec.argoApplicationFile,
    argoRepoUrl: spec.argoRepoUrl,
      gitUrl: spec.gitUrl,
      gitReadUrl: spec.gitReadUrl,
      gitWriteUrl: spec.gitWriteUrl,
      toolsImage: gitMirror.toolsImage,
      toolsImagePullPolicy: gitMirror.toolsImagePullPolicy,
      gitMirror: renderGitMirror,
    networkProfileId: spec.networkProfileId,
    downloadProfileId: spec.downloadProfileId,
    gitSshProxyHost: gitSshProxy?.host,
    gitSshProxyPort: gitSshProxy?.port,
    proxyHttp: spec.networkProfile.proxy.http,
    proxyHttps: spec.networkProfile.proxy.https,
    proxyAll: spec.networkProfile.proxy.all,
    noProxy: spec.networkProfile.proxy.noProxy.join(","),
    dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http,
    dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https,
    dockerProxyAll: spec.networkProfile.dockerBuildProxy.all,
    dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
    dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy,
    npmRegistry: spec.downloadProfile.npm.registry,
    npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000,
    npmRetries: spec.downloadProfile.npm.retries,
    stepEnv: spec.stepEnv,
    observability: spec.observability,
    runtimeStore: spec.runtimeStore,
    codeAgentRuntime: spec.codeAgentRuntime === undefined ? undefined : {
      enabled: spec.codeAgentRuntime.enabled,
      adapter: spec.codeAgentRuntime.adapter,
      managerUrl: spec.codeAgentRuntime.managerUrl,
      apiKeySecretName: spec.codeAgentRuntime.apiKeySecretName,
      apiKeySecretKey: spec.codeAgentRuntime.apiKeySecretKey,
      runnerNamespace: spec.codeAgentRuntime.runnerNamespace,
      secretNamespace: spec.codeAgentRuntime.secretNamespace,
      repoUrlFrom: spec.codeAgentRuntime.repoUrlFrom,
      repoUrl: spec.gitReadUrl,
      providerIdFrom: spec.codeAgentRuntime.providerIdFrom,
      providerId: spec.nodeId,
      defaultProviderProfile: spec.codeAgentRuntime.defaultProviderProfile,
      codexStdioSupervisor: spec.codeAgentRuntime.codexStdioSupervisor,
      valuesPrinted: false,
    },
    runtimeImageRewrites: spec.runtimeImageRewrites,
    runtimeImageBuilds: spec.runtimeImageBuilds,
    registryPrefix: spec.registryPrefix,
    buildkitSidecarImage: spec.buildkit?.sidecarImage,
    publicWebUrl: spec.publicWebUrl,
    publicApiUrl: spec.publicApiUrl,
    publicExposure: spec.publicExposure === null ? undefined : {
      enabled: spec.publicExposure.enabled,
      mode: spec.publicExposure.mode,
      publicBaseUrl: spec.publicExposure.publicBaseUrl,
      hostname: spec.publicExposure.hostname,
      expectedA: spec.publicExposure.expectedA,
      serverAddr: spec.publicExposure.serverAddr,
      serverPort: spec.publicExposure.serverPort,
      secretName: spec.publicExposure.secretName,
      secretKey: spec.publicExposure.secretKey,
      tokenKey: spec.publicExposure.tokenKey,
      webProxy: spec.publicExposure.webProxy,
      apiProxy: spec.publicExposure.apiProxy,
    },
    externalPostgres: activeExternalPostgres === undefined ? undefined : {
      enabled: true,
      serviceName: activeExternalPostgres.serviceName,
      endpointAddress: activeExternalPostgres.endpointAddress,
      port: activeExternalPostgres.port,
      runtimeAccess: activeExternalPostgres.runtimeAccess ?? null,
      sslmode: activeExternalPostgres.sslmode,
    },
  };
}

export function httpProxyEndpoint(value: string): { host: string; port: number } | null {
  let parsed: URL;
  try {
    parsed = new URL(value);
  } catch {
    return null;
  }
  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
  const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10);
  if (!Number.isInteger(port) || port <= 0) return null;
  return { host: parsed.hostname, port };
}

export function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
  return [
    `${renderDir}/devops-infra/git-mirror.yaml`,
    `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
    `${renderDir}/${spec.tektonDir}/rbac.yaml`,
    `${renderDir}/${spec.tektonDir}/pipeline.yaml`,
    `${renderDir}/argocd/project.yaml`,
    `${renderDir}/argocd/${spec.argoApplicationFile}`,
  ];
}

export function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
  const files = nodeRuntimeControlPlaneFiles(spec, renderDir);
  const args = [
    "kubectl",
    "apply",
    ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
    ...files.flatMap((file) => ["-f", file]),
  ];
  return runNodeK3sArgs(spec, args, timeoutSeconds);
}

export function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
  const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir)
    .map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`)
    .join("");
  const args = [
    "kubectl",
    "apply",
    ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
    "-f",
    "-",
  ];
  return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest);
}

export function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult {
  const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
  const script = [
    "set -eu",
    `render_dir=${shellQuote(renderDir)}`,
    `prefix=${shellQuote(prefix)}`,
    "case \"$render_dir\" in",
    "  \"$prefix\"*) rm -rf \"$render_dir\" ;;",
    "  *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;",
    "esac",
  ].join("\n");
  return runNodeHostScript(spec, script, 60);
}

export function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult {
  const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
  const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`;
  if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) {
    return {
      command: ["rm", "-rf", "<refused>"],
      cwd: repoRoot,
      exitCode: 44,
      stdout: "",
      stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`,
      signal: null,
      timedOut: false,
    };
  }
  return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 });
}

export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
  const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane);
  return {
    apiVersion: "tekton.dev/v1",
    kind: "PipelineRun",
    metadata: {
      name: pipelineRun,
      namespace: gitWorkspaceSecret.namespace,
      labels: {
        "app.kubernetes.io/part-of": "hwlab",
        "hwlab.pikastech.local/gitops-target": spec.lane,
        "hwlab.pikastech.local/source-commit": sourceCommit,
        "hwlab.pikastech.local/trigger": "unidesk-node-cli",
      },
      annotations: {
        "hwlab.pikastech.local/node": spec.nodeId,
        "hwlab.pikastech.local/source-branch": spec.sourceBranch,
        "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
        "hwlab.pikastech.local/runtime-path": spec.runtimePath,
        "hwlab.pikastech.local/network-profile": spec.networkProfileId,
        "hwlab.pikastech.local/download-profile": spec.downloadProfileId,
      },
    },
    spec: {
      pipelineRef: { name: spec.pipeline },
      taskRunTemplate: {
        serviceAccountName: spec.serviceAccountName,
        podTemplate: {
          hostNetwork: true,
          dnsPolicy: "ClusterFirstWithHostNet",
          securityContext: { fsGroup: 1000 },
        },
      },
      params: [
        { name: "git-url", value: spec.gitUrl },
        { name: "git-read-url", value: spec.gitReadUrl },
        { name: "git-write-url", value: spec.gitWriteUrl },
        { name: "source-branch", value: spec.sourceBranch },
        { name: "gitops-branch", value: spec.gitopsBranch },
        { name: "lane", value: spec.lane },
        { name: "catalog-path", value: spec.catalogPath },
        { name: "image-tag-mode", value: "full" },
        { name: "runtime-path", value: spec.runtimePath },
        { name: "source-commit", value: sourceCommit },
        { name: "revision", value: sourceCommit },
        { name: "registry-prefix", value: spec.registryPrefix },
        { name: "services", value: spec.serviceIds.join(",") },
        { name: "base-image", value: spec.baseImage },
      ],
      workspaces: [
        { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
        { name: "git-ssh", secret: { secretName: gitWorkspaceSecret.name } },
      ],
    },
  };
}

export function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult {
  const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane);
  const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64");
  const script = [
    "set -eu",
    `manifest_b64=${shellQuote(manifestB64)}`,
    `pipeline_run=${shellQuote(pipelineRun)}`,
    `ci_namespace=${shellQuote(gitWorkspaceSecret.namespace)}`,
    `git_workspace_secret=${shellQuote(gitWorkspaceSecret.name)}`,
    `rerun=${rerun ? "true" : "false"}`,
    "manifest_path=\"/tmp/$pipeline_run.json\"",
    "if ! kubectl -n \"$ci_namespace\" get secret \"$git_workspace_secret\" >/dev/null 2>&1; then",
    "  printf 'missing Tekton git workspace Secret %s/%s; run: bun scripts/cli.ts hwlab nodes control-plane infra apply --node %s --lane %s --confirm\\n' \"$ci_namespace\" \"$git_workspace_secret\" " + shellQuote(spec.nodeId) + " " + shellQuote(spec.lane) + " >&2",
    "  exit 44",
    "fi",
    "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
    "existing_status=$(kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
    "if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then",
    "  kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "  kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "  kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "  kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "elif [ \"$existing_status\" = False ]; then",
    "  kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "  kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "  kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "  kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "fi",
    "if kubectl create -f \"$manifest_path\"; then",
    "  :",
    "else",
    "  code=$?",
    "  if kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" >/dev/null 2>&1; then",
    "    printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2",
    "  else",
    "    exit \"$code\"",
    "  fi",
    "fi",
    "kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'",
  ].join("\n");
  return runNodeK3sScript(spec, script, timeoutSeconds);
}

export function waitForNodeRuntimePipelineRunTerminal(
  spec: HwlabRuntimeLaneSpec,
  pipelineRun: string,
  timeoutSeconds: number,
  options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
): Record<string, unknown> {
  const severeWarningThresholdMs = 120_000;
  const startedAt = Date.now();
  const deadline = startedAt + timeoutSeconds * 1000;
  let polls = 0;
  let last: Record<string, unknown> = { exists: false, name: pipelineRun };
  let lastOpportunisticPostFlushAt = 0;
  let severeWarningEmitted = false;
  let opportunisticPostSyncAttempted = false;
  const opportunisticPostFlushes: Record<string, unknown>[] = [];
  const opportunisticPostSyncs: Record<string, unknown>[] = [];
  printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds });
  while (Date.now() <= deadline) {
    polls += 1;
    last = getNodeRuntimePipelineRun(spec, pipelineRun);
    const status = typeof last.status === "string" ? last.status : null;
    const reason = typeof last.reason === "string" ? last.reason : null;
    const elapsedMs = Date.now() - startedAt;
    printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs });
    if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") {
      severeWarningEmitted = true;
      printNodeRuntimeTriggerProgress(spec, {
        stage: "pipelinerun-wait",
        status: "warning",
        warning: "pipelinerun-wait-severe-timeout",
        severity: "warning",
        pipelineRun,
        pipelineStatus: status,
        reason,
        polls,
        elapsedMs,
        thresholdMs: severeWarningThresholdMs,
        message: "PipelineRun has been non-terminal for more than 120s; this is a severe timeout and requires further investigation of Tekton taskRuns/pods instead of treating it as normal wait time.",
        next: {
          status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
        },
      });
    }
    if (status === "True" || status === "False") {
      const ok = status === "True";
      const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
      const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
      printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
      return {
        ok,
        status: ok ? "succeeded" : "failed",
        pipelineRun: last,
        diagnostics,
        failureSummary,
        polls,
        elapsedMs: Date.now() - startedAt,
        opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
        opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
        degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
      };
    }
    if (
      options.opportunisticPostSync !== undefined
      && !opportunisticPostSyncAttempted
      && nodeRuntimePipelineRunSourceCloneCompleted(spec, pipelineRun)
    ) {
      opportunisticPostSyncAttempted = true;
      const sync = options.opportunisticPostSync();
      if (sync !== null) opportunisticPostSyncs.push(sync);
    }
    if (options.opportunisticPostFlush !== undefined && Date.now() - lastOpportunisticPostFlushAt >= 15_000) {
      lastOpportunisticPostFlushAt = Date.now();
      const flush = options.opportunisticPostFlush();
      if (flush !== null) opportunisticPostFlushes.push(flush);
      last = getNodeRuntimePipelineRun(spec, pipelineRun);
      const refreshedStatus = typeof last.status === "string" ? last.status : null;
      const refreshedReason = typeof last.reason === "string" ? last.reason : null;
      const refreshedElapsedMs = Date.now() - startedAt;
      if (refreshedStatus === "True" || refreshedStatus === "False") {
        const ok = refreshedStatus === "True";
        const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
        const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
        printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: refreshedStatus, reason: refreshedReason, polls, elapsedMs: refreshedElapsedMs, observedAfterPostFlush: true });
        return {
          ok,
          status: ok ? "succeeded" : "failed",
          pipelineRun: last,
          diagnostics,
          failureSummary,
          polls,
          elapsedMs: refreshedElapsedMs,
          opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
          opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
          observedAfterPostFlush: true,
          degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
        };
      }
    }
    sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
  }
  const elapsedMs = Date.now() - startedAt;
  const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
  printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
  return {
    ok: true,
    status: "pending",
    pipelineRun: last,
    polls,
    elapsedMs,
    waitLimitSeconds: timeoutSeconds,
    warning,
    opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
    opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
    next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
  };
}

export function nodeRuntimePipelineRunSourceCloneCompleted(spec: HwlabRuntimeLaneSpec, pipelineRun: string): boolean {
  const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
  if (result.exitCode !== 0) return false;
  let parsed: unknown;
  try {
    parsed = JSON.parse(result.stdout);
  } catch {
    return false;
  }
  const items = Array.isArray(record(parsed).items) ? record(parsed).items as unknown[] : [];
  return items.some((item) => {
    const itemRecord = record(item);
    const metadata = record(itemRecord.metadata);
    const labels = record(metadata.labels);
    const name = typeof metadata.name === "string" ? metadata.name : "";
    const pipelineTask = typeof labels["tekton.dev/pipelineTask"] === "string" ? labels["tekton.dev/pipelineTask"] : "";
    const taskName = `${pipelineTask} ${name}`;
    if (!/(clone|checkout|local-git|source[-_ ]*worktree)/iu.test(taskName)) return false;
    const status = record(itemRecord.status);
    const conditions = Array.isArray(status.conditions) ? status.conditions as unknown[] : [];
    return conditions.some((condition) => {
      const conditionRecord = record(condition);
      return conditionRecord.type === "Succeeded" && conditionRecord.status === "True";
    });
  });
}

export function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
  process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`);
}

export function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
  const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60);
  if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) };
  const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u);
  return {
    exists: true,
    name: pipelineRun,
    status: status || null,
    reason: reason || null,
    message: message || null,
    createdAt: createdAt || null,
    result: compactRuntimeCommand(result),
  };
}

export function sleepSync(ms: number): void {
  const buffer = new SharedArrayBuffer(4);
  Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms));
}

export function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
  const pg = hwlabRuntimeActiveExternalPostgres(spec);
  if (pg === undefined) return null;
  const secretRoot = externalPostgresSecretSourceRoot(spec);
  const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey);
  const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey);
  const missing = [
    ...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]),
    ...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]),
  ];
  if (missing.length > 0) {
    if (dryRun) {
      return {
        ok: true,
        mode: "dry-run",
        mutation: false,
        skipped: true,
        reason: "external-postgres-secret-source-missing",
        missing,
        sourceRoot: secretRoot,
        valuesPrinted: false,
        next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
      };
    }
    return {
      ok: false,
      mode: dryRun ? "dry-run" : "confirmed-secret-sync",
      mutation: false,
      degradedReason: "external-postgres-secret-source-missing",
      missing,
      sourceRoot: secretRoot,
      valuesPrinted: false,
      next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
    };
  }
  const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds);
  if (!isCommandSuccess(setup)) {
    return {
      ok: false,
      mode: dryRun ? "dry-run" : "confirmed-secret-sync",
      mutation: false,
      phase: "namespace-authn-setup",
      setup: compactRuntimeCommand(setup),
      valuesPrinted: false,
    };
  }
  const manifest = {
    apiVersion: "v1",
    kind: "List",
    items: [
      {
        apiVersion: "v1",
        kind: "Secret",
        metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace },
        type: "Opaque",
        stringData: { [pg.cloudApi.secretKey]: cloudApi.value },
      },
      {
        apiVersion: "v1",
        kind: "Secret",
        metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace },
        type: "Opaque",
        stringData: { [pg.openfga.secretKey]: openfga.value },
      },
    ],
  };
  const applyScript = [
    "set -eu",
    [
      "kubectl",
      "apply",
      "--server-side",
      "--force-conflicts",
      "--field-manager=unidesk-hwlab-node-external-postgres-secret",
      ...(dryRun ? ["--dry-run=server"] : []),
      "-f",
      "-",
    ].map(shellQuote).join(" "),
  ].join("\n");
  const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest));
  return {
    ok: isCommandSuccess(apply),
    mode: dryRun ? "dry-run" : "confirmed-secret-sync",
    mutation: !dryRun && isCommandSuccess(apply),
    namespace: spec.runtimeNamespace,
    sourceRoot: secretRoot,
    secrets: [
      { name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey },
      { name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null },
    ],
    setup: compactRuntimeCommand(setup),
    apply: compactRuntimeCommand(apply),
    valuesPrinted: false,
  };
}

export function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
  const pg = spec.runtimeStore?.postgres;
  if (pg?.mode !== "local-k3s") return null;
  const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`;
  const sourceRef = pg.adminPasswordSourceRef ?? "";
  const sourceKey = pg.adminPasswordSourceKey ?? "";
  if (sourceRef.length === 0 || sourceKey.length === 0) {
    return {
      ok: false,
      mode: dryRun ? "dry-run" : "confirmed-secret-sync",
      mutation: false,
      namespace: spec.runtimeNamespace,
      secretName,
      secretKey: "POSTGRES_PASSWORD",
      degradedReason: "local-postgres-secret-source-not-configured",
      valuesPrinted: false,
      next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` },
    };
  }
  const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun });
  if (material.ok !== true) {
    return {
      ok: false,
      mode: dryRun ? "dry-run" : "confirmed-secret-sync",
      mutation: false,
      namespace: spec.runtimeNamespace,
      secretName,
      secretKey: "POSTGRES_PASSWORD",
      source: material,
      degradedReason: material.error ?? "local-postgres-secret-source-missing",
      valuesPrinted: false,
      next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=<redacted>` },
    };
  }
  const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? "");
  const fields = keyValueLinesFromText(statusText(result));
  const afterBytes = numericField(fields.afterPasswordBytes) ?? 0;
  const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0;
  return {
    ok,
    mode: dryRun ? "dry-run" : "confirmed-secret-sync",
    mutation: !dryRun && fields.mutation === "true",
    namespace: spec.runtimeNamespace,
    secretName,
    secretKey: "POSTGRES_PASSWORD",
    source: {
      ok: material.ok,
      sourceRef,
      sourceKey,
      sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath),
      generated: material.generated,
      fingerprint: material.fingerprint,
      valueBytes: material.value?.length ?? 0,
      valuesPrinted: false,
    },
    before: {
      namespaceExists: fields.beforeNamespaceExists === "yes",
      secretExists: fields.beforeSecretExists === "yes",
      passwordBytes: numericField(fields.beforePasswordBytes),
      fingerprint: fields.beforePasswordFingerprint || null,
    },
    after: {
      namespaceExists: fields.afterNamespaceExists === "yes",
      secretExists: fields.afterSecretExists === "yes",
      passwordBytes: numericField(fields.afterPasswordBytes),
      fingerprint: fields.afterPasswordFingerprint || null,
    },
    action: fields.action || null,
    applyExitCode: numericField(fields.applyExitCode),
    namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode),
    degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed",
    result: compactRuntimeCommand(result),
    valuesPrinted: false,
  };
}

export function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): {
  ok: boolean;
  sourceRef: string;
  sourceKey: string;
  sourcePath: string | null;
  checkedPaths: string[];
  value: string | null;
  fingerprint: string | null;
  generated: boolean;
  error?: string;
} {
  const checkedPaths = localSecretSourcePaths(input.sourceRef);
  const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null;
  const sourcePath = existingPath ?? checkedPaths[0] ?? null;
  if (sourcePath === null) {
    return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" };
  }
  if (existingPath === null && input.dryRun) {
    return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" };
  }
  if (existingPath === null) {
    const value = randomBytes(32).toString("base64url");
    mkdirSync(dirname(sourcePath), { recursive: true });
    writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 });
    return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
  }
  const text = readFileSync(existingPath, "utf8");
  const values = parseEnvFile(text);
  const existingValue = values[input.sourceKey];
  if (existingValue !== undefined && existingValue.length > 0) {
    return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false };
  }
  if (input.dryRun) {
    return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" };
  }
  const value = randomBytes(32).toString("base64url");
  const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n";
  writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 });
  return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
}

export function localSecretSourcePaths(sourceRef: string): string[] {
  const marker = "/.worktree/";
  const index = repoRoot.indexOf(marker);
  const paths = index >= 0
    ? [
      join(repoRoot.slice(0, index), ".state", "secrets", sourceRef),
      join(repoRoot, ".state", "secrets", sourceRef),
    ]
    : [join(repoRoot, ".state", "secrets", sourceRef)];
  return [...new Set(paths)];
}

export function shortSecretFingerprint(value: string): string {
  return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`;
}

export function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string {
  return [
    "set +e",
    `namespace=${shellQuote(spec.runtimeNamespace)}`,
    `secret=${shellQuote(secretName)}`,
    "secret_key=POSTGRES_PASSWORD",
    `source_ref=${shellQuote(sourceRef)}`,
    `source_key=${shellQuote(sourceKey)}`,
    `source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`,
    `dry_run=${shellQuote(dryRun ? "true" : "false")}`,
    "field_manager=unidesk-hwlab-node-local-postgres-secret",
    "password=$(cat)",
    "password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')",
    "exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
    "namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }",
    "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }",
    "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
    "decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }",
    "before_namespace_exists=$(namespace_exists_flag)",
    "before_secret_exists=no",
    "before_b64=",
    "if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi",
    "before_password_bytes=$(decoded_length \"$before_b64\")",
    "before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")",
    "after_namespace_exists=$before_namespace_exists",
    "after_secret_exists=$before_secret_exists",
    "after_password_bytes=$before_password_bytes",
    "after_password_fingerprint=$before_password_fingerprint",
    "action=observed",
    "mutation=false",
    "namespace_create_exit=",
    "apply_exit=",
    "if [ \"$dry_run\" = true ]; then",
    "  if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi",
    "elif [ \"$password_bytes\" -le 0 ]; then",
    "  action=source-empty",
    "  apply_exit=42",
    "elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then",
    "  action=source-mismatch",
    "  apply_exit=47",
    "else",
    "  if [ \"$before_namespace_exists\" != yes ]; then",
    "    kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err",
    "    namespace_create_exit=$?",
    "  else",
    "    namespace_create_exit=0",
    "  fi",
    "  if [ \"$namespace_create_exit\" = 0 ]; then",
    "    tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)",
    "    password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')",
    "    cat >\"$tmp\" <<EOF_SECRET",
    "apiVersion: v1",
    "kind: Secret",
    "metadata:",
    "  name: $secret",
    "  namespace: $namespace",
    "  labels:",
    "    app.kubernetes.io/part-of: hwlab",
    "    app.kubernetes.io/managed-by: unidesk",
    "  annotations:",
    "    hwlab.pikastech.local/secret-source-ref: $source_ref",
    "    hwlab.pikastech.local/secret-source-key: $source_key",
    "    hwlab.pikastech.local/secret-source-fingerprint: $source_fingerprint",
    "type: Opaque",
    "data:",
    "  $secret_key: $password_b64",
    "EOF_SECRET",
    "    kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err",
    "    apply_exit=$?",
    "    rm -f \"$tmp\"",
    "    if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
    "  else",
    "    action=namespace-create-failed",
    "    apply_exit=$namespace_create_exit",
    "  fi",
    "fi",
    "after_namespace_exists=$(namespace_exists_flag)",
    "if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi",
    "after_password_bytes=$(decoded_length \"$after_b64\")",
    "after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")",
    "printf 'namespace\\t%s\\n' \"$namespace\"",
    "printf 'secret\\t%s\\n' \"$secret\"",
    "printf 'key\\t%s\\n' \"$secret_key\"",
    "printf 'sourceRef\\t%s\\n' \"$source_ref\"",
    "printf 'sourceKey\\t%s\\n' \"$source_key\"",
    "printf 'dryRun\\t%s\\n' \"$dry_run\"",
    "printf 'action\\t%s\\n' \"$action\"",
    "printf 'mutation\\t%s\\n' \"$mutation\"",
    "printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"",
    "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
    "printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"",
    "printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"",
    "printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"",
    "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
    "printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"",
    "printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"",
    "printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"",
    "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
    "password=",
    "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
    "if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi",
  ].join("\n");
}

interface NodeRuntimeImageDependency {
  id: string;
  target: string;
  source: string;
  mode: "pull" | "build-moonbridge";
  sourceRepo?: string;
  sourceRef?: string;
  builderImage?: string;
  goProxy?: string;
  dockerNetworkMode?: "default" | "host";
}

function nodeRuntimeImageDependencies(spec: HwlabRuntimeLaneSpec): NodeRuntimeImageDependency[] {
  const images: NodeRuntimeImageDependency[] = [
    { id: "base", target: spec.baseImage, source: spec.baseImageSource ?? "", mode: "pull" },
  ];
  if (spec.buildkit !== undefined) {
    images.push({ id: "buildkit", target: spec.buildkit.sidecarImage, source: spec.buildkit.sourceImage, mode: "pull" });
  }
  spec.runtimeImageRewrites.forEach((rewrite, index) => {
    images.push({ id: `rewrite-${index + 1}`, target: rewrite.target, source: rewrite.source, mode: "pull" });
  });
  spec.runtimeImageBuilds.forEach((build) => {
    images.push({
      id: build.id,
      target: build.target,
      source: `${build.sourceRepo}#${build.sourceRef}`,
      mode: "build-moonbridge",
      sourceRepo: build.sourceRepo,
      sourceRef: build.sourceRef,
      builderImage: build.builderImage,
      goProxy: build.goProxy,
      dockerNetworkMode: build.dockerNetworkMode,
    });
  });
  return images;
}

function nodeRuntimeImageCalls(spec: HwlabRuntimeLaneSpec, functionName: string): string {
  return nodeRuntimeImageDependencies(spec)
    .map((image) => [
      functionName,
      image.id,
      image.target,
      image.source,
      image.mode,
      image.sourceRepo ?? "",
      image.sourceRef ?? "",
      image.builderImage ?? "",
      image.goProxy ?? "",
      image.dockerNetworkMode ?? "",
    ].map(shellQuote).join(" "))
    .join("\n");
}

function parseNodeRuntimeImageRows(text: string): Record<string, unknown>[] {
  return text.split(/\r?\n/u).flatMap((line) => {
    if (!line.startsWith("imageJson\t")) return [];
    try {
      return [JSON.parse(line.slice("imageJson\t".length)) as Record<string, unknown>];
    } catch {
      return [];
    }
  });
}

export function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
  const script = [
    "set -eu",
    "check_image() {",
    "  id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"",
    "  repo_tag=${target#*/}",
    "  repo=${repo_tag%:*}",
    "  tag=${repo_tag##*:}",
    "  if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
    "  registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
    "  registry_present=false",
    "  if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi",
    "  target_present=false",
    "  if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi",
    "  source_present=false",
    "  if [ \"$mode\" = build-moonbridge ]; then source_present=true; elif [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi",
    "  source_configured=false",
    "  if [ -n \"$source\" ]; then source_configured=true; fi",
    "  python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$source_configured\" \"$registry_url\" \"$registry_present\" \"$target_present\" \"$source_present\" <<'PY'",
    "import json, sys",
    "keys=['id','target','source','mode','sourceConfigured','registryUrl','registryTagPresent','targetImagePresent','sourceImagePresent']",
    "data=dict(zip(keys, sys.argv[1:]))",
    "for key in ['sourceConfigured','registryTagPresent','targetImagePresent','sourceImagePresent']:",
    "    data[key] = data[key] == 'true'",
    "print('imageJson\\t' + json.dumps(data, separators=(',', ':')))",
    "PY",
    "}",
    nodeRuntimeImageCalls(spec, "check_image"),
  ].join("\n");
  const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
  const imageRows = parseNodeRuntimeImageRows(statusText(result));
  const base = imageRows.find((image) => image.id === "base") ?? {};
  return {
    ok: isCommandSuccess(result) && imageRows.length === nodeRuntimeImageDependencies(spec).length && imageRows.every((image) => image.sourceConfigured === true && image.registryTagPresent === true),
    target: base.target ?? spec.baseImage,
    source: base.source || null,
    sourceConfigured: base.sourceConfigured === true,
    registryUrl: base.registryUrl ?? null,
    registryTagPresent: base.registryTagPresent === true,
    targetImagePresent: base.targetImagePresent === true,
    sourceImagePresent: base.sourceImagePresent === true,
    images: imageRows,
    result: compactRuntimeCommand(result),
  };
}

export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
  if (spec.baseImageSource === undefined) return null;
  const script = [
    "set -eu",
    `dry_run=${shellQuote(dryRun ? "true" : "false")}`,
    `pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`,
    `build_http_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.http)}`,
    `build_https_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.https)}`,
    `build_all_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.all)}`,
    `build_no_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.noProxy.join(","))}`,
    "resolve_host_proxy_url() {",
    "  raw_url=\"$1\"",
    "  if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi",
    "  info=$(python3 - \"$raw_url\" <<'PY'",
    "from urllib.parse import urlparse",
    "import sys",
    "url = sys.argv[1]",
    "parsed = urlparse(url)",
    "host = parsed.hostname or ''",
    "port = parsed.port",
    "scheme = parsed.scheme or 'http'",
    "parts = host.split('.')",
    "if len(parts) >= 4 and parts[2] == 'svc':",
    "    print('\\t'.join(['service', scheme, parts[0], parts[1], str(port or 80)]))",
    "else:",
    "    print('\\t'.join(['literal', url]))",
    "PY",
    "  )",
    "  kind=$(printf '%s' \"$info\" | cut -f1)",
    "  service_cluster_ip() {",
    "    service_namespace=\"$1\"; service_name=\"$2\"",
    "    cluster_ip=\"\"",
    "    if command -v kubectl >/dev/null 2>&1; then",
    "      cluster_ip=$(kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)",
    "    fi",
    "    if { [ -z \"$cluster_ip\" ] || [ \"$cluster_ip\" = None ]; } && command -v k3s >/dev/null 2>&1; then",
    "      cluster_ip=$(k3s kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)",
    "    fi",
    "    printf '%s\\n' \"$cluster_ip\"",
    "  }",
    "  if [ \"$kind\" = service ]; then",
    "    scheme=$(printf '%s' \"$info\" | cut -f2)",
    "    service_name=$(printf '%s' \"$info\" | cut -f3)",
    "    service_namespace=$(printf '%s' \"$info\" | cut -f4)",
    "    service_port=$(printf '%s' \"$info\" | cut -f5)",
    "    cluster_ip=$(service_cluster_ip \"$service_namespace\" \"$service_name\")",
    "    if [ -n \"$cluster_ip\" ] && [ \"$cluster_ip\" != None ]; then printf '%s://%s:%s\\n' \"$scheme\" \"$cluster_ip\" \"$service_port\"; else printf '%s\\n' \"$raw_url\"; fi",
    "  else",
    "    printf '%s\\n' \"$raw_url\"",
    "  fi",
    "}",
    "build_http_proxy=$(resolve_host_proxy_url \"$build_http_proxy\")",
    "build_https_proxy=$(resolve_host_proxy_url \"$build_https_proxy\")",
    "build_all_proxy=$(resolve_host_proxy_url \"$build_all_proxy\")",
    "build_container_proxy_url() {",
    "  raw_url=\"$1\"",
    "  if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi",
    "  python3 - \"$raw_url\" <<'PY'",
    "from urllib.parse import urlparse, urlunparse",
    "import sys",
    "url = sys.argv[1]",
    "parsed = urlparse(url)",
    "host = parsed.hostname or ''",
    "if host in {'127.0.0.1', 'localhost'}:",
    "    netloc = 'host.docker.internal'",
    "    if parsed.port is not None:",
    "        netloc = f'{netloc}:{parsed.port}'",
    "    print(urlunparse(parsed._replace(netloc=netloc)))",
    "else:",
    "    print(url)",
    "PY",
    "}",
    "build_container_http_proxy=$(build_container_proxy_url \"$build_http_proxy\")",
    "build_container_https_proxy=$(build_container_proxy_url \"$build_https_proxy\")",
    "build_container_all_proxy=$(build_container_proxy_url \"$build_all_proxy\")",
    "docker_build_add_host_args=\"\"",
    "case \"$build_container_http_proxy $build_container_https_proxy $build_container_all_proxy\" in",
    "  *host.docker.internal*) docker_build_add_host_args=\"--add-host host.docker.internal:host-gateway\" ;;",
    "esac",
    "failed=false",
    "ensure_image() {",
    "  id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"; source_repo=\"$5\"; source_ref=\"$6\"; builder_image=\"$7\"; go_proxy=\"$8\"; docker_network_mode=\"$9\"",
    "  repo_tag=${target#*/}",
    "  repo=${repo_tag%:*}",
    "  tag=${repo_tag##*:}",
    "  if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
    "  registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
    "  present=false",
    "  if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi",
    "  action=none",
    "  source_present_before=unknown",
    "  pulled_source=false",
    "  pull_attempts=0",
    "  if [ \"$present\" = false ]; then",
    "    action=seed",
    "    if [ \"$dry_run\" = false ]; then",
    "      if [ \"$mode\" = build-moonbridge ]; then",
    "        action=build",
    "        source_present_before=build-source",
    "        tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)",
    "        dockerfile=\"$tmpdir/Dockerfile\"",
    "        cat > \"$dockerfile\" <<'DOCKERFILE'",
    "ARG BUILDER_IMAGE",
    "FROM ${BUILDER_IMAGE} AS builder",
    "ARG MOONBRIDGE_REPO",
    "ARG MOONBRIDGE_REF",
    "ARG GOPROXY_VALUE",
    "ARG HTTP_PROXY",
    "ARG HTTPS_PROXY",
    "ARG ALL_PROXY",
    "ARG NO_PROXY",
    "ENV GOPROXY=${GOPROXY_VALUE}",
    "ENV HTTP_PROXY=${HTTP_PROXY}",
    "ENV HTTPS_PROXY=${HTTPS_PROXY}",
    "ENV ALL_PROXY=${ALL_PROXY}",
    "ENV NO_PROXY=${NO_PROXY}",
    "ENV http_proxy=${HTTP_PROXY}",
    "ENV https_proxy=${HTTPS_PROXY}",
    "ENV all_proxy=${ALL_PROXY}",
    "ENV no_proxy=${NO_PROXY}",
    "WORKDIR /src",
    "RUN git clone --filter=blob:none \"${MOONBRIDGE_REPO}\" moon-bridge",
    "WORKDIR /src/moon-bridge",
    "RUN git checkout \"${MOONBRIDGE_REF}\" \\",
    "  && go mod download \\",
    "  && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags=\"-s -w\" -o /out/moonbridge ./cmd/moonbridge",
    "FROM scratch",
    "WORKDIR /app",
    "COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt",
    "COPY --from=builder /out/moonbridge /app/moonbridge",
    "EXPOSE 4001",
    "USER 65532:65532",
    "ENTRYPOINT [\"/app/moonbridge\"]",
    "DOCKERFILE",
    "        if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$build_container_https_proxy\" --build-arg ALL_PROXY=\"$build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$build_container_http_proxy\" --build-arg https_proxy=\"$build_container_https_proxy\" --build-arg all_proxy=\"$build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then",
    "          docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }",
    "        else",
    "          cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true",
    "          failed=true",
    "        fi",
    "        rm -rf \"$tmpdir\"",
    "      else",
    "      source_present_before=true",
    "      if ! docker image inspect \"$source\" >/dev/null 2>&1; then",
    "        source_present_before=false",
    "        attempt=1",
    "        while [ \"$attempt\" -le \"$pull_retries\" ]; do",
    "          pull_attempts=$attempt",
    "          if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker pull \"$source\" >/tmp/hwlab-node-runtime-image-$id-pull.out 2>&1; then pulled_source=true; break; fi",
    "          attempt=$((attempt + 1))",
    "          sleep 2",
    "        done",
    "        if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-runtime-image-$id-pull.out >&2 2>/dev/null || true; failed=true; fi",
    "      fi",
    "      if [ \"$failed\" != true ]; then",
    "        if docker image inspect \"$source\" >/dev/null 2>&1; then",
    "          docker tag \"$source\" \"$target\"",
    "          docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || failed=true",
    "        else",
    "          failed=true",
    "        fi",
    "      fi",
    "      fi",
    "    fi",
    "  fi",
    "  after=false",
    "  if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi",
    "  python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$dry_run\" \"$present\" \"$after\" \"$action\" \"$pull_retries\" \"$source_present_before\" \"$pulled_source\" \"$pull_attempts\" <<'PY'",
    "import json, sys",
    "keys=['id','target','source','mode','dryRun','presentBefore','presentAfter','action','pullRetries','sourcePresentBefore','pulledSource','pullAttempts']",
    "data=dict(zip(keys, sys.argv[1:]))",
    "for key in ['dryRun','presentBefore','presentAfter','pulledSource']:",
    "    data[key] = data[key] == 'true'",
    "for key in ['pullRetries','pullAttempts']:",
    "    data[key] = int(data[key]) if str(data[key]).isdigit() else None",
    "print('imageJson\\t' + json.dumps(data, separators=(',', ':')))",
    "PY",
    "  if [ \"$dry_run\" = false ] && [ \"$after\" != true ]; then failed=true; fi",
    "}",
    nodeRuntimeImageCalls(spec, "ensure_image"),
    "if [ \"$failed\" = true ]; then exit 1; fi",
  ].join("\n");
  const hasBuild = nodeRuntimeImageDependencies(spec).some((image) => image.mode === "build-moonbridge");
  const result = hasBuild && !dryRun
    ? runNodeHostScriptAsync(spec, script, Math.min(timeoutSeconds, 600), "runtime-image-build")
    : runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
  const imageRows = parseNodeRuntimeImageRows(statusText(result));
  const base = imageRows.find((image) => image.id === "base") ?? {};
  return {
    ok: isCommandSuccess(result),
    dryRun,
    target: base.target ?? spec.baseImage,
    source: base.source ?? spec.baseImageSource,
    presentBefore: base.presentBefore === true,
    presentAfter: base.presentAfter === true,
    action: base.action ?? null,
    pullRetries: numericField(typeof base.pullRetries === "number" ? String(base.pullRetries) : undefined),
    sourcePresentBefore: typeof base.sourcePresentBefore === "string" ? base.sourcePresentBefore : null,
    pulledSource: base.pulledSource === true,
    pullAttempts: numericField(typeof base.pullAttempts === "number" ? String(base.pullAttempts) : undefined),
    images: imageRows,
    result: compactRuntimeCommand(result),
  };
}

export function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
  const pg = hwlabRuntimeActiveExternalPostgres(spec);
  if (pg === undefined) return "true";
  const authnKey = pg.openfga.authnKey ?? "authn-preshared-key";
  return [
    "set -eu",
    `namespace=${shellQuote(spec.runtimeNamespace)}`,
    `secret=${shellQuote(pg.openfga.secretName)}`,
    `authn_key=${shellQuote(authnKey)}`,
    `dry_run=${shellQuote(dryRun ? "true" : "false")}`,
    "namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"",
    "if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi",
    "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -",
    "authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')",
    "if [ \"${authn_present:-0}\" -gt 0 ]; then",
    "  action=kept",
    "elif [ \"$dry_run\" = true ]; then",
    "  action=would-generate-authn-key",
    "else",
    "  if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi",
    "  kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null",
    "  authn_value=",
    "  action=generated-authn-key",
    "fi",
    "printf 'namespace\\t%s\\n' \"$namespace\"",
    "printf 'secret\\t%s\\n' \"$secret\"",
    "printf 'authnKey\\t%s\\n' \"$authn_key\"",
    "printf 'action\\t%s\\n' \"$action\"",
    "printf 'dryRun\\t%s\\n' \"$dry_run\"",
    "printf 'valuesPrinted\\tfalse\\n'",
  ].join("\n");
}

export function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string {
  const configRef = spec.externalPostgres?.configRef;
  if (configRef === undefined) return join(repoRoot, ".state", "secrets");
  const configPath = rootPath(configRef);
  const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown;
  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`);
  const secrets = (parsed as Record<string, unknown>).secrets;
  if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`);
  const root = (secrets as Record<string, unknown>).root;
  if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`);
  return root.startsWith("/") ? root : rootPath(root);
}

export function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } {
  const sourcePath = join(secretRoot, sourceRef);
  if (!existsSync(sourcePath)) return { ok: false, sourcePath };
  const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
  const value = values[key];
  if (value === undefined || value.length === 0) return { ok: false, sourcePath };
  return { ok: true, value, sourcePath };
}

export function secretSourcePaths(sourceRef: string): string[] {
  const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
  const marker = "/.worktree/";
  const index = repoRoot.indexOf(marker);
  if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef));
  return [...new Set(paths)];
}

export function displayRepoPath(path: string): string {
  const normalizedRoot = repoRoot.replace(/\/+$/u, "");
  if (path === normalizedRoot) return ".";
  if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1);
  const marker = "/.worktree/";
  const index = normalizedRoot.indexOf(marker);
  if (index >= 0) {
    const mainRoot = normalizedRoot.slice(0, index);
    if (path === mainRoot) return ".";
    if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1);
  }
  return path;
}

export function hwlabPasswordHash(password: string): string {
  const salt = randomBytes(16).toString("hex");
  return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`;
}

export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial {
  const sourceRef = spec.bootstrapAdminPasswordSourceRef;
  const sourceKey = spec.bootstrapAdminPasswordSourceKey;
  if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
    return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "bootstrap-admin-yaml-source-missing" };
  }
  const paths = secretSourcePaths(sourceRef);
  const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
  if (!existsSync(sourcePath)) {
    return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "secret-source-missing" };
  }
  const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
  const password = values[sourceKey];
  if (password === undefined || password.length === 0) {
    return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, passwordHash: null, error: "secret-key-missing" };
  }
  return {
    ok: true,
    sourceRef,
    sourceKey,
    sourcePath,
    sourcePresent: true,
    sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
    passwordHash: hwlabPasswordHash(password),
    error: null,
  };
}

export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial {
  const sourceRef = spec.bootstrapAdminPasswordSourceRef;
  const sourceKey = spec.bootstrapAdminPasswordSourceKey;
  if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
    return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, password: null, error: "bootstrap-admin-yaml-source-missing" };
  }
  const paths = secretSourcePaths(sourceRef);
  const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
  const runtimePassword = process.env[sourceKey];
  if (!existsSync(sourcePath) && (runtimePassword === undefined || runtimePassword.length === 0)) {
    return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" };
  }
  const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
  const password = values[sourceKey] ?? runtimePassword;
  if (password === undefined || password.length === 0) {
    return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" };
  }
  return {
    ok: true,
    sourceRef,
    sourceKey,
    sourcePath,
    sourcePresent: true,
    sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
    password,
    error: null,
  };
}

export function parseEnvFile(text: string): Record<string, string> {
  const values: Record<string, string> = {};
  for (const rawLine of text.split(/\r?\n/u)) {
    const line = rawLine.trim();
    if (line.length === 0 || line.startsWith("#")) continue;
    const eq = line.indexOf("=");
    if (eq <= 0) continue;
    const key = line.slice(0, eq).trim();
    const rawValue = line.slice(eq + 1).trim();
    values[key] = rawValue.startsWith("'") && rawValue.endsWith("'")
      ? rawValue.slice(1, -1).replace(/'"'"'/gu, "'")
      : rawValue.startsWith("\"") && rawValue.endsWith("\"")
        ? rawValue.slice(1, -1)
        : rawValue;
  }
  return values;
}

export function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean {
  if (!/postgres|pgsql|pgdata/iu.test(name)) return false;
  const externalServiceName = spec.externalPostgres?.serviceName;
  if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false;
  return true;
}

export function nodeRuntimeCodeAgentRuntimeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
  const runtime = spec.codeAgentRuntime;
  if (runtime === undefined || runtime.enabled === false) return { required: false, ready: true, enabled: runtime?.enabled ?? false };
  if (!namespaceExists) return { required: true, ready: false, enabled: true, degradedReason: "runtime-namespace-missing" };
  const managerTarget = nodeRuntimeCodeAgentManagerTarget(runtime.managerUrl, runtime.runnerNamespace);
  const runnerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.runnerNamespace, "-o", "name"], 60);
  const secretNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.secretNamespace, "-o", "name"], 60);
  const managerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", managerTarget.namespace, "-o", "name"], 60);
  const managerService = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "service", managerTarget.serviceName, "-o", "name"], 60);
  const managerDeployment = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "deployment", managerTarget.serviceName, "-o", "jsonpath={.status.readyReplicas}{\"\\n\"}{.spec.replicas}{\"\\n\"}"], 60);
  const [managerReadyReplicasRaw = "", managerDesiredReplicasRaw = ""] = managerDeployment.stdout.split(/\r?\n/u);
  const managerReadyReplicas = numericField(managerReadyReplicasRaw);
  const managerDesiredReplicas = numericField(managerDesiredReplicasRaw);
  const managerDeploymentReady = managerDeployment.exitCode === 0 && (managerReadyReplicas ?? 0) >= Math.max(1, managerDesiredReplicas ?? 1);
  const apiKey = secretKeyStatus(spec, runtime.apiKeySecretName, runtime.apiKeySecretKey);
  const cloudApiEnv = nodeRuntimeCodeAgentCloudApiEnvStatus(spec, runtime);
  const ready = runnerNamespace.exitCode === 0
    && secretNamespace.exitCode === 0
    && managerNamespace.exitCode === 0
    && managerService.exitCode === 0
    && managerDeploymentReady
    && apiKey.present === true
    && cloudApiEnv.ready === true;
  const degradedReason = ready
    ? undefined
    : runnerNamespace.exitCode !== 0
      ? "agentrun-runner-namespace-missing"
      : secretNamespace.exitCode !== 0
        ? "agentrun-secret-namespace-missing"
        : managerNamespace.exitCode !== 0
          ? "agentrun-manager-namespace-missing"
          : managerService.exitCode !== 0
            ? "agentrun-manager-service-missing"
            : !managerDeploymentReady
              ? "agentrun-manager-deployment-not-ready"
              : apiKey.present !== true
                ? "agentrun-api-key-secret-missing"
                : "hwlab-cloud-api-code-agent-env-mismatch";
  return {
    required: true,
    enabled: true,
    ready,
    degradedReason,
    adapter: runtime.adapter,
    managerUrl: runtime.managerUrl,
    manager: {
      namespace: managerTarget.namespace,
      serviceName: managerTarget.serviceName,
      serviceExists: managerService.exitCode === 0,
      deploymentReady: managerDeploymentReady,
      readyReplicas: managerReadyReplicas,
      desiredReplicas: managerDesiredReplicas,
      namespaceResult: compactRuntimeCommand(managerNamespace),
      serviceResult: compactRuntimeCommand(managerService),
      deploymentResult: compactRuntimeCommand(managerDeployment),
    },
    runnerNamespace: {
      name: runtime.runnerNamespace,
      exists: runnerNamespace.exitCode === 0,
      result: compactRuntimeCommand(runnerNamespace),
    },
    secretNamespace: {
      name: runtime.secretNamespace,
      exists: secretNamespace.exitCode === 0,
      result: compactRuntimeCommand(secretNamespace),
    },
    apiKey,
    cloudApiEnv,
    repoUrlFrom: runtime.repoUrlFrom,
    providerIdFrom: runtime.providerIdFrom,
    valuesPrinted: false,
  };
}

function nodeRuntimeCodeAgentManagerTarget(managerUrl: string, fallbackNamespace: string): { namespace: string; serviceName: string } {
  let parsed: URL | null = null;
  try {
    parsed = new URL(managerUrl);
  } catch {
    parsed = null;
  }
  const host = parsed?.hostname ?? "";
  const parts = host.split(".").filter(Boolean);
  if (parts.length >= 3 && parts[2] === "svc") return { serviceName: parts[0], namespace: parts[1] };
  return { serviceName: parts[0] || "agentrun-mgr", namespace: parts[1] || fallbackNamespace };
}

function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable<HwlabRuntimeLaneSpec["codeAgentRuntime"]>): Record<string, unknown> {
  const deployment = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deployment", "hwlab-cloud-api", "-o", "json"], 60);
  if (deployment.exitCode !== 0) {
    return {
      ready: false,
      deploymentExists: false,
      mismatches: [{ name: "hwlab-cloud-api", reason: "deployment-missing" }],
      result: compactCodeAgentDeploymentProbeResult(deployment),
      valuesPrinted: false,
    };
  }
  let parsed: unknown;
  try {
    parsed = JSON.parse(deployment.stdout) as unknown;
  } catch (error) {
    return {
      ready: false,
      deploymentExists: true,
      mismatches: [{ name: "hwlab-cloud-api", reason: "deployment-json-invalid", message: error instanceof Error ? error.message : String(error) }],
      result: compactCodeAgentDeploymentProbeResult(deployment),
      valuesPrinted: false,
    };
  }
  const deploy = record(parsed);
  const specRecord = record(record(record(deploy.spec).template).spec);
  const containers = Array.isArray(specRecord.containers) ? specRecord.containers.map(record) : [];
  const container = containers.find((item) => item.name === "hwlab-cloud-api") ?? containers[0];
  const env = Array.isArray(container?.env) ? container.env.map(record) : [];
  const envByName = new Map(env.map((item) => [String(item.name || ""), item]));
  const mismatches: Array<Record<string, unknown>> = [];
  const expectValue = (name: string, expected: string): void => {
    const item = envByName.get(name);
    const value = typeof item?.value === "string" ? item.value : null;
    if (value !== expected) mismatches.push({ name, expected, value, kind: "value" });
  };
  const expectSecret = (name: string, expectedSecret: string, expectedKey: string): void => {
    const item = envByName.get(name);
    const secretRef = record(record(item?.valueFrom).secretKeyRef);
    if (secretRef.name !== expectedSecret || secretRef.key !== expectedKey) {
      mismatches.push({ name, expectedSecret, expectedKey, secret: secretRef.name ?? null, key: secretRef.key ?? null, kind: "secretKeyRef" });
    }
  };
  expectValue("HWLAB_CODE_AGENT_ADAPTER", runtime.adapter);
  expectValue("AGENTRUN_MGR_URL", runtime.managerUrl);
  expectSecret("AGENTRUN_API_KEY", runtime.apiKeySecretName, runtime.apiKeySecretKey);
  expectValue("HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE", runtime.runnerNamespace);
  expectValue("HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE", runtime.secretNamespace);
  expectValue("HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", spec.gitReadUrl);
  expectValue("HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", spec.nodeId);
  expectValue("HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE", runtime.defaultProviderProfile);
  expectValue("HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", runtime.codexStdioSupervisor);
  return {
    ready: mismatches.length === 0,
    deploymentExists: true,
    deploymentName: "hwlab-cloud-api",
    containerName: container?.name ?? null,
    checkedEnvCount: 8,
    mismatches,
    result: compactCodeAgentDeploymentProbeResult(deployment),
    valuesPrinted: false,
  };
}

function compactCodeAgentDeploymentProbeResult(result: CommandResult): Record<string, unknown> {
  return {
    exitCode: result.exitCode,
    stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
    stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
    stderrTail: result.stderr.slice(-600),
    timedOut: result.timedOut === true,
  };
}

export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
  const pg = hwlabRuntimeActiveExternalPostgres(spec);
  if (pg === undefined) return { required: false, ready: true };
  if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
  const runtimeAccess = pg.runtimeAccess ?? { endpointAddress: pg.endpointAddress, port: pg.port };
  const endpointIsIpv4 = /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/u.test(runtimeAccess.endpointAddress);
  const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "jsonpath={.spec.type}{\"\\n\"}{.spec.externalName}{\"\\n\"}{.spec.ports[0].port}{\"\\n\"}"], 60);
  const [serviceType = "", externalName = "", servicePort = ""] = service.stdout.split(/\r?\n/u);
  if (!endpointIsIpv4) {
    const ready = service.exitCode === 0 && serviceType === "ExternalName" && externalName === runtimeAccess.endpointAddress && Number(servicePort) === runtimeAccess.port;
    return {
      required: true,
      ready,
      serviceName: pg.serviceName,
      serviceType: serviceType || null,
      externalName: externalName || null,
      expectedExternalName: runtimeAccess.endpointAddress,
      port: numericField(servicePort),
      expectedPort: runtimeAccess.port,
      providerEndpointAddress: pg.endpointAddress,
      providerPort: pg.port,
      runtimeAccess: pg.runtimeAccess ?? null,
      service: compactRuntimeCommand(service),
    };
  }
  const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
  const [addressType = "", address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u);
  return {
    required: true,
    ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port,
    serviceName: pg.serviceName,
    serviceType: serviceType || null,
    addressType: addressType || null,
    endpointAddress: address || null,
    expectedEndpointAddress: runtimeAccess.endpointAddress,
    port: numericField(port),
    expectedPort: runtimeAccess.port,
    providerEndpointAddress: pg.endpointAddress,
    providerPort: pg.port,
    runtimeAccess: pg.runtimeAccess ?? null,
    service: compactRuntimeCommand(service),
    endpointSlice: compactRuntimeCommand(endpointSlice),
  };
}

export function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
  const pg = hwlabRuntimeActiveExternalPostgres(spec);
  if (pg === undefined) return { required: false, ready: true };
  if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
  const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey);
  const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey);
  const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key");
  return {
    required: true,
    ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present,
    valuesPrinted: false,
    cloudApi,
    openfga: {
      datastoreUri: openfgaUri,
      authnPresharedKey: openfgaAuthn,
    },
  };
}

export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec {
  const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
  const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
  const nodes = record(parsed.nodes);
  const node = record(nodes[spec.nodeId]);
  const targets = Array.isArray(parsed.targets) ? parsed.targets : [];
  const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane);
  if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
  const gitMirror = record(target.gitMirror);
  const gitMirrorEgressProxy = record(gitMirror.egressProxy);
  const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode");
  if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`);
  const nodeEgressProxy = gitMirrorEgressMode === "direct"
    ? { mode: "direct" as const, required: false as const }
    : nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
  if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
  const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
  const source = record(target.source);
  const gitops = record(target.gitops);
  const tekton = record(target.tekton);
  const toolsImage = record(tekton.toolsImage);
  const toolsImagePullPolicy = optionalStringValue(toolsImage.imagePullPolicy, "tekton.toolsImage.imagePullPolicy") ?? "IfNotPresent";
  if (toolsImagePullPolicy !== "Always" && toolsImagePullPolicy !== "IfNotPresent" && toolsImagePullPolicy !== "Never") throw new Error("tekton.toolsImage.imagePullPolicy must be Always, IfNotPresent, or Never");
  return {
    id: stringValue(target.id, "target.id"),
    node: stringValue(target.node, "target.node"),
    lane: stringValue(target.lane, "target.lane"),
    namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"),
    serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
    serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
    cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
    cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"),
    cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
    servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"),
    secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
    syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"),
    syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
    flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"),
    toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"),
    toolsImagePullPolicy,
    sourceRepository: stringValue(source.repository, "source.repository"),
    sourceBranch: stringValue(source.branch, "source.branch"),
    gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
    egressProxy: nodeEgressProxy,
    githubTransport,
  };
}

export function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorGithubTransportSpec {
  const mode = stringValue(raw.mode, `${path}.mode`);
  if (mode === "ssh") {
    const knownHostsSecretKey = optionalStringValue(raw.knownHostsSecretKey, `${path}.knownHostsSecretKey`);
    const knownHostsSourceRef = optionalStringValue(raw.knownHostsSourceRef, `${path}.knownHostsSourceRef`);
    const knownHostsSourceKey = optionalStringValue(raw.knownHostsSourceKey, `${path}.knownHostsSourceKey`);
    const knownHostsSourceEncoding = raw.knownHostsSourceEncoding === undefined ? null : gitMirrorSecretSourceEncoding(raw.knownHostsSourceEncoding, `${path}.knownHostsSourceEncoding`);
    const knownHostsValues = [knownHostsSecretKey, knownHostsSourceRef, knownHostsSourceKey, knownHostsSourceEncoding];
    if (knownHostsValues.some((value) => value !== null) && knownHostsValues.some((value) => value === null)) {
      throw new Error(`${path}.knownHostsSecretKey/sourceRef/sourceKey/sourceEncoding must be declared together`);
    }
    return {
      mode,
      privateKeySecretKey: stringValue(raw.privateKeySecretKey, `${path}.privateKeySecretKey`),
      privateKeySourceRef: stringValue(raw.privateKeySourceRef, `${path}.privateKeySourceRef`),
      privateKeySourceKey: stringValue(raw.privateKeySourceKey, `${path}.privateKeySourceKey`),
      privateKeySourceEncoding: gitMirrorSecretSourceEncoding(raw.privateKeySourceEncoding, `${path}.privateKeySourceEncoding`),
      knownHostsSecretKey,
      knownHostsSourceRef,
      knownHostsSourceKey,
      knownHostsSourceEncoding,
    };
  }
  if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
  return {
    mode,
    username: stringValue(raw.username, `${path}.username`),
    tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`),
    tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`),
    tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`),
    tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`),
  };
}

function gitMirrorSecretSourceEncoding(raw: unknown, path: string): "plain" | "base64" {
  const value = stringValue(raw, path);
  if (value !== "plain" && value !== "base64") throw new Error(`${path} must be plain or base64`);
  return value;
}

export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorEgressProxySpec {
  const mode = stringValue(raw.mode, `${path}.mode`);
  if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`);
  const port = Number(raw.port);
  if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`);
  const sourceConfigRef = optionalStringValue(raw.sourceConfigRef, `${path}.sourceConfigRef`) ?? null;
  const source = sourceConfigRef === null ? null : resolveEgressProxySourceRef(sourceConfigRef, `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.${path}.sourceConfigRef`);
  const sourceType = source?.sourceType ?? stringValue(raw.sourceType, `${path}.sourceType`);
  if (sourceType !== "subscription-url" && sourceType !== "master-shadowsocks") throw new Error(`${path}.sourceType must be subscription-url or master-shadowsocks`);
  if (raw.sourceType !== undefined && raw.sourceType !== sourceType) throw new Error(`${path}.sourceType must match sourceConfigRef value ${sourceType}`);
  const sourceRef = source?.sourceRef ?? stringValue(raw.sourceRef, `${path}.sourceRef`);
  const sourceKey = source?.sourceKey ?? stringValue(raw.sourceKey, `${path}.sourceKey`);
  if (source !== null) {
    if (raw.sourceRef !== undefined && raw.sourceRef !== source.sourceRef) throw new Error(`${path}.sourceRef must match sourceConfigRef value ${source.sourceRef}`);
    if (raw.sourceKey !== undefined && raw.sourceKey !== source.sourceKey) throw new Error(`${path}.sourceKey must match sourceConfigRef value ${source.sourceKey}`);
  }
  const preferredOutbound = source?.preferredOutbound ?? (raw.preferredOutbound === undefined ? null : stringValue(raw.preferredOutbound, `${path}.preferredOutbound`));
  if (preferredOutbound !== null && preferredOutbound !== "vless-reality" && preferredOutbound !== "hysteria2") throw new Error(`${path}.preferredOutbound must be vless-reality or hysteria2`);
  if (source !== null && source.preferredOutbound !== null && raw.preferredOutbound !== undefined && raw.preferredOutbound !== source.preferredOutbound) {
    throw new Error(`${path}.preferredOutbound must match sourceConfigRef value ${source.preferredOutbound}`);
  }
  const noProxyRaw = raw.noProxy;
  if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
  const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`));
  return {
    mode,
    clientName: stringValue(raw.clientName, `${path}.clientName`),
    namespace: stringValue(raw.namespace, `${path}.namespace`),
    serviceName: stringValue(raw.serviceName, `${path}.serviceName`),
    port,
    sourceConfigRef,
    sourceRef,
    sourceKey,
    sourceType,
    preferredOutbound,
    sourceFingerprint: source?.fingerprint ?? null,
    noProxy,
  };
}

export function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record<string, unknown> {
  const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60);
  return {
    secret,
    key,
    present: result.exitCode === 0 && result.stdout.trim().length > 0,
    valueBytesBase64: result.stdout.trim().length,
    exitCode: result.exitCode,
    stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500),
    valuesPrinted: false,
  };
}

export function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const commandArgs = [
    "hwlab",
    "nodes",
    options.domain,
    ...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
    "--node",
    options.node,
    "--lane",
    options.lane,
    "--confirm",
    "--timeout-seconds",
    String(options.timeoutSeconds),
    "--wait",
  ];
  const command = ["bun", "scripts/cli.ts", ...commandArgs];
  const job = startJob(
    `hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
    command,
    `Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
  );
  return {
    ok: true,
    command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
    node: options.node,
    lane: options.lane,
    mode: "async-job",
    reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
    job,
    statusCommand: `bun scripts/cli.ts job status ${job.id}`,
    waitCommand: command.join(" "),
  };
}

export function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const rewritten = rewriteDelegatedNodeValue(value, scoped);
  const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
  return {
    ...result,
    node: scoped.node,
    commandFamily: "hwlab nodes",
  };
}

export function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
  if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
  if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
  if (typeof value !== "object" || value === null) return value;
  return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
}

export function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
  const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
    const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
    return text
      .replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
      .replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
  };
  return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
}

export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
  const [actionRaw] = args;
  if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "screenshot" && actionRaw !== "observe" && actionRaw !== "sentinel") throw new Error("web-probe usage: run|script|screenshot|observe|sentinel --node NODE --lane vNN [--url URL]");
  if (actionRaw === "sentinel") return parseNodeWebProbeSentinelOptions(args.slice(1));
  if (actionRaw === "observe") {
    const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1));
    const explicitNode = optionValue(args, "--node");
    const explicitLane = optionValue(args, "--lane");
    const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null;
    const indexed = requestedId === null
      ? null
      : readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null);
    const node = explicitNode ?? indexed?.node;
    const lane = explicitLane ?? indexed?.lane;
    if (node === undefined || lane === undefined) {
      if (requestedId !== null) {
        throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`);
      }
      throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id");
    }
    assertNodeId(node);
    assertLane(lane);
    if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
    const spec = hwlabRuntimeLaneSpecForNode(lane, node);
    return parseNodeWebProbeObserveOptions(normalized.args, node, lane, spec, normalized.id, indexed);
  }
  const node = requiredOption(args, "--node");
  assertNodeId(node);
  const lane = requiredOption(args, "--lane");
  assertLane(lane);
  if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
  const spec = hwlabRuntimeLaneSpecForNode(lane, node);
  if (actionRaw === "screenshot") {
    assertKnownOptions(args.slice(1), new Set([
      "--node",
      "--lane",
      "--url",
      "--path",
      "--viewport",
      "--local-dir",
      "--name",
      "--timeout-ms",
      "--wait-until",
      "--selector",
      "--wait-timeout-ms",
      "--command-timeout-seconds",
    ]), new Set([
      "--full-page",
      "--no-full-page",
      "--keep-remote",
    ]));
    const url = optionValue(args, "--url");
    const targetPath = optionValue(args, "--path") ?? null;
    if (url !== undefined && targetPath !== null) throw new Error("web-probe screenshot accepts --url or --path, not both");
    const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000);
    const waitTimeoutMs = positiveIntegerOption(args, "--wait-timeout-ms", Math.max(90000, timeoutMs + 30000), 600000);
    const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.ceil(waitTimeoutMs / 1000) + 45, 900);
    return {
      action: "screenshot",
      node,
      lane,
      url: url ?? resolveWebProbeScreenshotUrl(spec, targetPath ?? "/"),
      path: targetPath,
      viewport: parseWebProbeScreenshotViewport(optionValue(args, "--viewport") ?? "1440x900"),
      localDir: optionValue(args, "--local-dir") ?? "/tmp",
      name: parseWebProbeScreenshotName(optionValue(args, "--name") ?? `web-probe-${node.toLowerCase()}-${lane}.png`),
      timeoutMs,
      waitUntil: parseWebProbeScreenshotWaitUntil(optionValue(args, "--wait-until") ?? "networkidle"),
      fullPage: !args.includes("--no-full-page"),
      selector: optionValue(args, "--selector") ?? null,
      keepRemote: args.includes("--keep-remote"),
      waitTimeoutMs,
      commandTimeoutSeconds,
    };
  }
  if (actionRaw === "script") {
    assertKnownOptions(args.slice(1), new Set([
      "--node",
      "--lane",
      "--url",
      "--timeout-ms",
      "--viewport",
      "--browser-proxy-mode",
      "--script-file",
      "--command-timeout-seconds",
    ]), new Set([]));
    const scriptFile = optionValue(args, "--script-file");
    if (scriptFile === undefined && process.stdin.isTTY) {
      throw new Error("web-probe script requires a stdin heredoc or --script-file <path>");
    }
    const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8");
    if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script");
    const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
    const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600);
    return {
      action: "script",
      node,
      lane,
      url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
      timeoutMs,
      viewport: optionValue(args, "--viewport") ?? "1440x900",
      browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode),
      commandTimeoutSeconds,
      scriptText,
      scriptSource: {
        kind: scriptFile === undefined ? "stdin" : "file",
        path: scriptFile ?? null,
        byteCount: Buffer.byteLength(scriptText),
        sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
      },
    };
  }
  assertKnownOptions(args.slice(1), new Set([
    "--node",
    "--lane",
    "--url",
    "--timeout-ms",
    "--wait-after-submit-ms",
    "--wait-messages-ms",
    "--wait-agent-terminal-ms",
    "--trace-sample-count",
    "--trace-sample-interval-ms",
    "--message",
    "--conversation-id",
    "--command-timeout-seconds",
  ]), new Set([
    "--fresh-session",
    "--no-cancel-running",
  ]));
  const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
  const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000);
  const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000);
  const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000);
  const message = optionValue(args, "--message") ?? null;
  const hasMessage = message !== null;
  const traceSampleCountDefault = hasMessage ? 2 : 0;
  const traceSampleIntervalDefault = hasMessage ? 1500 : 0;
  const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200);
  const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000);
  const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({
    timeoutMs,
    waitAfterSubmitMs,
    waitMessagesMs,
    waitAgentTerminalMs,
    traceSampleCount,
    traceSampleIntervalMs,
    freshSession: args.includes("--fresh-session"),
    hasMessage,
  });
  const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds");
  const commandTimeoutUserProvided = commandTimeoutRaw !== undefined;
  const commandTimeoutSeconds = commandTimeoutUserProvided
    ? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds)
    : commandTimeoutAutoSeconds;
  return {
    action: "run",
    node,
    lane,
    url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
    timeoutMs,
    waitAfterSubmitMs,
    waitMessagesMs,
    waitAgentTerminalMs,
    traceSampleCount,
    traceSampleIntervalMs,
    message,
    conversationId: optionValue(args, "--conversation-id") ?? null,
    freshSession: args.includes("--fresh-session"),
    cancelRunning: !args.includes("--no-cancel-running"),
    commandTimeoutSeconds,
    commandTimeoutAutoSeconds,
    commandTimeoutUserProvided,
  };
}

function resolveWebProbeScreenshotUrl(spec: HwlabRuntimeLaneSpec, targetPath: string): string {
  const origin = nodeWebProbeDefaultUrl(spec);
  try {
    return new URL(targetPath || "/", origin).toString();
  } catch {
    throw new Error(`web-probe screenshot --path cannot be resolved against ${origin}: ${targetPath}`);
  }
}

function parseWebProbeScreenshotViewport(value: string): string {
  if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(value)) throw new Error(`web-probe screenshot --viewport must look like 1440x900, got ${value}`);
  const [widthRaw, heightRaw] = value.split("x");
  const width = Number(widthRaw);
  const height = Number(heightRaw);
  if (width < 240 || width > 7680 || height < 240 || height > 4320) throw new Error(`web-probe screenshot --viewport out of range: ${value}`);
  return value;
}

function parseWebProbeScreenshotName(value: string): string {
  if (!/^[A-Za-z0-9._-]{1,120}$/u.test(value)) throw new Error("web-probe screenshot --name must contain only letters, numbers, dot, underscore, or dash, max 120 chars");
  return value.endsWith(".png") ? value : `${value}.png`;
}

function parseWebProbeScreenshotWaitUntil(value: string): "load" | "domcontentloaded" | "networkidle" | "commit" {
  if (value === "load" || value === "domcontentloaded" || value === "networkidle" || value === "commit") return value;
  throw new Error(`web-probe screenshot --wait-until must be load, domcontentloaded, networkidle, or commit; got ${value}`);
}
