// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. public-exposure module for scripts/src/hwlab-node-impl.ts.

// Moved mechanically from scripts/src/hwlab-node-impl.ts:11201-12084 for #903.

// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
import { createHash, randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import type { RenderedCliResult } from "../output";

import type { CodeAgentProviderSecretMaterial, NodePublicExposureOptions, NodeSecretOptions, RuntimeSecretSpec } from "./entry";
import { BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, BOOTSTRAP_ADMIN_SOURCE_NAMESPACE, BOOTSTRAP_ADMIN_SOURCE_SECRET, CLOUD_API_DB_KEY, CODE_AGENT_PROVIDER_OPENAI_KEY, CODE_AGENT_PROVIDER_OPENCODE_KEY, CODE_AGENT_PROVIDER_SOURCE_NAMESPACE, CODE_AGENT_PROVIDER_SOURCE_SECRET, MASTER_ADMIN_API_KEY_KEY, OPENFGA_AUTHN_KEY, OPENFGA_DATASTORE_URI_KEY, OPENFGA_POSTGRES_PASSWORD_KEY } from "./entry";
import { transPath } from "./runtime-common";
import { bootstrapAdminSecretScript, cloudApiDbSecretScript, codeAgentProviderSecretScript, masterAdminApiKeySecretScript, obsoletePlatformDbCleanupScript, obsoletePlatformDbStatusFromText, obsoleteSecretCleanupScript, openFgaSecretScript, ownedPostgresCleanupScript, platformDbSecretStatusScript, secretStatusFromText } from "./secret-scripts";
import { assertLane, assertNodeId, compactCommandResult, keyValueLinesFromText, numericField, optionValue, positiveIntegerOption, readMasterAdminApiKey, requiredOption, shellQuote, statusText } from "./utils";
import { displayRepoPath, localSecretSourcePaths, parseEnvFile, readBootstrapAdminSecretMaterial, shortSecretFingerprint, syncNodeExternalPostgresSecrets } from "./web-probe";
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";

export function isSafeWebProbeScriptRunDir(value: string | null): value is string {
  return typeof value === "string"
    && value.length > 0
    && !value.includes("\0")
    && !value.includes("..")
    && /\.state\/web-probe-script\/run\.[A-Za-z0-9]+$/u.test(value);
}

export function isSafeWebProbeScriptReportPath(value: string | null): value is string {
  return typeof value === "string"
    && isSafeWebProbeScriptArtifactPath(value)
    && value.endsWith("/web-probe-script-report.json");
}

export function isSafeWebProbeScriptArtifactPath(value: string): boolean {
  return typeof value === "string"
    && value.length > 0
    && !value.includes("\0")
    && !value.includes("..")
    && /\.state\/web-probe-script\/run\.[A-Za-z0-9]+\/[^/]+[.](?:png|json)$/u.test(value);
}

export function parseSecretOptions(args: string[]): NodeSecretOptions {
  const [actionRaw] = args;
  if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
    throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-vNN-bootstrap-admin|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]");
  }
  const node = requiredOption(args, "--node");
  assertNodeId(node);
  const lane = requiredOption(args, "--lane");
  assertLane(lane);
  const spec = runtimeSecretSpec({ node, lane });
  const name = optionValue(args, "--name") ?? spec.openFgaSecret;
  const key = optionValue(args, "--key");
  const confirm = args.includes("--confirm");
  const explicitDryRun = args.includes("--dry-run");
  if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
  if (actionRaw === "cleanup-owned-postgres") {
    if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
    if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
    if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`);
    return {
      action: actionRaw,
      node,
      lane,
      name: spec.postgresSecret,
      preset: "owned-postgres-cleanup",
      confirm,
      dryRun: explicitDryRun || !confirm,
      timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
    };
  }
  if (actionRaw === "cleanup-obsolete") {
    if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup");
    if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key");
    if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`);
    return {
      action: actionRaw,
      node,
      lane,
      name,
      preset: "obsolete-secret-cleanup",
      confirm,
      dryRun: explicitDryRun || !confirm,
      timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
    };
  }
  if (name === spec.masterAdminApiKeySecret) {
    if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
    return {
      action: actionRaw,
      node,
      lane,
      name,
      key: key ?? MASTER_ADMIN_API_KEY_KEY,
      preset: "master-server-admin-api-key",
      confirm,
      dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
      timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
    };
  }
  if (name === spec.bootstrapAdminSecret) {
    if (key !== undefined && key !== BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY) throw new Error(`secret ${name} supports only key ${BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY}`);
    const force = args.includes("--force");
    if (force && actionRaw !== "ensure") throw new Error("secret --force is only supported with ensure");
    return {
      action: actionRaw,
      node,
      lane,
      name,
      key: key ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
      preset: "bootstrap-admin",
      confirm,
      force,
      dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
      timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
    };
  }
  if (name === spec.codeAgentProviderSecret) {
    if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) {
      throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`);
    }
    return {
      action: actionRaw,
      node,
      lane,
      name,
      key,
      preset: "code-agent-provider",
      confirm,
      dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
      timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
    };
  }
  if (name === spec.cloudApiDbSecret) {
    if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`);
    if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
      throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
    }
    return {
      action: actionRaw,
      node,
      lane,
      name,
      key: key ?? spec.cloudApiDbKey,
      preset: "cloud-api-db",
      confirm,
      dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
      timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
    };
  }
  if (name !== spec.openFgaSecret) {
    throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.bootstrapAdminSecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`);
  }
  if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
    throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
  }
  if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
    throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
  }
  return {
    action: actionRaw,
    node,
    lane,
    name,
    key,
    preset: "openfga",
    confirm,
    dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
    timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
  };
}

export function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec {
  const namespace = `hwlab-${input.lane}`;
  const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined;
  const externalPostgres = runtimeLaneSpec === undefined ? undefined : hwlabRuntimeActiveExternalPostgres(runtimeLaneSpec);
  const postgresStore = runtimeLaneSpec?.runtimeStore?.postgres;
  const bootstrapAdmin = runtimeLaneSpec?.bootstrapAdmin;
  const platformDb = postgresStore?.mode === "platform-service";
  const localPostgresService = postgresStore?.serviceName ?? `${namespace}-postgres`;
  const platformPostgresService = externalPostgres?.serviceName ?? postgresStore?.serviceName ?? "g14-platform-postgres";
  const platformPostgresRuntimeAccess = externalPostgres?.runtimeAccess;
  const legacyPostgresHost = `${localPostgresService}.${namespace}.svc.cluster.local`;
  const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`;
  const platformPostgresEndpointSlice = `${platformPostgresService}-host`;
  const openFgaDbName = externalPostgres?.database ?? postgresStore?.openfga?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga");
  const openFgaDbUser = externalPostgres?.openfga.role ?? postgresStore?.openfga?.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga");
  const cloudApiDbName = externalPostgres?.database ?? postgresStore?.cloudApi?.database ?? `hwlab_${input.lane}`;
  const cloudApiDbUser = externalPostgres?.cloudApi.role ?? postgresStore?.cloudApi?.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`);
  return {
    node: input.node,
    lane: input.lane,
    namespace,
    platformDb,
    ...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }),
    ...(externalPostgres === undefined ? {} : { externalPostgres }),
    platformPostgresService,
    platformPostgresEndpointAddress: platformPostgresRuntimeAccess?.endpointAddress ?? externalPostgres?.endpointAddress,
    platformPostgresEndpointSlice,
    postgresSecret: postgresStore?.secretName ?? `${namespace}-postgres`,
    postgresStatefulSet: postgresStore?.statefulSet ?? postgresStore?.secretName ?? `${namespace}-postgres`,
    postgresAdminUser: postgresStore?.adminUser ?? `hwlab_${input.lane}`,
    openFgaSecret: externalPostgres?.openfga.secretName ?? postgresStore?.openfga?.secretName ?? `${namespace}-openfga`,
    openFgaDbName,
    openFgaDbUser,
    openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
    masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`,
    bootstrapAdminSecret: bootstrapAdmin?.secretName ?? `${namespace}-bootstrap-admin`,
    bootstrapAdminPasswordHashKey: bootstrapAdmin?.secretKey ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
    bootstrapAdminUsername: bootstrapAdmin?.username ?? "admin",
    bootstrapAdminDisplayName: bootstrapAdmin?.displayName ?? `HWLAB ${input.lane} Admin`,
    ...(bootstrapAdmin?.usernameSourceRef === undefined ? {} : { bootstrapAdminUsernameSourceRef: bootstrapAdmin.usernameSourceRef }),
    ...(bootstrapAdmin?.usernameSourceKey === undefined ? {} : { bootstrapAdminUsernameSourceKey: bootstrapAdmin.usernameSourceKey }),
    ...(bootstrapAdmin?.usernameSourceLine === undefined ? {} : { bootstrapAdminUsernameSourceLine: bootstrapAdmin.usernameSourceLine }),
    ...(bootstrapAdmin?.passwordSourceRef === undefined ? {} : { bootstrapAdminPasswordSourceRef: bootstrapAdmin.passwordSourceRef }),
    ...(bootstrapAdmin?.passwordSourceKey === undefined ? {} : { bootstrapAdminPasswordSourceKey: bootstrapAdmin.passwordSourceKey }),
    ...(bootstrapAdmin?.passwordSourceLine === undefined ? {} : { bootstrapAdminPasswordSourceLine: bootstrapAdmin.passwordSourceLine }),
    ...(bootstrapAdmin?.passwordHashTransform === undefined ? {} : { bootstrapAdminPasswordHashTransform: bootstrapAdmin.passwordHashTransform }),
    bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE,
    bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET,
    cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? postgresStore?.cloudApi?.secretName ?? `hwlab-cloud-api-${input.lane}-db`,
    cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? postgresStore?.cloudApi?.secretKey ?? CLOUD_API_DB_KEY,
    cloudApiDbName,
    cloudApiDbUser,
    cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
    cloudApiDeployment: "hwlab-cloud-api",
    obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`,
    obsoleteHwpodDbName: `hwpod_${input.lane}`,
    obsoleteHwpodDbUser: `hwpod_${input.lane}_app`,
    codeAgentProviderSecret: runtimeLaneSpec?.codeAgentProvider?.secretName ?? `${namespace}-code-agent-provider`,
    codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
    codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET,
    ...(runtimeLaneSpec?.codeAgentProvider?.sourceRef === undefined ? {} : { codeAgentProviderSourceRef: runtimeLaneSpec.codeAgentProvider.sourceRef }),
    ...(runtimeLaneSpec?.codeAgentProvider?.openaiSourceKey === undefined ? {} : { codeAgentProviderOpenaiSourceKey: runtimeLaneSpec.codeAgentProvider.openaiSourceKey }),
    ...(runtimeLaneSpec?.codeAgentProvider?.opencodeSourceKey === undefined ? {} : { codeAgentProviderOpencodeSourceKey: runtimeLaneSpec.codeAgentProvider.opencodeSourceKey }),
    fieldManager: `unidesk-hwlab-node-${input.lane}-secret`,
  };
}

export function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
  const spec = runtimeSecretSpec(options);
  if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec);
  if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
    return runExternalPostgresSecretEnsure(options, spec);
  }
  const bootstrapAdminMaterial = options.preset === "bootstrap-admin" ? readBootstrapAdminSecretMaterial(spec) : null;
  const codeAgentProviderMaterial = options.preset === "code-agent-provider" && spec.codeAgentProviderSourceRef !== undefined
    ? readCodeAgentProviderSecretMaterial(spec)
    : null;
  const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun
    ? readMasterAdminApiKey(spec).key
    : options.preset === "bootstrap-admin" && options.action === "ensure" && !options.dryRun && bootstrapAdminMaterial?.ok === true
      ? bootstrapAdminMaterial.passwordHash ?? ""
    : "";
  const script = options.preset === "openfga"
    ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
    : options.preset === "master-server-admin-api-key"
      ? masterAdminApiKeySecretScript(options, spec)
      : options.preset === "bootstrap-admin"
        ? bootstrapAdminSecretScript(options, spec, bootstrapAdminMaterial)
        : options.preset === "cloud-api-db"
          ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
          : options.preset === "owned-postgres-cleanup"
            ? ownedPostgresCleanupScript(options, spec)
            : codeAgentProviderSecretScript(options, spec, codeAgentProviderMaterial);
  const result = runTransScript(options.node, script, input, options.timeoutSeconds);
  const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
  const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
  const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
  const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true;
  const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true;
  return {
    ok,
    command: `hwlab nodes secret ${options.action}`,
    node: options.node,
    lane: options.lane,
    namespace: spec.namespace,
    secret: options.name,
    key: options.key ?? null,
    preset: options.preset,
    mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure",
    status,
    mutation: status.mutation === true,
    result: compactCommandResult(result),
    valuesRedacted: true,
    next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec),
  };
}

export function readCodeAgentProviderSecretMaterial(spec: RuntimeSecretSpec): CodeAgentProviderSecretMaterial {
  const sourceRef = spec.codeAgentProviderSourceRef ?? null;
  if (sourceRef === null) {
    return {
      ok: false,
      sourceRef,
      sourcePath: null,
      sourcePresent: false,
      openaiSourceKey: spec.codeAgentProviderOpenaiSourceKey ?? null,
      opencodeSourceKey: spec.codeAgentProviderOpencodeSourceKey ?? null,
      openaiValue: null,
      opencodeValue: null,
      openaiFingerprint: null,
      opencodeFingerprint: null,
      error: "code-agent-provider-source-ref-missing",
    };
  }
  const paths = localSecretSourcePaths(sourceRef);
  const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? null;
  const openaiSourceKey = spec.codeAgentProviderOpenaiSourceKey ?? null;
  const opencodeSourceKey = spec.codeAgentProviderOpencodeSourceKey ?? null;
  if (sourcePath === null) {
    return {
      ok: false,
      sourceRef,
      sourcePath,
      sourcePresent: false,
      openaiSourceKey,
      opencodeSourceKey,
      openaiValue: null,
      opencodeValue: null,
      openaiFingerprint: null,
      opencodeFingerprint: null,
      error: "code-agent-provider-source-path-unresolved",
    };
  }
  if (!existsSync(sourcePath)) {
    return {
      ok: false,
      sourceRef,
      sourcePath: displayRepoPath(sourcePath),
      sourcePresent: false,
      openaiSourceKey,
      opencodeSourceKey,
      openaiValue: null,
      opencodeValue: null,
      openaiFingerprint: null,
      opencodeFingerprint: null,
      error: "code-agent-provider-source-missing",
    };
  }
  const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
  const openaiValue = openaiSourceKey === null ? null : values[openaiSourceKey] ?? null;
  const opencodeValue = opencodeSourceKey === null ? null : values[opencodeSourceKey] ?? null;
  const ok = (openaiValue !== null && openaiValue.length > 0) || (opencodeValue !== null && opencodeValue.length > 0);
  return {
    ok,
    sourceRef,
    sourcePath: displayRepoPath(sourcePath),
    sourcePresent: true,
    openaiSourceKey,
    opencodeSourceKey,
    openaiValue,
    opencodeValue,
    openaiFingerprint: openaiValue === null || openaiValue.length === 0 ? null : shortSecretFingerprint(openaiValue),
    opencodeFingerprint: opencodeValue === null || opencodeValue.length === 0 ? null : shortSecretFingerprint(opencodeValue),
    error: ok ? null : "code-agent-provider-source-key-missing",
  };
}

export function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, string> {
  if (options.action === "cleanup-owned-postgres") {
    return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
  }
  if (options.action === "cleanup-obsolete") {
    return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` };
  }
  if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
    return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
  }
  if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
    return {
      status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
      controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
    };
  }
  return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
}

export function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
  const runtimeLaneSpec = spec.runtimeLaneSpec;
  if (runtimeLaneSpec === undefined) {
    return {
      ok: false,
      command: `hwlab nodes secret ${options.action}`,
      node: options.node,
      lane: options.lane,
      namespace: spec.namespace,
      secret: options.name,
      key: options.key ?? null,
      preset: options.preset,
      mode: options.dryRun ? "dry-run" : "confirmed-ensure",
      mutation: false,
      degradedReason: "external-postgres-runtime-lane-spec-missing",
      valuesRedacted: true,
    };
  }
  const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds);
  const shouldReadStatus = sync !== null && sync.ok === true;
  const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null;
  const status = statusResult === null
    ? null
    : secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec);
  const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true);
  return {
    ok,
    command: `hwlab nodes secret ${options.action}`,
    node: options.node,
    lane: options.lane,
    namespace: spec.namespace,
    secret: options.name,
    key: options.key ?? null,
    preset: options.preset,
    mode: options.dryRun ? "dry-run" : "confirmed-ensure",
    status: status ?? sync,
    externalPostgresSecretSync: sync,
    mutation: sync?.mutation === true,
    result: statusResult === null ? null : compactCommandResult(statusResult),
    valuesRedacted: true,
    next: ok ? undefined : nextSecretCommand(options, spec),
  };
}

export function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record<string, unknown> {
  return {
    mode: exposure.mode,
    publicBaseUrl: exposure.publicBaseUrl,
    hostname: exposure.hostname,
    expectedA: exposure.expectedA,
    frpc: {
      serverAddr: exposure.serverAddr,
      serverPort: exposure.serverPort,
      tokenSourceRef: exposure.tokenSourceRef,
      tokenSourceKey: exposure.tokenSourceKey,
      secretName: exposure.secretName,
      secretKey: exposure.secretKey,
      tokenKey: exposure.tokenKey,
      webProxy: exposure.webProxy,
      apiProxy: exposure.apiProxy,
      extraProxies: exposure.extraProxies,
    },
    caddy: {
      route: exposure.caddyRoute,
      configPath: exposure.caddyConfigPath,
      serviceName: exposure.caddyServiceName,
      tls: exposure.caddyTls,
      responseHeaderTimeoutSeconds: exposure.responseHeaderTimeoutSeconds,
    },
  };
}

export function runNodePublicExposure(options: NodePublicExposureOptions): Record<string, unknown> {
  const exposure = options.spec.publicExposure;
  if (exposure === null || !exposure.enabled) {
    return {
      ok: false,
      command: "hwlab nodes control-plane public-exposure",
      node: options.node,
      lane: options.lane,
      configPath: hwlabRuntimeLaneConfigPath(),
      error: "publicExposure is not configured for this node/lane target",
      mutation: false,
    };
  }
  const source = readPublicExposureTokenSource(exposure);
  if (!source.ok) {
    return {
      ok: false,
      command: "hwlab nodes control-plane public-exposure",
      node: options.node,
      lane: options.lane,
      mode: options.dryRun ? "dry-run" : "confirmed",
      configPath: hwlabRuntimeLaneConfigPath(),
      publicExposure: publicExposureSummary(exposure),
      source,
      mutation: false,
      valuesRedacted: true,
      next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=<redacted>` },
    };
  }
  const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds);
  const caddyFields = keyValueLinesFromText(statusText(caddyResult));
  const secretStatus = options.dryRun
    ? publicExposureSecretDryRunStatus(options, exposure, source.value?.length ?? 0)
    : publicExposureSecretApplyStatus(options, exposure, source.value ?? "");
  const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult);
  const ok = secretStatus.ok === true && caddyStatus.ok === true;
  return {
    ok,
    command: "hwlab nodes control-plane public-exposure",
    node: options.node,
    lane: options.lane,
    mode: options.dryRun ? "dry-run" : "confirmed",
    mutation: !options.dryRun && (secretStatus.mutation === true || caddyFields.mutation === "true"),
    configPath: hwlabRuntimeLaneConfigPath(),
    publicExposure: publicExposureSummary(exposure),
    source: {
      ok: source.ok,
      path: source.path,
      key: exposure.tokenSourceKey,
      bytes: source.value?.length ?? 0,
      fingerprint: source.fingerprint,
      valueRedacted: true,
    },
    secret: secretStatus,
    caddy: caddyStatus,
    valuesRedacted: true,
    next: ok
      ? {
          triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${options.node} --lane ${options.lane} --confirm`,
          status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
        }
      : { retry: `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${options.node} --lane ${options.lane} --confirm` },
  };
}

function publicExposureSecretDryRunStatus(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec, tokenBytes: number): Record<string, unknown> {
  return {
    ok: true,
    dryRun: true,
    mutation: false,
    planned: true,
    namespace: options.spec.runtimeNamespace,
    secret: exposure.secretName,
    deployment: `${options.spec.runtimeNamespace}-frpc`,
    beforeExists: null,
    afterExists: null,
    tokenBytes,
    applyExitCode: null,
    restartMode: null,
    strategyPatchExitCode: null,
    rolloutRestartExitCode: null,
    scaleDownExitCode: null,
    scaleDownWaitExitCode: null,
    scaleDownWaitPodCount: null,
    scaleUpExitCode: null,
    rolloutStatusExitCode: null,
    readyReplicas: null,
    availableReplicas: null,
    desiredReplicas: null,
    strategyPatchErrorPreview: null,
    restartErrorPreview: null,
    scaleDownErrorPreview: null,
    scaleUpErrorPreview: null,
    readyErrorPreview: null,
    exitCode: 0,
    stderr: "",
    valuesRedacted: true,
  };
}

function publicExposureSecretApplyStatus(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec, token: string): Record<string, unknown> {
  const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), token, options.timeoutSeconds);
  let secretFields = keyValueLinesFromText(statusText(secretApplyResult));
  let secretResult = secretApplyResult;
  if (secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") {
    const restartResult = runPublicExposureFrpcRecreate(options);
    secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) };
    secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult);
  }
  return publicExposureSecretStatus(secretFields, secretResult);
}

export function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } {
  const checkedPaths = publicExposureTokenSourcePaths(exposure);
  const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef);
  if (!existsSync(path)) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-source-missing" };
  const values = parseEnvFile(readFileSync(path, "utf8"));
  const value = values[exposure.tokenSourceKey];
  if (value === undefined || value.length === 0) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-key-missing" };
  return {
    ok: true,
    path,
    checkedPaths,
    key: exposure.tokenSourceKey,
    value,
    fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
  };
}

export function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec): string[] {
  const paths = [join(repoRoot, ".state", "secrets", exposure.tokenSourceRef)];
  const marker = "/.worktree/";
  const index = repoRoot.indexOf(marker);
  if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", exposure.tokenSourceRef));
  return [...new Set(paths)];
}

export function publicExposureSecretStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
  const dryRun = fields.dryRun === "true";
  return {
    ok: result.exitCode === 0 && (dryRun || (
      fields.afterSecretExists === "yes"
      && fields.strategyPatchExitCode === "0"
      && fields.rolloutRestartExitCode === "0"
      && fields.rolloutStatusExitCode === "0"
    )),
    dryRun,
    mutation: fields.mutation === "true",
    namespace: fields.namespace || null,
    secret: fields.secret || null,
    deployment: fields.deployment || null,
    beforeExists: fields.beforeSecretExists === "yes",
    afterExists: fields.afterSecretExists === "yes",
    tokenBytes: numericField(fields.tokenBytes),
    applyExitCode: numericField(fields.applyExitCode),
    restartMode: fields.restartMode || null,
    strategyPatchExitCode: numericField(fields.strategyPatchExitCode),
    rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
    scaleDownExitCode: numericField(fields.scaleDownExitCode),
    scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode),
    scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount),
    scaleUpExitCode: numericField(fields.scaleUpExitCode),
    rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
    readyReplicas: numericField(fields.readyReplicas),
    availableReplicas: numericField(fields.availableReplicas),
    desiredReplicas: numericField(fields.desiredReplicas),
    strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null,
    restartErrorPreview: fields.restartErrorPreview || null,
    scaleDownErrorPreview: fields.scaleDownErrorPreview || null,
    scaleUpErrorPreview: fields.scaleUpErrorPreview || null,
    readyErrorPreview: fields.readyErrorPreview || null,
    exitCode: result.exitCode,
    stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
  };
}

export function publicExposureCaddyStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
  const dryRun = fields.dryRun === "true";
  return {
    ok: result.exitCode === 0
      && fields.afterBlockPresent === "yes"
      && fields.validateExitCode === "0"
      && (dryRun || (
        fields.active === "active"
        && fields.localWebStatus === "200"
        && fields.localApiStatus === "200"
      )),
    dryRun,
    mutation: fields.mutation === "true",
    hostname: fields.hostname || null,
    configPath: fields.configPath || null,
    beforeBlockPresent: fields.beforeBlockPresent === "yes",
    afterBlockPresent: fields.afterBlockPresent === "yes",
    staleBlocksRemoved: numericField(fields.staleBlocksRemoved),
    pythonExitCode: numericField(fields.pythonExitCode),
    validateMode: fields.validateMode || null,
    validateExitCode: numericField(fields.validateExitCode),
    reloadExitCode: numericField(fields.reloadExitCode),
    active: fields.active || null,
    localWebStatus: fields.localWebStatus || null,
    localApiStatus: fields.localApiStatus || null,
    validateErrorPreview: fields.validateErrorPreview || null,
    localWebErrorPreview: fields.localWebErrorPreview || null,
    localApiErrorPreview: fields.localApiErrorPreview || null,
    exitCode: result.exitCode,
    stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
  };
}

export function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
  const deployment = `${options.spec.runtimeNamespace}-frpc`;
  return [
    "set +e",
    `namespace=${shellQuote(options.spec.runtimeNamespace)}`,
    `secret=${shellQuote(exposure.secretName)}`,
    `deployment=${shellQuote(deployment)}`,
    `token_key=${shellQuote(exposure.tokenKey)}`,
    `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
    "token=$(cat)",
    "before_secret_exists=no",
    "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes",
    "token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')",
    "apply_exit=",
    "restart_mode=recreate-rollout-restart",
    "strategy_patch_exit=",
    "rollout_restart_exit=",
    "scale_down_exit=",
    "scale_down_wait_exit=",
    "scale_down_wait_pod_count=",
    "scale_up_exit=",
    "rollout_status_exit=",
    "ready_replicas=",
    "available_replicas=",
    "desired_replicas=",
    "mutation=false",
    "if [ \"$dry_run\" = false ]; then",
    "  tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)",
    "  token_b64=$(printf '%s' \"$token\" | base64 | tr -d '\\n')",
    "  cat >\"$tmp\" <<EOF",
    "apiVersion: v1",
    "kind: Secret",
    "metadata:",
    "  name: $secret",
    "  namespace: $namespace",
    "  labels:",
    "    app.kubernetes.io/part-of: hwlab",
    "    app.kubernetes.io/managed-by: unidesk",
    "type: Opaque",
    "data:",
    "  $token_key: $token_b64",
    "EOF",
    "  kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-public-exposure -f \"$tmp\" >/tmp/hwlab-public-frpc-secret.apply.out 2>/tmp/hwlab-public-frpc-secret.apply.err",
    "  apply_exit=$?",
    "  rm -f \"$tmp\"",
    "  if [ \"$apply_exit\" = 0 ]; then mutation=true; fi",
    "fi",
    "after_secret_exists=no",
    "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes",
    "printf 'namespace\\t%s\\n' \"$namespace\"",
    "printf 'secret\\t%s\\n' \"$secret\"",
    "printf 'deployment\\t%s\\n' \"$deployment\"",
    "printf 'dryRun\\t%s\\n' \"$dry_run\"",
    "printf 'mutation\\t%s\\n' \"$mutation\"",
    "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
    "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
    "printf 'tokenBytes\\t%s\\n' \"$token_bytes\"",
    "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
    "printf 'restartMode\\t%s\\n' \"$restart_mode\"",
    "printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"",
    "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
    "printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"",
    "printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"",
    "printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"",
    "printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"",
    "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
    "printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"",
    "printf 'availableReplicas\\t%s\\n' \"$available_replicas\"",
    "printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"",
    "if [ \"$dry_run\" = true ]; then exit 0; fi",
    "[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]",
  ].join("\n");
}

export function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult {
  const namespace = options.spec.runtimeNamespace;
  const deployment = `${namespace}-frpc`;
  const fields: Record<string, string> = {
    deployment,
    restartMode: "recreate-rollout-restart",
    strategyPatchExitCode: "",
    rolloutRestartExitCode: "",
    scaleDownExitCode: "",
    scaleDownWaitExitCode: "",
    scaleDownWaitPodCount: "",
    scaleUpExitCode: "",
    rolloutStatusExitCode: "",
    readyReplicas: "",
    availableReplicas: "",
    desiredReplicas: "",
    strategyPatchErrorPreview: "",
    restartErrorPreview: "",
    scaleDownErrorPreview: "",
    scaleUpErrorPreview: "",
    readyErrorPreview: "",
  };
  const stdoutParts: string[] = [];
  const stderrParts: string[] = [];
  const addResult = (result: CommandResult): void => {
    if (result.stdout.trim()) stdoutParts.push(result.stdout.trim());
    if (result.stderr.trim()) stderrParts.push(result.stderr.trim());
  };
  const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55);
  const strategyPatch = runTransScript(options.node, [
    "set +e",
    `namespace=${shellQuote(namespace)}`,
    `deployment=${shellQuote(deployment)}`,
    "kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err",
    "code=$?",
    "printf 'strategyPatchExitCode\\t%s\\n' \"$code\"",
    "printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-strategy-patch.err | cut -c1-1000 || true)\"",
    "exit \"$code\"",
  ].join("\n"), "", shortTimeout);
  addResult(strategyPatch);
  Object.assign(fields, keyValueLinesFromText(statusText(strategyPatch)));

  if (strategyPatch.exitCode === 0) {
    const restart = runTransScript(options.node, [
      "set +e",
      `namespace=${shellQuote(namespace)}`,
      `deployment=${shellQuote(deployment)}`,
      "kubectl -n \"$namespace\" rollout restart deploy/\"$deployment\" >/tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err",
      "code=$?",
      "printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"",
      "printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-restart.err | cut -c1-1000 || true)\"",
      "exit \"$code\"",
    ].join("\n"), "", shortTimeout);
    addResult(restart);
    Object.assign(fields, keyValueLinesFromText(statusText(restart)));
  }

  if (fields.rolloutRestartExitCode === "0") {
    const deadline = Date.now() + 55_000;
    while (Date.now() <= deadline) {
      const probe = runTransScript(options.node, [
        "set +e",
        `namespace=${shellQuote(namespace)}`,
        `deployment=${shellQuote(deployment)}`,
        "kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.replicas}{\"\\t\"}{.status.readyReplicas}{\"\\t\"}{.status.availableReplicas}' >/tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err",
        "code=$?",
        "values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)",
        "desired=$(printf '%s' \"$values\" | cut -f1)",
        "ready=$(printf '%s' \"$values\" | cut -f2)",
        "available=$(printf '%s' \"$values\" | cut -f3)",
        "printf 'desiredReplicas\\t%s\\n' \"$desired\"",
        "printf 'readyReplicas\\t%s\\n' \"$ready\"",
        "printf 'availableReplicas\\t%s\\n' \"$available\"",
        "printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-ready.err | cut -c1-1000 || true)\"",
        "[ \"$code\" = 0 ] && [ \"$desired\" = 1 ] && [ \"$ready\" = 1 ] && [ \"$available\" = 1 ]",
      ].join("\n"), "", shortTimeout);
      addResult(probe);
      Object.assign(fields, keyValueLinesFromText(statusText(probe)));
      if (probe.exitCode === 0) {
        fields.rolloutStatusExitCode = "0";
        break;
      }
      runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
    }
  }
  if (fields.rolloutStatusExitCode === "") fields.rolloutStatusExitCode = "1";

  const ok = fields.strategyPatchExitCode === "0"
    && fields.rolloutRestartExitCode === "0"
    && fields.rolloutStatusExitCode === "0";
  const stdout = Object.entries(fields).map(([key, value]) => `${key}\t${value}`).join("\n") + "\n";
  return {
    command: [transPath(), `${options.node}:k3s`, "sh", "--", "<public-exposure-frpc-recreate>"],
    cwd: repoRoot,
    exitCode: ok ? 0 : 1,
    stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n",
    stderr: stderrParts.join("\n"),
    signal: null,
    timedOut: false,
  };
}

export function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult {
  return {
    command: [...first.command, "&&", ...second.command],
    cwd: repoRoot,
    exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode,
    stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n",
    stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"),
    signal: second.signal ?? first.signal,
    timedOut: first.timedOut || second.timedOut,
  };
}

export function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
  const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64");
  const marker = `unidesk managed ${exposure.hostname}`;
  return [
    "set +e",
    `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
    `hostname=${shellQuote(exposure.hostname)}`,
    `config_path=${shellQuote(exposure.caddyConfigPath)}`,
    `service=${shellQuote(exposure.caddyServiceName)}`,
    `marker=${shellQuote(marker)}`,
    `block_b64=${shellQuote(blockB64)}`,
    `expected_a=${shellQuote(exposure.expectedA)}`,
    `web_port=${shellQuote(String(exposure.webProxy.remotePort))}`,
    `api_port=${shellQuote(String(exposure.apiProxy.remotePort))}`,
    "rm -f /tmp/hwlab-public-caddy-validate.out /tmp/hwlab-public-caddy-validate.err /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-web.err /tmp/hwlab-public-caddy-api.err",
    "tmp=$(mktemp -d)",
    "block=\"$tmp/block\"",
    "next=\"$tmp/Caddyfile\"",
    "printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
    "before_present=no",
    "if [ -f \"$config_path\" ] && grep -Fq \"# BEGIN $marker\" \"$config_path\"; then before_present=yes; fi",
    "if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
    "stale_blocks_removed=$(python3 - \"$next\" \"$block\" \"$marker\" \"$web_port\" \"$api_port\" 2>/tmp/hwlab-public-caddy-python.err <<'PY'",
    "import pathlib, re, sys",
    "config = pathlib.Path(sys.argv[1])",
    "block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
    "marker = sys.argv[3]",
    "web_port = sys.argv[4]",
    "api_port = sys.argv[5]",
    "current_name = marker[len('unidesk managed '):] if marker.startswith('unidesk managed ') else marker",
    "text = config.read_text(encoding='utf-8') if config.exists() else ''",
    "begin = f'# BEGIN {marker}'",
    "end = f'# END {marker}'",
    "managed = f'{begin}\\n{block.rstrip()}\\n{end}\\n'",
    "stale_removed = [0]",
    "pattern = re.compile(r'(?ms)^# BEGIN unidesk managed (?P<name>[^\\n]+)\\n(?P<body>.*?)\\n# END unidesk managed (?P=name)\\n*')",
    "def keep(match):",
    "    name = match.group('name')",
    "    body = match.group('body')",
    "    if name != current_name and (f'127.0.0.1:{web_port}' in body or f'127.0.0.1:{api_port}' in body):",
    "        stale_removed[0] += 1",
    "        return ''",
    "    return match.group(0)",
    "text = pattern.sub(keep, text)",
    "if begin in text and end in text:",
    "    start = text.index(begin)",
    "    stop = text.index(end, start) + len(end)",
    "    text = text[:start].rstrip() + '\\n\\n' + managed.rstrip() + '\\n' + text[stop:].lstrip('\\n')",
    "else:",
    "    text = text.rstrip() + '\\n\\n' + managed",
    "config.write_text(text, encoding='utf-8')",
    "print(stale_removed[0])",
    "PY",
    ")",
    "python_exit=$?",
    "validate_exit=1",
    "validate_mode=validate",
    "if [ \"$python_exit\" != 0 ]; then",
    "  validate_mode=python",
    "  validate_exit=$python_exit",
    "else",
    "  if [ \"$dry_run\" = true ]; then",
    "    validate_mode=adapt",
    "    caddy adapt --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
    "  else",
    "    sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
    "  fi",
    "  validate_exit=$?",
    "fi",
    "reload_exit=",
    "mutation=false",
    "if [ \"$dry_run\" = false ] && [ \"$validate_exit\" = 0 ]; then",
    "  sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/hwlab-public-caddy-install.out 2>/tmp/hwlab-public-caddy-install.err",
    "  install_exit=$?",
    "  if [ \"$install_exit\" = 0 ]; then",
    "    mutation=true",
    "    sudo systemctl reload \"$service\" >/tmp/hwlab-public-caddy-reload.out 2>/tmp/hwlab-public-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/hwlab-public-caddy-reload.out 2>>/tmp/hwlab-public-caddy-reload.err",
    "    reload_exit=$?",
    "  else",
    "    reload_exit=$install_exit",
    "  fi",
    "fi",
    "active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
    "after_present=no",
    "if [ \"$dry_run\" = true ]; then grep -Fq \"# BEGIN $marker\" \"$next\" && after_present=yes; else grep -Fq \"# BEGIN $marker\" \"$config_path\" && after_present=yes; fi",
    "local_web_status=",
    "local_api_status=",
    "if [ \"$dry_run\" = false ]; then",
    "  local_web_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-web.err || true)",
    "  local_api_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/health/live\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-api.err || true)",
    "fi",
    "validate_err_preview=$(cat /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-validate.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
    "local_web_err_preview=$([ -f /tmp/hwlab-public-caddy-web.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-web.err 2>/dev/null | cut -c1-1000 || true)",
    "local_api_err_preview=$([ -f /tmp/hwlab-public-caddy-api.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-api.err 2>/dev/null | cut -c1-1000 || true)",
    "printf 'hostname\\t%s\\n' \"$hostname\"",
    "printf 'expectedA\\t%s\\n' \"$expected_a\"",
    "printf 'configPath\\t%s\\n' \"$config_path\"",
    "printf 'dryRun\\t%s\\n' \"$dry_run\"",
    "printf 'mutation\\t%s\\n' \"$mutation\"",
    "printf 'beforeBlockPresent\\t%s\\n' \"$before_present\"",
    "printf 'afterBlockPresent\\t%s\\n' \"$after_present\"",
    "printf 'staleBlocksRemoved\\t%s\\n' \"$stale_blocks_removed\"",
    "printf 'pythonExitCode\\t%s\\n' \"$python_exit\"",
    "printf 'validateMode\\t%s\\n' \"$validate_mode\"",
    "printf 'validateExitCode\\t%s\\n' \"$validate_exit\"",
    "printf 'reloadExitCode\\t%s\\n' \"$reload_exit\"",
    "printf 'active\\t%s\\n' \"$active\"",
    "printf 'localWebStatus\\t%s\\n' \"$local_web_status\"",
    "printf 'localApiStatus\\t%s\\n' \"$local_api_status\"",
    "printf 'validateErrorPreview\\t%s\\n' \"$validate_err_preview\"",
    "printf 'localWebErrorPreview\\t%s\\n' \"$local_web_err_preview\"",
    "printf 'localApiErrorPreview\\t%s\\n' \"$local_api_err_preview\"",
    "rm -rf \"$tmp\"",
    "[ \"$validate_exit\" = 0 ] && [ \"$after_present\" = yes ]",
  ].join("\n");
}

export function publicExposureCaddyBlock(exposure: HwlabRuntimePublicExposureSpec): string {
  const tlsLines = exposure.caddyTls === "internal" ? "    tls internal\n" : "";
  const mainBlock = `${exposure.hostname} {
${tlsLines}    @api path /health* /v1* /json-rpc* /openapi* /docs* /swagger*
    reverse_proxy @api 127.0.0.1:${exposure.apiProxy.remotePort} {
        transport http {
            response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
        }
    }
    reverse_proxy 127.0.0.1:${exposure.webProxy.remotePort} {
        transport http {
            response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
        }
    }
}`;
  const extraBlocks = exposure.extraProxies
    .filter((proxy) => proxy.hostname !== undefined)
    .map((proxy) => `${proxy.hostname} {
${tlsLines}    reverse_proxy 127.0.0.1:${proxy.remotePort} {
        transport http {
            response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
        }
    }
}`);
  return [mainBlock, ...extraBlocks].join("\n\n");
}

export function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
  return runCommand([transPath(), `${node}:k3s`, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}

export function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
  return runCommand([transPath(), node, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}

export function runTransWorkspaceStdinScript(node: string, workspace: string, script: string, timeoutSeconds: number): CommandResult {
  return runCommand([transPath(), `${node}:${workspace}`, "sh"], repoRoot, { input: script, timeoutMs: timeoutSeconds * 1000 });
}

export function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
  const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds);
  const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec);
  const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true };
  const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds);
  const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec);
  const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false);
  const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true;
  return {
    ok,
    command: `hwlab nodes secret ${options.action}`,
    node: options.node,
    lane: options.lane,
    namespace: spec.namespace,
    secret: options.name,
    key: null,
    preset: options.preset,
    mode: options.dryRun ? "dry-run" : "confirmed-delete",
    status: {
      ok,
      preset: "obsolete-hwpod-db-cleanup",
      dryRun: options.dryRun,
      mutation,
      kubernetesSecret: kubernetesStatus,
      platformDatabase: platformDbStatus,
      hostMutationSkipped: !options.dryRun && hostOptions.dryRun,
      summary: ok
        ? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove`
        : `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`,
    },
    mutation,
    result: {
      kubernetesSecret: compactCommandResult(kubernetesResult),
      platformDatabase: compactCommandResult(platformDbResult),
    },
    valuesRedacted: true,
    next: ok ? undefined : nextSecretCommand(options, spec),
  };
}