// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. control-actions module for scripts/src/hwlab-node-impl.ts.

// Moved mechanically from scripts/src/hwlab-node-impl.ts:2202-2799 for #903.

// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
import { createHash, randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import type { RenderedCliResult } from "../output";

import { isCommandSuccess, resolveNodeRuntimeLaneHead, runNodeK3sArgs, runNodeK3sScript, shortSha } from "./cleanup";
import { HWLAB_CI_NAMESPACE } from "./entry";
import { nodeRuntimeExpected, parseNodeScopedDelegatedOptions } from "./plan";
import { renderNodeRuntimeControlPlane } from "./render";
import { compactRuntimeCommand, sshTcpPoolDiagnosticsFromCommand } from "./runtime-common";
import { commaListField, keyValueLinesFromText, numericField, shellQuote, statusText } from "./utils";
import { applyLocalNodeRuntimeControlPlaneFiles, applyNodeRuntimeControlPlaneFiles, cleanupLocalNodeRuntimeRenderDir, cleanupNodeRuntimeRenderDir, ensureNodeBaseImage, nodeRuntimeBaseImageStatus, syncNodeExternalPostgresSecrets, syncNodeLocalPostgresBootstrapSecret } from "./web-probe";

export function nodeRuntimeCleanupOwnedTaskRunsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
  return text.split(/\r?\n/u).map((line) => {
    const [name = "", pipelineRun = "", status = "", reason = ""] = line.trim().split("\t");
    if (name.length === 0 || !wanted.has(pipelineRun)) return null;
    return {
      name,
      pipelineRun,
      status: status || null,
      reason: reason || null,
    };
  }).filter((item): item is Record<string, unknown> => item !== null);
}

export function nodeRuntimeCleanupOwnedPodsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
  return text.split(/\r?\n/u).map((line) => {
    const [name = "", pipelineRun = "", phase = "", nodeName = ""] = line.trim().split("\t");
    if (name.length === 0 || !wanted.has(pipelineRun)) return null;
    return {
      name,
      pipelineRun,
      phase: phase || null,
      nodeName: nodeName || null,
    };
  }).filter((item): item is Record<string, unknown> => item !== null);
}

export function nodeRuntimeCleanupOwnedPvcsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
  return text.split(/\r?\n/u).map((line) => {
    const [name = "", volumeName = "", phase = "", ownerKind = "", ownerName = "", storage = ""] = line.trim().split("\t");
    if (name.length === 0 || ownerKind !== "PipelineRun" || !wanted.has(ownerName)) return null;
    return {
      name,
      pipelineRun: ownerName,
      phase: phase || null,
      volumeName: volumeName || null,
      storage: storage || null,
    };
  }).filter((item): item is Record<string, unknown> => item !== null);
}

export function deleteNodeRuntimeCleanupRuns(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[], timeoutSeconds: number): CommandResult {
  if (pipelineRunNames.length === 0) {
    return {
      command: [],
      cwd: repoRoot,
      exitCode: 0,
      stdout: "no candidates",
      stderr: "",
      signal: null,
      timedOut: false,
    };
  }
  const script = [
    "set -eu",
    `namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
    "names_file=$(mktemp)",
    "cat > \"$names_file\"",
    "deleted_pipeline_runs=$(grep -c . \"$names_file\" | tr -d ' ')",
    "xargs -r -n 50 kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=false < \"$names_file\"",
    "deleted_pod_groups=0",
    "deleted_taskrun_groups=0",
    "explicit_owned_cleanup=skipped-large-batch",
    "if [ \"$deleted_pipeline_runs\" -le 40 ]; then",
    "  explicit_owned_cleanup=executed",
    "  while IFS= read -r pipeline_run; do",
    "    [ -n \"$pipeline_run\" ] || continue",
    "    kubectl -n \"$namespace\" delete pod -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "    deleted_pod_groups=$((deleted_pod_groups + 1))",
    "    kubectl -n \"$namespace\" delete taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
    "    deleted_taskrun_groups=$((deleted_taskrun_groups + 1))",
    "  done < \"$names_file\"",
    "fi",
    "printf 'deletedPipelineRunCount\\t%s\\n' \"$deleted_pipeline_runs\"",
    "printf 'deletedTaskRunLabelGroups\\t%s\\n' \"$deleted_taskrun_groups\"",
    "printf 'deletedPodLabelGroups\\t%s\\n' \"$deleted_pod_groups\"",
    "printf 'explicitOwnedCleanup\\t%s\\n' \"$explicit_owned_cleanup\"",
    "rm -f \"$names_file\"",
  ].join("\n");
  return runNodeK3sScript(spec, script, timeoutSeconds, pipelineRunNames.join("\n") + "\n");
}

export function nodeRuntimeCiObjectCounts(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
  const script = [
    "set +e",
    `namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
    "count_kind() { kubectl -n \"$namespace\" get \"$1\" -o name 2>/dev/null | wc -l | tr -d ' '; }",
    "printf 'pipelineRuns\\t%s\\n' \"$(count_kind pipelinerun)\"",
    "printf 'taskRuns\\t%s\\n' \"$(count_kind taskrun)\"",
    "printf 'pods\\t%s\\n' \"$(count_kind pod)\"",
    "printf 'pvcs\\t%s\\n' \"$(count_kind pvc)\"",
  ].join("\n");
  const result = runNodeK3sScript(spec, script, 30);
  const fields = keyValueLinesFromText(statusText(result));
  return {
    pipelineRuns: numericField(fields.pipelineRuns),
    taskRuns: numericField(fields.taskRuns),
    pods: numericField(fields.pods),
    pvcs: numericField(fields.pvcs),
    result: compactRuntimeCommand(result),
  };
}

export function nodeRuntimeBaseImageCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const action = scoped.runtimeImageAction;
  if (action === null) {
    return {
      ok: false,
      command: `hwlab nodes control-plane runtime-image --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mutation: false,
      degradedReason: "node-runtime-image-action-missing",
      message: "runtime-image requires one of: status, preload, build",
      expected: nodeRuntimeExpected(scoped.spec),
    };
  }
  if (action !== "status" && action !== "preload" && action !== "build") {
    return {
      ok: false,
      command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mutation: false,
      degradedReason: "unsupported-node-runtime-image-action",
      message: "runtime-image currently supports status/preload/build",
      expected: nodeRuntimeExpected(scoped.spec),
    };
  }
  const statusBefore = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds);
  if (action === "status") {
    return {
      ok: statusBefore.ok,
      command: `hwlab nodes control-plane runtime-image status --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mode: "status",
      mutation: false,
      status: statusBefore,
      degradedReason: statusBefore.ok ? undefined : "node-runtime-base-image-not-ready",
      next: statusBefore.ok ? undefined : {
        preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm`,
      },
    };
  }
  if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-image preload/build requires --dry-run or --confirm");
  const preload = ensureNodeBaseImage(scoped.spec, scoped.dryRun, scoped.timeoutSeconds);
  const statusAfter = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds);
  return {
    ok: preload !== null && preload.ok === true && (scoped.dryRun || statusAfter.ok === true),
    command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`,
    node: scoped.node,
    lane: scoped.lane,
    mode: scoped.dryRun ? "dry-run" : "confirmed-preload",
    requestedAction: action,
    effectiveAction: "preload",
    mutation: !scoped.dryRun && preload !== null && preload.ok === true,
    statusBefore,
    preload,
    statusAfter,
    degradedReason: preload === null
      ? "node-runtime-base-image-source-missing"
      : preload.ok === true && (scoped.dryRun || statusAfter.ok === true)
        ? undefined
        : "node-runtime-base-image-seed-failed",
    next: scoped.dryRun
      ? { preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm` }
      : { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun` },
  };
}

export function nodeRuntimeMigration(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const spec = scoped.spec;
  if (scoped.allowLiveDbRead && scoped.confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
  if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-migration requires --dry-run or --confirm");
  const head = resolveNodeRuntimeLaneHead(spec);
  const sourceCommit = head.sourceCommit;
  if (sourceCommit === null) {
    return {
      ok: false,
      command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      phase: "source-head",
      mutation: false,
      degradedReason: "node-runtime-source-head-unresolved",
      headProbe: compactRuntimeCommand(head.result),
    };
  }
  const reportPath = `/tmp/hwlab-${scoped.node.toLowerCase()}-${scoped.lane}-runtime-migration-${shortSha(sourceCommit)}.json`;
  const migrationArgs = scoped.dryRun
    ? [
      ...(scoped.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
      "--report",
      reportPath,
    ]
    : [
      "--apply",
      "--confirm-dev",
      "--confirmed-non-production",
      "--report",
      reportPath,
    ];
  const result = runNodeK3sArgs(spec, [
    "kubectl",
    "exec",
    "-n",
    spec.runtimeNamespace,
    "deployment/hwlab-cloud-api",
    "-c",
    "hwlab-cloud-api",
    "--",
    "sh",
    "-lc",
    `cd /workspace/hwlab-boot/repo && exec bun cmd/hwlab-cloud-api/migrate.ts ${migrationArgs.map(shellQuote).join(" ")}`,
  ], scoped.timeoutSeconds);
  const ok = isCommandSuccess(result);
  return {
    ok,
    command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
    node: scoped.node,
    lane: scoped.lane,
    mode: scoped.dryRun ? scoped.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
    sourceCommit,
    runtimeNamespace: spec.runtimeNamespace,
    target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
    workingDirectory: "/workspace/hwlab-boot/repo",
    migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
    reportPath,
    mutation: !scoped.dryRun && ok,
    result: compactRuntimeCommand(result),
    degradedReason: ok ? undefined : "node-runtime-migration-failed",
    next: scoped.dryRun
      ? {
        liveReadDryRun: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --allow-live-db-read --dry-run`,
        apply: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --confirm`,
      }
      : {
        health: `curl -fsS --max-time 20 ${spec.publicApiUrl}/health/live`,
        status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
      },
  };
}

export function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const spec = scoped.spec;
  const head = resolveNodeRuntimeLaneHead(spec);
  const sourceCommit = head.sourceCommit;
  if (sourceCommit === null) {
    return {
      ok: false,
      command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      phase: "source-head",
      degradedReason: "node-runtime-source-head-unresolved",
      headProbe: compactRuntimeCommand(head.result),
    };
  }
  const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
  if (localPostgres !== null && localPostgres.ok !== true) {
    return {
      ok: false,
      command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
      phase: "local-postgres-secret-sync",
      sourceCommit,
      localPostgres,
      degradedReason: "local-postgres-secret-sync-failed",
    };
  }
  const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds);
  if (secrets !== null && secrets.ok !== true) {
    return {
      ok: false,
      command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
      phase: "external-postgres-secret-sync",
      sourceCommit,
      localPostgres,
      secrets,
      degradedReason: "external-postgres-secret-sync-failed",
    };
  }
  const baseImage = ensureNodeBaseImage(spec, scoped.dryRun, scoped.timeoutSeconds);
  if (baseImage !== null && baseImage.ok !== true) {
    return {
      ok: false,
      command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
      phase: "base-image-seed",
      sourceCommit,
      localPostgres,
      secrets,
      baseImage,
      degradedReason: "node-runtime-base-image-seed-failed",
    };
  }
  const render = renderNodeRuntimeControlPlane(spec, sourceCommit, scoped.timeoutSeconds);
  if (!isCommandSuccess(render.result)) {
    return {
      ok: false,
      command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
      phase: "source-render",
      sourceCommit,
      localPostgres,
      secrets,
      baseImage,
      renderDir: render.renderDir,
      renderLocation: render.location,
      render: compactRuntimeCommand(render.result),
      degradedReason: "node-runtime-control-plane-render-failed",
    };
  }
  const apply = render.location === "local"
    ? applyLocalNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds)
    : applyNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds);
  const cleanup = render.location === "local"
    ? cleanupLocalNodeRuntimeRenderDir(spec, render)
    : cleanupNodeRuntimeRenderDir(spec, render.renderDir);
  const sshTcpPoolDiagnostics = sshTcpPoolDiagnosticsFromCommand(spec, apply);
  return {
    ok: isCommandSuccess(apply),
    command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
    node: scoped.node,
    lane: scoped.lane,
    mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
    mutation: !scoped.dryRun && isCommandSuccess(apply),
    sourceCommit,
    expected: nodeRuntimeExpected(spec),
    localPostgres,
    secrets,
    baseImage,
    renderDir: render.renderDir,
    renderLocation: render.location,
    render: compactRuntimeCommand(render.result),
    apply: compactRuntimeCommand(apply),
    cleanupRenderDir: compactRuntimeCommand(cleanup),
    diagnostics: sshTcpPoolDiagnostics,
    degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed",
    next: scoped.dryRun
      ? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` }
      : {
        triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
        ...(sshTcpPoolDiagnostics === null ? {} : { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` }),
      },
  };
}

export function nodeRuntimeRefresh(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const spec = scoped.spec;
  const script = [
    "set +e",
    `app=${shellQuote(spec.app)}`,
    "argo_namespace=argocd",
    `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
    "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
    "if [ \"$dry_run\" = true ]; then",
    "  output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)",
    "else",
    "  output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)",
    "fi",
    "code=$?",
    "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
    "printf 'app\\t%s\\n' \"$app\"",
    "printf 'dryRun\\t%s\\n' \"$dry_run\"",
    "printf 'before\\t%s\\n' \"$before\"",
    "printf 'after\\t%s\\n' \"$after\"",
    "printf 'annotateExitCode\\t%s\\n' \"$code\"",
    "printf 'annotateOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' '  ' | cut -c1-500)\"",
    "exit \"$code\"",
  ].join("\n");
  const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
  const fields = keyValueLinesFromText(statusText(result));
  return {
    ok: isCommandSuccess(result),
    command: `hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane}`,
    node: scoped.node,
    lane: scoped.lane,
    mode: scoped.dryRun ? "dry-run" : "confirmed-refresh",
    mutation: !scoped.dryRun && isCommandSuccess(result),
    argoApplication: fields.app || spec.app,
    before: fields.before || null,
    after: fields.after || null,
    annotateExitCode: numericField(fields.annotateExitCode),
    result: compactRuntimeCommand(result),
    next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
  };
}

export function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const spec = scoped.spec;
  const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
  if (localPostgres !== null && localPostgres.ok !== true) {
    return {
      ok: false,
      command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
      node: scoped.node,
      lane: scoped.lane,
      mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
      phase: "local-postgres-secret-sync",
      localPostgres,
      degradedReason: "local-postgres-secret-sync-failed",
      next: { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` },
    };
  }
  const operation = {
    operation: {
      initiatedBy: { username: "unidesk-hwlab-node-cli" },
      retry: { limit: 1 },
      sync: {
        prune: true,
        revision: spec.gitopsBranch,
        source: {
          repoURL: spec.argoRepoUrl,
          targetRevision: spec.gitopsBranch,
          path: spec.runtimePath,
        },
        syncOptions: ["CreateNamespace=true"],
        syncStrategy: { hook: {} },
      },
    },
  };
  const patchB64 = Buffer.from(JSON.stringify(operation), "utf8").toString("base64");
  const script = [
    "set +e",
    `app=${shellQuote(spec.app)}`,
    "argo_namespace=argocd",
    `runtime_namespace=${shellQuote(spec.runtimeNamespace)}`,
    `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
    `patch_b64=${shellQuote(patchB64)}`,
    "terminate_patch_file=$(mktemp /tmp/hwlab-node-argocd-terminate.XXXXXX.json)",
    "patch_file=$(mktemp /tmp/hwlab-node-argocd-sync.XXXXXX.json)",
    "printf '{\"operation\":null}' > \"$terminate_patch_file\"",
    "printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"",
    "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
    "operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)",
    "terminate_code=0",
    "terminate_output=",
    "terminated_operation=false",
    "if [ \"$operation_phase\" = Running ]; then",
    "  if [ \"$dry_run\" = true ]; then",
    "    terminated_operation=would-terminate",
    "  else",
    "    terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)",
    "    terminate_code=$?",
    "    if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi",
    "    sleep 2",
    "  fi",
    "fi",
    "failed_hook_count=0",
    "deleted_hook_count=0",
    "failed_hooks=",
    "deleted_hooks=",
    "hook_delete_errors=",
    "for job in $(kubectl -n \"$runtime_namespace\" get job -o name 2>/dev/null || true); do",
    "  tracking=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ index .metadata.annotations \"argocd.argoproj.io/tracking-id\" }}' 2>/dev/null || true)",
    "  failed=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ range .status.conditions }}{{ if or (eq .type \"Failed\") (eq .type \"FailureTarget\") }}{{ .status }} {{ end }}{{ end }}' 2>/dev/null || true)",
    "  case \"$tracking\" in \"$app:\"*) ;; *) continue ;; esac",
    "  case \"$failed\" in *True*) ;; *) continue ;; esac",
    "  hook_name=${job#job.batch/}",
    "  failed_hook_count=$((failed_hook_count + 1))",
    "  if [ -z \"$failed_hooks\" ]; then failed_hooks=$hook_name; else failed_hooks=$failed_hooks,$hook_name; fi",
    "  if [ \"$dry_run\" = false ]; then",
    "    delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$job\" --wait=false 2>&1)",
    "    delete_code=$?",
    "    if [ \"$delete_code\" -eq 0 ]; then",
    "      deleted_hook_count=$((deleted_hook_count + 1))",
    "      if [ -z \"$deleted_hooks\" ]; then deleted_hooks=$hook_name; else deleted_hooks=$deleted_hooks,$hook_name; fi",
    "    else",
    "      if [ -z \"$hook_delete_errors\" ]; then hook_delete_errors=\"$hook_name:$delete_code\"; else hook_delete_errors=\"$hook_delete_errors,$hook_name:$delete_code\"; fi",
    "    fi",
    "  fi",
    "done",
    "stale_statefulset_pod_count=0",
    "deleted_stale_statefulset_pod_count=0",
    "stale_statefulset_pods=",
    "deleted_stale_statefulset_pods=",
    "stale_statefulset_pod_delete_errors=",
    "for sts in $(kubectl -n \"$runtime_namespace\" get statefulset -o name 2>/dev/null || true); do",
    "  sts_name=${sts#statefulset.apps/}",
    "  current_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.currentRevision}' 2>/dev/null || true)",
    "  update_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.updateRevision}' 2>/dev/null || true)",
    "  if [ -z \"$current_revision\" ] || [ -z \"$update_revision\" ] || [ \"$current_revision\" = \"$update_revision\" ]; then continue; fi",
    "  for pod in $(kubectl -n \"$runtime_namespace\" get pod -l \"controller-revision-hash=$current_revision\" -o name 2>/dev/null || true); do",
    "    pod_name=${pod#pod/}",
    "    owner=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}' 2>/dev/null || true)",
    "    if [ \"$owner\" != \"StatefulSet/$sts_name\" ]; then continue; fi",
    "    ready=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.status.conditions[?(@.type==\"Ready\")].status}' 2>/dev/null || true)",
    "    waiting=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={range .status.containerStatuses[*]}{.state.waiting.reason}{\" \"}{end}' 2>/dev/null || true)",
    "    case \"$waiting\" in *ImagePullBackOff*|*ErrImagePull*|*CrashLoopBackOff*) ;; *) continue ;; esac",
    "    if [ \"$ready\" = True ]; then continue; fi",
    "    stale_statefulset_pod_count=$((stale_statefulset_pod_count + 1))",
    "    stale_item=\"$sts_name/$pod_name:$waiting\"",
    "    if [ -z \"$stale_statefulset_pods\" ]; then stale_statefulset_pods=$stale_item; else stale_statefulset_pods=$stale_statefulset_pods,$stale_item; fi",
    "    if [ \"$dry_run\" = false ]; then",
    "      delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$pod\" --wait=false 2>&1)",
    "      delete_code=$?",
    "      if [ \"$delete_code\" -eq 0 ]; then",
    "        deleted_stale_statefulset_pod_count=$((deleted_stale_statefulset_pod_count + 1))",
    "        if [ -z \"$deleted_stale_statefulset_pods\" ]; then deleted_stale_statefulset_pods=$pod_name; else deleted_stale_statefulset_pods=$deleted_stale_statefulset_pods,$pod_name; fi",
    "      else",
    "        if [ -z \"$stale_statefulset_pod_delete_errors\" ]; then stale_statefulset_pod_delete_errors=\"$pod_name:$delete_code\"; else stale_statefulset_pod_delete_errors=\"$stale_statefulset_pod_delete_errors,$pod_name:$delete_code\"; fi",
    "      fi",
    "    fi",
    "  done",
    "done",
    "if [ \"$dry_run\" = true ]; then",
    "  output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)",
    "else",
    "  output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" -o name 2>&1)",
    "fi",
    "code=$?",
    "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.operation.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
    "printf 'app\\t%s\\n' \"$app\"",
    "printf 'dryRun\\t%s\\n' \"$dry_run\"",
    "printf 'before\\t%s\\n' \"$before\"",
    "printf 'after\\t%s\\n' \"$after\"",
    "printf 'operationPhase\\t%s\\n' \"$operation_phase\"",
    "printf 'terminatedOperation\\t%s\\n' \"$terminated_operation\"",
    "printf 'terminateExitCode\\t%s\\n' \"$terminate_code\"",
    "printf 'terminateOutput\\t%s\\n' \"$(printf '%s' \"$terminate_output\" | tr '\\n\\t' '  ' | cut -c1-500)\"",
    "printf 'failedHookCount\\t%s\\n' \"$failed_hook_count\"",
    "printf 'failedHooks\\t%s\\n' \"$failed_hooks\"",
    "printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"",
    "printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"",
    "printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"",
    "printf 'staleStatefulSetPodCount\\t%s\\n' \"$stale_statefulset_pod_count\"",
    "printf 'staleStatefulSetPods\\t%s\\n' \"$stale_statefulset_pods\"",
    "printf 'deletedStaleStatefulSetPodCount\\t%s\\n' \"$deleted_stale_statefulset_pod_count\"",
    "printf 'deletedStaleStatefulSetPods\\t%s\\n' \"$deleted_stale_statefulset_pods\"",
    "printf 'staleStatefulSetPodDeleteErrors\\t%s\\n' \"$stale_statefulset_pod_delete_errors\"",
    "printf 'patchExitCode\\t%s\\n' \"$code\"",
    "printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' '  ' | cut -c1-500)\"",
    "rm -f \"$patch_file\" \"$terminate_patch_file\"",
    "exit \"$code\"",
  ].join("\n");
  const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
  const fields = keyValueLinesFromText(statusText(result));
  return {
    ok: isCommandSuccess(result),
    command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
    node: scoped.node,
    lane: scoped.lane,
    mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
    mutation: !scoped.dryRun && (isCommandSuccess(result) || localPostgres?.mutation === true),
    localPostgres,
    argoApplication: fields.app || spec.app,
    syncSource: {
      repoURL: spec.argoRepoUrl,
      targetRevision: spec.gitopsBranch,
      path: spec.runtimePath,
    },
    before: fields.before || null,
    after: fields.after || null,
    operationRecovery: {
      operationPhase: fields.operationPhase || null,
      terminatedOperation: fields.terminatedOperation || "false",
      terminateExitCode: numericField(fields.terminateExitCode),
      terminateOutput: fields.terminateOutput || null,
      failedHookCount: numericField(fields.failedHookCount),
      failedHooks: commaListField(fields.failedHooks),
      deletedHookCount: numericField(fields.deletedHookCount),
      deletedHooks: commaListField(fields.deletedHooks),
      hookDeleteErrors: fields.hookDeleteErrors || null,
      staleStatefulSetPodCount: numericField(fields.staleStatefulSetPodCount),
      staleStatefulSetPods: commaListField(fields.staleStatefulSetPods),
      deletedStaleStatefulSetPodCount: numericField(fields.deletedStaleStatefulSetPodCount),
      deletedStaleStatefulSetPods: commaListField(fields.deletedStaleStatefulSetPods),
      staleStatefulSetPodDeleteErrors: fields.staleStatefulSetPodDeleteErrors || null,
    },
    patchExitCode: numericField(fields.patchExitCode),
    patchOutput: fields.patchOutput || null,
    result: compactRuntimeCommand(result),
    next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
  };
}