fix: use k8s git mirror source snapshots

2026-07-01 10:32:05 +00:00
parent 79e419ce98
commit c90ad04bff
19 changed files with 481 additions and 168 deletions
@@ -103,10 +103,71 @@ export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
 }

 export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
-  const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 });
-  if (!isCommandSuccess(result)) return { sourceCommit: null, result };
-  const match = /[0-9a-f]{40}/iu.exec(statusText(result));
-  return { sourceCommit: match?.[0].toLowerCase() ?? null, result };
+  const mirror = nodeRuntimeSourceMirrorTarget(spec);
+  const script = [
+    "set +e",
+    `namespace=${shellQuote(mirror.namespace)}`,
+    `read_deploy=${shellQuote(mirror.serviceReadName)}`,
+    `repo_path=${shellQuote(`/cache/${mirror.sourceRepository}.git`)}`,
+    `source_branch=${shellQuote(mirror.sourceBranch)}`,
+    "read_ref() {",
+    "  ref=\"$1\"",
+    "  kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'repo_path=$1; ref=$2; git --git-dir=\"$repo_path\" rev-parse --verify \"$ref^{commit}\" 2>/dev/null' sh \"$repo_path\" \"$ref\" 2>/dev/null",
+    "}",
+    "mirror_stage=$(read_ref \"refs/mirror-stage/heads/$source_branch\")",
+    "mirror_stage_rc=$?",
+    "local_head=$(read_ref \"refs/heads/$source_branch\")",
+    "local_head_rc=$?",
+    "source_commit=\"$mirror_stage\"",
+    "source_ref=\"refs/mirror-stage/heads/$source_branch\"",
+    "if ! printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then",
+    "  source_commit=\"$local_head\"",
+    "  source_ref=\"refs/heads/$source_branch\"",
+    "fi",
+    "node - \"$mirror_stage_rc\" \"$local_head_rc\" \"$mirror_stage\" \"$local_head\" \"$source_commit\" \"$source_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'",
+    "const [mirrorStageRc, localHeadRc, mirrorStage, localHead, sourceCommit, sourceRef, repoPath, branch] = process.argv.slice(2);",
+    "const isSha = (value) => /^[0-9a-f]{40}$/i.test(value || '');",
+    "const ok = isSha(sourceCommit);",
+    "console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-cache', sourceAuthority: 'git-mirror-cache', sourceCommit: ok ? sourceCommit.toLowerCase() : null, sourceRef: ok ? sourceRef : null, mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, localHead: isSha(localHead) ? localHead.toLowerCase() : null, mirrorStageRc: Number(mirrorStageRc), localHeadRc: Number(localHeadRc), branch, repoPath, valuesRedacted: true }));",
+    "NODE",
+  ].join("\n");
+  const result = runNodeK3sScript(spec, script, 45);
+  const payload = parseJsonObject(result.stdout);
+  const payloadCommit = typeof payload.sourceCommit === "string" && /^[0-9a-f]{40}$/iu.test(payload.sourceCommit) ? payload.sourceCommit.toLowerCase() : null;
+  const match = payloadCommit ?? /[0-9a-f]{40}/iu.exec(statusText(result))?.[0].toLowerCase() ?? null;
+  return { sourceCommit: result.exitCode === 0 ? match : null, result };
+}
+
+interface NodeRuntimeSourceMirrorTarget {
+  readonly namespace: string;
+  readonly serviceReadName: string;
+  readonly sourceRepository: string;
+  readonly sourceBranch: string;
+}
+
+function nodeRuntimeSourceMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeSourceMirrorTarget {
+  const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
+  const parsed = runtimeRecord(Bun.YAML.parse(readFileSync(configPath, "utf8")));
+  const targets = Array.isArray(parsed.targets) ? parsed.targets.map((item) => runtimeRecord(item)) : [];
+  const target = targets.find((item) => item.node === spec.nodeId && item.lane === spec.lane);
+  if (target === undefined) throw new Error(`no control-plane target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
+  const gitMirror = runtimeRecord(target.gitMirror);
+  const source = runtimeRecord(target.source);
+  return {
+    namespace: runtimeString(gitMirror.namespace, "gitMirror.namespace"),
+    serviceReadName: runtimeString(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
+    sourceRepository: runtimeString(source.repository, "source.repository"),
+    sourceBranch: runtimeString(source.branch, "source.branch"),
+  };
+}
+
+function runtimeRecord(value: unknown): Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
+function runtimeString(value: unknown, path: string): string {
+  if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
+  return value;
 }

 export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
@@ -52,7 +52,7 @@ export type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cle

 export type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";

-export type NodeRuntimeRenderLocation = "node-host" | "local";
+export type NodeRuntimeRenderLocation = "node-host";

 export type WebProbeBrowserProxyMode = "auto" | "direct";

@@ -34,7 +34,7 @@ import { NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, NODE_RUNTIME_TRIGGER_SEVERE_WAR
 import { parseNodeScopedDelegatedOptions } from "./plan";
 import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimePipelineFailureSummary } from "./render";
 import { compactRuntimeCommand } from "./runtime-common";
-import { compactNodeRuntimeGitMirrorObservation, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
+import { compactNodeRuntimeGitMirrorObservation, compactNodeRuntimeGitMirrorRun, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
 import { record } from "./utils";
 import { webObserveTable } from "./web-observe-render";
 import { createNodeRuntimePipelineRun, getNodeRuntimePipelineRun, nodeRuntimePipelineRunManifest, printNodeRuntimeTriggerProgress, waitForNodeRuntimePipelineRunTerminal } from "./web-probe";
@@ -51,6 +51,28 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
  const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
  const triggerStartedAt = Date.now();
  const triggerElapsedMs = () => Date.now() - triggerStartedAt;
+  const sourceSnapshotSync = scoped.dryRun
+    ? null
+    : nodeRuntimeGitMirrorRun({
+      ...scoped,
+      domain: "git-mirror",
+      action: "sync",
+      confirm: true,
+      dryRun: false,
+      wait: true,
+      discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true,
+    });
+  if (sourceSnapshotSync !== null && sourceSnapshotSync.ok !== true) {
+    return {
+      ok: false,
+      command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      phase: "source-snapshot-sync",
+      degradedReason: "node-runtime-source-snapshot-sync-failed",
+      sourceSnapshotSync: nodeScopedFullOutput(scoped) ? sourceSnapshotSync : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
+    };
+  }
  printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
  const head = resolveNodeRuntimeLaneHead(spec);
  const sourceCommit = head.sourceCommit;
@@ -64,6 +86,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
      phase: "source-head",
      degradedReason: "node-runtime-source-head-unresolved",
      headProbe: compactRuntimeCommand(head.result),
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
    };
  }
  const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
@@ -86,6 +109,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
      rerun: scoped.rerun,
      before,
      gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
      manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
      next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
    };
@@ -122,6 +146,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
      pipelineWait,
      pipelineFailureSummary,
      postFlush,
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
      skipped: true,
      reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
      skipPolicy: "source-commit-only",
@@ -149,6 +174,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
      pipelineRun,
      before,
      gitMirror,
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
      degradedReason: "node-runtime-git-mirror-pre-sync-failed",
    };
  }
@@ -175,6 +201,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
      sourceCommit,
      pipelineRun,
      refresh,
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
      diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null,
      degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
      next: {
@@ -220,6 +247,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
    mutation: createOk,
    sourceCommit,
    pipelineRun,
+    sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
    rerunOf: scoped.rerun ? basePipelineRun : null,
    rerun: scoped.rerun,
    before,
@@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command";
 import { startJob } from "../jobs";
 import { classifySshTcpPoolFailure } from "../ssh";
 import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
-import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
+import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
 import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
 import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
 import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -971,14 +971,9 @@ export function nodeRuntimeRenderToken(): string {
 }

 export function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
-  if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds);
  return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds);
 }

-export function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean {
-  return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
-}
-
 export function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
  const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
  const retryCount = Math.max(0, Math.floor(retries));
@@ -1138,92 +1133,6 @@ export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec,
  return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
 }

-export function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
-  const token = nodeRuntimeRenderToken();
-  const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
-  const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
-  const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
-  const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
-  const script = [
-    "set -eu",
-    `source_url=${shellQuote(spec.gitUrl)}`,
-    `source_branch=${shellQuote(spec.sourceBranch)}`,
-    `source_commit=${shellQuote(sourceCommit)}`,
-    `render_dir=${shellQuote(renderDir)}`,
-    `worktree_dir=${shellQuote(worktreeDir)}`,
-    `overlay_b64=${shellQuote(overlay)}`,
-    `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
-    "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
-    "rm -rf \"$render_dir\" \"$worktree_dir\"",
-    "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
-    "echo \"phase=local-git-clone-worktree\" >&2",
-    "run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
-    "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
-    "cd \"$worktree_dir\"",
-    "echo \"phase=local-install-yaml\" >&2",
-    ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
-    "node - \"$overlay_b64\" <<'NODE'",
-    "const fs = require('fs');",
-    "const YAML = require('yaml');",
-    "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
-    "const path = 'deploy/deploy.yaml';",
-    "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
-    "doc.nodes = doc.nodes || {};",
-    "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
-    "doc.lanes = doc.lanes || {};",
-    "const lane = doc.lanes[overlay.lane] || {};",
-    "const downloadStack = {",
-    "  ...(lane.envRecipe?.downloadStack || {}),",
-    "  httpProxy: overlay.dockerProxyHttp,",
-    "  httpsProxy: overlay.dockerProxyHttps,",
-    "  noProxy: overlay.dockerNoProxyList,",
-    "};",
-    "doc.lanes[overlay.lane] = {",
-    "  ...lane,",
-    "  node: overlay.nodeId,",
-    "  sourceBranch: overlay.sourceBranch,",
-    "  gitopsBranch: overlay.gitopsBranch,",
-    "  namespace: overlay.runtimeNamespace,",
-    "  endpoint: overlay.publicApiUrl,",
-    "  publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
-    "  artifactCatalog: overlay.catalogPath,",
-    "  runtimePath: overlay.runtimePath,",
-    "  imageTagMode: 'full',",
-    "  sourceRepo: overlay.gitUrl,",
-    "  externalPostgres: overlay.externalPostgres,",
-    "  observability: overlay.observability,",
-    "  envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
-    "};",
-    "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
-    "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
-    "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
-    "fs.writeFileSync(path, YAML.stringify(doc));",
-    "NODE",
-    "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
-    "echo \"phase=local-gitops-render\" >&2",
-    [
-      "node scripts/run-bun.mjs \"$render_script\"",
-      `--lane ${shellQuote(spec.lane)}`,
-      `--node ${shellQuote(spec.nodeId)}`,
-      `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
-      `--catalog-path ${shellQuote(spec.catalogPath)}`,
-      "--image-tag-mode full",
-      `--source-revision ${shellQuote(sourceCommit)}`,
-      `--source-repo ${shellQuote(spec.gitUrl)}`,
-      `--source-branch ${shellQuote(spec.sourceBranch)}`,
-      `--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
-      `--git-read-url ${shellQuote(spec.gitReadUrl)}`,
-      `--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
-      `--registry-prefix ${shellQuote(spec.registryPrefix)}`,
-      `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
-      `--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
-      `--out ${shellQuote(renderDir)}`,
-    ].join(" "),
-    ...nodeRuntimePipelinePostprocessScript(),
-  ].join("\n");
-  return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" };
-}
-
 export function nodeRuntimePipelinePostprocessScript(): string[] {
  return [
    "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'",
@@ -72,7 +72,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
    "--timeout-ms",
    "--wait-timeout-ms",
    "--command-timeout-seconds",
-  ]), new Set(["--dry-run", "--confirm", "--wait", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"]));
+  ]), new Set(["--dry-run", "--confirm", "--wait", "--rerun", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"]));
  const node = requiredOption(args, "--node");
  assertNodeId(node);
  const lane = requiredOption(args, "--lane");
@@ -96,9 +96,9 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
    if (controlPlaneAction !== "plan" && controlPlaneAction !== "apply" && controlPlaneAction !== "status" && controlPlaneAction !== "trigger-current") {
      throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]");
    }
-    sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds };
+    sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") };
  } else if (sentinelActionRaw === "publish-current") {
-    sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds };
+    sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") };
  } else if (sentinelActionRaw === "maintenance") {
    const maintenanceAction = args[1];
    if (maintenanceAction !== "status" && maintenanceAction !== "start" && maintenanceAction !== "stop") {