fix: use k8s git mirror source snapshots

scripts/src/hwlab-node-web-sentinel-cicd.ts

@@ -7,6 +7,7 @@
 // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries.
 // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility.
 // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
+// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot.
 // Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
 import { createHash, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
@@ -61,6 +62,7 @@ export type WebProbeSentinelOptions =
     readonly confirm: boolean;
     readonly wait: boolean;
     readonly timeoutSeconds: number;
+    readonly rerun: boolean;
   }
   | {
     readonly kind: "publish";
@@ -72,6 +74,7 @@ export type WebProbeSentinelOptions =
     readonly confirm: boolean;
     readonly wait: boolean;
     readonly timeoutSeconds: number;
+    readonly rerun: boolean;
   }
   | {
     readonly kind: "maintenance";
@@ -156,7 +159,10 @@ interface SourceHead {
   readonly repository: string;
   readonly branch: string;
   readonly commit: string | null;
-  readonly localHead: string | null;
+  readonly stageRef: string | null;
+  readonly mirrorCommit: string | null;
+  readonly sourceAuthority: "git-mirror-cache" | "git-mirror-snapshot";
+  readonly latestDrift: boolean;
   readonly result: CompactCommandResult;
 }
 
@@ -218,12 +224,14 @@ export interface ChildCliResult {
   readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string };
 }
 
-const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel";
+const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot";
+
+type SourceResolveMode = "cached" | "sync";
 
 export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult {
   if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId));
   requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
-  const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds);
+  const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options));
   if (options.kind === "image") return runSentinelImage(state, options);
   if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
   if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
@@ -233,6 +241,13 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
   return runSentinelReport(state, options);
 }
 
+function sentinelSourceResolveMode(options: WebProbeSentinelOptions): SourceResolveMode {
+  if (options.kind === "image" && options.action === "build" && options.confirm && options.wait) return "sync";
+  if (options.kind === "control-plane" && options.action === "trigger-current" && options.confirm && options.wait) return "sync";
+  if (options.kind === "publish" && options.confirm && options.wait) return "sync";
+  return "cached";
+}
+
 function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
   const command = `web-probe sentinel image ${options.action}`;
   if (options.action === "build" && options.confirm) {
@@ -280,7 +295,7 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
   const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null;
   const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
   const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
-  const pipelineRun = sentinelPipelineRunName(state);
+  const pipelineRun = sentinelPipelineRunName(state, options.rerun);
   const result = {
     ok: state.configReady && state.sourceHead.ok && observedReady,
     command,
@@ -354,7 +369,7 @@ function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<We
     specRef: SPEC_REF,
     source: state.sourceHead,
     image: state.image,
-    pipelineRun: sentinelPipelineRunName(state),
+    pipelineRun: sentinelPipelineRunName(state, options.rerun),
     gitops: {
       path: stringAt(state.cicd, "gitopsPath"),
       targetRevision: stringAt(state.cicd, "argo.targetRevision"),
@@ -408,6 +423,7 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
     confirm: true,
     wait: true,
     timeoutSeconds: Math.max(1, remainingBudgetSeconds()),
+    rerun: options.rerun,
   });
   let health: Record<string, unknown>;
   let healthElapsedMs: number | null = null;
@@ -439,7 +455,7 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
     specRef: SPEC_REF,
     source: state.sourceHead,
     image: state.image,
-    pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state),
+    pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state, options.rerun),
     controlPlane: controlResult,
     health,
     budget,
@@ -571,7 +587,7 @@ function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed:
   };
 }
 
-function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState {
+function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number, sourceResolveMode: SourceResolveMode): SentinelCicdState {
   const sentinel = resolveWebProbeSentinel(spec, sentinelId);
   const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
   const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime);
@@ -585,7 +601,8 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
   const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef));
   const nodeId = stringField(controlPlaneTarget, "node");
   const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
-  const sourceHead = resolveSourceHead(cicd, timeoutSeconds);
+  validateSentinelSourceAuthority(cicd);
+  const sourceHead = resolveSourceHead(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds, sourceResolveMode);
   const image = sentinelImagePlan(spec, cicd, sourceHead);
   const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image);
   const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
@@ -609,23 +626,145 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
   };
 }
 
-function resolveSourceHead(cicd: Record<string, unknown>, timeoutSeconds: number): SourceHead {
+function resolveSourceHead(
+  spec: HwlabRuntimeLaneSpec,
+  cicd: Record<string, unknown>,
+  controlPlaneTarget: Record<string, unknown>,
+  controlPlaneNode: Record<string, unknown>,
+  timeoutSeconds: number,
+  mode: SourceResolveMode,
+): SourceHead {
   const repository = stringAt(cicd, "source.repository");
   const branch = stringAt(cicd, "source.branch");
-  const remote = runCommand(["git", "ls-remote", "origin", `refs/heads/${branch}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
-  const local = runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 10_000 });
-  const commit = /^[0-9a-f]{40}\b/iu.exec(remote.stdout.trim())?.[0].toLowerCase() ?? null;
-  const localHead = /^[0-9a-f]{40}$/iu.test(local.stdout.trim()) ? local.stdout.trim().toLowerCase() : null;
+  const resolved = mode === "sync"
+    ? resolveSourceHeadWithK8sSnapshot(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds)
+    : probeSourceMirrorCache(cicd, controlPlaneNode, timeoutSeconds, null);
+  const probe = record(resolved.probe);
+  const commit = nonEmptyString(probe.sourceCommit) ?? nonEmptyString(probe.commit) ?? nonEmptyString(probe.mirrorCommit);
+  const stageRef = nonEmptyString(probe.stageRef) ?? (commit === null ? null : sentinelSourceSnapshotRef(cicd, commit));
+  const mirrorCommit = nonEmptyString(probe.mirrorCommit) ?? nonEmptyString(probe.commit);
   return {
-    ok: remote.exitCode === 0 && commit !== null,
+    ok: resolved.ok === true && commit !== null,
     repository,
     branch,
     commit,
-    localHead,
-    result: compactCommand(remote),
+    stageRef,
+    mirrorCommit,
+    sourceAuthority: mode === "sync" ? "git-mirror-snapshot" : "git-mirror-cache",
+    latestDrift: commit !== null && mirrorCommit !== null && commit !== mirrorCommit,
+    result: compactCommand(resolved.result),
   };
 }
 
+function resolveSourceHeadWithK8sSnapshot(
+  spec: HwlabRuntimeLaneSpec,
+  cicd: Record<string, unknown>,
+  controlPlaneTarget: Record<string, unknown>,
+  controlPlaneNode: Record<string, unknown>,
+  timeoutSeconds: number,
+): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
+  const namespace = stringAt(cicd, "builder.namespace");
+  const prefix = `${stringAt(cicd, "builder.jobPrefix")}-source-resolve`;
+  const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
+  const manifest = sentinelSourceMirrorResolveJobManifest(spec, cicd, controlPlaneTarget, controlPlaneNode, jobName);
+  const created = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
+  if (created.exitCode !== 0) return { ok: false, probe: { ok: false, status: "create-failed", jobName, valuesRedacted: true }, result: created };
+  const startedAt = Date.now();
+  const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, 120_000));
+  let lastCapture = created;
+  while (Date.now() - startedAt < timeoutMs) {
+    const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
+    lastCapture = probeCapture;
+    const probe = parseJsonObject(probeCapture.stdout) ?? {};
+    const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
+    if (probe.succeeded === true) return { ok: payload.ok === true, probe: payload, result: probeCapture };
+    if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, valuesRedacted: true } : payload, result: probeCapture };
+    runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
+  }
+  return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture };
+}
+
+function sentinelSourceMirrorResolveJobManifest(
+  spec: HwlabRuntimeLaneSpec,
+  cicd: Record<string, unknown>,
+  controlPlaneTarget: Record<string, unknown>,
+  controlPlaneNode: Record<string, unknown>,
+  jobName: string,
+): Record<string, unknown> {
+  const namespace = stringAt(cicd, "builder.namespace");
+  const labels = {
+    "app.kubernetes.io/name": "web-probe-sentinel-source-resolve",
+    "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
+    "unidesk.ai/spec-ref": "PJ2026-01060508",
+    "unidesk.ai/node": spec.nodeId,
+    "unidesk.ai/lane": spec.lane,
+  };
+  return {
+    apiVersion: "batch/v1",
+    kind: "Job",
+    metadata: { name: jobName, namespace, labels },
+    spec: {
+      backoffLimit: 0,
+      activeDeadlineSeconds: numberAt(cicd, "builder.activeDeadlineSeconds"),
+      ttlSecondsAfterFinished: numberAt(cicd, "builder.ttlSecondsAfterFinished"),
+      template: {
+        metadata: { labels },
+        spec: {
+          restartPolicy: "Never",
+          volumes: [
+            sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget),
+            { name: "git-ssh", secret: { secretName: stringAt(cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
+          ],
+          containers: [{
+            name: "resolve",
+            image: sentinelSourceResolverImage(spec, cicd),
+            imagePullPolicy: "IfNotPresent",
+            command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShellFromConfig(cicd, controlPlaneNode, jobName, null)],
+            volumeMounts: [
+              { name: "cache", mountPath: "/cache" },
+              { name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
+            ],
+          }],
+        },
+      },
+    },
+  };
+}
+
+function sentinelSourceResolverImage(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>): string {
+  const baseImageRef = stringAt(cicd, "image.baseImageRef");
+  return stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef);
+}
+
+function validateSentinelSourceAuthority(cicd: Record<string, unknown>): void {
+  const mode = stringAt(cicd, "sourceAuthority.mode");
+  const resolver = stringAt(cicd, "sourceAuthority.resolver");
+  const allowHostGit = booleanAt(cicd, "sourceAuthority.allowHostGit");
+  const allowGithubDirectInPipeline = booleanAt(cicd, "sourceAuthority.allowGithubDirectInPipeline");
+  const missingObjectPolicy = stringAt(cicd, "sourceSnapshot.missingObjectPolicy");
+  if (mode !== "gitMirrorSnapshot") throw new Error("sourceAuthority.mode must be gitMirrorSnapshot");
+  if (resolver !== "k8s-git-mirror") throw new Error("sourceAuthority.resolver must be k8s-git-mirror");
+  if (allowHostGit !== false) throw new Error("sourceAuthority.allowHostGit must be false");
+  if (allowGithubDirectInPipeline !== false) throw new Error("sourceAuthority.allowGithubDirectInPipeline must be false");
+  if (missingObjectPolicy !== "fail-fast") throw new Error("sourceSnapshot.missingObjectPolicy must be fail-fast");
+  sentinelSourceSnapshotStageRefPrefix(cicd);
+}
+
+function sentinelSourceSnapshotStageRefPrefix(cicd: Record<string, unknown>): string {
+  const branch = stringAt(cicd, "source.branch");
+  const repository = stringAt(cicd, "source.repository");
+  const prefix = stringAt(cicd, "sourceSnapshot.stageRefPrefix")
+    .replaceAll("{branch}", branch)
+    .replaceAll("{repository}", repository)
+    .replace(/\/+$/u, "");
+  if (!prefix.startsWith("refs/")) throw new Error("sourceSnapshot.stageRefPrefix must resolve to a git ref prefix");
+  return prefix;
+}
+
+function sentinelSourceSnapshotRef(cicd: Record<string, unknown>, commit: string): string {
+  return `${sentinelSourceSnapshotStageRefPrefix(cicd)}/${commit}`;
+}
+
 function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>, sourceHead: SourceHead): SentinelImagePlan {
   const repository = stringAt(cicd, "image.repository");
   const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12);
@@ -1409,7 +1548,7 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra
   const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds);
   const sourceMirrorReady = sourceMirrorSync.ok === true;
   const publish = sourceMirrorReady
-    ? runSentinelPublishJob(state, false, options.timeoutSeconds)
+    ? runSentinelPublishJob(state, false, options.timeoutSeconds, false)
     : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
   const registry = probeImageRegistry(state, options.timeoutSeconds);
   const registryReady = record(registry.probe).present === true;
@@ -1474,7 +1613,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
   const publish = applyOnly
     ? null
     : sourceMirrorReady
-      ? runSentinelPublishJob(state, true, remainingCommandSeconds())
+      ? runSentinelPublishJob(state, true, remainingCommandSeconds(), options.rerun)
       : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
   const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8;
   const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true
@@ -1523,6 +1662,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
           ? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
           : "one or more publicExposure, Argo or runtime observation checks did not pass",
   };
+  const publishPipelineRun = applyOnly ? sentinelPipelineRunName(state, options.rerun) : record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun);
   const result = {
     ok,
     command,
@@ -1533,7 +1673,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
     specRef: SPEC_REF,
     source: state.sourceHead,
     image: state.image,
-    pipelineRun: sentinelPipelineRunName(state),
+    pipelineRun: publishPipelineRun,
     gitops: {
       path: stringAt(state.cicd, "gitopsPath"),
       targetRevision: stringAt(state.cicd, "argo.targetRevision"),
@@ -1732,19 +1872,26 @@ function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObser
 }
 
 function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
-  const namespace = stringAt(state.cicd, "builder.namespace");
-  const repository = stringAt(state.cicd, "source.repository");
-  const branch = stringAt(state.cicd, "source.branch");
-  const expectedCommit = state.sourceHead.commit;
+  const result = probeSourceMirrorCache(state.cicd, state.controlPlaneNode, timeoutSeconds, state.sourceHead.commit);
+  return { ...result, result: compactCommand(result.result) };
+}
+
+function probeSourceMirrorCache(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, timeoutSeconds: number, expectedCommit: string | null): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
+  const namespace = stringAt(cicd, "builder.namespace");
+  const repository = stringAt(cicd, "source.repository");
+  const branch = stringAt(cicd, "source.branch");
+  const stageRef = expectedCommit === null ? "" : sentinelSourceSnapshotRef(cicd, expectedCommit);
   const script = [
     "set +e",
     `repo_path=${shellQuote(`/cache/${repository}.git`)}`,
     `branch=${shellQuote(branch)}`,
     `expected=${shellQuote(expectedCommit ?? "")}`,
+    `stage_ref=${shellQuote(stageRef)}`,
     "commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)",
     "rc=$?",
     "object_rc=1",
     "expected_object_rc=1",
+    "stage_object_rc=1",
     "contains_rc=1",
     "if [ \"$rc\" -eq 0 ]; then",
     "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
@@ -1754,22 +1901,27 @@ function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Re
     "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$expected^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
     "  expected_object_rc=$?",
     "fi",
+    "if [ -n \"$stage_ref\" ]; then",
+    "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse --verify \\\"$stage_ref^{commit}\\\" >/dev/null 2>&1\" >/dev/null 2>&1",
+    "  stage_object_rc=$?",
+    "fi",
     "if [ \"$rc\" -eq 0 ] && [ \"$expected_object_rc\" -eq 0 ]; then",
     "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" merge-base --is-ancestor \\\"$expected\\\" \\\"$commit\\\" 2>/dev/null\" >/dev/null 2>&1",
     "  contains_rc=$?",
     "fi",
-    "node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$repo_path\" \"$branch\" <<'NODE'",
-    "const [rc, objectRc, expectedObjectRc, containsRc, commit, expected, repoPath, branch] = process.argv.slice(2);",
+    "node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$stage_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$stage_ref\" \"$repo_path\" \"$branch\" <<'NODE'",
+    "const [rc, objectRc, expectedObjectRc, stageObjectRc, containsRc, commit, expected, stageRef, repoPath, branch] = process.argv.slice(2);",
     "const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');",
     "const objectPresent = present && Number(objectRc) === 0;",
     "const expectedObjectPresent = !expected || Number(expectedObjectRc) === 0;",
+    "const stageObjectPresent = !stageRef || Number(stageObjectRc) === 0;",
     "const containsExpected = !expected || commit === expected || Number(containsRc) === 0;",
     "const relation = !expected ? 'unconstrained' : commit === expected ? 'equal' : containsExpected ? 'mirror-ahead' : expectedObjectPresent ? 'diverged-or-behind' : 'expected-object-missing';",
-    "console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror', present, objectPresent, expectedObjectPresent, containsExpected, relation, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
+    "console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror-cache', present, objectPresent, expectedObjectPresent, stageObjectPresent, containsExpected, relation, commit: present ? commit : null, sourceCommit: expected || (present ? commit : null), mirrorCommit: present ? commit : null, expectedCommit: expected || null, stageRef: stageRef || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
     "NODE",
   ].join("\n");
-  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
-  return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) };
+  const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
+  return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout) ?? {}, result };
 }
 
 function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record<string, unknown> {
@@ -2130,18 +2282,22 @@ function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName:
 }
 
 function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string {
+  return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit);
+}
+
+function sentinelSourceMirrorSyncShellFromConfig(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, jobName: string, selectedCommit: string | null): string {
   return [
     "set -eu",
     `job_name=${shellQuote(jobName)}`,
-    `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
-    `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
-    `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`,
-    `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
+    `source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`,
+    `source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`,
+    `source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`,
+    `source_commit=${shellQuote(selectedCommit ?? "")}`,
+    `source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`,
     "started_ms=$(node -e 'console.log(Date.now())')",
     "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
     "trap emit_failed EXIT",
-    "test -n \"$source_commit\"",
-    ...sentinelSourceMirrorSshSetupShellLines(state),
+    ...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode),
     "repo=\"/cache/${source_repository}.git\"",
     "mkdir -p \"$(dirname \"$repo\")\"",
     "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
@@ -2164,20 +2320,28 @@ function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string
     "done",
     "test \"$fetch_ok\" = 1",
     "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")",
+    "if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi",
+    "git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"",
     "test \"$mirror_commit\" = \"$source_commit\"",
+    "stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"",
     "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"",
+    "git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"",
     "git --git-dir=\"$repo\" update-server-info",
     "finished_ms=$(node -e 'console.log(Date.now())')",
-    "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
-    "const [jobName, repository, branch, sourceCommit, mirrorCommit, startedMs, finishedMs] = process.argv.slice(2);",
-    "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
+    "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
+    "const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);",
+    "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
     "NODE",
     "trap - EXIT",
   ].join("\n");
 }
 
 function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] {
-  const proxy = record(valueAtPath(state.controlPlaneNode, "egressProxy"));
+  return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode);
+}
+
+function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record<string, unknown>): string[] {
+  const proxy = record(valueAtPath(controlPlaneNode, "egressProxy"));
   const serviceName = nonEmptyString(proxy.serviceName);
   const namespace = nonEmptyString(proxy.namespace);
   const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null;
@@ -2299,8 +2463,8 @@ function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): strin
   ];
 }
 
-function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult {
-  const pipelineRunName = sentinelPipelineRunName(state);
+function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult {
+  const pipelineRunName = sentinelPipelineRunName(state, rerun);
   const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops);
   const namespace = stringAt(state.cicd, "builder.namespace");
   sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
@@ -2371,7 +2535,9 @@ function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRu
       namespace,
       labels,
       annotations: {
-        "unidesk.ai/source-commit": state.sourceHead.commit,
+        "unidesk.ai/source-commit": state.sourceHead.commit ?? "",
+        "unidesk.ai/source-authority": state.sourceHead.sourceAuthority,
+        "unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "",
         "unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"),
         "unidesk.ai/publish-gitops": publishGitops ? "true" : "false",
       },
@@ -2459,9 +2625,13 @@ function tektonShellScript(body: string): string {
 }
 
 function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string, unknown> {
-  const hostPath = nonEmptyString(valueAtPath(state.controlPlaneTarget, "gitMirror.cacheHostPath"));
+  return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget);
+}
+
+function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record<string, unknown>): Record<string, unknown> {
+  const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath"));
   if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } };
-  return { name: "cache", persistentVolumeClaim: { claimName: stringAt(state.controlPlaneTarget, "gitMirror.cachePvcName") } };
+  return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } };
 }
 
 function sentinelBuildkitStateVolume(state: SentinelCicdState): Record<string, unknown> {
@@ -2524,6 +2694,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
     `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
     `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
     `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
+    `source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`,
     `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
     `dockerfile_b64=${shellQuote(dockerfileB64)}`,
     `env_reuse_mode=${shellQuote(envReuseMode)}`,
@@ -2542,6 +2713,9 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
     "started_ms=$(now_ms)",
     "write_meta started_ms \"$started_ms\"",
     "write_meta source_commit \"$source_commit\"",
+    "write_meta source_stage_ref \"$source_stage_ref\"",
+    "test -n \"$source_commit\"",
+    "test -n \"$source_stage_ref\"",
     "mkdir -p /root/.ssh",
     "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
     "chmod 0400 /root/.ssh/id_rsa",
@@ -2561,7 +2735,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
     "source_fetch_started_ms=$(now_ms)",
     "write_meta source_fetch_started_ms \"$source_fetch_started_ms\"",
     "emit_stage source-fetch running \"$source_fetch_started_ms\"",
-    "git fetch --depth=1 --filter=blob:none origin \"+refs/heads/$source_branch:refs/remotes/origin/$source_branch\"",
+    "git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"",
     "git checkout --detach \"$source_commit\"",
     "mirror_commit=$(git rev-parse HEAD)",
     "test \"$mirror_commit\" = \"$source_commit\"",
@@ -2720,6 +2894,7 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
     "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
     "started_ms=$(read_meta started_ms)",
     "source_commit=$(read_meta source_commit)",
+    "source_stage_ref=$(read_meta source_stage_ref)",
     "mirror_commit=$(read_meta mirror_commit)",
     "source_fetch_started_ms=$(read_meta source_fetch_started_ms)",
     "source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)",
@@ -2781,11 +2956,11 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
     "fi",
     "gitops_finished_ms=$(now_ms)",
     "finished_ms=$(now_ms)",
-    "node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
-    "const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
+    "node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
+    "const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
     "const elapsed = (start, finish) => Number(finish) - Number(start);",
     "const cacheHits = Number(imageBuildCacheHits || 0);",
-    "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
+    "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
     "NODE",
     "trap - EXIT",
   ].join("\n");
@@ -2862,7 +3037,7 @@ function probeK8sJobScript(namespace: string, jobName: string): string {
     "pod_phase=''",
     "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi",
     "logs_tail=''",
-    "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=80 2>/dev/null || true; done; } | tail -c 16000 | base64 | tr -d '\\n'); fi",
+    "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
     "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
     "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);",
     "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
@@ -2878,12 +3053,15 @@ function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string
     "condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)",
     "condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)",
     "condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)",
+    "if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi",
+    "if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi",
+    "if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi",
     "task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
     "pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
     "pod_phase=''",
     "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi",
     "logs_tail=''",
-    "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=180 2>/dev/null || true; } | tail -c 24000 | base64 | tr -d '\\n'); fi",
+    "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
     "node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
     "const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);",
     "const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');",
@@ -3082,7 +3260,9 @@ function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, prob
       ok: true,
       status: "already-present",
       sourceCommit: state.sourceHead.commit,
-      mirrorCommit: state.sourceHead.commit,
+      mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit,
+      stageRef: state.sourceHead.stageRef,
+      sourceAuthority: state.sourceHead.sourceAuthority,
       valuesRedacted: true,
     },
     polls: 0,
@@ -3653,9 +3833,12 @@ export function displayPath(pathValue: string): string {
   return pathValue;
 }
 
-function sentinelPipelineRunName(state: SentinelCicdState): string {
+function sentinelPipelineRunName(state: SentinelCicdState, rerun = false): string {
   const commit = state.sourceHead.commit ?? "source";
-  return `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`;
+  const base = `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`;
+  if (!rerun) return base;
+  const suffix = `-r${Date.now().toString(36)}`;
+  return `${base.slice(0, Math.max(1, 63 - suffix.length)).replace(/-+$/u, "")}${suffix}`;
 }
 
 export function sentinelCliSuffix(state: SentinelCicdState): string {
@@ -3795,9 +3978,11 @@ function renderPublishCurrentResult(result: Record<string, unknown>): string {
       finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000),
     ]]),
     "",
-    table(["SOURCE", "COMMIT", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[
+    table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[
       `${source.repository ?? "-"}@${source.branch ?? "-"}`,
       short(source.commit),
+      source.sourceAuthority ?? "-",
+      short(source.stageRef),
       image.ref ?? "-",
       short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest),
       result.pipelineRun ?? publish.jobName ?? "-",
@@ -3902,7 +4087,7 @@ function renderImageResult(result: Record<string, unknown>): string {
     "",
     table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]),
     "",
-    table(["SOURCE_REPO", "BRANCH", "COMMIT", "LOCAL_HEAD"], [[source.repository, source.branch, short(source.commit), short(source.localHead)]]),
+    table(["SOURCE_REPO", "BRANCH", "COMMIT", "AUTHORITY", "STAGE_REF", "MIRROR"], [[source.repository, source.branch, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), short(source.mirrorCommit)]]),
     "",
     Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]),
     "",
@@ -3912,7 +4097,7 @@ function renderImageResult(result: Record<string, unknown>): string {
     "",
     Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
     "",
-    Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
+    Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]),
     "",
     Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish),
     "",
@@ -3958,7 +4143,7 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
     "",
     table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]),
     "",
-    table(["SOURCE", "COMMIT", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), image.ref, short(gitops.manifestSha256)]]),
+    table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref, short(gitops.manifestSha256)]]),
     "",
     table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]),
     "",
@@ -3968,7 +4153,7 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
     "",
     renderObservedStatus(observed),
     "",
-    Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
+    Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]),
     "",
     Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[
       targetValidation.ok,

scripts/src/hwlab-node-web-sentinel-config.ts

@@ -138,6 +138,14 @@ const REQUIRED_TARGET_SHAPES: Record<HwlabRuntimeWebProbeSentinelConfigRefKey, R
       "source.gitMirrorReadUrl",
       "source.buildContext",
       "source.entrypoint",
+      "sourceAuthority.mode",
+      "sourceAuthority.resolver",
+      "sourceAuthority.allowHostGit",
+      "sourceAuthority.allowGithubDirectInPipeline",
+      "sourceSnapshot.stageRefPrefix",
+      "sourceSnapshot.missingObjectPolicy",
+      "sourceSnapshot.refreshPolicy",
+      "sourceSnapshot.cacheRef",
       "gitopsPath",
       "argo.namespace",
       "argo.projectName",

scripts/src/hwlab-node/cleanup.ts

@@ -103,10 +103,71 @@ export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
 }
 
 export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
-  const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 });
-  if (!isCommandSuccess(result)) return { sourceCommit: null, result };
-  const match = /[0-9a-f]{40}/iu.exec(statusText(result));
-  return { sourceCommit: match?.[0].toLowerCase() ?? null, result };
+  const mirror = nodeRuntimeSourceMirrorTarget(spec);
+  const script = [
+    "set +e",
+    `namespace=${shellQuote(mirror.namespace)}`,
+    `read_deploy=${shellQuote(mirror.serviceReadName)}`,
+    `repo_path=${shellQuote(`/cache/${mirror.sourceRepository}.git`)}`,
+    `source_branch=${shellQuote(mirror.sourceBranch)}`,
+    "read_ref() {",
+    "  ref=\"$1\"",
+    "  kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'repo_path=$1; ref=$2; git --git-dir=\"$repo_path\" rev-parse --verify \"$ref^{commit}\" 2>/dev/null' sh \"$repo_path\" \"$ref\" 2>/dev/null",
+    "}",
+    "mirror_stage=$(read_ref \"refs/mirror-stage/heads/$source_branch\")",
+    "mirror_stage_rc=$?",
+    "local_head=$(read_ref \"refs/heads/$source_branch\")",
+    "local_head_rc=$?",
+    "source_commit=\"$mirror_stage\"",
+    "source_ref=\"refs/mirror-stage/heads/$source_branch\"",
+    "if ! printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then",
+    "  source_commit=\"$local_head\"",
+    "  source_ref=\"refs/heads/$source_branch\"",
+    "fi",
+    "node - \"$mirror_stage_rc\" \"$local_head_rc\" \"$mirror_stage\" \"$local_head\" \"$source_commit\" \"$source_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'",
+    "const [mirrorStageRc, localHeadRc, mirrorStage, localHead, sourceCommit, sourceRef, repoPath, branch] = process.argv.slice(2);",
+    "const isSha = (value) => /^[0-9a-f]{40}$/i.test(value || '');",
+    "const ok = isSha(sourceCommit);",
+    "console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-cache', sourceAuthority: 'git-mirror-cache', sourceCommit: ok ? sourceCommit.toLowerCase() : null, sourceRef: ok ? sourceRef : null, mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, localHead: isSha(localHead) ? localHead.toLowerCase() : null, mirrorStageRc: Number(mirrorStageRc), localHeadRc: Number(localHeadRc), branch, repoPath, valuesRedacted: true }));",
+    "NODE",
+  ].join("\n");
+  const result = runNodeK3sScript(spec, script, 45);
+  const payload = parseJsonObject(result.stdout);
+  const payloadCommit = typeof payload.sourceCommit === "string" && /^[0-9a-f]{40}$/iu.test(payload.sourceCommit) ? payload.sourceCommit.toLowerCase() : null;
+  const match = payloadCommit ?? /[0-9a-f]{40}/iu.exec(statusText(result))?.[0].toLowerCase() ?? null;
+  return { sourceCommit: result.exitCode === 0 ? match : null, result };
+}
+
+interface NodeRuntimeSourceMirrorTarget {
+  readonly namespace: string;
+  readonly serviceReadName: string;
+  readonly sourceRepository: string;
+  readonly sourceBranch: string;
+}
+
+function nodeRuntimeSourceMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeSourceMirrorTarget {
+  const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
+  const parsed = runtimeRecord(Bun.YAML.parse(readFileSync(configPath, "utf8")));
+  const targets = Array.isArray(parsed.targets) ? parsed.targets.map((item) => runtimeRecord(item)) : [];
+  const target = targets.find((item) => item.node === spec.nodeId && item.lane === spec.lane);
+  if (target === undefined) throw new Error(`no control-plane target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
+  const gitMirror = runtimeRecord(target.gitMirror);
+  const source = runtimeRecord(target.source);
+  return {
+    namespace: runtimeString(gitMirror.namespace, "gitMirror.namespace"),
+    serviceReadName: runtimeString(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
+    sourceRepository: runtimeString(source.repository, "source.repository"),
+    sourceBranch: runtimeString(source.branch, "source.branch"),
+  };
+}
+
+function runtimeRecord(value: unknown): Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
+function runtimeString(value: unknown, path: string): string {
+  if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
+  return value;
 }
 
 export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {

scripts/src/hwlab-node/entry.ts

@@ -52,7 +52,7 @@ export type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cle
 
 export type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
 
-export type NodeRuntimeRenderLocation = "node-host" | "local";
+export type NodeRuntimeRenderLocation = "node-host";
 
 export type WebProbeBrowserProxyMode = "auto" | "direct";
 

scripts/src/hwlab-node/git-mirror.ts

@@ -34,7 +34,7 @@ import { NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, NODE_RUNTIME_TRIGGER_SEVERE_WAR
 import { parseNodeScopedDelegatedOptions } from "./plan";
 import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimePipelineFailureSummary } from "./render";
 import { compactRuntimeCommand } from "./runtime-common";
-import { compactNodeRuntimeGitMirrorObservation, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
+import { compactNodeRuntimeGitMirrorObservation, compactNodeRuntimeGitMirrorRun, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
 import { record } from "./utils";
 import { webObserveTable } from "./web-observe-render";
 import { createNodeRuntimePipelineRun, getNodeRuntimePipelineRun, nodeRuntimePipelineRunManifest, printNodeRuntimeTriggerProgress, waitForNodeRuntimePipelineRunTerminal } from "./web-probe";
@@ -51,6 +51,28 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
   const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
   const triggerStartedAt = Date.now();
   const triggerElapsedMs = () => Date.now() - triggerStartedAt;
+  const sourceSnapshotSync = scoped.dryRun
+    ? null
+    : nodeRuntimeGitMirrorRun({
+      ...scoped,
+      domain: "git-mirror",
+      action: "sync",
+      confirm: true,
+      dryRun: false,
+      wait: true,
+      discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true,
+    });
+  if (sourceSnapshotSync !== null && sourceSnapshotSync.ok !== true) {
+    return {
+      ok: false,
+      command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      phase: "source-snapshot-sync",
+      degradedReason: "node-runtime-source-snapshot-sync-failed",
+      sourceSnapshotSync: nodeScopedFullOutput(scoped) ? sourceSnapshotSync : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
+    };
+  }
   printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
   const head = resolveNodeRuntimeLaneHead(spec);
   const sourceCommit = head.sourceCommit;
@@ -64,6 +86,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
       phase: "source-head",
       degradedReason: "node-runtime-source-head-unresolved",
       headProbe: compactRuntimeCommand(head.result),
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
     };
   }
   const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
@@ -86,6 +109,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
       rerun: scoped.rerun,
       before,
       gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
       manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
       next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
     };
@@ -122,6 +146,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
       pipelineWait,
       pipelineFailureSummary,
       postFlush,
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
       skipped: true,
       reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
       skipPolicy: "source-commit-only",
@@ -149,6 +174,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
       pipelineRun,
       before,
       gitMirror,
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
       degradedReason: "node-runtime-git-mirror-pre-sync-failed",
     };
   }
@@ -175,6 +201,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
       sourceCommit,
       pipelineRun,
       refresh,
+      sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
       diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null,
       degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
       next: {
@@ -220,6 +247,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
     mutation: createOk,
     sourceCommit,
     pipelineRun,
+    sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
     rerunOf: scoped.rerun ? basePipelineRun : null,
     rerun: scoped.rerun,
     before,

scripts/src/hwlab-node/render.ts

@@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command";
 import { startJob } from "../jobs";
 import { classifySshTcpPoolFailure } from "../ssh";
 import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
-import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
+import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
 import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
 import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
 import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -971,14 +971,9 @@ export function nodeRuntimeRenderToken(): string {
 }
 
 export function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
-  if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds);
   return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds);
 }
 
-export function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean {
-  return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
-}
-
 export function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
   const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
   const retryCount = Math.max(0, Math.floor(retries));
@@ -1138,92 +1133,6 @@ export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec,
   return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
 }
 
-export function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
-  const token = nodeRuntimeRenderToken();
-  const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
-  const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
-  const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
-  const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
-  const script = [
-    "set -eu",
-    `source_url=${shellQuote(spec.gitUrl)}`,
-    `source_branch=${shellQuote(spec.sourceBranch)}`,
-    `source_commit=${shellQuote(sourceCommit)}`,
-    `render_dir=${shellQuote(renderDir)}`,
-    `worktree_dir=${shellQuote(worktreeDir)}`,
-    `overlay_b64=${shellQuote(overlay)}`,
-    `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
-    "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
-    "rm -rf \"$render_dir\" \"$worktree_dir\"",
-    "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
-    "echo \"phase=local-git-clone-worktree\" >&2",
-    "run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
-    "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
-    "cd \"$worktree_dir\"",
-    "echo \"phase=local-install-yaml\" >&2",
-    ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
-    "node - \"$overlay_b64\" <<'NODE'",
-    "const fs = require('fs');",
-    "const YAML = require('yaml');",
-    "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
-    "const path = 'deploy/deploy.yaml';",
-    "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
-    "doc.nodes = doc.nodes || {};",
-    "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
-    "doc.lanes = doc.lanes || {};",
-    "const lane = doc.lanes[overlay.lane] || {};",
-    "const downloadStack = {",
-    "  ...(lane.envRecipe?.downloadStack || {}),",
-    "  httpProxy: overlay.dockerProxyHttp,",
-    "  httpsProxy: overlay.dockerProxyHttps,",
-    "  noProxy: overlay.dockerNoProxyList,",
-    "};",
-    "doc.lanes[overlay.lane] = {",
-    "  ...lane,",
-    "  node: overlay.nodeId,",
-    "  sourceBranch: overlay.sourceBranch,",
-    "  gitopsBranch: overlay.gitopsBranch,",
-    "  namespace: overlay.runtimeNamespace,",
-    "  endpoint: overlay.publicApiUrl,",
-    "  publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
-    "  artifactCatalog: overlay.catalogPath,",
-    "  runtimePath: overlay.runtimePath,",
-    "  imageTagMode: 'full',",
-    "  sourceRepo: overlay.gitUrl,",
-    "  externalPostgres: overlay.externalPostgres,",
-    "  observability: overlay.observability,",
-    "  envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
-    "};",
-    "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
-    "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
-    "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
-    "fs.writeFileSync(path, YAML.stringify(doc));",
-    "NODE",
-    "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
-    "echo \"phase=local-gitops-render\" >&2",
-    [
-      "node scripts/run-bun.mjs \"$render_script\"",
-      `--lane ${shellQuote(spec.lane)}`,
-      `--node ${shellQuote(spec.nodeId)}`,
-      `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
-      `--catalog-path ${shellQuote(spec.catalogPath)}`,
-      "--image-tag-mode full",
-      `--source-revision ${shellQuote(sourceCommit)}`,
-      `--source-repo ${shellQuote(spec.gitUrl)}`,
-      `--source-branch ${shellQuote(spec.sourceBranch)}`,
-      `--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
-      `--git-read-url ${shellQuote(spec.gitReadUrl)}`,
-      `--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
-      `--registry-prefix ${shellQuote(spec.registryPrefix)}`,
-      `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
-      `--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
-      `--out ${shellQuote(renderDir)}`,
-    ].join(" "),
-    ...nodeRuntimePipelinePostprocessScript(),
-  ].join("\n");
-  return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" };
-}
-
 export function nodeRuntimePipelinePostprocessScript(): string[] {
   return [
     "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'",

scripts/src/hwlab-node/web-probe-observe.ts

@@ -72,7 +72,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
     "--timeout-ms",
     "--wait-timeout-ms",
     "--command-timeout-seconds",
-  ]), new Set(["--dry-run", "--confirm", "--wait", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"]));
+  ]), new Set(["--dry-run", "--confirm", "--wait", "--rerun", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"]));
   const node = requiredOption(args, "--node");
   assertNodeId(node);
   const lane = requiredOption(args, "--lane");
@@ -96,9 +96,9 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
     if (controlPlaneAction !== "plan" && controlPlaneAction !== "apply" && controlPlaneAction !== "status" && controlPlaneAction !== "trigger-current") {
       throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]");
     }
-    sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds };
+    sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") };
   } else if (sentinelActionRaw === "publish-current") {
-    sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds };
+    sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") };
   } else if (sentinelActionRaw === "maintenance") {
     const maintenanceAction = args[1];
     if (maintenanceAction !== "status" && maintenanceAction !== "start" && maintenanceAction !== "stop") {