// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot.
// Responsibility: source mirror and publish remote Job/PipelineRun orchestration for web-probe sentinel CI/CD.
import { runCommand } from "./command";
import { repoRoot, rootPath } from "./config";
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
import { emitWebProbeSentinelSpan } from "./hwlab-node-web-sentinel-otel";
import {
  arrayAt,
  compactCommand,
  finiteNumberOrNull,
  monitorWebBuildkitStatePlan,
  nonEmptyString,
  numberAt,
  numberAtNullable,
  parseJsonObject,
  record,
  resolveSentinelChildJson,
  safeKubernetesSegment,
  sentinelCliSuffix,
  sentinelPipelineRunName,
  sentinelProgressEvent,
  sentinelSourceSnapshotStageRefPrefix,
  shellQuote,
  short,
  stringAt,
  stringAtNullable,
  text,
  valueAtPath,
  type SentinelCicdState,
  type SentinelRemoteJobResult,
} from "./hwlab-node-web-sentinel-cicd-shared";

export function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult {
  const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`;
  const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
  const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName);
  const namespace = stringAt(state.cicd, "builder.namespace");
  sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
  const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  if (created.exitCode !== 0) {
    sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane });
    return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "source-mirror");
  }
  const startedAt = Date.now();
  const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
  const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
  let slowWarningSent = false;
  let polls = 0;
  let lastProbe: Record<string, unknown> = {};
  while (Date.now() - startedAt < timeoutMs) {
    polls += 1;
    const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
    const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-source-mirror-job-probe");
    const probe = probeResolution.parsed ?? {};
    lastProbe = { ...probe, stdoutRecovery: probeResolution.diagnostics, capture: compactCommand(probeCapture) };
    const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
    sentinelProgressEvent("sentinel.source-mirror.progress", {
      phase: "remote-job",
      status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
      jobName,
      polls,
      elapsedMs: Date.now() - startedAt,
      pod: probe.pod ?? null,
      sourceCommit: state.sourceHead.commit,
      node: state.spec.nodeId,
      lane: state.spec.lane,
    });
    if (probe.succeeded === true) {
      const ok = payload.ok === true;
      return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
    }
    if (probe.failed === true) {
      return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
    }
    if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
      slowWarningSent = true;
      sentinelProgressEvent("sentinel.source-mirror.warning", { warning: `source mirror sync exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
    }
    runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
  }
  return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
}

export function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult {
  return {
    ok: false,
    phase,
    jobName: "-",
    payload: { ok: false, status: phase, reason, valuesRedacted: true },
    valuesRedacted: true,
  };
}

function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record<string, unknown> {
  const namespace = stringAt(state.cicd, "builder.namespace");
  const labels = {
    "app.kubernetes.io/name": "web-probe-sentinel-source-mirror",
    "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
    "unidesk.ai/spec-ref": "PJ2026-01060508",
    "unidesk.ai/node": state.spec.nodeId,
    "unidesk.ai/lane": state.spec.lane,
  };
  return {
    apiVersion: "batch/v1",
    kind: "Job",
    metadata: { name: jobName, namespace, labels },
    spec: {
      backoffLimit: 0,
      activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"),
      ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"),
      template: {
        metadata: { labels },
        spec: {
          restartPolicy: "Never",
          volumes: [
            sentinelGitMirrorCacheVolume(state),
            { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
          ],
          containers: [{
            name: "sync",
            image: state.image.baseImage,
            imagePullPolicy: "IfNotPresent",
            command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)],
            volumeMounts: [
              { name: "cache", mountPath: "/cache" },
              { name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
            ],
          }],
        },
      },
    },
  };
}

function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string {
  return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit);
}

export function sentinelSourceMirrorSyncShellFromConfig(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, jobName: string, selectedCommit: string | null): string {
  return [
    "set -eu",
    `job_name=${shellQuote(jobName)}`,
    `source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`,
    `source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`,
    `source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`,
    `source_commit=${shellQuote(selectedCommit ?? "")}`,
    `source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`,
    "started_ms=$(node -e 'console.log(Date.now())')",
    "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
    "trap emit_failed EXIT",
    ...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode),
    "repo=\"/cache/${source_repository}.git\"",
    "mkdir -p \"$(dirname \"$repo\")\"",
    "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
    "  git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
    "else",
    "  rm -rf \"$repo\"",
    "  git init --bare \"$repo\"",
    "  git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
    "fi",
    "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
    "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
    "git --git-dir=\"$repo\" config http.uploadpack true",
    "git --git-dir=\"$repo\" config http.receivepack true",
    "fetch_ok=0",
    "for attempt in 1 2 3; do",
    "  if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi",
    "  code=$?",
    "  printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2",
    "  sleep $((attempt * 5))",
    "done",
    "test \"$fetch_ok\" = 1",
    "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")",
    "if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi",
    "git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"",
    "test \"$mirror_commit\" = \"$source_commit\"",
    "stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"",
    "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"",
    "git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"",
    "git --git-dir=\"$repo\" update-server-info",
    "finished_ms=$(node -e 'console.log(Date.now())')",
    "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
    "const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);",
    "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
    "NODE",
    "trap - EXIT",
  ].join("\n");
}

function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] {
  return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode);
}

function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record<string, unknown>): string[] {
  const proxy = record(valueAtPath(controlPlaneNode, "egressProxy"));
  const serviceName = nonEmptyString(proxy.serviceName);
  const namespace = nonEmptyString(proxy.namespace);
  const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null;
  const hostRouteProxyUrl = nonEmptyString(proxy.proxyUrl);
  const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : "";
  let proxyHost: string | null = null;
  let proxyPort: number | null = null;
  let proxyUrl: string | null = null;
  if (serviceName !== null && namespace !== null && port !== null) {
    proxyHost = `${serviceName}.${namespace}.svc.cluster.local`;
    proxyPort = port;
    proxyUrl = `http://${proxyHost}:${proxyPort}`;
  } else if (hostRouteProxyUrl !== null) {
    try {
      const parsed = new URL(hostRouteProxyUrl);
      const parsedPort = Number.parseInt(parsed.port || "80", 10);
      if ((parsed.protocol === "http:" || parsed.protocol === "https:") && parsed.hostname.length > 0 && Number.isInteger(parsedPort)) {
        proxyHost = parsed.hostname;
        proxyPort = parsedPort;
        proxyUrl = hostRouteProxyUrl;
      }
    } catch {
      proxyHost = null;
      proxyPort = null;
      proxyUrl = null;
    }
  }
  const useProxy = proxyHost !== null && proxyPort !== null && proxyUrl !== null;
  if (!useProxy) {
    return [
      "mkdir -p /root/.ssh",
      "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
      "chmod 0400 /root/.ssh/id_rsa",
      "printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2",
      "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy",
      "export NO_PROXY='*'",
      "export no_proxy='*'",
      "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
      "#!/bin/sh",
      "exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"",
      "SH_PROXY",
      "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
      "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
      "unset GIT_SSH_COMMAND",
    ];
  }
  const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
  return [
    "mkdir -p /root/.ssh",
    "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
    "chmod 0400 /root/.ssh/id_rsa",
    `printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`,
    `export HTTP_PROXY=${shellQuote(proxyUrl)}`,
    `export HTTPS_PROXY=${shellQuote(proxyUrl)}`,
    `export ALL_PROXY=${shellQuote(proxyUrl)}`,
    `export http_proxy=${shellQuote(proxyUrl)}`,
    `export https_proxy=${shellQuote(proxyUrl)}`,
    `export all_proxy=${shellQuote(proxyUrl)}`,
    `export NO_PROXY=${shellQuote(noProxy)}`,
    `export no_proxy=${shellQuote(noProxy)}`,
    "cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'",
    "#!/usr/bin/env node",
    "const net = require('node:net');",
    "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
    "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
    "const targetPort = Number.parseInt(targetPortRaw || '', 10);",
    "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
    "  console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');",
    "  process.exit(64);",
    "}",
    "let settled = false;",
    "let tunnelEstablished = false;",
    "function finish(code, message) {",
    "  if (settled) return;",
    "  settled = true;",
    "  if (message) console.error('sentinel source-mirror proxy-connect: ' + message);",
    "  process.exit(code);",
    "}",
    "const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
    "let buffer = Buffer.alloc(0);",
    "socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
    "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
    "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
    "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
    "function onData(chunk) {",
    "  buffer = Buffer.concat([buffer, chunk]);",
    "  const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
    "  if (headerEnd === -1 && buffer.length < 8192) return;",
    "  if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
    "  const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
    "  const statusLine = head.split('\\r\\n', 1)[0] || '';",
    "  const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
    "  if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
    "    const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
    "    socket.destroy();",
    "    finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
    "    return;",
    "  }",
    "  socket.off('data', onData);",
    "  socket.setTimeout(0);",
    "  tunnelEstablished = true;",
    "  const rest = buffer.slice(headerEnd + 4);",
    "  if (rest.length) process.stdout.write(rest);",
    "  process.stdin.on('error', () => {});",
    "  process.stdout.on('error', () => {});",
    "  process.stdin.pipe(socket);",
    "  socket.pipe(process.stdout);",
    "}",
    "socket.on('data', onData);",
    "NODE_PROXY",
    "chmod 0700 /tmp/sentinel-github-proxy-connect.cjs",
    "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
    "#!/bin/sh",
    `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`,
    "SH_PROXY",
    "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
    "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
    "unset GIT_SSH_COMMAND",
  ];
}

export function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult {
  const pipelineRunName = sentinelPipelineRunName(state, rerun);
  const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops);
  const namespace = stringAt(state.cicd, "builder.namespace");
  sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
  const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createTektonPipelineRunScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  if (created.exitCode !== 0) {
    sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "failed", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
    return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-pipelinerun", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish");
  }
  sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "succeeded", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
  const startedAt = Date.now();
  const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
  const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
  let slowWarningSent = false;
  let polls = 0;
  let lastProbe: Record<string, unknown> = {};
  while (Date.now() - startedAt < timeoutMs) {
    polls += 1;
    const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeTektonPipelineRunScript(namespace, pipelineRunName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
    const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-publish-pipelinerun-probe");
    const probe = probeResolution.parsed ?? {};
    lastProbe = { ...probe, stdoutRecovery: probeResolution.diagnostics, capture: compactCommand(probeCapture) };
    const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
    sentinelProgressEvent("sentinel.publish.progress", {
      phase: "remote-pipelinerun",
      status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
      pipelineRun: pipelineRunName,
      publishGitops,
      polls,
      elapsedMs: Date.now() - startedAt,
      taskRun: probe.taskRun ?? null,
      pod: probe.pod ?? null,
      sourceCommit: state.sourceHead.commit,
      node: state.spec.nodeId,
      lane: state.spec.lane,
    });
    if (probe.succeeded === true) {
      const ok = payload.ok === true;
      return withSentinelRemoteJobDiagnostics(state, { ok, phase: "pipelinerun-succeeded", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
    }
    if (probe.failed === true) {
      return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-failed", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
    }
    if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
      slowWarningSent = true;
      sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish PipelineRun exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, pipelineRun: pipelineRunName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
    }
    runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
  }
  return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-timeout", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
}

export function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRunName: string, publishGitops: boolean): Record<string, unknown> {
  const namespace = stringAt(state.cicd, "builder.namespace");
  const buildkitImage = requireSentinelBuildkitImage(state);
  const proxyEnv = sentinelImageBuildProxyEnv(state);
  const labels = {
    "app.kubernetes.io/name": "web-probe-sentinel-publish",
    "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
    "unidesk.ai/spec-ref": "PJ2026-01060508",
    "unidesk.ai/node": state.spec.nodeId,
    "unidesk.ai/lane": state.spec.lane,
    "unidesk.ai/ci-system": "tekton",
  };
  return {
    apiVersion: "tekton.dev/v1",
    kind: "PipelineRun",
    metadata: {
      name: pipelineRunName,
      namespace,
      labels,
      annotations: {
        "unidesk.ai/source-commit": state.sourceHead.commit ?? "",
        "unidesk.ai/source-authority": state.sourceHead.sourceAuthority,
        "unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "",
        "unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"),
        "unidesk.ai/publish-gitops": publishGitops ? "true" : "false",
      },
    },
    spec: {
      timeouts: { pipeline: `${numberAt(state.cicd, "builder.activeDeadlineSeconds")}s` },
      taskRunTemplate: {
        podTemplate: {
          hostNetwork: true,
          dnsPolicy: "ClusterFirstWithHostNet",
          securityContext: { fsGroup: 1000 },
        },
      },
      pipelineSpec: {
        tasks: [{
          name: "publish",
          taskSpec: {
            volumes: [
              sentinelGitMirrorCacheVolume(state),
              { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
              { name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
              sentinelBuildkitStateVolume(state),
              { name: "tmp", emptyDir: {} },
            ],
            steps: [
              {
                name: "source",
                image: state.image.baseImage,
                imagePullPolicy: "IfNotPresent",
                env: proxyEnv,
                script: tektonShellScript(sentinelPublishSourceShell(state, pipelineRunName)),
                volumeMounts: [
                  { name: "workspace", mountPath: "/workspace" },
                  { name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
                ],
              },
              {
                name: "prepare-buildkit-state",
                image: state.image.baseImage,
                imagePullPolicy: "IfNotPresent",
                script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"),
                securityContext: { runAsUser: 0, runAsGroup: 0 },
                volumeMounts: [
                  { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
                ],
              },
              {
                name: "image-build",
                image: buildkitImage,
                imagePullPolicy: "IfNotPresent",
                env: [
                  ...proxyEnv,
                  { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
                ],
                script: tektonShellScript(sentinelPublishImageBuildShell(state, pipelineRunName)),
                securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
                volumeMounts: [
                  { name: "workspace", mountPath: "/workspace" },
                  { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
                  { name: "tmp", mountPath: "/tmp" },
                ],
              },
              {
                name: "publish",
                image: state.image.baseImage,
                imagePullPolicy: "IfNotPresent",
                env: proxyEnv,
                script: tektonShellScript(sentinelPublishShell(state, pipelineRunName, publishGitops)),
                volumeMounts: [
                  { name: "workspace", mountPath: "/workspace" },
                  { name: "cache", mountPath: "/cache" },
                  { name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
                ],
              },
            ],
          },
        }],
      },
    },
  };
}

function tektonShellScript(body: string): string {
  return `#!/bin/sh\n${body}`;
}

function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string, unknown> {
  return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget);
}

export function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record<string, unknown>): Record<string, unknown> {
  const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath"));
  if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } };
  return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } };
}

function sentinelBuildkitStateVolume(state: SentinelCicdState): Record<string, unknown> {
  const buildkitState = monitorWebBuildkitStatePlan(state.cicd);
  const mode = stringAt(buildkitState, "mode");
  if (mode === "hostPath") {
    return {
      name: "buildkit-state",
      hostPath: {
        path: stringAt(buildkitState, "path"),
        type: stringAt(buildkitState, "type"),
      },
    };
  }
  if (mode === "persistentVolumeClaim") {
    return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } };
  }
  if (mode === "emptyDir") {
    return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } };
  }
  throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`);
}

function requireSentinelBuildkitImage(state: SentinelCicdState): string {
  const image = state.spec.buildkit?.sidecarImage;
  if (typeof image !== "string" || image.length === 0) {
    throw new Error(`config/hwlab-node-lanes.yaml ${state.spec.nodeId}/${state.spec.lane} buildkit.sidecarImage is required for pure k8s web-probe sentinel image publish`);
  }
  return image;
}

function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> {
  const proxy = state.spec.networkProfile.imageBuildProxy;
  const noProxy = proxy.noProxy.join(",");
  return [
    { name: "HTTP_PROXY", value: proxy.http },
    { name: "http_proxy", value: proxy.http },
    { name: "HTTPS_PROXY", value: proxy.https },
    { name: "https_proxy", value: proxy.https },
    { name: "ALL_PROXY", value: proxy.all },
    { name: "all_proxy", value: proxy.all },
    { name: "NO_PROXY", value: noProxy },
    { name: "no_proxy", value: noProxy },
  ];
}

function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string {
  const monitorWeb = record(state.image.monitorWeb);
  const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => {
    if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths");
    return item;
  })), "utf8").toString("base64");
  const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64");
  const envReuseMode = stringAt(monitorWeb, "envReuseMode");
  const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath");
  return [
    "set -eu",
    `job_name=${shellQuote(jobName)}`,
    `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
    `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
    `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
    `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
    `source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`,
    `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
    `dockerfile_b64=${shellQuote(dockerfileB64)}`,
    `env_reuse_mode=${shellQuote(envReuseMode)}`,
    `env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`,
    "meta_dir=/workspace/meta",
    "event_log=/workspace/publish-events.log",
    "worktree=/workspace/source",
    "rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json",
    "mkdir -p \"$meta_dir\"",
    ": > \"$event_log\"",
    "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
    "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
    "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
    "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
    "trap emit_failed EXIT",
    "started_ms=$(now_ms)",
    "write_meta started_ms \"$started_ms\"",
    "write_meta source_commit \"$source_commit\"",
    "write_meta source_stage_ref \"$source_stage_ref\"",
    "test -n \"$source_commit\"",
    "test -n \"$source_stage_ref\"",
    "mkdir -p /root/.ssh",
    "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
    "chmod 0400 /root/.ssh/id_rsa",
    "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
    "git init \"$worktree\"",
    "cd \"$worktree\"",
    "git remote add origin \"$source_git_url\"",
    "git config core.sparseCheckout true",
    "git config remote.origin.promisor true",
    "git config remote.origin.partialclonefilter blob:none",
    "CHECKOUT_PATHS_B64=\"$checkout_paths_b64\" node <<'NODE'",
    "const fs = require('node:fs');",
    "const paths = JSON.parse(Buffer.from(process.env.CHECKOUT_PATHS_B64 || '', 'base64').toString('utf8'));",
    "fs.mkdirSync('.git/info', { recursive: true });",
    "fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');",
    "NODE",
    "source_fetch_started_ms=$(now_ms)",
    "write_meta source_fetch_started_ms \"$source_fetch_started_ms\"",
    "emit_stage source-fetch running \"$source_fetch_started_ms\"",
    "git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"",
    "git checkout --detach \"$source_commit\"",
    "mirror_commit=$(git rev-parse HEAD)",
    "test \"$mirror_commit\" = \"$source_commit\"",
    "write_meta mirror_commit \"$mirror_commit\"",
    "source_fetch_finished_ms=$(now_ms)",
    "write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"",
    "emit_stage source-fetch succeeded \"$source_fetch_started_ms\"",
    "env_reuse_node_deps_present=false",
    "env_reuse_node_deps_entries=0",
    "if [ -d \"$env_reuse_node_deps_path\" ]; then env_reuse_node_deps_present=true; env_reuse_node_deps_entries=$(find \"$env_reuse_node_deps_path\" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' '); fi",
    "env_reuse_linked_node_deps=0",
    "rm -rf node_modules",
    "if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi",
    "write_meta env_reuse_mode \"$env_reuse_mode\"",
    "write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"",
    "write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"",
    "write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"",
    "write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"",
    "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'",
    "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
    "NODE",
    "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"",
    "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
    "NODE",
    "monitor_web_verify_started_ms=$(now_ms)",
    "write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"",
    "emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"",
    "if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi",
    "cat /tmp/web-probe-sentinel-monitor-web-verify.log",
    "monitor_web_verify_finished_ms=$(now_ms)",
    "write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"",
    "emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"",
    "mkdir -p .unidesk-sentinel-bin",
    "cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'",
    "#!/bin/sh",
    "exec bun /app/scripts/ssh-cli.ts \"$@\"",
    "SH_TRANS",
    "chmod 0755 .unidesk-sentinel-bin/trans",
    "DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'",
    "const fs = require('node:fs');",
    "fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));",
    "NODE",
    "cat > .dockerignore <<'EOF_DOCKERIGNORE'",
    ".git",
    ".git/**",
    ".state",
    ".state/**",
    "logs",
    "logs/**",
    "node_modules",
    "node_modules/**",
    "**/node_modules",
    "**/node_modules/**",
    "**/dist",
    "**/dist/**",
    "**/target",
    "**/target/**",
    "**/coverage",
    "**/coverage/**",
    "npm-debug.log*",
    ".env",
    ".env.*",
    "EOF_DOCKERIGNORE",
    "context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')",
    "write_meta context_ignore_entries \"$context_ignore_entries\"",
    "chmod -R a+rwX /workspace",
    "trap - EXIT",
  ].join("\n");
}

function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string {
  const monitorWeb = record(state.image.monitorWeb);
  const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode");
  const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode");
  const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource");
  const imageBuildProxy = state.spec.networkProfile.imageBuildProxy;
  const imageBuildNoProxy = imageBuildProxy.noProxy.join(",");
  return [
    "set -eu",
    `job_name=${shellQuote(jobName)}`,
    `image_ref=${shellQuote(state.image.ref)}`,
    `image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`,
    `image_build_package_mode=${shellQuote(imageBuildPackageMode)}`,
    `image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`,
    `image_build_proxy_source=${shellQuote(imageBuildProxySource)}`,
    `image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`,
    `image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`,
    `image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`,
    `image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`,
    "meta_dir=/workspace/meta",
    "event_log=/workspace/publish-events.log",
    "build_log=/workspace/image-build.log",
    "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
    "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
    "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
    "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
    "trap emit_failed EXIT",
    "cd /workspace/source",
    "image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi",
    "image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi",
    "image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi",
    "image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi",
    "write_meta image_build_builder \"$image_build_builder\"",
    "write_meta image_build_package_mode \"$image_build_package_mode\"",
    "write_meta image_build_network_mode \"$image_build_network_mode\"",
    "write_meta image_build_proxy_source \"$image_build_proxy_source\"",
    "write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"",
    "write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"",
    "write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"",
    "write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"",
    "image_build_started_ms=$(now_ms)",
    "write_meta image_build_started_ms \"$image_build_started_ms\"",
    "emit_stage image-build running \"$image_build_started_ms\"",
    "if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi",
    "cat \"$build_log\"",
    "image_build_finished_ms=$(now_ms)",
    "write_meta image_build_finished_ms \"$image_build_finished_ms\"",
    "metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)",
    "digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)",
    "test -n \"$digest\"",
    "repo_no_tag=${image_ref%:*}",
    "digest_ref=\"$repo_no_tag@$digest\"",
    "write_meta digest \"$digest\"",
    "write_meta digest_ref \"$digest_ref\"",
    "emit_stage image-build succeeded \"$image_build_started_ms\"",
    "trap - EXIT",
  ].join("\n");
}

function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
  const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : [];
  const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({
    path: file.path,
    contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
  }))), "utf8").toString("base64");
  return [
    "set -eu",
    `job_name=${shellQuote(jobName)}`,
    `image_ref=${shellQuote(state.image.ref)}`,
    `image_repository=${shellQuote(state.image.repository)}`,
    `gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`,
    `gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`,
    `files_b64=${shellQuote(filesB64)}`,
    "meta_dir=/workspace/meta",
    "event_log=/workspace/publish-events.log",
    "build_log=/workspace/image-build.log",
    "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
    "read_meta() { cat \"$meta_dir/$1\"; }",
    "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }",
    "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }",
    "trap emit_failed EXIT",
    "if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi",
    "mkdir -p /root/.ssh",
    "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
    "chmod 0400 /root/.ssh/id_rsa",
    "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
    "started_ms=$(read_meta started_ms)",
    "source_commit=$(read_meta source_commit)",
    "source_stage_ref=$(read_meta source_stage_ref)",
    "mirror_commit=$(read_meta mirror_commit)",
    "source_fetch_started_ms=$(read_meta source_fetch_started_ms)",
    "source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)",
    "monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)",
    "monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)",
    "image_build_started_ms=$(read_meta image_build_started_ms)",
    "image_build_finished_ms=$(read_meta image_build_finished_ms)",
    "digest_ref=$(read_meta digest_ref)",
    "env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')",
    "env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')",
    "env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)",
    "env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)",
    "env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)",
    "image_build_builder=$(read_meta image_build_builder)",
    "image_build_package_mode=$(read_meta image_build_package_mode)",
    "image_build_network_mode=$(read_meta image_build_network_mode)",
    "image_build_proxy_source=$(read_meta image_build_proxy_source)",
    "image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)",
    "image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)",
    "image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)",
    "image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)",
    "context_ignore_entries=$(read_meta context_ignore_entries)",
    "image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)",
    "image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)",
    "image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')",
    "gitops_commit=''",
    "changed=false",
    "file_count=0",
    "gitops_started_ms=$(now_ms)",
    "if [ \"$files_b64\" != \"W10=\" ]; then",
    "  emit_stage gitops running \"$gitops_started_ms\"",
    "  gitops_cache=\"/cache/${gitops_repository}.git\"",
    "  gitops_worktree=\"/tmp/$job_name/gitops\"",
    "  git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"",
    "  cd \"$gitops_worktree\"",
    "  git fetch origin \"$gitops_branch\" || true",
    "  if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"; else git checkout --orphan \"$gitops_branch\"; git rm -rf . >/dev/null 2>&1 || true; fi",
    "  FILES_B64=\"$files_b64\" IMAGE_REF=\"$image_ref\" DIGEST_REF=\"$digest_ref\" node <<'NODE'",
    "const fs = require('node:fs');",
    "const path = require('node:path');",
    "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
    "for (const file of files) {",
    "  const target = path.resolve(process.cwd(), file.path);",
    "  if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);",
    "  fs.mkdirSync(path.dirname(target), { recursive: true });",
    "  const text = Buffer.from(file.contentBase64, 'base64').toString('utf8').split(process.env.IMAGE_REF).join(process.env.DIGEST_REF);",
    "  fs.writeFileSync(target, text);",
    "}",
    "console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));",
    "NODE",
    "  git add .",
    "  file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')",
    "  if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi",
    "  git push origin \"HEAD:refs/heads/$gitops_branch\"",
    "  gitops_commit=$(git rev-parse HEAD)",
    "  emit_stage gitops succeeded \"$gitops_started_ms\"",
    "else",
    "  emit_stage gitops skipped \"$gitops_started_ms\"",
    "fi",
    "gitops_finished_ms=$(now_ms)",
    "finished_ms=$(now_ms)",
    "node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
    "const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
    "const elapsed = (start, finish) => Number(finish) - Number(start);",
    "const cacheHits = Number(imageBuildCacheHits || 0);",
    "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
    "NODE",
    "trap - EXIT",
  ].join("\n");
}

function sentinelGitopsFiles(state: SentinelCicdState): readonly { path: string; content: string }[] {
  const runtimeManifests = state.manifests.filter((item) => item.kind !== "Application");
  return [{
    path: `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`,
    content: `${runtimeManifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`,
  }];
}

export function applySentinelArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
  const app = state.manifests.find((item) => item.kind === "Application");
  if (app === undefined) return { ok: false, reason: "application-manifest-missing", valuesRedacted: true };
  const yaml = `${Bun.YAML.stringify(app).trim()}\n`;
  const namespace = stringAt(state.cicd, "argo.namespace");
  const applicationName = stringAt(state.cicd, "argo.applicationName");
  const script = [
    "set -eu",
    "tmp=$(mktemp)",
    `cat >"$tmp" <<'YAML'\n${yaml}YAML`,
    "kubectl apply -f \"$tmp\"",
    `kubectl -n ${shellQuote(namespace)} annotate application ${shellQuote(applicationName)} argocd.argoproj.io/refresh=hard --overwrite`,
  ].join("\n");
  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  return { ok: result.exitCode === 0, result: compactCommand(result), valuesRedacted: true };
}

export function createK8sJobScript(namespace: string, manifest: Record<string, unknown>): string {
  const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
  return [
    "set -eu",
    `kubectl -n ${shellQuote(namespace)} delete job ${shellQuote(stringAt(manifest, "metadata.name"))} --ignore-not-found=true >/dev/null 2>&1 || true`,
    "tmp=$(mktemp)",
    `cat >"$tmp" <<'YAML'\n${yaml}YAML`,
    "kubectl apply -f \"$tmp\"",
  ].join("\n");
}

function createTektonPipelineRunScript(namespace: string, manifest: Record<string, unknown>): string {
  const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
  const pipelineRunName = stringAt(manifest, "metadata.name");
  return [
    "set -eu",
    `pipeline_run=${shellQuote(pipelineRunName)}`,
    `namespace=${shellQuote(namespace)}`,
    "tmp=$(mktemp)",
    `cat >"$tmp" <<'YAML'\n${yaml}YAML`,
    "if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then",
    "  printf '%s\\n' \"sentinel publish PipelineRun already exists; reusing $pipeline_run\"",
    "  exit 0",
    "fi",
    "if ! kubectl create -f \"$tmp\"; then",
    "  if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then",
    "    printf '%s\\n' \"sentinel publish PipelineRun appeared concurrently; reusing $pipeline_run\"",
    "    exit 0",
    "  fi",
    "  exit 1",
    "fi",
  ].join("\n");
}

export function probeK8sJobScript(namespace: string, jobName: string): string {
  return [
    "set +e",
    `namespace=${shellQuote(namespace)}`,
    `job=${shellQuote(jobName)}`,
    "succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)",
    "failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)",
    "active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)",
    "pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)",
    "pod_phase=''",
    "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi",
    "logs_tail=''",
    "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
    "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
    "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);",
    "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
    "NODE",
  ].join("\n");
}

function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string): string {
  return [
    "set +e",
    `namespace=${shellQuote(namespace)}`,
    `pipeline_run=${shellQuote(pipelineRunName)}`,
    "condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)",
    "condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)",
    "condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)",
    "if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi",
    "if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi",
    "if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi",
    "task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
    "pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
    "pod_phase=''",
    "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi",
    "logs_tail=''",
    "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
    "node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
    "const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);",
    "const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');",
    "const active = conditionStatus === 'Unknown' || (!conditionStatus && (podPhase === 'Pending' || podPhase === 'Running'));",
    "console.log(JSON.stringify({ succeeded: conditionStatus === 'True', failed: conditionStatus === 'False', active, conditionStatus: conditionStatus || null, conditionReason: conditionReason || null, conditionMessage: message || null, taskRun: taskRun || null, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
    "NODE",
  ].join("\n");
}

export function sentinelPayloadFromLogs(logsTail: string): Record<string, unknown> {
  const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
  for (let index = lines.length - 1; index >= 0; index -= 1) {
    const line = lines[index];
    if (!line.startsWith("{") || !line.endsWith("}")) continue;
    const parsed = parseJsonObject(line);
    if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed;
  }
  return {};
}

function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult {
  return { ...result, diagnostics: sentinelRemoteJobDiagnostics(state, result, domain), valuesRedacted: true };
}

function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): Record<string, unknown> {
  const namespace = stringAt(state.cicd, "builder.namespace");
  const probe = record(result.probe);
  const logsTail = typeof probe.logsTail === "string" ? probe.logsTail : "";
  const events = sentinelStageEventsFromLogs(logsTail, domain);
  const envReuse = sentinelEnvReuseFromLogs(logsTail);
  const completedStages = sentinelCompletedStages(events, record(result.payload));
  const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs);
  const currentPhase = sentinelCurrentRemotePhase(result, events, domain);
  const isPipelineRun = result.resourceKind === "PipelineRun";
  const commands = {
    cliStatus: domain === "publish"
      ? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`
      : `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
    logs: result.jobName === "-"
      ? "-"
      : isPipelineRun
        ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs -l tekton.dev/pipelineRun=${result.jobName} --all-containers=true --tail=120`
        : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`,
    describe: result.jobName === "-"
      ? "-"
      : isPipelineRun
        ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe pipelinerun/${result.jobName}`
        : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`,
    gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
    gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`,
    gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`,
    controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
    publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
    valuesRedacted: true,
  };
  return {
    domain,
    resourceKind: result.resourceKind ?? "Job",
    pipelineRun: isPipelineRun ? result.jobName : null,
    taskRun: probe.taskRun ?? null,
    currentPhase,
    completedStages,
    stageTimings,
    envReuse,
    pod: probe.pod ?? null,
    podPhase: probe.podPhase ?? null,
    active: probe.active ?? null,
    conditionStatus: probe.conditionStatus ?? null,
    conditionReason: probe.conditionReason ?? null,
    recentLogSummary: sentinelRecentLogSummary(logsTail),
    commands,
    valuesRedacted: true,
  };
}

function sentinelEnvReuseFromLogs(logsTail: string): Record<string, unknown> | null {
  const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
  for (let index = lines.length - 1; index >= 0; index -= 1) {
    const parsed = parseJsonObject(lines[index]);
    if (parsed !== null && parsed.event === "sentinel-publish-env-reuse") return { ...parsed, valuesRedacted: true };
  }
  return null;
}

function sentinelStageEventsFromLogs(logsTail: string, domain: "source-mirror" | "publish"): Record<string, unknown>[] {
  const expectedEvent = domain === "publish" ? "sentinel-publish-stage" : "sentinel-source-mirror-stage";
  return logsTail
    .split(/\r?\n/u)
    .map((line) => parseJsonObject(line.trim()))
    .filter((item): item is Record<string, unknown> => item !== null && item.event === expectedEvent);
}

function sentinelCompletedStages(events: readonly Record<string, unknown>[], payload: Record<string, unknown>): string[] {
  const completed = events
    .filter((event) => event.status === "succeeded" || event.status === "skipped")
    .map((event) => `${text(event.stage)}:${text(event.status)}`);
  const payloadStages = Array.isArray(payload.completedStages) ? payload.completedStages.map(text) : [];
  return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-");
}

function sentinelStageTimingSummary(events: readonly Record<string, unknown>[], payload: Record<string, unknown>, fallbackTotalMs: unknown): Record<string, unknown> {
  const payloadTimings = record(payload.stageTimings);
  const eventElapsed = (stage: string): number | null => {
    const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed"));
    return event === undefined ? null : finiteNumberOrNull(event.elapsedMs);
  };
  const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch");
  const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify");
  const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build");
  const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops");
  const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null);
  const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0);
  const totalMs = finiteNumberOrNull(payloadTimings.totalMs)
    ?? finiteNumberOrNull(payload.elapsedMs)
    ?? finiteNumberOrNull(fallbackTotalMs)
    ?? summedTotalMs;
  const result = {
    sourceFetchMs,
    monitorWebVerifyMs,
    imageBuildMs,
    gitopsMs,
    totalMs,
    valuesRedacted: true,
  };
  return Object.values(result).some((item) => item !== null && item !== true) ? result : {};
}

function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record<string, unknown>[], domain: "source-mirror" | "publish"): string {
  if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed";
  if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase;
  const reversed = [...events].reverse();
  const failed = reversed.find((event) => event.status === "failed");
  if (failed !== undefined) return text(failed.stage);
  const running = reversed.find((event) => event.status === "running");
  if (running !== undefined) return text(running.stage);
  const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage)));
  const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"];
  const next = order.find((stage) => !completed.has(stage));
  return next ?? result.phase;
}

function sentinelRecentLogSummary(logsTail: string): string {
  const lines = logsTail
    .split(/\r?\n/u)
    .map((line) => line.trim())
    .filter((line) => line.length > 0 && !line.startsWith("{"))
    .slice(-5)
    .map((line) => short(line));
  return lines.length === 0 ? "-" : lines.join(" | ");
}

export function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] {
  const remote = record(job);
  if (remote.phase !== "job-timeout" && remote.phase !== "pipelinerun-timeout") return [];
  const diagnostics = record(remote.diagnostics);
  const commands = record(diagnostics.commands);
  return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`];
}

export function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] {
  if (!observedReady) return [];
  const warnings: string[] = [];
  const publishRecord = record(publish);
  if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) {
    warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`);
  }
  const flushRecord = record(flush);
  if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) {
    warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout.");
  }
  return warnings;
}

export function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number {
  return numberAt(state.cicd, "confirmWait.maxSeconds");
}

export function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
  const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
  const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
  if (elapsedMs === null || elapsedMs <= budgetMs) return [];
  return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`];
}

export function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] {
  const sync = record(sourceMirrorSync);
  if (sync.ok === true || state.sourceHead.ok !== true) return [];
  return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`];
}

export function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, probe: unknown): Record<string, unknown> {
  return {
    ok: true,
    phase: "already-present",
    jobName: null,
    probe: record(probe),
    payload: {
      ok: true,
      status: "already-present",
      sourceCommit: state.sourceHead.commit,
      mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit,
      stageRef: state.sourceHead.stageRef,
      sourceAuthority: state.sourceHead.sourceAuthority,
      valuesRedacted: true,
    },
    polls: 0,
    elapsedMs: 0,
    valuesRedacted: true,
  };
}