// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p8-web-probe-sentinel-recovery. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p10-monitor-web-aggregation. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot. // SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower. // Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel. import { createHash, randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { repoRoot, rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered } from "./hwlab-node-web-sentinel-config"; import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import type { RenderedCliResult } from "./output"; import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5"; import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; import { arrayAt, arrayAtNullable, booleanAt, booleanAtNullable, compactCommand, configRefFile, displayPath, finiteNumberOrNull, isRecord, manifestObjectSummary, monitorWebCicdPlan, nonEmptyString, numberAt, numberAtNullable, parseEnvFile, parseJsonObject, readConfigFile, record, recordTarget, rendered, renderAsyncJobResult, renderControlPlaneResult, renderImageResult, renderPublishCurrentResult, renderPublishResult, resolveSentinelChildJson, safeJobSegment, safeKubernetesSegment, secretSourcePaths, sentinelCliSuffix, sentinelPipelineRunName, sentinelProgressEvent, sentinelSourceSnapshotRef, sentinelSourceSnapshotStageRefPrefix, sha256, shellQuote, short, stringAt, stringAtNullable, stringField, stringTarget, table, text, tomlEscape, valueAtPath, type ChildCliResult, type CompactCommandResult, type SentinelCicdState, type SentinelImagePlan, type SentinelObservedExpectation, type SentinelObservedStatus, type SentinelRemoteJobResult, type SourceHead, type WebProbeSentinelDashboardAction, type WebProbeSentinelOptions, type WebProbeSentinelReportView, } from "./hwlab-node-web-sentinel-cicd-shared"; import { applySentinelArgoApplication, controlPlaneWaitWarningSeconds, createK8sJobScript, probeK8sJobScript, publishSatisfiedByObservedWarnings, runSentinelPublishJob, runSentinelSourceMirrorSyncJob, sentinelBlockedRemoteResult, sentinelCicdElapsedWarnings, sentinelGitMirrorCacheVolumeFromTarget, sentinelPayloadFromLogs, sentinelRemoteJobTimeoutWarnings, sentinelSourceMirrorSyncShellFromConfig, sentinelSourceMirrorAlreadyPresentResult, sourceMirrorAlreadyReadyWarnings, } from "./hwlab-node-web-sentinel-cicd-jobs"; export type { ChildCliResult, CompactCommandResult, SentinelCicdState, WebProbeSentinelDashboardAction, WebProbeSentinelOptions, WebProbeSentinelReportView, } from "./hwlab-node-web-sentinel-cicd-shared"; export { arrayAt, compactCommand, displayPath, isRecord, nonEmptyString, numberAt, numberAtNullable, parseEnvFile, parseJsonObject, record, recordTarget, rendered, renderAsyncJobResult, safeJobSegment, secretSourcePaths, sentinelCliSuffix, shellQuote, short, stringAt, stringAtNullable, table, text, }; const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot"; export type SourceResolveMode = "cached" | "sync"; export interface SentinelSourceOverride { readonly commit: string; readonly stageRef?: string | null; readonly mirrorCommit?: string | null; readonly sourceAuthority: "git-mirror-snapshot"; } export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult { if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId)); requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`); const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options)); if (options.kind === "image") return runSentinelImage(state, options); if (options.kind === "control-plane") return runSentinelControlPlane(state, options); if (options.kind === "publish") return runSentinelPublishCurrent(state, options); if (options.kind === "maintenance") return runSentinelMaintenance(state, options); if (options.kind === "validate") return runSentinelValidate(state, options); if (options.kind === "dashboard") return runSentinelDashboard(state, options); return runSentinelReport(state, options); } function sentinelSourceResolveMode(options: WebProbeSentinelOptions): SourceResolveMode { if (options.kind === "image" && options.action === "build" && options.confirm && options.wait) return "sync"; if (options.kind === "control-plane" && options.action === "trigger-current" && options.confirm && options.wait) return "sync"; if (options.kind === "publish" && options.confirm && options.wait) return "sync"; return "cached"; } function runSentinelImage(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel image ${options.action}`; if (options.action === "build" && options.confirm) { if (!options.wait) return renderAsyncSentinelJob(state, "image", "build", options.timeoutSeconds); return runSentinelImageBuildConfirmed(state, options); } const sourceMirror = options.action === "status" ? probeSourceMirror(state, options.timeoutSeconds) : null; const registry = options.action === "status" ? probeImageRegistry(state, options.timeoutSeconds) : null; const sourceMirrorReady = options.action !== "status" || record(sourceMirror).ok === true; const registryReady = options.action !== "status" || record(registry?.probe).present === true; const result = { ok: state.configReady && state.sourceHead.ok && sourceMirrorReady && registryReady, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run", mutation: false, specRef: SPEC_REF, source: state.sourceHead, sourceMirror, image: state.image, registry, blocker: sourceMirrorReady ? registryReady ? null : { code: "sentinel-image-missing", reason: "expected sentinel image tag is not present in the node-local registry" } : { code: "sentinel-source-mirror-not-ready", reason: "source.gitMirrorReadUrl does not expose the selected source commit yet" }, next: { status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, dryRun: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`, confirm: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`, controlPlanePlan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`, }, valuesRedacted: true, }; return rendered(result.ok, command, renderImageResult(result)); } function runSentinelControlPlane(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel control-plane ${options.action}`; const mutationAction = options.action === "apply" || options.action === "trigger-current"; if (options.confirm && mutationAction) { if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds); return runSentinelControlPlaneConfirmed(state, options); } const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null; const observedReady = options.action !== "status" || sentinelObservedReady(record(observed)); const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : []; const pipelineRun = sentinelPipelineRunName(state, options.rerun); const statusDiagnosis = options.action === "status" && !observedReady ? sentinelObservedStatusDiagnosis(state, observed, pipelineRun) : null; const result = { ok: state.configReady && state.sourceHead.ok && observedReady, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run", mutation: false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun, gitops: { path: stringField(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestObjects: state.manifests.length, manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), projectName: stringAt(state.cicd, "argo.projectName"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, maintenance: { startCommand: stringAt(state.cicd, "maintenance.startCommand"), stopCommand: stringAt(state.cicd, "maintenance.stopCommand"), serviceUnavailablePolicy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"), }, validation: { scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"), controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state), quickVerifyMode: "manual-validate", automaticSecondPath: false, }, manifests: { objects: manifestObjectSummary(state.manifests), sha256: state.manifestSha256, }, observability: webProbeSentinelOtelSummary({ node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, namespace: stringAt(state.runtime, "namespace"), runtime: state.runtime, cicd: state.cicd, }), observed, statusDiagnosis, warnings: mergeWarnings(observedWarnings, record(statusDiagnosis).warning), blocker: observedReady ? null : record(statusDiagnosis).blocker ?? { code: "sentinel-control-plane-observed-not-ready", reason: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass", valuesRedacted: true }, recoveryNext: record(statusDiagnosis).recoveryNext ?? null, next: controlPlaneNext(state, options.action), valuesRedacted: true, }; return rendered(result.ok, command, renderControlPlaneResult(result)); } function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = "web-probe sentinel publish-current"; if (options.confirm) { if (!options.wait) return renderAsyncSentinelJob(state, "publish", options.action, options.timeoutSeconds); return runSentinelPublishCurrentConfirmed(state, options); } const result = { ok: state.configReady && state.sourceHead.ok, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: "dry-run", mutation: false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: sentinelPipelineRunName(state, options.rerun), gitops: { path: stringAt(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, budget: publishCurrentBudget(state), validationPlan: publishCurrentHealthValidationPlan(state), stageBudgets: publishCurrentStageBudgets(state), blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" }, next: publishCurrentNext(state), valuesRedacted: true, }; return rendered(result.ok === true, command, renderPublishCurrentResult(result)); } function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const interruptContext: Record = { phase: "starting", pipelineRun: sentinelPipelineRunName(state, options.rerun), sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, valuesRedacted: true, }; const uninstallInterruptHandler = installSentinelPublishInterruptHandler(state, interruptContext); try { return runSentinelPublishCurrentConfirmedInner(state, options, interruptContext); } finally { uninstallInterruptHandler(); } } function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, options: Extract, interruptContext: Record): RenderedCliResult { const startedAt = Date.now(); const command = "web-probe sentinel publish-current"; const budget = publishCurrentBudget(state); const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds")); const deadline = startedAt + budgetSeconds * 1000; const remainingBudgetSeconds = () => strictRemainingSeconds(deadline, budgetSeconds); let controlResult: Record | null = null; sentinelProgressEvent("sentinel.publish.progress", { phase: "publish-current-start", status: "running", pipelineRun: interruptContext.pipelineRun, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, }); if (state.configReady && state.sourceHead.ok && remainingBudgetSeconds() >= 5) { interruptContext.phase = "already-current-registry-probe"; const registryProbe = probeImageRegistry(state, Math.max(1, Math.min(remainingBudgetSeconds(), 5))); if (record(record(registryProbe).probe).present === true && remainingBudgetSeconds() >= 5) { const preflightStartedAt = Date.now(); const preflightTimeoutSeconds = Math.max(1, Math.min(remainingBudgetSeconds(), 10)); interruptContext.phase = "already-current-observed-probe"; const preflightObserved = withObservedWait( collectSentinelObservedStatus(state, preflightTimeoutSeconds, undefined, true), preflightStartedAt, preflightTimeoutSeconds, true, ); if (sentinelObservedReady(preflightObserved)) { controlResult = sentinelAlreadyCurrentControlResult(state, preflightObserved, Date.now() - preflightStartedAt); } } } interruptContext.phase = "control-plane-trigger-current"; controlResult ??= sentinelControlPlaneConfirmedResult(state, { kind: "control-plane", action: "trigger-current", node: options.node, lane: options.lane, sentinelId: options.sentinelId, dryRun: false, confirm: true, wait: true, timeoutSeconds: Math.max(1, remainingBudgetSeconds()), rerun: options.rerun, }); let health: Record; let healthElapsedMs: number | null = null; if (controlResult.ok !== true) { health = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true }; } else if (remainingBudgetSeconds() < 2) { health = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-health", valuesRedacted: true }; } else { interruptContext.phase = "health-endpoint-validation"; const healthStartedAt = Date.now(); health = probeSentinelRuntimeHealthEndpoint(state, remainingBudgetSeconds()); healthElapsedMs = Date.now() - healthStartedAt; health = { ...health, elapsedMs: healthElapsedMs, valuesRedacted: true }; } const elapsedMs = Date.now() - startedAt; const timings = publishCurrentStageTimings(controlResult, health, elapsedMs); const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds); const withinBudget = elapsedMs <= budgetSeconds * 1000; const healthOk = health.ok === true; const ok = controlResult.ok === true && healthOk && withinBudget; const blocker = ok ? null : publishCurrentBlocker(controlResult, health, withinBudget); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: controlResult.mode === "already-current" ? "already-current" : "confirm-wait", mutation: controlResult.mutation !== false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state, options.rerun), controlPlane: controlResult, health, budget, validationPlan: publishCurrentHealthValidationPlan(state), stageBudgets: publishCurrentStageBudgets(state), elapsedMs, withinBudget, timings, slowStages, warnings: mergeWarnings(controlResult.warnings, publishCurrentBudgetWarnings(slowStages, withinBudget, budgetSeconds, elapsedMs)), blocker, next: publishCurrentNext(state), valuesRedacted: true, }; interruptContext.phase = "completed"; return rendered(ok, command, renderPublishCurrentResult(result)); } function strictRemainingSeconds(deadline: number, cap: number): number { return Math.max(0, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000))); } function withObservedWait(observed: SentinelObservedStatus, startedAt: number, timeoutSeconds: number, includeGitMirror: boolean): SentinelObservedStatus { const elapsedMs = Date.now() - startedAt; return { ...observed, wait: { polls: 1, elapsedMs, timeoutMs: Math.max(1, timeoutSeconds) * 1000, ready: sentinelObservedReady(observed), includeGitMirror, valuesRedacted: true, }, }; } function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed: SentinelObservedStatus, elapsedMs: number): Record { const registryProbe = record(record(observed.registry).probe); const gitops = record(observed.gitops); const argo = record(observed.argo); const digest = text(registryProbe.digest); const digestRef = digest === "-" ? null : `${state.image.repository}@${digest}`; const stageTimings = { sourceFetchMs: 0, monitorWebVerifyMs: 0, imageBuildMs: 0, gitopsMs: 0, totalMs: 0, valuesRedacted: true, }; return { ok: true, command: "web-probe sentinel control-plane trigger-current", node: state.spec.nodeId, lane: state.spec.lane, mode: "already-current", mutation: false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: "already-current", gitops: { path: stringAt(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestObjects: state.manifests.length, manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), projectName: stringAt(state.cicd, "argo.projectName"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, validation: { scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"), controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state), quickVerifyMode: "manual-validate", automaticSecondPath: false, }, manifests: { objects: manifestObjectSummary(state.manifests), sha256: state.manifestSha256, }, sourceMirrorSync: { ok: true, phase: "already-current", jobName: "-", elapsedMs: 0, valuesRedacted: true }, publish: { ok: true, phase: "already-current", resourceKind: "PipelineRun", jobName: "already-current", elapsedMs: 0, payload: { ok: true, status: "already-current", sourceCommit: state.sourceHead.commit, imageRef: state.image.ref, digestRef, gitopsCommit: gitops.revision ?? argo.revision ?? null, stageTimings, completedStages: ["already-current"], valuesRedacted: true, }, diagnostics: { domain: "publish", resourceKind: "PipelineRun", pipelineRun: "already-current", currentPhase: "already-current", completedStages: ["already-current:skipped"], stageTimings, valuesRedacted: true, }, valuesRedacted: true, }, flush: { ok: true, skipped: true, reason: "already-current", valuesRedacted: true }, runtimeSecretsApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, publicExposureApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, argoApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, observed, targetValidation: null, elapsedMs, warnings: [ "publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used health endpoint validation only.", ...sentinelObservedWarnings(observed), ...targetValidationDeferredWarnings(state, false, controlPlaneWaitWarningSeconds(state)), ], blocker: null, recoveryNext: controlPlaneRecoveryNext(state, true, {}, { ok: true }, observed), next: controlPlaneNext(state, "trigger-current"), valuesRedacted: true, }; } export function loadSentinelCicdState( spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number, sourceResolveMode: SourceResolveMode, sourceOverride: SentinelSourceOverride | null = null, ): SentinelCicdState { const sentinel = resolveWebProbeSentinel(spec, sentinelId); const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime); const cicd = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.cicd), sentinel.configRefs.cicd); const scenarios = readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.scenarios); const rawPublicExposure = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure); const publicExposure = effectiveWebProbeSentinelPublicExposure(spec, sentinel.id, rawPublicExposure); const secrets = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.secrets), sentinel.configRefs.secrets); const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef"); const controlPlaneTarget = recordTarget(readWebProbeSentinelConfigRefTarget(spec, controlPlaneRef), controlPlaneRef); const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef)); const nodeId = stringField(controlPlaneTarget, "node"); const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`); validateSentinelSourceAuthority(cicd); const sourceHead = sourceOverride === null ? resolveSourceHead(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds, sourceResolveMode) : sourceHeadFromOverride(cicd, sourceOverride); const image = sentinelImagePlan(spec, cicd, sourceHead); const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image, sourceHead); const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; return { spec, sentinelId: sentinel.id, configRefs: sentinel.configRefs, configReady: configPlan.ok, runtime, cicd, scenarios, publicExposure, secrets, controlPlaneTarget, controlPlaneNode, sourceHead, image, manifests, manifestSha256: sha256(manifestYaml), valuesRedacted: true, }; } function sourceHeadFromOverride(cicd: Record, override: SentinelSourceOverride): SourceHead { if (!/^[0-9a-f]{40}$/iu.test(override.commit)) throw new Error(`sentinel source override commit must be a full sha, got ${override.commit}`); const stageRef = override.stageRef ?? sentinelSourceSnapshotRef(cicd, override.commit); if (!stageRef.startsWith("refs/")) throw new Error(`sentinel source override stageRef must be a git ref, got ${stageRef}`); const mirrorCommit = override.mirrorCommit ?? override.commit; if (!/^[0-9a-f]{40}$/iu.test(mirrorCommit)) throw new Error(`sentinel source override mirrorCommit must be a full sha, got ${mirrorCommit}`); return { ok: true, repository: stringAt(cicd, "source.repository"), branch: stringAt(cicd, "source.branch"), commit: override.commit, stageRef, mirrorCommit, sourceAuthority: override.sourceAuthority, latestDrift: mirrorCommit !== override.commit, result: { exitCode: 0, timedOut: false, stdoutBytes: 0, stderrBytes: 0, stdoutPreview: "source supplied by cicd branch-follower k8s git-mirror snapshot", stderrPreview: "", }, }; } function resolveSourceHead( spec: HwlabRuntimeLaneSpec, cicd: Record, controlPlaneTarget: Record, controlPlaneNode: Record, timeoutSeconds: number, mode: SourceResolveMode, ): SourceHead { const repository = stringAt(cicd, "source.repository"); const branch = stringAt(cicd, "source.branch"); const resolved = mode === "sync" ? resolveSourceHeadWithK8sSnapshot(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds) : probeSourceMirrorCache(cicd, controlPlaneNode, timeoutSeconds, null); const probe = record(resolved.probe); const commit = nonEmptyString(probe.sourceCommit) ?? nonEmptyString(probe.commit) ?? nonEmptyString(probe.mirrorCommit); const stageRef = nonEmptyString(probe.stageRef) ?? (commit === null ? null : sentinelSourceSnapshotRef(cicd, commit)); const mirrorCommit = nonEmptyString(probe.mirrorCommit) ?? nonEmptyString(probe.commit); return { ok: resolved.ok === true && commit !== null, repository, branch, commit, stageRef, mirrorCommit, sourceAuthority: mode === "sync" ? "git-mirror-snapshot" : "git-mirror-cache", latestDrift: commit !== null && mirrorCommit !== null && commit !== mirrorCommit, result: compactCommand(resolved.result), }; } function resolveSourceHeadWithK8sSnapshot( spec: HwlabRuntimeLaneSpec, cicd: Record, controlPlaneTarget: Record, controlPlaneNode: Record, timeoutSeconds: number, ): { ok: boolean; probe: Record; result: CommandResult } { const namespace = stringAt(cicd, "builder.namespace"); const prefix = `${stringAt(cicd, "builder.jobPrefix")}-source-resolve`; const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); const manifest = sentinelSourceMirrorResolveJobManifest(spec, cicd, controlPlaneTarget, controlPlaneNode, jobName); const created = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); if (created.exitCode !== 0) return { ok: false, probe: { ok: false, status: "create-failed", jobName, valuesRedacted: true }, result: created }; const startedAt = Date.now(); const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, 120_000)); let lastCapture = created; while (Date.now() - startedAt < timeoutMs) { const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); lastCapture = probeCapture; const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-source-resolve-job-probe"); const probe = probeResolution.parsed ?? {}; const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); if (probe.succeeded === true) return { ok: payload.ok === true, probe: { ...payload, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result: probeCapture }; if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true } : { ...payload, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result: probeCapture }; runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture }; } function sentinelSourceMirrorResolveJobManifest( spec: HwlabRuntimeLaneSpec, cicd: Record, controlPlaneTarget: Record, controlPlaneNode: Record, jobName: string, ): Record { const namespace = stringAt(cicd, "builder.namespace"); const labels = { "app.kubernetes.io/name": "web-probe-sentinel-source-resolve", "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": spec.nodeId, "unidesk.ai/lane": spec.lane, }; return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace, labels }, spec: { backoffLimit: 0, activeDeadlineSeconds: numberAt(cicd, "builder.activeDeadlineSeconds"), ttlSecondsAfterFinished: numberAt(cicd, "builder.ttlSecondsAfterFinished"), template: { metadata: { labels }, spec: { restartPolicy: "Never", volumes: [ sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget), { name: "git-ssh", secret: { secretName: stringAt(cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, ], containers: [{ name: "resolve", image: sentinelSourceResolverImage(spec, cicd), imagePullPolicy: "IfNotPresent", command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShellFromConfig(cicd, controlPlaneNode, jobName, null)], volumeMounts: [ { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }], }, }, }, }; } function sentinelSourceResolverImage(spec: HwlabRuntimeLaneSpec, cicd: Record): string { const baseImageRef = stringAt(cicd, "image.baseImageRef"); return stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef); } function validateSentinelSourceAuthority(cicd: Record): void { const mode = stringAt(cicd, "sourceAuthority.mode"); const resolver = stringAt(cicd, "sourceAuthority.resolver"); const allowHostGit = booleanAt(cicd, "sourceAuthority.allowHostGit"); const allowGithubDirectInPipeline = booleanAt(cicd, "sourceAuthority.allowGithubDirectInPipeline"); const missingObjectPolicy = stringAt(cicd, "sourceSnapshot.missingObjectPolicy"); if (mode !== "gitMirrorSnapshot") throw new Error("sourceAuthority.mode must be gitMirrorSnapshot"); if (resolver !== "k8s-git-mirror") throw new Error("sourceAuthority.resolver must be k8s-git-mirror"); if (allowHostGit !== false) throw new Error("sourceAuthority.allowHostGit must be false"); if (allowGithubDirectInPipeline !== false) throw new Error("sourceAuthority.allowGithubDirectInPipeline must be false"); if (missingObjectPolicy !== "fail-fast") throw new Error("sourceSnapshot.missingObjectPolicy must be fail-fast"); sentinelSourceSnapshotStageRefPrefix(cicd); } function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record, sourceHead: SourceHead): SentinelImagePlan { const repository = stringAt(cicd, "image.repository"); const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12); const baseImageRef = stringAt(cicd, "image.baseImageRef"); const baseImage = stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef); const entrypoint = stringAt(cicd, "source.entrypoint"); const monitorWeb = monitorWebCicdPlan(spec, cicd); const dockerfile = sentinelDockerfile(baseImage, entrypoint); return { repository, tag, ref: `${repository}:${tag}`, digestRef: null, baseImage, buildContext: stringAt(cicd, "source.buildContext"), entrypoint, dockerfileSha256: sha256(dockerfile), dockerfilePreview: dockerfile, monitorWeb, }; } function sentinelDockerfile(baseImage: string, entrypoint: string): string { return [ `FROM ${baseImage}`, "WORKDIR /app", "ENV NODE_ENV=production", "COPY .unidesk-sentinel-bin/trans /usr/local/bin/trans", "COPY . /app", `ENTRYPOINT ["bun", "${entrypoint}"]`, "", ].join("\n"); } function publishCurrentBudget(state: SentinelCicdState): Record { const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget"); return { maxSeconds: numberAt(budget, "maxSeconds"), valuesRedacted: true, }; } function publishCurrentStageBudgets(state: SentinelCicdState): Record { const budgets = recordTarget(valueAtPath(state.cicd, "publishCurrent.stageBudgets"), "publishCurrent.stageBudgets"); return { sourceSyncSeconds: numberAt(budgets, "sourceSyncSeconds"), sourceFetchSeconds: numberAt(budgets, "sourceFetchSeconds"), monitorWebVerifySeconds: numberAt(budgets, "monitorWebVerifySeconds"), imageBuildSeconds: numberAt(budgets, "imageBuildSeconds"), gitopsSeconds: numberAt(budgets, "gitopsSeconds"), argoRuntimeSeconds: numberAt(budgets, "argoRuntimeSeconds"), dashboardVerifySeconds: numberAt(budgets, "dashboardVerifySeconds"), valuesRedacted: true, }; } function publishCurrentDashboardPlan(state: SentinelCicdState): Record { const dashboard = recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"); const viewport = stringAt(dashboard, "viewport"); if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(viewport)) throw new Error(`publishCurrent.dashboard.viewport must look like 1440x900, got ${viewport}`); return { enabled: booleanAt(dashboard, "enabled"), required: booleanAt(dashboard, "required"), viewport, timeoutMs: numberAt(dashboard, "timeoutMs"), waitTimeoutMs: numberAt(dashboard, "waitTimeoutMs"), commandTimeoutSeconds: numberAt(dashboard, "commandTimeoutSeconds"), fullPage: booleanAt(dashboard, "fullPage"), valuesRedacted: true, }; } function publishCurrentDashboardEnabled(state: SentinelCicdState): boolean { return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "enabled"); } function publishCurrentDashboardRequired(state: SentinelCicdState): boolean { return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required"); } function publishCurrentHealthValidationPlan(state: SentinelCicdState): Record { return { enabled: true, required: true, endpoint: stringAt(state.runtime, "healthPath"), source: "runtime.healthPath", browser: false, playwright: false, webProbe: false, valuesRedacted: true, }; } function publishCurrentDashboardReserveSeconds(state: SentinelCicdState): number { if (!publishCurrentDashboardEnabled(state)) return 0; const dashboard = publishCurrentDashboardPlan(state); const budgets = publishCurrentStageBudgets(state); return Math.max(0, Math.min( numberAt(budgets, "dashboardVerifySeconds"), numberAt(dashboard, "commandTimeoutSeconds"), Math.ceil(numberAt(dashboard, "waitTimeoutMs") / 1000), )); } function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract { const dashboard = publishCurrentDashboardPlan(state); const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000)); const waitTimeoutMs = Math.min(numberAt(dashboard, "waitTimeoutMs"), remainingMs); const timeoutMs = Math.min(numberAt(dashboard, "timeoutMs"), waitTimeoutMs); return { kind: "dashboard", action: "verify", node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, viewport: stringAt(dashboard, "viewport"), localDir: "/tmp", name: null, runId: null, timeoutMs, waitTimeoutMs, timeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))), commandTimeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))), fullPage: booleanAt(dashboard, "fullPage"), raw: false, }; } function publishCurrentStageTimings(controlResult: Record, health: Record, elapsedMs: number): Record { const publish = record(controlResult.publish); const payload = record(publish.payload); const payloadStageTimings = record(payload.stageTimings); const diagnosticStageTimings = record(record(publish.diagnostics).stageTimings); const stageTimings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings; const observedWait = record(record(controlResult.observed).wait); return { sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs), sourceFetchMs: finiteNumberOrNull(stageTimings.sourceFetchMs), monitorWebVerifyMs: finiteNumberOrNull(stageTimings.monitorWebVerifyMs), imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs), gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs), argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs), healthValidationMs: finiteNumberOrNull(health.elapsedMs), totalMs: elapsedMs, valuesRedacted: true, }; } function publishCurrentSlowStages(state: SentinelCicdState, timings: Record, budgetSeconds: number): Record[] { const budgets = publishCurrentStageBudgets(state); const stageMap: Array<[string, string, string, string]> = [ ["source-sync", "sourceSyncMs", "sourceSyncSeconds", "check git-mirror pre-sync, node-local mirror health and SSH-over-proxy latency"], ["source-fetch", "sourceFetchMs", "sourceFetchSeconds", "keep sparse checkout paths narrow and verify node-local git mirror object availability"], ["monitor-web-verify", "monitorWebVerifyMs", "monitorWebVerifySeconds", "keep monitor-web verification copy-only and avoid rebuilding frontend assets during publish"], ["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"], ["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"], ["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"], ["health-validation", "healthValidationMs", "dashboardVerifySeconds", "inspect runtime health endpoint latency and service routing"], ]; const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => { const elapsed = finiteNumberOrNull(timings[timingKey]); const stageBudgetSeconds = finiteNumberOrNull(budgets[budgetKey]); if (elapsed === null || stageBudgetSeconds === null || elapsed <= stageBudgetSeconds * 1000) return []; return [{ stage, elapsedMs: elapsed, budgetSeconds: stageBudgetSeconds, suggestion, valuesRedacted: true }]; }); const total = finiteNumberOrNull(timings.totalMs); if (total !== null && total > budgetSeconds * 1000) { slow.push({ stage: "total", elapsedMs: total, budgetSeconds, suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or health endpoint segment before rerun", valuesRedacted: true, }); } return slow; } function publishCurrentBudgetWarnings(slowStages: readonly Record[], withinBudget: boolean, budgetSeconds: number, elapsedMs: number): string[] { const warnings = slowStages.map((stage) => `${text(stage.stage)} exceeded configured ${text(stage.budgetSeconds)}s budget (${Math.round((finiteNumberOrNull(stage.elapsedMs) ?? 0) / 1000)}s): ${text(stage.suggestion)}`); if (!withinBudget) warnings.unshift(`publish-current exceeded configured ${budgetSeconds}s end-to-end budget (${Math.round(elapsedMs / 1000)}s); no further blind wait was attempted.`); return warnings; } function publishCurrentBlocker(controlResult: Record, health: Record, withinBudget: boolean): Record { if (controlResult.ok !== true) { const blocker = record(controlResult.blocker); return { code: blocker.code ?? "sentinel-publish-current-control-plane-blocked", reason: blocker.reason ?? "control-plane publish did not converge", valuesRedacted: true, }; } if (health.ok !== true) { const degradedReason = text(health.degradedReason); return { code: health.skipped === true ? text(health.reason) : "sentinel-publish-current-health-endpoint-failed", reason: health.skipped === true ? "health endpoint validation did not run" : degradedReason === "-" ? "health endpoint validation did not pass" : `health endpoint validation did not pass: ${degradedReason}`, valuesRedacted: true, }; } if (!withinBudget) { return { code: "sentinel-publish-current-over-budget", reason: "runtime and health endpoint converged, but the one-click CI/CD path exceeded the YAML end-to-end budget", valuesRedacted: true, }; } return { code: "sentinel-publish-current-blocked", reason: "publish-current did not satisfy all checks", valuesRedacted: true }; } function publishCurrentNext(state: SentinelCicdState): Record { const node = state.spec.nodeId; const lane = state.spec.lane; const suffix = sentinelCliSuffix(state); return { publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait`, controlPlaneStatus: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`, dashboardVerify: `bun scripts/cli.ts web-probe sentinel dashboard verify --node ${node} --lane ${lane}${suffix}`, gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`, gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm`, gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`, }; } function renderSentinelManifests( spec: HwlabRuntimeLaneSpec, sentinelId: string, runtime: Record, cicd: Record, scenarios: unknown, publicExposure: Record, secrets: Record, image: SentinelImagePlan, sourceHead: SourceHead, ): readonly Record[] { const namespace = stringAt(runtime, "namespace"); const labels = { "app.kubernetes.io/name": stringAt(runtime, "deploymentName"), "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": spec.nodeId, "unidesk.ai/lane": spec.lane, "unidesk.ai/web-probe-sentinel-id": sentinelId, }; const deploymentName = stringAt(runtime, "deploymentName"); const serviceName = stringAt(runtime, "serviceName"); const servicePort = numberAt(runtime, "servicePort"); const pvcStorage = stringAt(runtime, "pvcStorage"); const stateRoot = stringAt(runtime, "stateRoot"); const sourceCommitAnnotations = sentinelSourceCommitAnnotations(sourceHead.commit); const sourceCommitMetadata = sourceCommitAnnotations === null ? {} : { annotations: sourceCommitAnnotations }; const sentinelEnv = sentinelContainerEnv(sentinelId, runtime, cicd, secrets, sourceHead.commit); const kubernetesApiEgress = sentinelKubernetesApiEgress(runtime); const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv); if (cadenceJob !== null) { emitWebProbeSentinelSpan({ node: spec.nodeId, lane: spec.lane, sentinelId, namespace, runtime, cicd, }, "web_probe_sentinel.cadence.cronjob_rendered", { cronJobName: record(cadenceJob.metadata).name ?? null, namespace, cadence: record(cadenceJob.metadata).annotations === undefined ? null : record(record(cadenceJob.metadata).annotations)["unidesk.ai/cadence"], schedule: record(cadenceJob.spec).schedule ?? null, valuesRedacted: true, }); } return [ { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels }, }, { apiVersion: "rbac.authorization.k8s.io/v1", kind: "Role", metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels }, rules: [ { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get"] }, { apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "list"] }, ], }, { apiVersion: "rbac.authorization.k8s.io/v1", kind: "RoleBinding", metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels }, subjects: [{ kind: "ServiceAccount", name: stringAt(runtime, "serviceAccountName"), namespace }], roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${deploymentName}-manual-trigger` }, }, { apiVersion: "v1", kind: "PersistentVolumeClaim", metadata: { name: stringAt(runtime, "pvcName"), namespace, labels }, spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: pvcStorage } } }, }, { apiVersion: "v1", kind: "ConfigMap", metadata: { name: `${deploymentName}-config`, namespace, labels }, data: { "config-summary.json": JSON.stringify({ specRef: SPEC_REF, node: spec.nodeId, lane: spec.lane, sentinelId, publicBaseUrl: stringAt(publicExposure, "publicBaseUrl"), routePrefix: stringAtNullable(publicExposure, "routePrefix") ?? "/", gitopsPath: stringAt(cicd, "gitopsPath"), valuesRedacted: true, }, null, 2), }, }, { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels, ...sourceCommitMetadata }, spec: { replicas: numberAt(runtime, "replicas"), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, template: { metadata: { labels, ...sourceCommitMetadata }, spec: { serviceAccountName: stringAt(runtime, "serviceAccountName"), containers: [{ name: "sentinel", image: image.ref, imagePullPolicy: "IfNotPresent", args: [ "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", sentinelId, "--state-root", stateRoot, "--host", stringAt(runtime, "listenHost"), "--port", String(servicePort), ], env: sentinelEnv, ports: [{ name: "http", containerPort: servicePort }], readinessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } }, livenessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } }, volumeMounts: [{ name: "state", mountPath: stateRoot }], }], volumes: [{ name: "state", persistentVolumeClaim: { claimName: stringAt(runtime, "pvcName") } }], }, }, }, }, { apiVersion: "v1", kind: "Service", metadata: { name: serviceName, namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] }, }, ...(cadenceJob === null ? [] : [cadenceJob]), { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: stringAt(publicExposure, "frpc.deploymentName"), namespace, labels: { ...labels, "app.kubernetes.io/component": "tunnel" } }, spec: { replicas: 1, selector: { matchLabels: { "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName") } }, template: { metadata: { labels: { ...labels, "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName"), "app.kubernetes.io/component": "tunnel" }, annotations: { "unidesk.ai/public-base-url": stringAt(publicExposure, "publicBaseUrl"), "unidesk.ai/frp-server": `${stringAt(publicExposure, "frpc.serverAddr")}:${numberAt(publicExposure, "frpc.serverPort")}`, "unidesk.ai/frp-remote-port": String(numberAt(publicExposure, "frpc.httpProxy.remotePort")), }, }, spec: { containers: [{ name: "frpc", image: stringAt(publicExposure, "frpc.image"), imagePullPolicy: "IfNotPresent", args: ["-c", "/etc/frp/frpc.toml"], volumeMounts: [{ name: "frpc-config", mountPath: "/etc/frp/frpc.toml", subPath: stringAt(publicExposure, "frpc.secretKey"), readOnly: true }], }], volumes: [{ name: "frpc-config", secret: { secretName: stringAt(publicExposure, "frpc.secretName") } }], }, }, }, }, { apiVersion: "networking.k8s.io/v1", kind: "NetworkPolicy", metadata: { name: `${deploymentName}-egress`, namespace, labels }, spec: { podSelector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, policyTypes: ["Ingress", "Egress"], ingress: [{ from: [{ namespaceSelector: {} }], ports: [{ protocol: "TCP", port: servicePort }] }], egress: [ { to: [{ namespaceSelector: {} }] }, ...kubernetesApiEgress, ], }, }, { apiVersion: "argoproj.io/v1alpha1", kind: "Application", metadata: { name: stringAt(cicd, "argo.applicationName"), namespace: stringAt(cicd, "argo.namespace"), labels }, spec: { project: stringAt(cicd, "argo.projectName"), source: { repoURL: stringAt(cicd, "argo.repoURL"), targetRevision: stringAt(cicd, "argo.targetRevision"), path: stringAt(cicd, "gitopsPath"), }, destination: { server: "https://kubernetes.default.svc", namespace }, syncPolicy: { automated: { prune: true, selfHeal: true } }, }, }, ]; } function sentinelKubernetesApiEgress(runtime: Record): readonly Record[] { if (booleanAtNullable(runtime, "kubernetesApi.egress.enabled") !== true) return []; const rules = arrayAtNullable(runtime, "kubernetesApi.egress.rules"); const rows = rules.length > 0 ? rules : [{ cidr: stringAt(runtime, "kubernetesApi.egress.cidr"), port: numberAt(runtime, "kubernetesApi.egress.port") }]; return rows.map((rule) => ({ to: [{ ipBlock: { cidr: stringAt(rule, "cidr") } }], ports: [{ protocol: "TCP", port: numberAt(rule, "port") }], })); } function sentinelSourceCommitAnnotations(sourceCommit: string | null): Record | null { if (sourceCommit === null) return null; return { "unidesk.ai/source-commit": sourceCommit, "hwlab.pikastech.local/source-commit": sourceCommit, }; } function sentinelContainerEnv(sentinelId: string, runtime: Record, cicd: Record, secrets: Record, sourceCommit: string | null): readonly Record[] { const env: Record[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }]; const otelEnabled = booleanAtNullable(runtime, "observability.otel.enabled") ?? booleanAtNullable(cicd, "observability.otel.enabled") ?? false; const otelEndpoint = stringAtNullable(runtime, "observability.otel.tracesEndpoint") ?? stringAtNullable(runtime, "observability.otel.endpoint") ?? stringAtNullable(cicd, "observability.otel.tracesEndpoint") ?? stringAtNullable(cicd, "observability.otel.endpoint"); const otelServiceName = stringAtNullable(runtime, "observability.otel.serviceName") ?? stringAtNullable(cicd, "observability.otel.serviceName"); const otelSampler = stringAtNullable(runtime, "observability.otel.sampler") ?? stringAtNullable(cicd, "observability.otel.sampler"); const otelSamplerArg = stringAtNullable(runtime, "observability.otel.samplerArg") ?? stringAtNullable(cicd, "observability.otel.samplerArg"); const sourcesByPurpose = new Map>(); for (const source of arrayAt(secrets, "sources").map(record)) { const purpose = stringAtNullable(source, "purpose"); if (purpose !== null) sourcesByPurpose.set(purpose, source); } const used = new Set(env.map((item) => String(item.name ?? ""))); const pushEnv = (item: Record): void => { const name = String(item.name ?? ""); if (name.length === 0 || used.has(name)) return; used.add(name); env.push(item); }; if (sourceCommit !== null) { pushEnv({ name: "UNIDESK_SOURCE_COMMIT", value: sourceCommit }); pushEnv({ name: "WEB_PROBE_SENTINEL_SOURCE_COMMIT", value: sourceCommit }); } if (otelEnabled) { if (otelEndpoint !== null) pushEnv({ name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: otelEndpoint }); if (otelServiceName !== null) pushEnv({ name: "OTEL_SERVICE_NAME", value: otelServiceName }); if (otelSampler !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER", value: otelSampler }); if (otelSamplerArg !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER_ARG", value: otelSamplerArg }); } for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) { const secretName = stringAtNullable(runtimeSecret, "name"); if (secretName === null) continue; for (const item of arrayAt(runtimeSecret, "data").map(record)) { const targetKey = stringAtNullable(item, "targetKey"); const sourcePurpose = stringAtNullable(item, "sourcePurpose"); const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey"); const sourceKeyEnvName = sourcePurpose === "bootstrap-admin" || sourcePurpose === "prompt-set" ? sourceKey : null; if (targetKey !== null && sourceKeyEnvName !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKeyEnvName)) { pushEnv({ name: sourceKeyEnvName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } }); } const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey); if (envName === null) continue; pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } }); } } return env; } function sentinelCadenceCronJobPlan( spec: HwlabRuntimeLaneSpec, sentinelId: string, runtime: Record, cicd: Record, scenarios: unknown, imageRef: string, sentinelEnv: readonly Record[], ): Record | null { const scheduler = record(valueAtPath(cicd, "targetValidation.cadenceScheduler")); const cadenceSchedulerEnabled = booleanAtNullable(cicd, "targetValidation.cadenceScheduler.enabled") === true; if (!cadenceSchedulerEnabled) return null; const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId"); if (scenarioId === null) return null; const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null; if (scenario === null) return null; const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null; const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds); if (schedule === null) return null; const namespace = stringAt(runtime, "namespace"); const deploymentName = stringAt(runtime, "deploymentName"); const serviceAccountName = stringAt(runtime, "serviceAccountName"); const timeoutSeconds = numberAt(cicd, "targetValidation.maxSeconds"); const activeDeadlineSlackSeconds = numberAt(scheduler, "activeDeadlineSlackSeconds"); const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost"); const name = sentinelCadenceCronJobName(deploymentName); const concurrencyPolicy = stringAt(scheduler, "concurrencyPolicy"); if (!["Allow", "Forbid", "Replace"].includes(concurrencyPolicy)) throw new Error("targetValidation.cadenceScheduler.concurrencyPolicy must be Allow, Forbid or Replace"); const labels = { "app.kubernetes.io/name": name, "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "app.kubernetes.io/component": "cadence-scheduler", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": spec.nodeId, "unidesk.ai/lane": spec.lane, "unidesk.ai/web-probe-sentinel-id": sentinelId, }; return { apiVersion: "batch/v1", kind: "CronJob", metadata: { name, namespace, labels, annotations: { "unidesk.ai/cadence": String(scenario.cadence), "unidesk.ai/target-validation-max-seconds": String(timeoutSeconds), "unidesk.ai/source": "targetValidation.cadenceScheduler", }, }, spec: { schedule, concurrencyPolicy, successfulJobsHistoryLimit: numberAt(scheduler, "successfulJobsHistoryLimit"), failedJobsHistoryLimit: numberAt(scheduler, "failedJobsHistoryLimit"), startingDeadlineSeconds: numberAt(scheduler, "startingDeadlineSeconds"), jobTemplate: { spec: { activeDeadlineSeconds: timeoutSeconds + activeDeadlineSlackSeconds, ttlSecondsAfterFinished: numberAt(scheduler, "ttlSecondsAfterFinished"), backoffLimit: numberAt(scheduler, "backoffLimit"), template: { metadata: { labels }, spec: { restartPolicy: "Never", serviceAccountName, containers: [{ name: "quick-verify", image: imageRef, imagePullPolicy: "IfNotPresent", command: ["bun", "scripts/cli.ts"], args: [ "web-probe", "sentinel", "validate", "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", sentinelId, "--quick-verify", "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds), ], env: [ ...sentinelEnv, { name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" }, ...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]), ], }], }, }, }, }, }, }; } function sentinelCadenceCronJobName(deploymentName: string): string { return safeKubernetesSegment(`${deploymentName}-quick-verify`, 52); } function scenarioRows(value: unknown): Record[] { if (Array.isArray(value)) return value.map(record); if (!isRecord(value)) return []; if (Array.isArray(value.scenarios)) return value.scenarios.map(record); if (isRecord(value.workflow)) return [value.workflow]; return [value]; } function parseDurationSeconds(value: string): number | null { const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim()); if (match === null) return null; const amount = Number(match[1]); const unit = match[2]; if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000)); if (unit === "s") return Math.max(60, amount); if (unit === "m") return amount * 60; if (unit === "h") return amount * 3600; return null; } function cronScheduleForCadenceSeconds(seconds: number): string | null { if (!Number.isFinite(seconds) || seconds <= 0) return null; if (seconds <= 60) return "* * * * *"; if (seconds % 60 === 0) { const minutes = Math.trunc(seconds / 60); if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`; if (minutes % 60 === 0) { const hours = Math.trunc(minutes / 60); if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`; if (hours === 24) return "0 0 * * *"; } } return null; } function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null { if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null; const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, ""); return segment.length === 0 ? null : `HWLAB_WEB_${segment}_JSON`; } export function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record } { const sourcesByPurpose = new Map>(); for (const source of arrayAt(state.secrets, "sources").map(record)) { const purpose = stringAtNullable(source, "purpose"); if (purpose !== null) sourcesByPurpose.set(purpose, source); } const env: NodeJS.ProcessEnv = {}; const items: Record[] = []; const missing: Record[] = []; for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) { const secretName = stringAtNullable(runtimeSecret, "name"); for (const item of arrayAt(runtimeSecret, "data").map(record)) { const targetKey = stringAtNullable(item, "targetKey"); const sourcePurpose = stringAtNullable(item, "sourcePurpose"); const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey); if (envName === null || sourcePurpose === null || targetKey === null) continue; const source = sourcesByPurpose.get(sourcePurpose); const runtimeValue = process.env[envName]; if (source === undefined) { if (runtimeValue !== undefined && runtimeValue.length > 0) { env[envName] = runtimeValue; items.push({ envName, secretName, targetKey, sourcePurpose, sourceMode: "runtime-env", fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`, valuesRedacted: true, }); continue; } missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true }); continue; } const sourceRef = stringAt(source, "sourceRef"); const sourceKey = stringAt(source, "sourceKey"); const material = readSentinelSecretSourceValue(source); if (!material.ok) { if (runtimeValue !== undefined && runtimeValue.length > 0) { env[envName] = runtimeValue; items.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, sourceMode: "runtime-env", fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`, valuesRedacted: true, }); continue; } missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true }); continue; } const value = stringAt(material, "value"); env[envName] = value; items.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`, valuesRedacted: true, }); } } const summary = { ok: missing.length === 0, envCount: items.length, items, missing, valuesRedacted: true, }; return { ok: missing.length === 0, env, summary }; } function normalizeRoutePrefix(value: string | null): string { if (value === null || value.trim() === "" || value.trim() === "/") return "/"; const prefixed = value.trim().startsWith("/") ? value.trim() : `/${value.trim()}`; return prefixed.replace(/\/+$/u, "") || "/"; } function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): Record { const endpoint = stringAt(state.controlPlaneNode, "registry.endpoint"); const repoTag = state.image.ref.replace(`${endpoint}/`, ""); const repo = repoTag.slice(0, repoTag.lastIndexOf(":")); const tag = repoTag.slice(repoTag.lastIndexOf(":") + 1); const registryMode = stringAtNullable(state.controlPlaneNode, "registry.mode") ?? "host-docker"; const endpointIsLoopback = endpoint === "127.0.0.1:5000" || endpoint.startsWith("127.0.0.1:"); const probeMode = registryMode === "k8s-workload" && !endpointIsLoopback ? "k8s-service" : "node-loopback"; const url = probeMode === "k8s-service" ? `http://${stringAt(state.controlPlaneNode, "registry.serviceName")}.${stringAt(state.controlPlaneNode, "registry.namespace")}.svc.cluster.local:${numberAtNullable(state.controlPlaneNode, "registry.containerPort") ?? 5000}/v2/${repo}/manifests/${tag}` : `http://${endpoint}/v2/${repo}/manifests/${tag}`; const script = [ "set +e", `url=${shellQuote(url)}`, `probe_mode=${shellQuote(probeMode)}`, "headers=$(mktemp)", "accept='application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json'", "if command -v curl >/dev/null 2>&1 && curl -fsSI -H \"Accept: $accept\" --max-time 5 \"$url\" >\"$headers\" 2>/tmp/web-probe-sentinel-image.err; then present=true; else present=false; fi", "digest=$(awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}' \"$headers\" 2>/dev/null)", "python3 - \"$present\" \"$digest\" \"$url\" \"$probe_mode\" <<'PY'", "import json, sys", "print(json.dumps({'present': sys.argv[1] == 'true', 'digest': sys.argv[2] or None, 'url': sys.argv[3], 'mode': sys.argv[4], 'valuesRedacted': True}))", "PY", ].join("\n"); const result = probeMode === "k8s-service" ? runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", stringAt(state.cicd, "builder.namespace"), "exec", "deploy/git-mirror-http", "--", "sh", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }) : runCommand(["trans", stringAt(state.controlPlaneNode, "route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-image-registry-probe"); return { ok: result.exitCode === 0, probe: probeResolution.parsed, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result), }; } function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const startedAt = Date.now(); const command = "web-probe sentinel image build"; const sourceMirrorProbe = probeSourceMirror(state, Math.min(options.timeoutSeconds, 20)); const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds); const sourceMirrorReady = sourceMirrorSync.ok === true; const publish = sourceMirrorReady ? runSentinelPublishJob(state, false, options.timeoutSeconds, false) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const registry = probeImageRegistry(state, options.timeoutSeconds); const registryReady = record(registry.probe).present === true; const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publish.ok === true && registryReady; const elapsedMs = Date.now() - startedAt; const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: true, specRef: SPEC_REF, source: state.sourceHead, image: state.image, registry, sourceMirrorSync, publish, elapsedMs, warnings: [ ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel image build confirm-wait", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"), ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ], blocker: ok ? null : !sourceMirrorReady ? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" } : publish.ok !== true ? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" } : { code: "sentinel-image-registry-missing", reason: "image publish completed but expected registry tag is not visible" }, next: { status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, controlPlaneTrigger: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`, }, valuesRedacted: true, }; return rendered(result.ok, command, renderImageResult(result)); } function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const result = sentinelControlPlaneConfirmedResult(state, options); return rendered(result.ok === true, String(result.command), renderControlPlaneResult(result)); } function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: Extract): Record { const startedAt = Date.now(); const command = `web-probe sentinel control-plane ${options.action}`; const applyOnly = options.action === "apply"; const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state); const waitBudgetSeconds = Math.max(1, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds)); const deadline = startedAt + waitBudgetSeconds * 1000; const remainingCicdWaitSeconds = () => strictRemainingSeconds(deadline, waitBudgetSeconds); const remainingCommandSeconds = () => Math.max(1, remainingCicdWaitSeconds()); const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCommandSeconds(), 20)); const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCommandSeconds()); const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true; const publish = applyOnly ? null : sourceMirrorReady ? runSentinelPublishJob(state, true, remainingCommandSeconds(), options.rerun) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8; const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true ? startSentinelGitMirrorFlushAsync(state) : null; const runtimeSecretsApply = publishWaitBudgetExhausted ? sentinelSkippedControlStep("publish-wait-budget-exhausted") : applySentinelRuntimeSecrets(state, remainingCommandSeconds()); const publicExposureApply = publishWaitBudgetExhausted ? sentinelSkippedControlStep("publish-wait-budget-exhausted") : applySentinelPublicExposure(state, remainingCommandSeconds()); const argoApply = publishWaitBudgetExhausted ? sentinelSkippedControlStep("publish-wait-budget-exhausted") : applySentinelArgoApplication(state, remainingCommandSeconds()); const observed = publishWaitBudgetExhausted ? sentinelSkippedObservedStatus("publish-wait-budget-exhausted") : waitForSentinelObservedStatus(state, remainingCommandSeconds(), undefined, false); const observedReady = sentinelObservedReady(observed); const publishReady = applyOnly || record(publish).ok === true || observedReady; const flushReady = applyOnly || record(flush).ok === true || observedReady; const targetValidation = null; const targetValidationBlocked = false; const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publishReady && flushReady && record(runtimeSecretsApply).ok === true && record(publicExposureApply).ok === true && record(argoApply).ok === true && observedReady; const elapsedMs = Date.now() - startedAt; const blocker = ok ? null : { code: !sourceMirrorReady ? "sentinel-source-mirror-sync-failed" : !publishReady ? "sentinel-image-gitops-publish-not-ready" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready", reason: !sourceMirrorReady ? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish" : !publishReady ? "Tekton publish did not finish and runtime observation has not proven the selected source/image/GitOps state yet" : record(runtimeSecretsApply).ok === false ? "one or more YAML-declared runtime Secrets were not synced from sourceRef" : "one or more publicExposure, Argo or runtime observation checks did not pass", }; const publishPipelineRun = applyOnly ? sentinelPipelineRunName(state, options.rerun) : record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: true, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: publishPipelineRun, gitops: { path: stringAt(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestObjects: state.manifests.length, manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), projectName: stringAt(state.cicd, "argo.projectName"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, validation: { scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"), controlPlaneWaitMaxSeconds: cicdWaitWarningSeconds, quickVerifyMode: applyOnly ? "not-applicable" : "manual-validate", automaticSecondPath: false, }, manifests: { objects: manifestObjectSummary(state.manifests), sha256: state.manifestSha256, }, sourceMirrorSync, publish, flush, runtimeSecretsApply, publicExposureApply, argoApply, observed, targetValidation, elapsedMs, warnings: Array.from(new Set([ ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel control-plane confirm-wait", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"), ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds), ...asyncGitMirrorFlushWarnings(flush, cicdWaitWarningSeconds), ...publishSatisfiedByObservedWarnings(publish, flush, observedReady), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ...sentinelObservedWarnings(observed), ...(publishWaitBudgetExhausted ? [`sentinel publish consumed the configured ${waitBudgetSeconds}s confirm-wait budget; skipped runtime Secret, public exposure, Argo apply and observed wait to avoid blind over-budget waiting. Use the reported status/log drill-down after the PipelineRun advances.`] : []), ...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds), ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []), ...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []), ])), blocker, recoveryNext: controlPlaneRecoveryNext(state, ok, publish, flush, observed), next: controlPlaneNext(state, options.action), valuesRedacted: true, }; return result; } function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane" | "publish", action: string, timeoutSeconds: number): RenderedCliResult { const args = domain === "image" ? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)] : domain === "control-plane" ? ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)] : ["web-probe", "sentinel", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`); const command = domain === "publish" ? `web-probe sentinel ${action}` : `web-probe sentinel ${domain} ${action}`; const result = { ok: true, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "async-job", mutation: true, reason: "confirmed sentinel publish/build can exceed the short interactive window; use job status for bounded progress.", job, next: { status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, wait: ["bun", "scripts/cli.ts", ...args].join(" "), }, valuesRedacted: true, }; return rendered(true, command, renderAsyncJobResult(result)); } function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record { const args = ["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"]; const job = startJob( `hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_git_mirror_flush`, ["bun", "scripts/cli.ts", ...args], `Flush HWLAB ${state.spec.lane} git mirror after web-probe sentinel ${state.sentinelId} GitOps publish for node ${state.spec.nodeId}`, ); return { ok: true, mode: "async-job", job, next: { status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, wait: ["bun", "scripts/cli.ts", ...args].join(" "), gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, }, valuesRedacted: true, }; } function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): string[] { const item = record(flush); if (item.mode !== "async-job") return []; const next = record(item.next); return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`]; } function sentinelSkippedControlStep(reason: string): Record { return { ok: false, skipped: true, reason, valuesRedacted: true }; } function sentinelSkippedObservedStatus(reason: string): SentinelObservedStatus { const skipped = { ok: false, skipped: true, reason, valuesRedacted: true }; return { sourceMirror: skipped, registry: skipped, gitMirror: { skipped: true, reason, valuesRedacted: true }, gitops: skipped, argo: skipped, runtime: skipped, cadence: skipped, wait: { polls: 0, elapsedMs: 0, timeoutMs: 0, ready: false, includeGitMirror: false, reason, valuesRedacted: true, }, }; } function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus { const registry = probeImageRegistry(state, timeoutSeconds); const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds); const effectiveExpectation = { gitopsRevision: expectation?.gitopsRevision ?? nonEmptyString(gitops.revision), runtimeImage: expectation?.runtimeImage ?? nonEmptyString(gitops.image) ?? expectedRuntimeImageFromRegistry(state, registry), }; return { sourceMirror: probeSourceMirror(state, timeoutSeconds), registry, gitMirror: includeGitMirror ? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds) : { ok: true, skipped: true, reason: "deferred-to-async-flush", valuesRedacted: true }, gitops, argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision), runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage), cadence: probeCadenceCronJob(state, timeoutSeconds), }; } function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus { const startedAt = Date.now(); const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror); let polls = 1; while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) { runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror); polls += 1; } return { ...observed, wait: { polls, elapsedMs: Date.now() - startedAt, timeoutMs, ready: sentinelObservedReady(observed), includeGitMirror, valuesRedacted: true, }, }; } function sentinelObservedReady(value: Record | SentinelObservedStatus): boolean { const observed = record(value); const gitMirror = record(observed.gitMirror); const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true; return record(observed.sourceMirror).ok === true && record(record(observed.registry).probe).present === true && gitMirrorReady && record(observed.gitops).ok === true && record(observed.argo).ok === true && record(observed.runtime).ok === true && record(observed.cadence).ok === true; } function sentinelObservedWarnings(value: Record | SentinelObservedStatus | null): string[] { const observed = record(value); const argo = record(observed.argo); const cadence = record(observed.cadence); return mergeWarnings(argo.warning, cadence.warning); } function sentinelObservedStatusDiagnosis(state: SentinelCicdState, value: unknown, pipelineRun: string): Record | null { const observed = record(value); if (sentinelObservedReady(observed)) return null; const sourceMirror = record(observed.sourceMirror); const registryProbe = record(record(observed.registry).probe); const gitMirror = record(observed.gitMirror); const gitops = record(observed.gitops); const argo = record(observed.argo); const runtimeDeployment = record(record(record(observed.runtime).probe).deployment); const cadence = record(observed.cadence); const sourceReady = sourceMirror.ok === true; const registryPresent = registryProbe.present === true; const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true; const gitopsReady = gitops.ok === true; const argoReady = argo.ok === true; const runtimeReady = record(observed.runtime).ok === true; const cadenceReady = cadence.ok === true; const code = sourceReady && !registryPresent ? "sentinel-publish-half-state-registry-missing" : registryPresent && !gitopsReady ? "sentinel-gitops-manifest-not-updated" : gitopsReady && (!argoReady || !runtimeReady) ? "sentinel-runtime-not-aligned" : !gitMirrorReady ? "sentinel-git-mirror-not-in-sync" : !cadenceReady ? "sentinel-cadence-not-ready" : "sentinel-control-plane-observed-not-ready"; const reason = code === "sentinel-publish-half-state-registry-missing" ? "source mirror contains the selected commit, but the expected registry tag is missing; retry publish-current so Tekton builds/pushes the image and advances GitOps/runtime" : code === "sentinel-gitops-manifest-not-updated" ? "registry contains the selected image, but GitOps manifest is not yet updated to a digest-pinned runtime image" : code === "sentinel-runtime-not-aligned" ? "GitOps manifest is present, but Argo or runtime objects have not converged to the selected image yet" : code === "sentinel-git-mirror-not-in-sync" ? "runtime is aligned, but git-mirror status is not in sync; run the controlled git-mirror sync/status path before treating the control plane as fully healthy" : code === "sentinel-cadence-not-ready" ? "runtime is aligned, but cadence CronJob validation did not pass" : "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass"; const next = publishCurrentNext(state); const controlNext = controlPlaneNext(state, "trigger-current"); const shouldRetryPublish = code === "sentinel-publish-half-state-registry-missing" || code === "sentinel-gitops-manifest-not-updated"; return { code, phase: code === "sentinel-publish-half-state-registry-missing" ? "source-ready-registry-missing" : code === "sentinel-gitops-manifest-not-updated" ? "registry-ready-gitops-pending" : code === "sentinel-runtime-not-aligned" ? "gitops-ready-runtime-pending" : code === "sentinel-git-mirror-not-in-sync" ? "runtime-ready-git-mirror-pending" : code === "sentinel-cadence-not-ready" ? "runtime-ready-cadence-pending" : "observed-not-ready", reason, sourceMirror: sourceReady ? `ready ${short(record(sourceMirror.probe).commit ?? record(sourceMirror.probe).expectedCommit)}` : `blocked ${short(record(sourceMirror.probe).commit)}/${short(record(sourceMirror.probe).expectedCommit)}`, registry: registryPresent ? `present ${short(registryProbe.digest)}` : "missing -", gitMirror: gitMirrorReady ? "ready" : "pending", gitops: gitopsReady ? `ready ${short(gitops.image)}` : `pending ${short(gitops.image)}`, argo: `${argo.syncStatus ?? "-"} ${argo.healthStatus ?? "-"} ${short(argo.revision)}/${short(argo.expectedRevision)}`, runtime: `ready=${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"} image=${short(runtimeDeployment.image)} expected=${short(runtimeDeployment.expectedImage)}`, pipelineRun, warning: code === "sentinel-publish-half-state-registry-missing" ? `source mirror already exposes ${short(state.sourceHead.commit)} but registry tag ${state.image.tag} is missing; rerun ${next.publishCurrent} and then recheck ${next.controlPlaneStatus}.` : code === "sentinel-git-mirror-not-in-sync" ? `runtime is aligned but git-mirror is not in sync; run ${next.gitMirrorSync} and then recheck ${next.controlPlaneStatus}.` : null, blocker: { code, reason, valuesRedacted: true }, recoveryNext: { reason, pipelineRun, digestRef: registryPresent ? expectedRuntimeImageFromRegistry(state, record(observed.registry)) : null, gitopsCommit: gitops.revision ?? null, publishCurrent: shouldRetryPublish ? next.publishCurrent : null, nextStatus: next.controlPlaneStatus, gitMirrorStatus: next.gitMirrorStatus, gitMirrorSync: !gitMirrorReady ? next.gitMirrorSync : null, gitMirrorFlush: null, controlPlaneApply: shouldRetryPublish || code === "sentinel-runtime-not-aligned" ? controlNext.apply : null, valuesRedacted: true, }, valuesRedacted: true, }; } function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record { const result = probeSourceMirrorCache(state.cicd, state.controlPlaneNode, timeoutSeconds, state.sourceHead.commit); return { ...result, result: compactCommand(result.result) }; } function probeSourceMirrorCache(cicd: Record, controlPlaneNode: Record, timeoutSeconds: number, expectedCommit: string | null): { ok: boolean; probe: Record; result: CommandResult } { const namespace = stringAt(cicd, "builder.namespace"); const repository = stringAt(cicd, "source.repository"); const branch = stringAt(cicd, "source.branch"); const stageRef = expectedCommit === null ? "" : sentinelSourceSnapshotRef(cicd, expectedCommit); const script = [ "set +e", `repo_path=${shellQuote(`/cache/${repository}.git`)}`, `branch=${shellQuote(branch)}`, `expected=${shellQuote(expectedCommit ?? "")}`, `stage_ref=${shellQuote(stageRef)}`, "commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)", "rc=$?", "object_rc=1", "expected_object_rc=1", "stage_object_rc=1", "contains_rc=1", "if [ \"$rc\" -eq 0 ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1", " object_rc=$?", "fi", "if [ -n \"$expected\" ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$expected^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1", " expected_object_rc=$?", "fi", "if [ -n \"$stage_ref\" ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse --verify \\\"$stage_ref^{commit}\\\" >/dev/null 2>&1\" >/dev/null 2>&1", " stage_object_rc=$?", "fi", "if [ \"$rc\" -eq 0 ] && [ \"$expected_object_rc\" -eq 0 ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" merge-base --is-ancestor \\\"$expected\\\" \\\"$commit\\\" 2>/dev/null\" >/dev/null 2>&1", " contains_rc=$?", "fi", "node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$stage_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$stage_ref\" \"$repo_path\" \"$branch\" <<'NODE'", "const [rc, objectRc, expectedObjectRc, stageObjectRc, containsRc, commit, expected, stageRef, repoPath, branch] = process.argv.slice(2);", "const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');", "const objectPresent = present && Number(objectRc) === 0;", "const expectedObjectPresent = !expected || Number(expectedObjectRc) === 0;", "const stageObjectPresent = !stageRef || Number(stageObjectRc) === 0;", "const containsExpected = !expected || commit === expected || Number(containsRc) === 0;", "const relation = !expected ? 'unconstrained' : commit === expected ? 'equal' : containsExpected ? 'mirror-ahead' : expectedObjectPresent ? 'diverged-or-behind' : 'expected-object-missing';", "console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror-cache', present, objectPresent, expectedObjectPresent, stageObjectPresent, containsExpected, relation, commit: present ? commit : null, sourceCommit: expected || (present ? commit : null), mirrorCommit: present ? commit : null, expectedCommit: expected || null, stageRef: stageRef || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-source-mirror-cache-probe"); const probe = probeResolution.parsed ?? {}; return { ok: result.exitCode === 0 && probe.ok === true, probe: { ...probe, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result }; } function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record { const namespace = stringAt(state.cicd, "argo.namespace"); const applicationName = stringAt(state.cicd, "argo.applicationName"); const jsonpath = "{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.sync.revision}{\"\\n\"}"; const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", `jsonpath=${jsonpath}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const [syncStatusRaw, healthStatusRaw, revisionRaw] = result.stdout.trim().split(/\r?\n/u); const syncStatus = nonEmptyString(syncStatusRaw); const healthStatus = nonEmptyString(healthStatusRaw); const revision = nonEmptyString(revisionRaw); const revisionMatches = expectedRevision === null || revision === expectedRevision; const healthy = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy"; const diagnostics = result.exitCode === 0 && !healthy ? probeArgoApplicationDiagnostics(state, timeoutSeconds, namespace, applicationName) : null; return { ok: healthy, present: result.exitCode === 0, syncStatus, healthStatus, revision, expectedRevision, revisionMatches, revisionPolicy: "non-blocking-branch-head-drift", warning: healthy && !revisionMatches ? "Argo app is Synced/Healthy but status.sync.revision differs from current GitOps branch HEAD; in multi-sentinel GitOps this can happen when another sentinel path advances the branch. Runtime image/manifest checks remain authoritative for rollout readiness." : null, diagnostics, result: compactCommand(result), }; } function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSeconds: number, namespace: string, applicationName: string): Record { const route = stringAt(state.controlPlaneNode, "kubeRoute"); const appResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const appResolution = resolveSentinelChildJson(appResult, "web-probe-sentinel-argo-application-json"); const parsed = appResolution.parsed; const status = record(parsed?.status); const resourcesRaw = Array.isArray(status.resources) ? status.resources : []; const resources = resourcesRaw .filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) .map((item) => { const health = record(item.health); return { kind: item.kind ?? null, namespace: item.namespace ?? null, name: item.name ?? null, status: item.status ?? null, healthStatus: health.status ?? null, healthMessage: short(health.message), }; }); const problemResources = resources .filter((item) => { const sync = nonEmptyString(item.status); const health = nonEmptyString(item.healthStatus); return (sync !== null && sync !== "Synced") || (health !== null && health !== "Healthy"); }) .slice(0, 12); const conditions = (Array.isArray(status.conditions) ? status.conditions : []) .filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) .slice(-8) .map((item) => ({ type: item.type ?? null, message: short(item.message), lastTransitionTime: item.lastTransitionTime ?? null, })); const operationState = record(status.operationState); const eventResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "events", "--field-selector", `involvedObject.name=${applicationName}`, "--sort-by=.lastTimestamp", "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const eventResolution = resolveSentinelChildJson(eventResult, "web-probe-sentinel-argo-events-json"); const eventsJson = eventResolution.parsed; const events = (Array.isArray(eventsJson?.items) ? eventsJson.items : []) .filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) .slice(-8) .map((item) => ({ type: item.type ?? null, reason: item.reason ?? null, message: short(item.message), count: item.count ?? null, lastTimestamp: item.lastTimestamp ?? item.eventTime ?? null, })); return { ok: appResult.exitCode === 0, resourceCount: resources.length, problemResourceCount: problemResources.length, problemResources, conditions, operationState: { phase: operationState.phase ?? null, message: short(operationState.message), startedAt: operationState.startedAt ?? null, finishedAt: operationState.finishedAt ?? null, }, events, result: compactCommand(appResult), eventsResult: compactCommand(eventResult), stdoutRecovery: { application: appResolution.diagnostics, events: eventResolution.diagnostics, valuesRedacted: true, }, drillDown: { application: `trans ${route} kubectl -n ${namespace} get application ${applicationName} -o json`, events: `trans ${route} kubectl -n ${namespace} get events --field-selector involvedObject.name=${applicationName} --sort-by=.lastTimestamp`, }, }; } function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record { const namespace = stringAt(state.cicd, "builder.namespace"); const repository = stringAt(state.controlPlaneTarget, "source.repository"); const branch = stringAt(state.cicd, "argo.targetRevision"); const manifestPath = `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`; const repoPath = `/cache/${repository}.git`; const inner = [ "set -eu", `git --git-dir=${shellQuote(repoPath)} rev-parse ${shellQuote(`refs/heads/${branch}`)}`, `git --git-dir=${shellQuote(repoPath)} show ${shellQuote(`refs/heads/${branch}:${manifestPath}`)}`, ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "exec", "deploy/git-mirror-http", "--", "sh", "-lc", inner], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const [revisionLine, ...manifestLines] = result.stdout.split(/\r?\n/u); const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null; const manifest = manifestLines.join("\n"); const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]); const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`); const compact = compactCommand(result); return { ok: result.exitCode === 0 && revision !== null && imageMatchesRepository, revision, branch, manifestPath, image, imageMatchesRepository, result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` }, valuesRedacted: true, }; } function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, expectedImage: string | null): Record { const namespace = stringAt(state.runtime, "namespace"); const deploymentName = stringAt(state.runtime, "deploymentName"); const serviceName = stringAt(state.runtime, "serviceName"); const pvcName = stringAt(state.runtime, "pvcName"); const configMapName = `${deploymentName}-config`; const serviceAccountName = stringAt(state.runtime, "serviceAccountName"); const script = [ "set +e", `namespace=${shellQuote(namespace)}`, `deployment=${shellQuote(deploymentName)}`, `service=${shellQuote(serviceName)}`, `pvc=${shellQuote(pvcName)}`, `configmap=${shellQuote(configMapName)}`, `serviceaccount=${shellQuote(serviceAccountName)}`, "tmp=$(mktemp -d)", "kubectl -n \"$namespace\" get deploy \"$deployment\" -o json >\"$tmp/deploy.json\" 2>/dev/null; echo $? >\"$tmp/deploy.rc\"", "kubectl -n \"$namespace\" get svc \"$service\" -o json >\"$tmp/svc.json\" 2>/dev/null; echo $? >\"$tmp/svc.rc\"", "kubectl -n \"$namespace\" get pvc \"$pvc\" -o json >\"$tmp/pvc.json\" 2>/dev/null; echo $? >\"$tmp/pvc.rc\"", "kubectl -n \"$namespace\" get cm \"$configmap\" -o json >\"$tmp/cm.json\" 2>/dev/null; echo $? >\"$tmp/cm.rc\"", "kubectl -n \"$namespace\" get sa \"$serviceaccount\" -o json >\"$tmp/sa.json\" 2>/dev/null; echo $? >\"$tmp/sa.rc\"", `expected_image=${shellQuote(expectedImage ?? "")}`, "node - \"$tmp\" \"$expected_image\" <<'NODE'", "const fs = require('node:fs');", "const dir = process.argv[2];", "const expectedImage = process.argv[3] || null;", "function rc(name){ return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); }", "function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }", "const dep = json('deploy');", "const deploymentPresent = rc('deploy') === 0;", "const desired = Number(dep?.spec?.replicas ?? 0);", "const ready = Number(dep?.status?.readyReplicas ?? 0);", "const updated = Number(dep?.status?.updatedReplicas ?? 0);", "const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;", "const imageMatches = expectedImage === null || image === expectedImage;", "const payload = {", " deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },", " service: { present: rc('svc') === 0 },", " pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },", " configMap: { present: rc('cm') === 0 },", " serviceAccount: { present: rc('sa') === 0 },", " valuesRedacted: true", "};", "payload.ok = payload.deployment.present && imageMatches && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;", "console.log(JSON.stringify(payload));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-runtime-objects-probe"); const probe = probeResolution.parsed; return { ok: result.exitCode === 0 && probe?.ok === true, probe, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result) }; } function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record { const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null; if (expected === null) { return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true }; } const metadata = record(expected.metadata); const spec = record(expected.spec); const namespace = stringAt(metadata, "namespace"); const name = stringAt(metadata, "name"); const expectedSchedule = stringAt(spec, "schedule"); const script = [ "set +e", `namespace=${shellQuote(namespace)}`, `cronjob=${shellQuote(name)}`, `sentinel=${shellQuote(state.sentinelId)}`, `expected_schedule=${shellQuote(expectedSchedule)}`, "tmp=$(mktemp -d)", "kubectl -n \"$namespace\" get cronjob \"$cronjob\" -o json >\"$tmp/cronjob.json\" 2>/dev/null; echo $? >\"$tmp/cronjob.rc\"", "kubectl -n \"$namespace\" get jobs -l \"unidesk.ai/web-probe-sentinel-id=$sentinel,app.kubernetes.io/component=cadence-scheduler\" -o json >\"$tmp/jobs.json\" 2>/dev/null; echo $? >\"$tmp/jobs.rc\"", "node - \"$tmp\" \"$namespace\" \"$cronjob\" \"$expected_schedule\" <<'NODE'", "const fs = require('node:fs');", "const [dir, namespace, cronJobName, expectedSchedule] = process.argv.slice(2);", "function rc(name){ try { return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); } catch { return 1; } }", "function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }", "const cron = json('cronjob');", "const jobs = Array.isArray(json('jobs')?.items) ? json('jobs').items : [];", "const present = rc('cronjob') === 0 && !!cron;", "const schedule = cron?.spec?.schedule || null;", "const scheduleMatches = present && schedule === expectedSchedule;", "const suspended = cron?.spec?.suspend === true;", "const active = Array.isArray(cron?.status?.active) ? cron.status.active.length : 0;", "const sortedJobs = jobs.slice().sort((a,b)=>String(b?.metadata?.creationTimestamp||'').localeCompare(String(a?.metadata?.creationTimestamp||''))).slice(0,8);", "let code = null;", "if (!present) code = 'sentinel-cadence-cronjob-missing';", "else if (!scheduleMatches) code = 'sentinel-cadence-cronjob-schedule-mismatch';", "else if (suspended) code = 'sentinel-cadence-cronjob-suspended';", "const latestJob = sortedJobs[0] || null;", "console.log(JSON.stringify({ ok: code === null, code, present, namespace, name: cronJobName, schedule, expectedSchedule, scheduleMatches, suspended, lastScheduleTime: cron?.status?.lastScheduleTime || null, lastSuccessfulTime: cron?.status?.lastSuccessfulTime || null, active, jobCount: jobs.length, latestJobs: sortedJobs.map((job)=>({ name: job?.metadata?.name || null, createdAt: job?.metadata?.creationTimestamp || null, active: Number(job?.status?.active || 0), succeeded: Number(job?.status?.succeeded || 0), failed: Number(job?.status?.failed || 0), completionTime: job?.status?.completionTime || null, valuesRedacted:true })), latestJobName: latestJob?.metadata?.name || null, valuesRedacted: true }));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-cadence-cronjob-probe"); const probe = probeResolution.parsed; const ok = result.exitCode === 0 && probe?.ok === true; emitWebProbeSentinelSpan({ node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, namespace, runtime: state.runtime, cicd: state.cicd, }, "web_probe_sentinel.cadence.cronjob_observed", { cronJobName: name, namespace, schedule: expectedSchedule, status: ok ? "ok" : text(probe?.code ?? "unknown"), jobName: probe?.latestJobName ?? null, failureKind: probe?.code ?? null, valuesRedacted: true, }, ok); return { ok, probe, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result), warning: ok ? null : `cadence CronJob is not ready: ${text(probe?.code ?? "probe-failed")}`, valuesRedacted: true, }; } function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record): string | null { const digest = nonEmptyString(record(record(registry).probe).digest); if (digest === null) return null; return `${state.image.repository}@${digest}`; } function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] { if (applyOnly) return []; const next = sentinelP5Next(state); return [`targetValidation quick verify is outside the CI/CD validation gate; run ${next.quickVerify} only as separate post-deploy evidence if needed.`]; } export function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] { const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; if (elapsedMs === null || elapsedMs <= budgetMs) return []; return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, only Code Agent multi-round business failures should block acceptance.`]; } export function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] { const warnings: string[] = []; for (const item of items) { const values = Array.isArray(item) ? item : [item]; for (const value of values) { if (value === undefined || value === null || value === "") continue; const warning = text(value).trim(); if (warning.length > 0 && warning !== "-" && !warnings.includes(warning)) warnings.push(warning); } } return warnings; } export function withWarnings(payload: Record, warnings: readonly unknown[]): Record { const merged = mergeWarnings(payload.warnings, warnings); return merged.length === 0 ? payload : { ...payload, warnings: merged, valuesRedacted: true }; } function installSentinelPublishInterruptHandler(state: SentinelCicdState, context: Record): () => void { let handled = false; const handler = (signal: string) => { if (handled) return; handled = true; const exitCode = signal === "SIGINT" ? 130 : 143; const next = publishCurrentNext(state); sentinelProgressEvent("sentinel.publish.interrupted", { signal, exitCode, phase: context.phase ?? "unknown", pipelineRun: context.pipelineRun ?? sentinelPipelineRunName(state, false), sourceCommit: context.sourceCommit ?? state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, recoveryNext: { status: next.controlPlaneStatus, retry: next.publishCurrent, gitMirrorStatus: next.gitMirrorStatus, gitMirrorSync: next.gitMirrorSync, gitMirrorFlush: next.gitMirrorFlush, valuesRedacted: true, }, valuesRedacted: true, }); process.exit(exitCode); }; process.once("SIGTERM", handler); process.once("SIGINT", handler); return () => { process.off("SIGTERM", handler); process.off("SIGINT", handler); }; } function confirmBlocked(action: string, state: SentinelCicdState): Record { return { code: "sentinel-cicd-confirm-requires-tekton-pipelinerun", action, reason: "Confirmed publish uses a Tekton PipelineRun for image build and GitOps writeback; dry-run refuses to report a deployment mutation before that CI run is submitted.", sourceGitMirrorReadUrl: stringAt(state.cicd, "source.gitMirrorReadUrl"), requiredNextImplementation: [ "clone source from source.gitMirrorReadUrl at selected commit", "build and push digest-pinned image through Tekton PipelineRun on the selected node", "publish manifests to the HWLAB gitops branch/path through git-mirror", "flush/recheck git-mirror and let Argo reconcile the Application", ], valuesRedacted: true, }; } function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelControlPlaneAction): Record { const node = state.spec.nodeId; const lane = state.spec.lane; const suffix = sentinelCliSuffix(state); return { plan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${node} --lane ${lane}${suffix} --dry-run`, status: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`, image: `bun scripts/cli.ts web-probe sentinel image status --node ${node} --lane ${lane}${suffix}`, triggerCurrent: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${node} --lane ${lane}${suffix} --dry-run`, apply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${node} --lane ${lane}${suffix} --confirm --wait`, validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`, quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`, gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`, gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm`, gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`, issue: "https://github.com/pikasTech/unidesk/issues/1285", currentAction: action, }; } function controlPlaneRecoveryNext(state: SentinelCicdState, ok: boolean, publish: unknown, flush: unknown, observed: unknown): Record | null { const payload = record(record(publish).payload); if (ok || nonEmptyString(payload.digestRef) === null) return null; const next = controlPlaneNext(state, "apply"); const flushRecord = record(flush); const observedRecord = record(observed); return { reason: "publish produced an image digest, but GitOps/git-mirror/Argo/runtime alignment is not complete yet", pipelineRun: record(publish).jobName ?? null, digestRef: payload.digestRef, gitopsCommit: payload.gitopsCommit ?? null, flushMode: flushRecord.mode ?? null, observedReady: sentinelObservedReady(observedRecord), publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, nextStatus: next.status, gitMirrorStatus: next.gitMirrorStatus, gitMirrorSync: next.gitMirrorSync, gitMirrorFlush: next.gitMirrorFlush, controlPlaneApply: next.apply, valuesRedacted: true, }; } function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record { const sourcesByPurpose = new Map>(); for (const source of arrayAt(state.secrets, "sources").map(record)) { const purpose = stringAtNullable(source, "purpose"); if (purpose !== null) sourcesByPurpose.set(purpose, source); } const desired: Array<{ namespace: string; name: string; data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> }> = []; const missing: Record[] = []; const skipped: Record[] = []; for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) { const name = stringAt(runtimeSecret, "name"); const namespace = stringAt(runtimeSecret, "namespace"); const data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> = []; for (const item of arrayAt(runtimeSecret, "data").map(record)) { const sourcePurpose = stringAt(item, "sourcePurpose"); const targetKey = stringAt(item, "targetKey"); if (sourcePurpose === "frp-token") { skipped.push({ name, namespace, targetKey, sourcePurpose, reason: "managed-by-publicExposure-frpc", valuesRedacted: true }); continue; } const source = sourcesByPurpose.get(sourcePurpose); if (source === undefined) { missing.push({ name, namespace, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true }); continue; } const sourceRef = stringAt(source, "sourceRef"); const sourceKey = stringAt(source, "sourceKey"); const material = readSentinelSecretSourceValue(source); if (!material.ok) { missing.push({ name, namespace, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true }); continue; } const value = stringAt(material, "value"); data.push({ key: targetKey, value, sourcePurpose, sourceRef, sourceKey, fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`, }); } if (data.length > 0) desired.push({ namespace, name, data }); } if (missing.length > 0) return { ok: false, phase: "local-source", missing, skipped, valuesRedacted: true }; if (desired.length === 0) return { ok: true, phase: "skipped-no-runtime-secrets", secretCount: 0, keyCount: 0, skippedKeyCount: skipped.length, skipped, valuesRedacted: true }; const manifests = desired.map((secret) => ({ apiVersion: "v1", kind: "Secret", metadata: { name: secret.name, namespace: secret.namespace, labels: { "app.kubernetes.io/managed-by": "unidesk", "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "unidesk.ai/node": state.spec.nodeId, "unidesk.ai/lane": state.spec.lane, "unidesk.ai/web-probe-sentinel-id": state.sentinelId, }, }, type: "Opaque", data: Object.fromEntries(secret.data.map((item) => [item.key, Buffer.from(item.value, "utf8").toString("base64")])), })); const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; const summary = desired.map((secret) => ({ name: secret.name, namespace: secret.namespace, keys: secret.data.map((item) => item.key).sort(), sources: secret.data.map((item) => ({ key: item.key, sourcePurpose: item.sourcePurpose, sourceRef: item.sourceRef, sourceKey: item.sourceKey, fingerprint: item.fingerprint, valuesRedacted: true })), valuesRedacted: true, })); const summaryB64 = Buffer.from(JSON.stringify(summary), "utf8").toString("base64"); const script = [ "set +e", `summary_b64=${shellQuote(summaryB64)}`, "tmp=$(mktemp -d)", "trap 'rm -rf \"$tmp\"' EXIT", "manifest=\"$tmp/runtime-secrets.yaml\"", "cat >\"$manifest\"", "kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-runtime-secrets -f \"$manifest\" >/tmp/web-probe-sentinel-runtime-secrets.out 2>/tmp/web-probe-sentinel-runtime-secrets.err", "apply_rc=$?", "python3 - \"$summary_b64\" \"$apply_rc\" <<'PY'", "import base64, json, subprocess, sys", "summary = json.loads(base64.b64decode(sys.argv[1]).decode('utf-8'))", "apply_rc = int(sys.argv[2])", "items = []", "ok = apply_rc == 0", "for secret in summary:", " proc = subprocess.run(['kubectl', '-n', secret['namespace'], 'get', 'secret', secret['name'], '-o', 'json'], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)", " data = json.loads(proc.stdout).get('data', {}) if proc.returncode == 0 and proc.stdout else {}", " keys = {key: key in data for key in secret['keys']}", " present = proc.returncode == 0 and all(keys.values())", " ok = ok and present", " items.append({'name': secret['name'], 'namespace': secret['namespace'], 'present': present, 'keys': keys, 'sources': secret['sources'], 'valuesRedacted': True})", "print(json.dumps({'ok': ok, 'applyExitCode': apply_rc, 'secretCount': len(summary), 'keyCount': sum(len(item['keys']) for item in summary), 'items': items, 'valuesRedacted': True}, ensure_ascii=False))", "PY", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: manifestYaml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-runtime-secrets-apply"); const parsed = parsedResolution.parsed; return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true }; } function readSentinelSecretSourceValue(source: Record): Record { const sourceRef = stringAt(source, "sourceRef"); const sourceKey = stringAt(source, "sourceKey"); const sourceLine = numberAtNullable(source, "sourceLine"); const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, error: "secret-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const textValue = readFileSync(sourcePath, "utf8"); const value = sourceLine === null ? parseEnvFile(textValue)[sourceKey] : textValue.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, ""); if (value === undefined || value.length === 0) return { ok: false, error: sourceLine === null ? "secret-source-key-missing" : "secret-source-line-missing", sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const format = stringAtNullable(source, "format"); if (format === null) return { ok: true, value, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; if (format === "web-account-json") { const username = readSentinelWebAccountUsername(source); if (!username.ok) return { ok: false, error: username.error, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; return { ok: true, value: JSON.stringify({ username: username.value, password: value }), sourceRef, sourceKey, sourceLine, format, sourcePath: displayPath(sourcePath), valuesRedacted: true }; } return { ok: false, error: "unsupported-secret-source-format", sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true }; } function readSentinelWebAccountUsername(source: Record): { ok: true; value: string } | { ok: false; error: string } { const username = stringAtNullable(source, "username"); if (username !== null) return { ok: true, value: username }; const sourceRef = stringAtNullable(source, "usernameSourceRef"); const sourceLine = numberAtNullable(source, "usernameSourceLine"); if (sourceRef === null || sourceLine === null) return { ok: false, error: "web-account-json-username-missing" }; const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, error: "web-account-json-username-source-missing" }; const value = readFileSync(sourcePath, "utf8").split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? ""; return value.length === 0 ? { ok: false, error: "web-account-json-username-line-missing" } : { ok: true, value }; } function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { const material = readSentinelFrpcMaterial(state); if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true }; const secret = applySentinelFrpcSecret(state, stringAt(material, "frpcToml"), timeoutSeconds); const caddy = applySentinelCaddyBlock(state, timeoutSeconds); return { ok: secret.ok === true && caddy.ok === true, hostname: stringAt(state.publicExposure, "hostname"), publicBaseUrl: stringAt(state.publicExposure, "publicBaseUrl"), material: { ok: true, sourceRef: material.sourceRef, sourcePath: material.sourcePath, fingerprint: material.fingerprint, valuesRedacted: true, }, secret, caddy, valuesRedacted: true, }; } function readSentinelFrpcMaterial(state: SentinelCicdState): Record { const sourceRef = stringAt(state.publicExposure, "frpc.tokenSourceRef"); const sourceKey = stringAt(state.publicExposure, "frpc.tokenSourceKey"); const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, error: "frp-token-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const token = values[sourceKey]; if (token === undefined || token.length === 0) return { ok: false, error: "frp-token-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const proxy = record(valueAtPath(state.publicExposure, "frpc.httpProxy")); const frpcToml = [ `serverAddr = "${tomlEscape(stringAt(state.publicExposure, "frpc.serverAddr"))}"`, `serverPort = ${numberAt(state.publicExposure, "frpc.serverPort")}`, "loginFailExit = true", `auth.token = "${tomlEscape(token)}"`, "", "[[proxies]]", `name = "${tomlEscape(stringAt(proxy, "name"))}"`, 'type = "tcp"', `localIP = "${tomlEscape(stringAt(proxy, "localIP"))}"`, `localPort = ${numberAt(proxy, "localPort")}`, `remotePort = ${numberAt(proxy, "remotePort")}`, "", ].join("\n"); return { ok: true, sourceRef, sourceKey, sourcePath: displayPath(sourcePath), frpcToml, fingerprint: `sha256:${createHash("sha256").update(`${token}\n${frpcToml}`).digest("hex").slice(0, 16)}`, valuesRedacted: true, }; } function applySentinelFrpcSecret(state: SentinelCicdState, frpcToml: string, timeoutSeconds: number): Record { const namespace = stringAt(state.runtime, "namespace"); const secretName = stringAt(state.publicExposure, "frpc.secretName"); const secretKey = stringAt(state.publicExposure, "frpc.secretKey"); const script = [ "set +e", `namespace=${shellQuote(namespace)}`, `secret=${shellQuote(secretName)}`, `key=${shellQuote(secretKey)}`, "tmp=$(mktemp -d)", "trap 'rm -rf \"$tmp\"' EXIT", "cat >\"$tmp/frpc.toml\"", "kubectl -n \"$namespace\" create secret generic \"$secret\" --from-file=\"$key=$tmp/frpc.toml\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-public-exposure -f - >/tmp/web-probe-sentinel-frpc-secret.out 2>/tmp/web-probe-sentinel-frpc-secret.err", "rc=$?", "present=no", "bytes=0", "if kubectl -n \"$namespace\" get secret \"$secret\" -o jsonpath=\"{.data.$key}\" >/tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null; then present=yes; bytes=$(base64 -d /dev/null | wc -c | tr -d ' '); fi", "node - \"$rc\" \"$namespace\" \"$secret\" \"$key\" \"$present\" \"$bytes\" <<'NODE'", "const [rc, namespace, secret, key, present, bytes] = process.argv.slice(2);", "console.log(JSON.stringify({ok:Number(rc)===0&&present==='yes',namespace,secret,key,present:present==='yes',bytes:Number(bytes||0),applyExitCode:Number(rc),valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: frpcToml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-frpc-secret-apply"); const parsed = parsedResolution.parsed; return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true }; } function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: number): Record { const hostname = stringAt(state.publicExposure, "hostname"); const owner = stringAt(state.publicExposure, "caddy.managedBlockOwner"); const configPath = stringAt(state.publicExposure, "caddy.configPath"); const serviceName = stringAt(state.publicExposure, "caddy.serviceName"); const responseHeaderTimeoutSeconds = numberAt(state.publicExposure, "caddy.responseHeaderTimeoutSeconds"); const remotePort = numberAt(state.publicExposure, "frpc.httpProxy.remotePort"); const routePrefix = normalizeRoutePrefix(stringAtNullable(state.publicExposure, "routePrefix")); const rootOrder = stringAtNullable(state.publicExposure, "caddy.rootOrder") ?? "normal"; const monitorRoot = record(state.publicExposure.monitorRoot); const cleanupOwner = monitorRoot.enabled === false ? stringAtNullable(monitorRoot, "caddyManagedBlockOwner") : null; const proxyLines = [ `reverse_proxy 127.0.0.1:${remotePort} {`, " transport http {", ` response_header_timeout ${responseHeaderTimeoutSeconds}s`, " }", "}", ]; const block = [ routePrefix === "/" ? "handle {" : `handle_path ${routePrefix}* {`, ...proxyLines.map((line) => ` ${line}`), "}", "", ].join("\n"); const blockB64 = Buffer.from(block, "utf8").toString("base64"); const script = [ "set +e", `hostname=${shellQuote(hostname)}`, `owner=${shellQuote(owner)}`, `config_path=${shellQuote(configPath)}`, `service=${shellQuote(serviceName)}`, `route_prefix=${shellQuote(routePrefix)}`, `root_order=${shellQuote(rootOrder)}`, `cleanup_owner=${shellQuote(cleanupOwner ?? "")}`, `block_b64=${shellQuote(blockB64)}`, "marker=\"unidesk managed $owner\"", "tmp=$(mktemp -d)", "trap 'rm -rf \"$tmp\"' EXIT", "block=\"$tmp/block\"", "next=\"$tmp/Caddyfile\"", "printf '%s' \"$block_b64\" | base64 -d >\"$block\"", "if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi", "python3 - \"$next\" \"$block\" \"$marker\" \"$hostname\" \"$route_prefix\" \"$root_order\" \"$cleanup_owner\" <<'PY' >/tmp/web-probe-sentinel-caddy-python.out 2>/tmp/web-probe-sentinel-caddy-python.err", "import pathlib, re, sys", "config = pathlib.Path(sys.argv[1])", "block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')", "marker = sys.argv[3]", "hostname = sys.argv[4]", "route_prefix = sys.argv[5]", "root_order = sys.argv[6]", "cleanup_owner = sys.argv[7]", "text = config.read_text(encoding='utf-8') if config.exists() else ''", "begin = f'# BEGIN {marker}'", "end = f'# END {marker}'", "def managed_pattern(marker_text):", " return re.compile(rf'(?ms)^[ \\t]*# BEGIN {re.escape(marker_text)}\\n.*?^[ \\t]*# END {re.escape(marker_text)}\\n*')", "pattern = managed_pattern(marker)", "def collect_nested_managed(segment):", " preserved = []", " lines = segment.splitlines()", " index = 0", " while index < len(lines):", " stripped = lines[index].strip()", " if stripped.startswith('# BEGIN ') and stripped != begin:", " owner_text = stripped[len('# BEGIN '):]", " end_line = '# END ' + owner_text", " block_lines = [lines[index]]", " index += 1", " while index < len(lines):", " block_lines.append(lines[index])", " if lines[index].strip() == end_line:", " break", " index += 1", " preserved.append('\\n'.join(block_lines).rstrip() + '\\n')", " index += 1", " return preserved", "preserved_blocks = []", "for match in pattern.finditer(text):", " preserved_blocks.extend(collect_nested_managed(match.group(0)))", "text = pattern.sub('', text)", "if cleanup_owner:", " cleanup_marker = f'unidesk managed {cleanup_owner}'", " if cleanup_marker != marker:", " text = managed_pattern(cleanup_marker).sub('', text)", "def site_span(src, host):", " match = re.search(rf'(?m)^([ \\t]*){re.escape(host)}[ \\t]*\\{{[ \\t]*\\n', src)", " if not match:", " return None", " depth = 1", " index = match.end()", " while index < len(src):", " char = src[index]", " if char == '{':", " depth += 1", " elif char == '}':", " depth -= 1", " if depth == 0:", " end_index = index + 1", " if end_index < len(src) and src[end_index] == '\\n':", " end_index += 1", " return match.start(), end_index, index, match.end()", " index += 1", " raise ValueError(f'unclosed Caddy site block for {host}')", "handler = '\\n'.join((' ' + line) if line else '' for line in block.rstrip().splitlines())", "managed = f' {begin}\\n{handler}\\n {end}\\n'", "def fallback_insert_pos(site, relative_open, close_rel):", " for match in re.finditer(r'(?m)^[ \\t]*handle[ \\t]*\\{[ \\t]*\\n', site[relative_open:close_rel]):", " pos = relative_open + match.start()", " prev_end = pos - 1", " if prev_end >= 0 and site[prev_end] == '\\n':", " prev_end -= 1", " if prev_end >= 0:", " prev_start = site.rfind('\\n', 0, prev_end + 1) + 1", " if site[prev_start:prev_end + 1].strip().startswith('# BEGIN '):", " return prev_start", " return pos", " return relative_open", "def append_before_close(site, close_rel, addition):", " prefix = site[:close_rel]", " suffix = site[close_rel:]", " if prefix and not prefix.endswith('\\n'):", " prefix += '\\n'", " return prefix + addition + suffix", "span = site_span(text, hostname)", "if span is None:", " body = ''.join(preserved_blocks) + managed", " text = text.rstrip() + '\\n\\n' + f'{hostname} {{\\n{body}}}\\n'", "else:", " start, stop, close_index, open_end = span", " site = text[start:stop]", " relative_open = open_end - start", " close_rel = close_index - start", " additions = ''.join(preserved_blocks) + managed", " if route_prefix == '/' and root_order == 'active':", " replacement = site[:relative_open] + additions + site[relative_open:]", " elif route_prefix == '/':", " replacement = append_before_close(site, close_rel, additions)", " else:", " insert_at = fallback_insert_pos(site, relative_open, close_rel)", " replacement = site[:insert_at] + additions + site[insert_at:]", " text = text[:start] + replacement + text[stop:]", "config.write_text(text, encoding='utf-8')", "PY", "python_rc=$?", "validate_rc=1", "reload_rc=", "if [ \"$python_rc\" = 0 ]; then sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/web-probe-sentinel-caddy-validate.out 2>/tmp/web-probe-sentinel-caddy-validate.err; validate_rc=$?; fi", "if [ \"$validate_rc\" = 0 ]; then sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/web-probe-sentinel-caddy-install.out 2>/tmp/web-probe-sentinel-caddy-install.err && (sudo systemctl reload \"$service\" >/tmp/web-probe-sentinel-caddy-reload.out 2>/tmp/web-probe-sentinel-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/web-probe-sentinel-caddy-reload.out 2>>/tmp/web-probe-sentinel-caddy-reload.err); reload_rc=$?; fi", "probe_rc=1", "probe_status=", "if [ \"$reload_rc\" = 0 ]; then", " probe_path=\"$route_prefix\"", " if [ \"$probe_path\" = \"/\" ]; then probe_url=\"https://$hostname/\"; else probe_url=\"https://$hostname$probe_path/\"; fi", " probe_status=$(curl -k -sS -o /tmp/web-probe-sentinel-caddy-probe.out -w '%{http_code}' --max-time 10 --resolve \"$hostname:443:127.0.0.1\" \"$probe_url\" 2>/tmp/web-probe-sentinel-caddy-probe.err)", " probe_rc=$?", "fi", "after_present=no", "grep -Fq \"# BEGIN $marker\" \"$config_path\" 2>/dev/null && after_present=yes", "active=$(systemctl is-active \"$service\" 2>/dev/null || true)", "err=$(cat /tmp/web-probe-sentinel-caddy-python.err /tmp/web-probe-sentinel-caddy-validate.err /tmp/web-probe-sentinel-caddy-install.err /tmp/web-probe-sentinel-caddy-reload.err /tmp/web-probe-sentinel-caddy-probe.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)", "python3 - \"$python_rc\" \"$validate_rc\" \"$reload_rc\" \"$probe_rc\" \"$probe_status\" \"$after_present\" \"$active\" \"$hostname\" \"$config_path\" \"$err\" <<'PY'", "import json, sys", "python_rc, validate_rc, reload_rc, probe_rc, probe_status, after_present, active, hostname, config_path, error_preview = sys.argv[1:11]", "def num(value):", " if value == '':", " return None", " try:", " return int(value)", " except ValueError:", " return None", "http_status = num(probe_status)", "probe_ok = num(probe_rc) == 0 and http_status is not None and 200 <= http_status < 400", "payload = {", " 'ok': num(python_rc) == 0 and num(validate_rc) == 0 and num(reload_rc) == 0 and probe_ok and after_present == 'yes',", " 'hostname': hostname,", " 'configPath': config_path,", " 'pythonExitCode': num(python_rc),", " 'validateExitCode': num(validate_rc),", " 'reloadExitCode': num(reload_rc),", " 'routeProbeExitCode': num(probe_rc),", " 'routeProbeHttpStatus': http_status,", " 'routeProbeOk': probe_ok,", " 'afterBlockPresent': after_present == 'yes',", " 'active': active,", " 'errorPreview': error_preview,", " 'valuesRedacted': True,", "}", "print(json.dumps(payload, ensure_ascii=False))", "PY", ].join("\n"); const result = runCommand(["trans", stringAt(state.publicExposure, "caddy.route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-caddy-apply"); const parsed = parsedResolution.parsed; return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, rootOrder, ...record(parsed), stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true }; }