// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p8-web-probe-sentinel-recovery. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p10-monitor-web-aggregation. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot. // Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel. import { createHash, randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { repoRoot, rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered } from "./hwlab-node-web-sentinel-config"; import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import type { RenderedCliResult } from "./output"; import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5"; import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; export type WebProbeSentinelConfigAction = "plan" | "status"; export type WebProbeSentinelImageAction = "status" | "build"; export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current"; export type WebProbeSentinelPublishAction = "publish-current"; export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop"; export type WebProbeSentinelDashboardAction = "verify" | "screenshot" | "trigger"; export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary"; export type WebProbeSentinelOptions = | { readonly kind: "config"; readonly action: WebProbeSentinelConfigAction; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly dryRun: boolean; } | { readonly kind: "image"; readonly action: WebProbeSentinelImageAction; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly dryRun: boolean; readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; } | { readonly kind: "control-plane"; readonly action: WebProbeSentinelControlPlaneAction; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly dryRun: boolean; readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; readonly rerun: boolean; } | { readonly kind: "publish"; readonly action: WebProbeSentinelPublishAction; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly dryRun: boolean; readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; readonly rerun: boolean; } | { readonly kind: "maintenance"; readonly action: WebProbeSentinelMaintenanceAction; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly dryRun: boolean; readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; readonly releaseId: string | null; readonly reason: string | null; readonly quickVerify: boolean; } | { readonly kind: "validate"; readonly action: "validate"; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly dryRun: boolean; readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; readonly quickVerify: boolean; } | { readonly kind: "report"; readonly action: "report"; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly view: WebProbeSentinelReportView; readonly runId: string | null; readonly latest: boolean; readonly traceId: string | null; readonly sampleSeq: number | null; readonly raw: boolean; readonly full: boolean; readonly timeoutSeconds: number; } | { readonly kind: "dashboard"; readonly action: WebProbeSentinelDashboardAction; readonly node: string; readonly lane: string; readonly sentinelId: string | null; readonly viewport: string; readonly localDir: string; readonly name: string | null; readonly runId: string | null; readonly timeoutMs: number; readonly waitTimeoutMs: number; readonly timeoutSeconds: number; readonly commandTimeoutSeconds: number; readonly fullPage: boolean; readonly raw: boolean; }; export interface SentinelCicdState { readonly spec: HwlabRuntimeLaneSpec; readonly sentinelId: string; readonly configRefs: Record; readonly configReady: boolean; readonly runtime: Record; readonly cicd: Record; readonly scenarios: unknown; readonly publicExposure: Record; readonly secrets: Record; readonly controlPlaneTarget: Record; readonly controlPlaneNode: Record; readonly sourceHead: SourceHead; readonly image: SentinelImagePlan; readonly manifests: readonly Record[]; readonly manifestSha256: string; readonly valuesRedacted: true; } interface SourceHead { readonly ok: boolean; readonly repository: string; readonly branch: string; readonly commit: string | null; readonly stageRef: string | null; readonly mirrorCommit: string | null; readonly sourceAuthority: "git-mirror-cache" | "git-mirror-snapshot"; readonly latestDrift: boolean; readonly result: CompactCommandResult; } interface SentinelImagePlan { readonly repository: string; readonly tag: string; readonly ref: string; readonly digestRef: string | null; readonly baseImage: string; readonly buildContext: string; readonly entrypoint: string; readonly dockerfileSha256: string; readonly dockerfilePreview: string; readonly monitorWeb: Record; } interface SentinelObservedStatus { readonly sourceMirror: Record; readonly registry: Record; readonly gitMirror: Record; readonly gitops: Record; readonly argo: Record; readonly runtime: Record; readonly cadence: Record; readonly wait?: Record; } interface SentinelObservedExpectation { readonly gitopsRevision: string | null; readonly runtimeImage: string | null; } interface SentinelRemoteJobResult { readonly ok: boolean; readonly phase: string; readonly resourceKind?: "Job" | "PipelineRun"; readonly jobName: string; readonly payload: Record; readonly polls?: number; readonly elapsedMs?: number; readonly create?: Record; readonly probe?: Record; readonly diagnostics?: Record; readonly valuesRedacted: true; } export interface CompactCommandResult { readonly exitCode: number | null; readonly timedOut: boolean; readonly stdoutBytes: number; readonly stderrBytes: number; readonly stdoutPreview: string; readonly stderrPreview: string; } export interface ChildCliResult { readonly ok: boolean; readonly parsed: Record | null; readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string }; } const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot"; type SourceResolveMode = "cached" | "sync"; export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult { if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId)); requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`); const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options)); if (options.kind === "image") return runSentinelImage(state, options); if (options.kind === "control-plane") return runSentinelControlPlane(state, options); if (options.kind === "publish") return runSentinelPublishCurrent(state, options); if (options.kind === "maintenance") return runSentinelMaintenance(state, options); if (options.kind === "validate") return runSentinelValidate(state, options); if (options.kind === "dashboard") return runSentinelDashboard(state, options); return runSentinelReport(state, options); } function sentinelSourceResolveMode(options: WebProbeSentinelOptions): SourceResolveMode { if (options.kind === "image" && options.action === "build" && options.confirm && options.wait) return "sync"; if (options.kind === "control-plane" && options.action === "trigger-current" && options.confirm && options.wait) return "sync"; if (options.kind === "publish" && options.confirm && options.wait) return "sync"; return "cached"; } function runSentinelImage(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel image ${options.action}`; if (options.action === "build" && options.confirm) { if (!options.wait) return renderAsyncSentinelJob(state, "image", "build", options.timeoutSeconds); return runSentinelImageBuildConfirmed(state, options); } const sourceMirror = options.action === "status" ? probeSourceMirror(state, options.timeoutSeconds) : null; const registry = options.action === "status" ? probeImageRegistry(state, options.timeoutSeconds) : null; const sourceMirrorReady = options.action !== "status" || record(sourceMirror).ok === true; const registryReady = options.action !== "status" || record(registry?.probe).present === true; const result = { ok: state.configReady && state.sourceHead.ok && sourceMirrorReady && registryReady, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run", mutation: false, specRef: SPEC_REF, source: state.sourceHead, sourceMirror, image: state.image, registry, blocker: sourceMirrorReady ? registryReady ? null : { code: "sentinel-image-missing", reason: "expected sentinel image tag is not present in the node-local registry" } : { code: "sentinel-source-mirror-not-ready", reason: "source.gitMirrorReadUrl does not expose the selected source commit yet" }, next: { status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, dryRun: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`, confirm: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`, controlPlanePlan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`, }, valuesRedacted: true, }; return rendered(result.ok, command, renderImageResult(result)); } function runSentinelControlPlane(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel control-plane ${options.action}`; const mutationAction = options.action === "apply" || options.action === "trigger-current"; if (options.confirm && mutationAction) { if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds); return runSentinelControlPlaneConfirmed(state, options); } const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null; const observedReady = options.action !== "status" || sentinelObservedReady(record(observed)); const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : []; const pipelineRun = sentinelPipelineRunName(state, options.rerun); const statusDiagnosis = options.action === "status" && !observedReady ? sentinelObservedStatusDiagnosis(state, observed, pipelineRun) : null; const result = { ok: state.configReady && state.sourceHead.ok && observedReady, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run", mutation: false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun, gitops: { path: stringField(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestObjects: state.manifests.length, manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), projectName: stringAt(state.cicd, "argo.projectName"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, maintenance: { startCommand: stringAt(state.cicd, "maintenance.startCommand"), stopCommand: stringAt(state.cicd, "maintenance.stopCommand"), serviceUnavailablePolicy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"), }, validation: { scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"), controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state), quickVerifyMode: "manual-validate", automaticSecondPath: false, }, manifests: { objects: manifestObjectSummary(state.manifests), sha256: state.manifestSha256, }, observability: webProbeSentinelOtelSummary({ node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, namespace: stringAt(state.runtime, "namespace"), runtime: state.runtime, cicd: state.cicd, }), observed, statusDiagnosis, warnings: mergeWarnings(observedWarnings, record(statusDiagnosis).warning), blocker: observedReady ? null : record(statusDiagnosis).blocker ?? { code: "sentinel-control-plane-observed-not-ready", reason: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass", valuesRedacted: true }, recoveryNext: record(statusDiagnosis).recoveryNext ?? null, next: controlPlaneNext(state, options.action), valuesRedacted: true, }; return rendered(result.ok, command, renderControlPlaneResult(result)); } function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = "web-probe sentinel publish-current"; if (options.confirm) { if (!options.wait) return renderAsyncSentinelJob(state, "publish", options.action, options.timeoutSeconds); return runSentinelPublishCurrentConfirmed(state, options); } const result = { ok: state.configReady && state.sourceHead.ok, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: "dry-run", mutation: false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: sentinelPipelineRunName(state, options.rerun), gitops: { path: stringAt(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, budget: publishCurrentBudget(state), validationPlan: publishCurrentHealthValidationPlan(state), stageBudgets: publishCurrentStageBudgets(state), blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" }, next: publishCurrentNext(state), valuesRedacted: true, }; return rendered(result.ok === true, command, renderPublishCurrentResult(result)); } function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const interruptContext: Record = { phase: "starting", pipelineRun: sentinelPipelineRunName(state, options.rerun), sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, valuesRedacted: true, }; const uninstallInterruptHandler = installSentinelPublishInterruptHandler(state, interruptContext); try { return runSentinelPublishCurrentConfirmedInner(state, options, interruptContext); } finally { uninstallInterruptHandler(); } } function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, options: Extract, interruptContext: Record): RenderedCliResult { const startedAt = Date.now(); const command = "web-probe sentinel publish-current"; const budget = publishCurrentBudget(state); const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds")); const deadline = startedAt + budgetSeconds * 1000; const remainingBudgetSeconds = () => strictRemainingSeconds(deadline, budgetSeconds); let controlResult: Record | null = null; sentinelProgressEvent("sentinel.publish.progress", { phase: "publish-current-start", status: "running", pipelineRun: interruptContext.pipelineRun, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, }); if (state.configReady && state.sourceHead.ok && remainingBudgetSeconds() >= 5) { interruptContext.phase = "already-current-registry-probe"; const registryProbe = probeImageRegistry(state, Math.max(1, Math.min(remainingBudgetSeconds(), 5))); if (record(record(registryProbe).probe).present === true && remainingBudgetSeconds() >= 5) { const preflightStartedAt = Date.now(); const preflightTimeoutSeconds = Math.max(1, Math.min(remainingBudgetSeconds(), 10)); interruptContext.phase = "already-current-observed-probe"; const preflightObserved = withObservedWait( collectSentinelObservedStatus(state, preflightTimeoutSeconds, undefined, true), preflightStartedAt, preflightTimeoutSeconds, true, ); if (sentinelObservedReady(preflightObserved)) { controlResult = sentinelAlreadyCurrentControlResult(state, preflightObserved, Date.now() - preflightStartedAt); } } } interruptContext.phase = "control-plane-trigger-current"; controlResult ??= sentinelControlPlaneConfirmedResult(state, { kind: "control-plane", action: "trigger-current", node: options.node, lane: options.lane, sentinelId: options.sentinelId, dryRun: false, confirm: true, wait: true, timeoutSeconds: Math.max(1, remainingBudgetSeconds()), rerun: options.rerun, }); let health: Record; let healthElapsedMs: number | null = null; if (controlResult.ok !== true) { health = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true }; } else if (remainingBudgetSeconds() < 2) { health = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-health", valuesRedacted: true }; } else { interruptContext.phase = "health-endpoint-validation"; const healthStartedAt = Date.now(); health = probeSentinelRuntimeHealthEndpoint(state, remainingBudgetSeconds()); healthElapsedMs = Date.now() - healthStartedAt; health = { ...health, elapsedMs: healthElapsedMs, valuesRedacted: true }; } const elapsedMs = Date.now() - startedAt; const timings = publishCurrentStageTimings(controlResult, health, elapsedMs); const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds); const withinBudget = elapsedMs <= budgetSeconds * 1000; const healthOk = health.ok === true; const ok = controlResult.ok === true && healthOk && withinBudget; const blocker = ok ? null : publishCurrentBlocker(controlResult, health, withinBudget); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, mode: controlResult.mode === "already-current" ? "already-current" : "confirm-wait", mutation: controlResult.mutation !== false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state, options.rerun), controlPlane: controlResult, health, budget, validationPlan: publishCurrentHealthValidationPlan(state), stageBudgets: publishCurrentStageBudgets(state), elapsedMs, withinBudget, timings, slowStages, warnings: mergeWarnings(controlResult.warnings, publishCurrentBudgetWarnings(slowStages, withinBudget, budgetSeconds, elapsedMs)), blocker, next: publishCurrentNext(state), valuesRedacted: true, }; interruptContext.phase = "completed"; return rendered(ok, command, renderPublishCurrentResult(result)); } function strictRemainingSeconds(deadline: number, cap: number): number { return Math.max(0, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000))); } function withObservedWait(observed: SentinelObservedStatus, startedAt: number, timeoutSeconds: number, includeGitMirror: boolean): SentinelObservedStatus { const elapsedMs = Date.now() - startedAt; return { ...observed, wait: { polls: 1, elapsedMs, timeoutMs: Math.max(1, timeoutSeconds) * 1000, ready: sentinelObservedReady(observed), includeGitMirror, valuesRedacted: true, }, }; } function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed: SentinelObservedStatus, elapsedMs: number): Record { const registryProbe = record(record(observed.registry).probe); const gitops = record(observed.gitops); const argo = record(observed.argo); const digest = text(registryProbe.digest); const digestRef = digest === "-" ? null : `${state.image.repository}@${digest}`; const stageTimings = { sourceFetchMs: 0, monitorWebVerifyMs: 0, imageBuildMs: 0, gitopsMs: 0, totalMs: 0, valuesRedacted: true, }; return { ok: true, command: "web-probe sentinel control-plane trigger-current", node: state.spec.nodeId, lane: state.spec.lane, mode: "already-current", mutation: false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: "already-current", gitops: { path: stringAt(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestObjects: state.manifests.length, manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), projectName: stringAt(state.cicd, "argo.projectName"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, validation: { scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"), controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state), quickVerifyMode: "manual-validate", automaticSecondPath: false, }, manifests: { objects: manifestObjectSummary(state.manifests), sha256: state.manifestSha256, }, sourceMirrorSync: { ok: true, phase: "already-current", jobName: "-", elapsedMs: 0, valuesRedacted: true }, publish: { ok: true, phase: "already-current", resourceKind: "PipelineRun", jobName: "already-current", elapsedMs: 0, payload: { ok: true, status: "already-current", sourceCommit: state.sourceHead.commit, imageRef: state.image.ref, digestRef, gitopsCommit: gitops.revision ?? argo.revision ?? null, stageTimings, completedStages: ["already-current"], valuesRedacted: true, }, diagnostics: { domain: "publish", resourceKind: "PipelineRun", pipelineRun: "already-current", currentPhase: "already-current", completedStages: ["already-current:skipped"], stageTimings, valuesRedacted: true, }, valuesRedacted: true, }, flush: { ok: true, skipped: true, reason: "already-current", valuesRedacted: true }, runtimeSecretsApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, publicExposureApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, argoApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, observed, targetValidation: null, elapsedMs, warnings: [ "publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used health endpoint validation only.", ...sentinelObservedWarnings(observed), ...targetValidationDeferredWarnings(state, false, controlPlaneWaitWarningSeconds(state)), ], blocker: null, recoveryNext: controlPlaneRecoveryNext(state, true, {}, { ok: true }, observed), next: controlPlaneNext(state, "trigger-current"), valuesRedacted: true, }; } function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number, sourceResolveMode: SourceResolveMode): SentinelCicdState { const sentinel = resolveWebProbeSentinel(spec, sentinelId); const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime); const cicd = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.cicd), sentinel.configRefs.cicd); const scenarios = readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.scenarios); const rawPublicExposure = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure); const publicExposure = effectiveWebProbeSentinelPublicExposure(spec, sentinel.id, rawPublicExposure); const secrets = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.secrets), sentinel.configRefs.secrets); const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef"); const controlPlaneTarget = recordTarget(readWebProbeSentinelConfigRefTarget(spec, controlPlaneRef), controlPlaneRef); const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef)); const nodeId = stringField(controlPlaneTarget, "node"); const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`); validateSentinelSourceAuthority(cicd); const sourceHead = resolveSourceHead(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds, sourceResolveMode); const image = sentinelImagePlan(spec, cicd, sourceHead); const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image); const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; return { spec, sentinelId: sentinel.id, configRefs: sentinel.configRefs, configReady: configPlan.ok, runtime, cicd, scenarios, publicExposure, secrets, controlPlaneTarget, controlPlaneNode, sourceHead, image, manifests, manifestSha256: sha256(manifestYaml), valuesRedacted: true, }; } function resolveSourceHead( spec: HwlabRuntimeLaneSpec, cicd: Record, controlPlaneTarget: Record, controlPlaneNode: Record, timeoutSeconds: number, mode: SourceResolveMode, ): SourceHead { const repository = stringAt(cicd, "source.repository"); const branch = stringAt(cicd, "source.branch"); const resolved = mode === "sync" ? resolveSourceHeadWithK8sSnapshot(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds) : probeSourceMirrorCache(cicd, controlPlaneNode, timeoutSeconds, null); const probe = record(resolved.probe); const commit = nonEmptyString(probe.sourceCommit) ?? nonEmptyString(probe.commit) ?? nonEmptyString(probe.mirrorCommit); const stageRef = nonEmptyString(probe.stageRef) ?? (commit === null ? null : sentinelSourceSnapshotRef(cicd, commit)); const mirrorCommit = nonEmptyString(probe.mirrorCommit) ?? nonEmptyString(probe.commit); return { ok: resolved.ok === true && commit !== null, repository, branch, commit, stageRef, mirrorCommit, sourceAuthority: mode === "sync" ? "git-mirror-snapshot" : "git-mirror-cache", latestDrift: commit !== null && mirrorCommit !== null && commit !== mirrorCommit, result: compactCommand(resolved.result), }; } function resolveSourceHeadWithK8sSnapshot( spec: HwlabRuntimeLaneSpec, cicd: Record, controlPlaneTarget: Record, controlPlaneNode: Record, timeoutSeconds: number, ): { ok: boolean; probe: Record; result: CommandResult } { const namespace = stringAt(cicd, "builder.namespace"); const prefix = `${stringAt(cicd, "builder.jobPrefix")}-source-resolve`; const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); const manifest = sentinelSourceMirrorResolveJobManifest(spec, cicd, controlPlaneTarget, controlPlaneNode, jobName); const created = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); if (created.exitCode !== 0) return { ok: false, probe: { ok: false, status: "create-failed", jobName, valuesRedacted: true }, result: created }; const startedAt = Date.now(); const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, 120_000)); let lastCapture = created; while (Date.now() - startedAt < timeoutMs) { const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); lastCapture = probeCapture; const probe = parseJsonObject(probeCapture.stdout) ?? {}; const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); if (probe.succeeded === true) return { ok: payload.ok === true, probe: payload, result: probeCapture }; if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, valuesRedacted: true } : payload, result: probeCapture }; runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture }; } function sentinelSourceMirrorResolveJobManifest( spec: HwlabRuntimeLaneSpec, cicd: Record, controlPlaneTarget: Record, controlPlaneNode: Record, jobName: string, ): Record { const namespace = stringAt(cicd, "builder.namespace"); const labels = { "app.kubernetes.io/name": "web-probe-sentinel-source-resolve", "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": spec.nodeId, "unidesk.ai/lane": spec.lane, }; return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace, labels }, spec: { backoffLimit: 0, activeDeadlineSeconds: numberAt(cicd, "builder.activeDeadlineSeconds"), ttlSecondsAfterFinished: numberAt(cicd, "builder.ttlSecondsAfterFinished"), template: { metadata: { labels }, spec: { restartPolicy: "Never", volumes: [ sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget), { name: "git-ssh", secret: { secretName: stringAt(cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, ], containers: [{ name: "resolve", image: sentinelSourceResolverImage(spec, cicd), imagePullPolicy: "IfNotPresent", command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShellFromConfig(cicd, controlPlaneNode, jobName, null)], volumeMounts: [ { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }], }, }, }, }; } function sentinelSourceResolverImage(spec: HwlabRuntimeLaneSpec, cicd: Record): string { const baseImageRef = stringAt(cicd, "image.baseImageRef"); return stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef); } function validateSentinelSourceAuthority(cicd: Record): void { const mode = stringAt(cicd, "sourceAuthority.mode"); const resolver = stringAt(cicd, "sourceAuthority.resolver"); const allowHostGit = booleanAt(cicd, "sourceAuthority.allowHostGit"); const allowGithubDirectInPipeline = booleanAt(cicd, "sourceAuthority.allowGithubDirectInPipeline"); const missingObjectPolicy = stringAt(cicd, "sourceSnapshot.missingObjectPolicy"); if (mode !== "gitMirrorSnapshot") throw new Error("sourceAuthority.mode must be gitMirrorSnapshot"); if (resolver !== "k8s-git-mirror") throw new Error("sourceAuthority.resolver must be k8s-git-mirror"); if (allowHostGit !== false) throw new Error("sourceAuthority.allowHostGit must be false"); if (allowGithubDirectInPipeline !== false) throw new Error("sourceAuthority.allowGithubDirectInPipeline must be false"); if (missingObjectPolicy !== "fail-fast") throw new Error("sourceSnapshot.missingObjectPolicy must be fail-fast"); sentinelSourceSnapshotStageRefPrefix(cicd); } function sentinelSourceSnapshotStageRefPrefix(cicd: Record): string { const branch = stringAt(cicd, "source.branch"); const repository = stringAt(cicd, "source.repository"); const prefix = stringAt(cicd, "sourceSnapshot.stageRefPrefix") .replaceAll("{branch}", branch) .replaceAll("{repository}", repository) .replace(/\/+$/u, ""); if (!prefix.startsWith("refs/")) throw new Error("sourceSnapshot.stageRefPrefix must resolve to a git ref prefix"); return prefix; } function sentinelSourceSnapshotRef(cicd: Record, commit: string): string { return `${sentinelSourceSnapshotStageRefPrefix(cicd)}/${commit}`; } function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record, sourceHead: SourceHead): SentinelImagePlan { const repository = stringAt(cicd, "image.repository"); const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12); const baseImageRef = stringAt(cicd, "image.baseImageRef"); const baseImage = stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef); const entrypoint = stringAt(cicd, "source.entrypoint"); const monitorWeb = monitorWebCicdPlan(spec, cicd); const dockerfile = sentinelDockerfile(baseImage, entrypoint); return { repository, tag, ref: `${repository}:${tag}`, digestRef: null, baseImage, buildContext: stringAt(cicd, "source.buildContext"), entrypoint, dockerfileSha256: sha256(dockerfile), dockerfilePreview: dockerfile, monitorWeb, }; } function sentinelDockerfile(baseImage: string, entrypoint: string): string { return [ `FROM ${baseImage}`, "WORKDIR /app", "ENV NODE_ENV=production", "COPY .unidesk-sentinel-bin/trans /usr/local/bin/trans", "COPY . /app", `ENTRYPOINT ["bun", "${entrypoint}"]`, "", ].join("\n"); } function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record): Record { return { stack: stringAtNullable(cicd, "monitorWeb.frontendStack") ?? "vue3-vendored-browser-build", runtimeMode: stringAtNullable(cicd, "monitorWeb.runtimeMode") ?? "runner-served-bridge", assetRoot: stringAtNullable(cicd, "monitorWeb.assetRoot") ?? "scripts/assets/web-probe-sentinel-monitor-web", verifyCommand: "bun scripts/verify-web-probe-sentinel-monitor-web.ts", gitMirrorReadUrl: stringAt(cicd, "source.gitMirrorReadUrl"), sourceMode: stringAt(cicd, "builder.sourceMode"), envReuseMode: stringAtNullable(cicd, "monitorWeb.envReuse.mode") ?? "k8s-buildkit-and-ci-node-deps", envReuseNodeDepsPath: stringAtNullable(cicd, "monitorWeb.envReuse.nodeDepsPath") ?? "/opt/hwlab-ci-node-deps/node_modules", verifyPhase: stringAtNullable(cicd, "monitorWeb.imageBuild.verifyPhase") ?? "pre-image-build", imageBuildBuilder: spec.buildkit?.sidecarImage ?? null, imageBuildPackageMode: stringAtNullable(cicd, "monitorWeb.imageBuild.packageMode") ?? "copy-only-containerfile", imageBuildNetworkMode: monitorWebImageBuildNetworkMode(cicd), imageBuildProxySource: stringAtNullable(cicd, "monitorWeb.imageBuild.proxySource") ?? "node.networkProfile.imageBuildProxy", imageBuildContextIgnore: stringAtNullable(cicd, "monitorWeb.imageBuild.contextIgnore") ?? "generated", imageBuildState: monitorWebBuildkitStatePlan(cicd), ciBudgetSeconds: numberAtNullable(cicd, "monitorWeb.ciBudget.maxSeconds") ?? numberAt(cicd, "confirmWait.maxSeconds"), valuesRedacted: true, }; } function monitorWebImageBuildNetworkMode(cicd: Record): "default" | "host" { const value = stringAtNullable(cicd, "monitorWeb.imageBuild.networkMode") ?? "host"; if (value !== "default" && value !== "host") throw new Error(`monitorWeb.imageBuild.networkMode must be default or host, got ${value}`); return value; } function monitorWebBuildkitStatePlan(cicd: Record): Record { const state = recordTarget(valueAtPath(cicd, "monitorWeb.imageBuild.buildkitState"), "monitorWeb.imageBuild.buildkitState"); const mode = stringAt(state, "mode"); if (mode === "hostPath") { return { mode, path: stringAt(state, "path"), type: stringAt(state, "type"), valuesRedacted: true, }; } if (mode === "persistentVolumeClaim") { return { mode, claimName: stringAt(state, "claimName"), valuesRedacted: true, }; } if (mode === "emptyDir") { return { mode, sizeLimit: stringAt(state, "sizeLimit"), valuesRedacted: true, }; } throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); } function publishCurrentBudget(state: SentinelCicdState): Record { const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget"); return { maxSeconds: numberAt(budget, "maxSeconds"), valuesRedacted: true, }; } function publishCurrentStageBudgets(state: SentinelCicdState): Record { const budgets = recordTarget(valueAtPath(state.cicd, "publishCurrent.stageBudgets"), "publishCurrent.stageBudgets"); return { sourceSyncSeconds: numberAt(budgets, "sourceSyncSeconds"), sourceFetchSeconds: numberAt(budgets, "sourceFetchSeconds"), monitorWebVerifySeconds: numberAt(budgets, "monitorWebVerifySeconds"), imageBuildSeconds: numberAt(budgets, "imageBuildSeconds"), gitopsSeconds: numberAt(budgets, "gitopsSeconds"), argoRuntimeSeconds: numberAt(budgets, "argoRuntimeSeconds"), dashboardVerifySeconds: numberAt(budgets, "dashboardVerifySeconds"), valuesRedacted: true, }; } function publishCurrentDashboardPlan(state: SentinelCicdState): Record { const dashboard = recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"); const viewport = stringAt(dashboard, "viewport"); if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(viewport)) throw new Error(`publishCurrent.dashboard.viewport must look like 1440x900, got ${viewport}`); return { enabled: booleanAt(dashboard, "enabled"), required: booleanAt(dashboard, "required"), viewport, timeoutMs: numberAt(dashboard, "timeoutMs"), waitTimeoutMs: numberAt(dashboard, "waitTimeoutMs"), commandTimeoutSeconds: numberAt(dashboard, "commandTimeoutSeconds"), fullPage: booleanAt(dashboard, "fullPage"), valuesRedacted: true, }; } function publishCurrentDashboardEnabled(state: SentinelCicdState): boolean { return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "enabled"); } function publishCurrentDashboardRequired(state: SentinelCicdState): boolean { return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required"); } function publishCurrentHealthValidationPlan(state: SentinelCicdState): Record { return { enabled: true, required: true, endpoint: stringAt(state.runtime, "healthPath"), source: "runtime.healthPath", browser: false, playwright: false, webProbe: false, valuesRedacted: true, }; } function publishCurrentDashboardReserveSeconds(state: SentinelCicdState): number { if (!publishCurrentDashboardEnabled(state)) return 0; const dashboard = publishCurrentDashboardPlan(state); const budgets = publishCurrentStageBudgets(state); return Math.max(0, Math.min( numberAt(budgets, "dashboardVerifySeconds"), numberAt(dashboard, "commandTimeoutSeconds"), Math.ceil(numberAt(dashboard, "waitTimeoutMs") / 1000), )); } function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract { const dashboard = publishCurrentDashboardPlan(state); const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000)); const waitTimeoutMs = Math.min(numberAt(dashboard, "waitTimeoutMs"), remainingMs); const timeoutMs = Math.min(numberAt(dashboard, "timeoutMs"), waitTimeoutMs); return { kind: "dashboard", action: "verify", node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, viewport: stringAt(dashboard, "viewport"), localDir: "/tmp", name: null, runId: null, timeoutMs, waitTimeoutMs, timeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))), commandTimeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))), fullPage: booleanAt(dashboard, "fullPage"), raw: false, }; } function publishCurrentStageTimings(controlResult: Record, health: Record, elapsedMs: number): Record { const publish = record(controlResult.publish); const payload = record(publish.payload); const payloadStageTimings = record(payload.stageTimings); const diagnosticStageTimings = record(record(publish.diagnostics).stageTimings); const stageTimings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings; const observedWait = record(record(controlResult.observed).wait); return { sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs), sourceFetchMs: finiteNumberOrNull(stageTimings.sourceFetchMs), monitorWebVerifyMs: finiteNumberOrNull(stageTimings.monitorWebVerifyMs), imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs), gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs), argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs), healthValidationMs: finiteNumberOrNull(health.elapsedMs), totalMs: elapsedMs, valuesRedacted: true, }; } function publishCurrentSlowStages(state: SentinelCicdState, timings: Record, budgetSeconds: number): Record[] { const budgets = publishCurrentStageBudgets(state); const stageMap: Array<[string, string, string, string]> = [ ["source-sync", "sourceSyncMs", "sourceSyncSeconds", "check git-mirror pre-sync, node-local mirror health and SSH-over-proxy latency"], ["source-fetch", "sourceFetchMs", "sourceFetchSeconds", "keep sparse checkout paths narrow and verify node-local git mirror object availability"], ["monitor-web-verify", "monitorWebVerifyMs", "monitorWebVerifySeconds", "keep monitor-web verification copy-only and avoid rebuilding frontend assets during publish"], ["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"], ["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"], ["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"], ["health-validation", "healthValidationMs", "dashboardVerifySeconds", "inspect runtime health endpoint latency and service routing"], ]; const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => { const elapsed = finiteNumberOrNull(timings[timingKey]); const stageBudgetSeconds = finiteNumberOrNull(budgets[budgetKey]); if (elapsed === null || stageBudgetSeconds === null || elapsed <= stageBudgetSeconds * 1000) return []; return [{ stage, elapsedMs: elapsed, budgetSeconds: stageBudgetSeconds, suggestion, valuesRedacted: true }]; }); const total = finiteNumberOrNull(timings.totalMs); if (total !== null && total > budgetSeconds * 1000) { slow.push({ stage: "total", elapsedMs: total, budgetSeconds, suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or health endpoint segment before rerun", valuesRedacted: true, }); } return slow; } function publishCurrentBudgetWarnings(slowStages: readonly Record[], withinBudget: boolean, budgetSeconds: number, elapsedMs: number): string[] { const warnings = slowStages.map((stage) => `${text(stage.stage)} exceeded configured ${text(stage.budgetSeconds)}s budget (${Math.round((finiteNumberOrNull(stage.elapsedMs) ?? 0) / 1000)}s): ${text(stage.suggestion)}`); if (!withinBudget) warnings.unshift(`publish-current exceeded configured ${budgetSeconds}s end-to-end budget (${Math.round(elapsedMs / 1000)}s); no further blind wait was attempted.`); return warnings; } function publishCurrentBlocker(controlResult: Record, health: Record, withinBudget: boolean): Record { if (controlResult.ok !== true) { const blocker = record(controlResult.blocker); return { code: blocker.code ?? "sentinel-publish-current-control-plane-blocked", reason: blocker.reason ?? "control-plane publish did not converge", valuesRedacted: true, }; } if (health.ok !== true) { const degradedReason = text(health.degradedReason); return { code: health.skipped === true ? text(health.reason) : "sentinel-publish-current-health-endpoint-failed", reason: health.skipped === true ? "health endpoint validation did not run" : degradedReason === "-" ? "health endpoint validation did not pass" : `health endpoint validation did not pass: ${degradedReason}`, valuesRedacted: true, }; } if (!withinBudget) { return { code: "sentinel-publish-current-over-budget", reason: "runtime and health endpoint converged, but the one-click CI/CD path exceeded the YAML end-to-end budget", valuesRedacted: true, }; } return { code: "sentinel-publish-current-blocked", reason: "publish-current did not satisfy all checks", valuesRedacted: true }; } function publishCurrentNext(state: SentinelCicdState): Record { const node = state.spec.nodeId; const lane = state.spec.lane; const suffix = sentinelCliSuffix(state); return { publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait`, controlPlaneStatus: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`, dashboardVerify: `bun scripts/cli.ts web-probe sentinel dashboard verify --node ${node} --lane ${lane}${suffix}`, gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`, gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm`, gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`, }; } function renderSentinelManifests( spec: HwlabRuntimeLaneSpec, sentinelId: string, runtime: Record, cicd: Record, scenarios: unknown, publicExposure: Record, secrets: Record, image: SentinelImagePlan, ): readonly Record[] { const namespace = stringAt(runtime, "namespace"); const labels = { "app.kubernetes.io/name": stringAt(runtime, "deploymentName"), "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": spec.nodeId, "unidesk.ai/lane": spec.lane, "unidesk.ai/web-probe-sentinel-id": sentinelId, }; const deploymentName = stringAt(runtime, "deploymentName"); const serviceName = stringAt(runtime, "serviceName"); const servicePort = numberAt(runtime, "servicePort"); const pvcStorage = stringAt(runtime, "pvcStorage"); const stateRoot = stringAt(runtime, "stateRoot"); const sentinelEnv = sentinelContainerEnv(sentinelId, runtime, cicd, secrets); const kubernetesApiEgress = sentinelKubernetesApiEgress(runtime); const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv); if (cadenceJob !== null) { emitWebProbeSentinelSpan({ node: spec.nodeId, lane: spec.lane, sentinelId, namespace, runtime, cicd, }, "web_probe_sentinel.cadence.cronjob_rendered", { cronJobName: record(cadenceJob.metadata).name ?? null, namespace, cadence: record(cadenceJob.metadata).annotations === undefined ? null : record(record(cadenceJob.metadata).annotations)["unidesk.ai/cadence"], schedule: record(cadenceJob.spec).schedule ?? null, valuesRedacted: true, }); } return [ { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels }, }, { apiVersion: "rbac.authorization.k8s.io/v1", kind: "Role", metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels }, rules: [ { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get"] }, { apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "list"] }, ], }, { apiVersion: "rbac.authorization.k8s.io/v1", kind: "RoleBinding", metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels }, subjects: [{ kind: "ServiceAccount", name: stringAt(runtime, "serviceAccountName"), namespace }], roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${deploymentName}-manual-trigger` }, }, { apiVersion: "v1", kind: "PersistentVolumeClaim", metadata: { name: stringAt(runtime, "pvcName"), namespace, labels }, spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: pvcStorage } } }, }, { apiVersion: "v1", kind: "ConfigMap", metadata: { name: `${deploymentName}-config`, namespace, labels }, data: { "config-summary.json": JSON.stringify({ specRef: SPEC_REF, node: spec.nodeId, lane: spec.lane, sentinelId, publicBaseUrl: stringAt(publicExposure, "publicBaseUrl"), routePrefix: stringAtNullable(publicExposure, "routePrefix") ?? "/", gitopsPath: stringAt(cicd, "gitopsPath"), valuesRedacted: true, }, null, 2), }, }, { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels }, spec: { replicas: numberAt(runtime, "replicas"), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, template: { metadata: { labels }, spec: { serviceAccountName: stringAt(runtime, "serviceAccountName"), containers: [{ name: "sentinel", image: image.ref, imagePullPolicy: "IfNotPresent", args: [ "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", sentinelId, "--state-root", stateRoot, "--host", stringAt(runtime, "listenHost"), "--port", String(servicePort), ], env: sentinelEnv, ports: [{ name: "http", containerPort: servicePort }], readinessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } }, livenessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } }, volumeMounts: [{ name: "state", mountPath: stateRoot }], }], volumes: [{ name: "state", persistentVolumeClaim: { claimName: stringAt(runtime, "pvcName") } }], }, }, }, }, { apiVersion: "v1", kind: "Service", metadata: { name: serviceName, namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] }, }, ...(cadenceJob === null ? [] : [cadenceJob]), { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: stringAt(publicExposure, "frpc.deploymentName"), namespace, labels: { ...labels, "app.kubernetes.io/component": "tunnel" } }, spec: { replicas: 1, selector: { matchLabels: { "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName") } }, template: { metadata: { labels: { ...labels, "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName"), "app.kubernetes.io/component": "tunnel" }, annotations: { "unidesk.ai/public-base-url": stringAt(publicExposure, "publicBaseUrl"), "unidesk.ai/frp-server": `${stringAt(publicExposure, "frpc.serverAddr")}:${numberAt(publicExposure, "frpc.serverPort")}`, "unidesk.ai/frp-remote-port": String(numberAt(publicExposure, "frpc.httpProxy.remotePort")), }, }, spec: { containers: [{ name: "frpc", image: stringAt(publicExposure, "frpc.image"), imagePullPolicy: "IfNotPresent", args: ["-c", "/etc/frp/frpc.toml"], volumeMounts: [{ name: "frpc-config", mountPath: "/etc/frp/frpc.toml", subPath: stringAt(publicExposure, "frpc.secretKey"), readOnly: true }], }], volumes: [{ name: "frpc-config", secret: { secretName: stringAt(publicExposure, "frpc.secretName") } }], }, }, }, }, { apiVersion: "networking.k8s.io/v1", kind: "NetworkPolicy", metadata: { name: `${deploymentName}-egress`, namespace, labels }, spec: { podSelector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, policyTypes: ["Ingress", "Egress"], ingress: [{ from: [{ namespaceSelector: {} }], ports: [{ protocol: "TCP", port: servicePort }] }], egress: [ { to: [{ namespaceSelector: {} }] }, ...kubernetesApiEgress, ], }, }, { apiVersion: "argoproj.io/v1alpha1", kind: "Application", metadata: { name: stringAt(cicd, "argo.applicationName"), namespace: stringAt(cicd, "argo.namespace"), labels }, spec: { project: stringAt(cicd, "argo.projectName"), source: { repoURL: stringAt(cicd, "argo.repoURL"), targetRevision: stringAt(cicd, "argo.targetRevision"), path: stringAt(cicd, "gitopsPath"), }, destination: { server: "https://kubernetes.default.svc", namespace }, syncPolicy: { automated: { prune: true, selfHeal: true } }, }, }, ]; } function sentinelKubernetesApiEgress(runtime: Record): readonly Record[] { if (booleanAtNullable(runtime, "kubernetesApi.egress.enabled") !== true) return []; const rules = arrayAtNullable(runtime, "kubernetesApi.egress.rules"); const rows = rules.length > 0 ? rules : [{ cidr: stringAt(runtime, "kubernetesApi.egress.cidr"), port: numberAt(runtime, "kubernetesApi.egress.port") }]; return rows.map((rule) => ({ to: [{ ipBlock: { cidr: stringAt(rule, "cidr") } }], ports: [{ protocol: "TCP", port: numberAt(rule, "port") }], })); } function sentinelContainerEnv(sentinelId: string, runtime: Record, cicd: Record, secrets: Record): readonly Record[] { const env: Record[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }]; const otelEnabled = booleanAtNullable(runtime, "observability.otel.enabled") ?? booleanAtNullable(cicd, "observability.otel.enabled") ?? false; const otelEndpoint = stringAtNullable(runtime, "observability.otel.tracesEndpoint") ?? stringAtNullable(runtime, "observability.otel.endpoint") ?? stringAtNullable(cicd, "observability.otel.tracesEndpoint") ?? stringAtNullable(cicd, "observability.otel.endpoint"); const otelServiceName = stringAtNullable(runtime, "observability.otel.serviceName") ?? stringAtNullable(cicd, "observability.otel.serviceName"); const otelSampler = stringAtNullable(runtime, "observability.otel.sampler") ?? stringAtNullable(cicd, "observability.otel.sampler"); const otelSamplerArg = stringAtNullable(runtime, "observability.otel.samplerArg") ?? stringAtNullable(cicd, "observability.otel.samplerArg"); const sourcesByPurpose = new Map>(); for (const source of arrayAt(secrets, "sources").map(record)) { const purpose = stringAtNullable(source, "purpose"); if (purpose !== null) sourcesByPurpose.set(purpose, source); } const used = new Set(env.map((item) => String(item.name ?? ""))); const pushEnv = (item: Record): void => { const name = String(item.name ?? ""); if (name.length === 0 || used.has(name)) return; used.add(name); env.push(item); }; if (otelEnabled) { if (otelEndpoint !== null) pushEnv({ name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: otelEndpoint }); if (otelServiceName !== null) pushEnv({ name: "OTEL_SERVICE_NAME", value: otelServiceName }); if (otelSampler !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER", value: otelSampler }); if (otelSamplerArg !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER_ARG", value: otelSamplerArg }); } for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) { const secretName = stringAtNullable(runtimeSecret, "name"); if (secretName === null) continue; for (const item of arrayAt(runtimeSecret, "data").map(record)) { const targetKey = stringAtNullable(item, "targetKey"); const sourcePurpose = stringAtNullable(item, "sourcePurpose"); const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey"); const sourceKeyEnvName = sourcePurpose === "bootstrap-admin" || sourcePurpose === "prompt-set" ? sourceKey : null; if (targetKey !== null && sourceKeyEnvName !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKeyEnvName)) { pushEnv({ name: sourceKeyEnvName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } }); } const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey); if (envName === null) continue; pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } }); } } return env; } function sentinelCadenceCronJobPlan( spec: HwlabRuntimeLaneSpec, sentinelId: string, runtime: Record, cicd: Record, scenarios: unknown, imageRef: string, sentinelEnv: readonly Record[], ): Record | null { const scheduler = record(valueAtPath(cicd, "targetValidation.cadenceScheduler")); const cadenceSchedulerEnabled = booleanAtNullable(cicd, "targetValidation.cadenceScheduler.enabled") === true; if (!cadenceSchedulerEnabled) return null; const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId"); if (scenarioId === null) return null; const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null; if (scenario === null) return null; const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null; const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds); if (schedule === null) return null; const namespace = stringAt(runtime, "namespace"); const deploymentName = stringAt(runtime, "deploymentName"); const serviceAccountName = stringAt(runtime, "serviceAccountName"); const timeoutSeconds = numberAt(cicd, "targetValidation.maxSeconds"); const activeDeadlineSlackSeconds = numberAt(scheduler, "activeDeadlineSlackSeconds"); const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost"); const name = sentinelCadenceCronJobName(deploymentName); const concurrencyPolicy = stringAt(scheduler, "concurrencyPolicy"); if (!["Allow", "Forbid", "Replace"].includes(concurrencyPolicy)) throw new Error("targetValidation.cadenceScheduler.concurrencyPolicy must be Allow, Forbid or Replace"); const labels = { "app.kubernetes.io/name": name, "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "app.kubernetes.io/component": "cadence-scheduler", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": spec.nodeId, "unidesk.ai/lane": spec.lane, "unidesk.ai/web-probe-sentinel-id": sentinelId, }; return { apiVersion: "batch/v1", kind: "CronJob", metadata: { name, namespace, labels, annotations: { "unidesk.ai/cadence": String(scenario.cadence), "unidesk.ai/target-validation-max-seconds": String(timeoutSeconds), "unidesk.ai/source": "targetValidation.cadenceScheduler", }, }, spec: { schedule, concurrencyPolicy, successfulJobsHistoryLimit: numberAt(scheduler, "successfulJobsHistoryLimit"), failedJobsHistoryLimit: numberAt(scheduler, "failedJobsHistoryLimit"), startingDeadlineSeconds: numberAt(scheduler, "startingDeadlineSeconds"), jobTemplate: { spec: { activeDeadlineSeconds: timeoutSeconds + activeDeadlineSlackSeconds, ttlSecondsAfterFinished: numberAt(scheduler, "ttlSecondsAfterFinished"), backoffLimit: numberAt(scheduler, "backoffLimit"), template: { metadata: { labels }, spec: { restartPolicy: "Never", serviceAccountName, containers: [{ name: "quick-verify", image: imageRef, imagePullPolicy: "IfNotPresent", command: ["bun", "scripts/cli.ts"], args: [ "web-probe", "sentinel", "validate", "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", sentinelId, "--quick-verify", "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds), ], env: [ ...sentinelEnv, { name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" }, ...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]), ], }], }, }, }, }, }, }; } function sentinelCadenceCronJobName(deploymentName: string): string { return safeKubernetesSegment(`${deploymentName}-quick-verify`, 52); } function scenarioRows(value: unknown): Record[] { if (Array.isArray(value)) return value.map(record); if (!isRecord(value)) return []; if (Array.isArray(value.scenarios)) return value.scenarios.map(record); if (isRecord(value.workflow)) return [value.workflow]; return [value]; } function parseDurationSeconds(value: string): number | null { const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim()); if (match === null) return null; const amount = Number(match[1]); const unit = match[2]; if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000)); if (unit === "s") return Math.max(60, amount); if (unit === "m") return amount * 60; if (unit === "h") return amount * 3600; return null; } function cronScheduleForCadenceSeconds(seconds: number): string | null { if (!Number.isFinite(seconds) || seconds <= 0) return null; if (seconds <= 60) return "* * * * *"; if (seconds % 60 === 0) { const minutes = Math.trunc(seconds / 60); if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`; if (minutes % 60 === 0) { const hours = Math.trunc(minutes / 60); if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`; if (hours === 24) return "0 0 * * *"; } } return null; } function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null { if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null; const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, ""); return segment.length === 0 ? null : `HWLAB_WEB_${segment}_JSON`; } export function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record } { const sourcesByPurpose = new Map>(); for (const source of arrayAt(state.secrets, "sources").map(record)) { const purpose = stringAtNullable(source, "purpose"); if (purpose !== null) sourcesByPurpose.set(purpose, source); } const env: NodeJS.ProcessEnv = {}; const items: Record[] = []; const missing: Record[] = []; for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) { const secretName = stringAtNullable(runtimeSecret, "name"); for (const item of arrayAt(runtimeSecret, "data").map(record)) { const targetKey = stringAtNullable(item, "targetKey"); const sourcePurpose = stringAtNullable(item, "sourcePurpose"); const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey); if (envName === null || sourcePurpose === null || targetKey === null) continue; const source = sourcesByPurpose.get(sourcePurpose); const runtimeValue = process.env[envName]; if (source === undefined) { if (runtimeValue !== undefined && runtimeValue.length > 0) { env[envName] = runtimeValue; items.push({ envName, secretName, targetKey, sourcePurpose, sourceMode: "runtime-env", fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`, valuesRedacted: true, }); continue; } missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true }); continue; } const sourceRef = stringAt(source, "sourceRef"); const sourceKey = stringAt(source, "sourceKey"); const material = readSentinelSecretSourceValue(source); if (!material.ok) { if (runtimeValue !== undefined && runtimeValue.length > 0) { env[envName] = runtimeValue; items.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, sourceMode: "runtime-env", fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`, valuesRedacted: true, }); continue; } missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true }); continue; } const value = stringAt(material, "value"); env[envName] = value; items.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`, valuesRedacted: true, }); } } const summary = { ok: missing.length === 0, envCount: items.length, items, missing, valuesRedacted: true, }; return { ok: missing.length === 0, env, summary }; } function normalizeRoutePrefix(value: string | null): string { if (value === null || value.trim() === "" || value.trim() === "/") return "/"; const prefixed = value.trim().startsWith("/") ? value.trim() : `/${value.trim()}`; return prefixed.replace(/\/+$/u, "") || "/"; } function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): Record { const endpoint = stringAt(state.controlPlaneNode, "registry.endpoint"); const repoTag = state.image.ref.replace(`${endpoint}/`, ""); const repo = repoTag.slice(0, repoTag.lastIndexOf(":")); const tag = repoTag.slice(repoTag.lastIndexOf(":") + 1); const registryMode = stringAtNullable(state.controlPlaneNode, "registry.mode") ?? "host-docker"; const endpointIsLoopback = endpoint === "127.0.0.1:5000" || endpoint.startsWith("127.0.0.1:"); const probeMode = registryMode === "k8s-workload" && !endpointIsLoopback ? "k8s-service" : "node-loopback"; const url = probeMode === "k8s-service" ? `http://${stringAt(state.controlPlaneNode, "registry.serviceName")}.${stringAt(state.controlPlaneNode, "registry.namespace")}.svc.cluster.local:${numberAtNullable(state.controlPlaneNode, "registry.containerPort") ?? 5000}/v2/${repo}/manifests/${tag}` : `http://${endpoint}/v2/${repo}/manifests/${tag}`; const script = [ "set +e", `url=${shellQuote(url)}`, `probe_mode=${shellQuote(probeMode)}`, "headers=$(mktemp)", "accept='application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json'", "if command -v curl >/dev/null 2>&1 && curl -fsSI -H \"Accept: $accept\" --max-time 5 \"$url\" >\"$headers\" 2>/tmp/web-probe-sentinel-image.err; then present=true; else present=false; fi", "digest=$(awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}' \"$headers\" 2>/dev/null)", "python3 - \"$present\" \"$digest\" \"$url\" \"$probe_mode\" <<'PY'", "import json, sys", "print(json.dumps({'present': sys.argv[1] == 'true', 'digest': sys.argv[2] or None, 'url': sys.argv[3], 'mode': sys.argv[4], 'valuesRedacted': True}))", "PY", ].join("\n"); const result = probeMode === "k8s-service" ? runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", stringAt(state.cicd, "builder.namespace"), "exec", "deploy/git-mirror-http", "--", "sh", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }) : runCommand(["trans", stringAt(state.controlPlaneNode, "route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); return { ok: result.exitCode === 0, probe: parseJsonObject(result.stdout), result: compactCommand(result), }; } function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const startedAt = Date.now(); const command = "web-probe sentinel image build"; const sourceMirrorProbe = probeSourceMirror(state, Math.min(options.timeoutSeconds, 20)); const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds); const sourceMirrorReady = sourceMirrorSync.ok === true; const publish = sourceMirrorReady ? runSentinelPublishJob(state, false, options.timeoutSeconds, false) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const registry = probeImageRegistry(state, options.timeoutSeconds); const registryReady = record(registry.probe).present === true; const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publish.ok === true && registryReady; const elapsedMs = Date.now() - startedAt; const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: true, specRef: SPEC_REF, source: state.sourceHead, image: state.image, registry, sourceMirrorSync, publish, elapsedMs, warnings: [ ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel image build confirm-wait", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"), ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ], blocker: ok ? null : !sourceMirrorReady ? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" } : publish.ok !== true ? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" } : { code: "sentinel-image-registry-missing", reason: "image publish completed but expected registry tag is not visible" }, next: { status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, controlPlaneTrigger: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`, }, valuesRedacted: true, }; return rendered(result.ok, command, renderImageResult(result)); } function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const result = sentinelControlPlaneConfirmedResult(state, options); return rendered(result.ok === true, String(result.command), renderControlPlaneResult(result)); } function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: Extract): Record { const startedAt = Date.now(); const command = `web-probe sentinel control-plane ${options.action}`; const applyOnly = options.action === "apply"; const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state); const waitBudgetSeconds = Math.max(1, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds)); const deadline = startedAt + waitBudgetSeconds * 1000; const remainingCicdWaitSeconds = () => strictRemainingSeconds(deadline, waitBudgetSeconds); const remainingCommandSeconds = () => Math.max(1, remainingCicdWaitSeconds()); const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCommandSeconds(), 20)); const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCommandSeconds()); const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true; const publish = applyOnly ? null : sourceMirrorReady ? runSentinelPublishJob(state, true, remainingCommandSeconds(), options.rerun) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8; const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true ? startSentinelGitMirrorFlushAsync(state) : null; const runtimeSecretsApply = publishWaitBudgetExhausted ? sentinelSkippedControlStep("publish-wait-budget-exhausted") : applySentinelRuntimeSecrets(state, remainingCommandSeconds()); const publicExposureApply = publishWaitBudgetExhausted ? sentinelSkippedControlStep("publish-wait-budget-exhausted") : applySentinelPublicExposure(state, remainingCommandSeconds()); const argoApply = publishWaitBudgetExhausted ? sentinelSkippedControlStep("publish-wait-budget-exhausted") : applySentinelArgoApplication(state, remainingCommandSeconds()); const observed = publishWaitBudgetExhausted ? sentinelSkippedObservedStatus("publish-wait-budget-exhausted") : waitForSentinelObservedStatus(state, remainingCommandSeconds(), undefined, false); const observedReady = sentinelObservedReady(observed); const publishReady = applyOnly || record(publish).ok === true || observedReady; const flushReady = applyOnly || record(flush).ok === true || observedReady; const targetValidation = null; const targetValidationBlocked = false; const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publishReady && flushReady && record(runtimeSecretsApply).ok === true && record(publicExposureApply).ok === true && record(argoApply).ok === true && observedReady; const elapsedMs = Date.now() - startedAt; const blocker = ok ? null : { code: !sourceMirrorReady ? "sentinel-source-mirror-sync-failed" : !publishReady ? "sentinel-image-gitops-publish-not-ready" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready", reason: !sourceMirrorReady ? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish" : !publishReady ? "Tekton publish did not finish and runtime observation has not proven the selected source/image/GitOps state yet" : record(runtimeSecretsApply).ok === false ? "one or more YAML-declared runtime Secrets were not synced from sourceRef" : "one or more publicExposure, Argo or runtime observation checks did not pass", }; const publishPipelineRun = applyOnly ? sentinelPipelineRunName(state, options.rerun) : record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: true, specRef: SPEC_REF, source: state.sourceHead, image: state.image, pipelineRun: publishPipelineRun, gitops: { path: stringAt(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), manifestObjects: state.manifests.length, manifestSha256: state.manifestSha256, }, argo: { namespace: stringAt(state.cicd, "argo.namespace"), projectName: stringAt(state.cicd, "argo.projectName"), applicationName: stringAt(state.cicd, "argo.applicationName"), }, validation: { scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"), controlPlaneWaitMaxSeconds: cicdWaitWarningSeconds, quickVerifyMode: applyOnly ? "not-applicable" : "manual-validate", automaticSecondPath: false, }, manifests: { objects: manifestObjectSummary(state.manifests), sha256: state.manifestSha256, }, sourceMirrorSync, publish, flush, runtimeSecretsApply, publicExposureApply, argoApply, observed, targetValidation, elapsedMs, warnings: Array.from(new Set([ ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel control-plane confirm-wait", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"), ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds), ...asyncGitMirrorFlushWarnings(flush, cicdWaitWarningSeconds), ...publishSatisfiedByObservedWarnings(publish, flush, observedReady), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ...sentinelObservedWarnings(observed), ...(publishWaitBudgetExhausted ? [`sentinel publish consumed the configured ${waitBudgetSeconds}s confirm-wait budget; skipped runtime Secret, public exposure, Argo apply and observed wait to avoid blind over-budget waiting. Use the reported status/log drill-down after the PipelineRun advances.`] : []), ...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds), ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []), ...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []), ])), blocker, recoveryNext: controlPlaneRecoveryNext(state, ok, publish, flush, observed), next: controlPlaneNext(state, options.action), valuesRedacted: true, }; return result; } function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane" | "publish", action: string, timeoutSeconds: number): RenderedCliResult { const args = domain === "image" ? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)] : domain === "control-plane" ? ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)] : ["web-probe", "sentinel", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`); const command = domain === "publish" ? `web-probe sentinel ${action}` : `web-probe sentinel ${domain} ${action}`; const result = { ok: true, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "async-job", mutation: true, reason: "confirmed sentinel publish/build can exceed the short interactive window; use job status for bounded progress.", job, next: { status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, wait: ["bun", "scripts/cli.ts", ...args].join(" "), }, valuesRedacted: true, }; return rendered(true, command, renderAsyncJobResult(result)); } function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record { const args = ["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"]; const job = startJob( `hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_git_mirror_flush`, ["bun", "scripts/cli.ts", ...args], `Flush HWLAB ${state.spec.lane} git mirror after web-probe sentinel ${state.sentinelId} GitOps publish for node ${state.spec.nodeId}`, ); return { ok: true, mode: "async-job", job, next: { status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, wait: ["bun", "scripts/cli.ts", ...args].join(" "), gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, }, valuesRedacted: true, }; } function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): string[] { const item = record(flush); if (item.mode !== "async-job") return []; const next = record(item.next); return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`]; } function sentinelSkippedControlStep(reason: string): Record { return { ok: false, skipped: true, reason, valuesRedacted: true }; } function sentinelSkippedObservedStatus(reason: string): SentinelObservedStatus { const skipped = { ok: false, skipped: true, reason, valuesRedacted: true }; return { sourceMirror: skipped, registry: skipped, gitMirror: { skipped: true, reason, valuesRedacted: true }, gitops: skipped, argo: skipped, runtime: skipped, cadence: skipped, wait: { polls: 0, elapsedMs: 0, timeoutMs: 0, ready: false, includeGitMirror: false, reason, valuesRedacted: true, }, }; } function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus { const registry = probeImageRegistry(state, timeoutSeconds); const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds); const effectiveExpectation = { gitopsRevision: expectation?.gitopsRevision ?? nonEmptyString(gitops.revision), runtimeImage: expectation?.runtimeImage ?? nonEmptyString(gitops.image) ?? expectedRuntimeImageFromRegistry(state, registry), }; return { sourceMirror: probeSourceMirror(state, timeoutSeconds), registry, gitMirror: includeGitMirror ? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds) : { ok: true, skipped: true, reason: "deferred-to-async-flush", valuesRedacted: true }, gitops, argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision), runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage), cadence: probeCadenceCronJob(state, timeoutSeconds), }; } function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus { const startedAt = Date.now(); const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror); let polls = 1; while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) { runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror); polls += 1; } return { ...observed, wait: { polls, elapsedMs: Date.now() - startedAt, timeoutMs, ready: sentinelObservedReady(observed), includeGitMirror, valuesRedacted: true, }, }; } function sentinelObservedReady(value: Record | SentinelObservedStatus): boolean { const observed = record(value); const gitMirror = record(observed.gitMirror); const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true; return record(observed.sourceMirror).ok === true && record(record(observed.registry).probe).present === true && gitMirrorReady && record(observed.gitops).ok === true && record(observed.argo).ok === true && record(observed.runtime).ok === true && record(observed.cadence).ok === true; } function sentinelObservedWarnings(value: Record | SentinelObservedStatus | null): string[] { const observed = record(value); const argo = record(observed.argo); const cadence = record(observed.cadence); return mergeWarnings(argo.warning, cadence.warning); } function sentinelObservedStatusDiagnosis(state: SentinelCicdState, value: unknown, pipelineRun: string): Record | null { const observed = record(value); if (sentinelObservedReady(observed)) return null; const sourceMirror = record(observed.sourceMirror); const registryProbe = record(record(observed.registry).probe); const gitMirror = record(observed.gitMirror); const gitops = record(observed.gitops); const argo = record(observed.argo); const runtimeDeployment = record(record(record(observed.runtime).probe).deployment); const cadence = record(observed.cadence); const sourceReady = sourceMirror.ok === true; const registryPresent = registryProbe.present === true; const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true; const gitopsReady = gitops.ok === true; const argoReady = argo.ok === true; const runtimeReady = record(observed.runtime).ok === true; const cadenceReady = cadence.ok === true; const code = sourceReady && !registryPresent ? "sentinel-publish-half-state-registry-missing" : registryPresent && !gitopsReady ? "sentinel-gitops-manifest-not-updated" : gitopsReady && (!argoReady || !runtimeReady) ? "sentinel-runtime-not-aligned" : !gitMirrorReady ? "sentinel-git-mirror-not-in-sync" : !cadenceReady ? "sentinel-cadence-not-ready" : "sentinel-control-plane-observed-not-ready"; const reason = code === "sentinel-publish-half-state-registry-missing" ? "source mirror contains the selected commit, but the expected registry tag is missing; retry publish-current so Tekton builds/pushes the image and advances GitOps/runtime" : code === "sentinel-gitops-manifest-not-updated" ? "registry contains the selected image, but GitOps manifest is not yet updated to a digest-pinned runtime image" : code === "sentinel-runtime-not-aligned" ? "GitOps manifest is present, but Argo or runtime objects have not converged to the selected image yet" : code === "sentinel-git-mirror-not-in-sync" ? "runtime is aligned, but git-mirror status is not in sync; run the controlled git-mirror sync/status path before treating the control plane as fully healthy" : code === "sentinel-cadence-not-ready" ? "runtime is aligned, but cadence CronJob validation did not pass" : "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass"; const next = publishCurrentNext(state); const controlNext = controlPlaneNext(state, "trigger-current"); const shouldRetryPublish = code === "sentinel-publish-half-state-registry-missing" || code === "sentinel-gitops-manifest-not-updated"; return { code, phase: code === "sentinel-publish-half-state-registry-missing" ? "source-ready-registry-missing" : code === "sentinel-gitops-manifest-not-updated" ? "registry-ready-gitops-pending" : code === "sentinel-runtime-not-aligned" ? "gitops-ready-runtime-pending" : code === "sentinel-git-mirror-not-in-sync" ? "runtime-ready-git-mirror-pending" : code === "sentinel-cadence-not-ready" ? "runtime-ready-cadence-pending" : "observed-not-ready", reason, sourceMirror: sourceReady ? `ready ${short(record(sourceMirror.probe).commit ?? record(sourceMirror.probe).expectedCommit)}` : `blocked ${short(record(sourceMirror.probe).commit)}/${short(record(sourceMirror.probe).expectedCommit)}`, registry: registryPresent ? `present ${short(registryProbe.digest)}` : "missing -", gitMirror: gitMirrorReady ? "ready" : "pending", gitops: gitopsReady ? `ready ${short(gitops.image)}` : `pending ${short(gitops.image)}`, argo: `${argo.syncStatus ?? "-"} ${argo.healthStatus ?? "-"} ${short(argo.revision)}/${short(argo.expectedRevision)}`, runtime: `ready=${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"} image=${short(runtimeDeployment.image)} expected=${short(runtimeDeployment.expectedImage)}`, pipelineRun, warning: code === "sentinel-publish-half-state-registry-missing" ? `source mirror already exposes ${short(state.sourceHead.commit)} but registry tag ${state.image.tag} is missing; rerun ${next.publishCurrent} and then recheck ${next.controlPlaneStatus}.` : code === "sentinel-git-mirror-not-in-sync" ? `runtime is aligned but git-mirror is not in sync; run ${next.gitMirrorSync} and then recheck ${next.controlPlaneStatus}.` : null, blocker: { code, reason, valuesRedacted: true }, recoveryNext: { reason, pipelineRun, digestRef: registryPresent ? expectedRuntimeImageFromRegistry(state, record(observed.registry)) : null, gitopsCommit: gitops.revision ?? null, publishCurrent: shouldRetryPublish ? next.publishCurrent : null, nextStatus: next.controlPlaneStatus, gitMirrorStatus: next.gitMirrorStatus, gitMirrorSync: !gitMirrorReady ? next.gitMirrorSync : null, gitMirrorFlush: null, controlPlaneApply: shouldRetryPublish || code === "sentinel-runtime-not-aligned" ? controlNext.apply : null, valuesRedacted: true, }, valuesRedacted: true, }; } function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record { const result = probeSourceMirrorCache(state.cicd, state.controlPlaneNode, timeoutSeconds, state.sourceHead.commit); return { ...result, result: compactCommand(result.result) }; } function probeSourceMirrorCache(cicd: Record, controlPlaneNode: Record, timeoutSeconds: number, expectedCommit: string | null): { ok: boolean; probe: Record; result: CommandResult } { const namespace = stringAt(cicd, "builder.namespace"); const repository = stringAt(cicd, "source.repository"); const branch = stringAt(cicd, "source.branch"); const stageRef = expectedCommit === null ? "" : sentinelSourceSnapshotRef(cicd, expectedCommit); const script = [ "set +e", `repo_path=${shellQuote(`/cache/${repository}.git`)}`, `branch=${shellQuote(branch)}`, `expected=${shellQuote(expectedCommit ?? "")}`, `stage_ref=${shellQuote(stageRef)}`, "commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)", "rc=$?", "object_rc=1", "expected_object_rc=1", "stage_object_rc=1", "contains_rc=1", "if [ \"$rc\" -eq 0 ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1", " object_rc=$?", "fi", "if [ -n \"$expected\" ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$expected^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1", " expected_object_rc=$?", "fi", "if [ -n \"$stage_ref\" ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse --verify \\\"$stage_ref^{commit}\\\" >/dev/null 2>&1\" >/dev/null 2>&1", " stage_object_rc=$?", "fi", "if [ \"$rc\" -eq 0 ] && [ \"$expected_object_rc\" -eq 0 ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" merge-base --is-ancestor \\\"$expected\\\" \\\"$commit\\\" 2>/dev/null\" >/dev/null 2>&1", " contains_rc=$?", "fi", "node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$stage_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$stage_ref\" \"$repo_path\" \"$branch\" <<'NODE'", "const [rc, objectRc, expectedObjectRc, stageObjectRc, containsRc, commit, expected, stageRef, repoPath, branch] = process.argv.slice(2);", "const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');", "const objectPresent = present && Number(objectRc) === 0;", "const expectedObjectPresent = !expected || Number(expectedObjectRc) === 0;", "const stageObjectPresent = !stageRef || Number(stageObjectRc) === 0;", "const containsExpected = !expected || commit === expected || Number(containsRc) === 0;", "const relation = !expected ? 'unconstrained' : commit === expected ? 'equal' : containsExpected ? 'mirror-ahead' : expectedObjectPresent ? 'diverged-or-behind' : 'expected-object-missing';", "console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror-cache', present, objectPresent, expectedObjectPresent, stageObjectPresent, containsExpected, relation, commit: present ? commit : null, sourceCommit: expected || (present ? commit : null), mirrorCommit: present ? commit : null, expectedCommit: expected || null, stageRef: stageRef || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout) ?? {}, result }; } function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record { const namespace = stringAt(state.cicd, "argo.namespace"); const applicationName = stringAt(state.cicd, "argo.applicationName"); const jsonpath = "{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.sync.revision}{\"\\n\"}"; const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", `jsonpath=${jsonpath}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const [syncStatusRaw, healthStatusRaw, revisionRaw] = result.stdout.trim().split(/\r?\n/u); const syncStatus = nonEmptyString(syncStatusRaw); const healthStatus = nonEmptyString(healthStatusRaw); const revision = nonEmptyString(revisionRaw); const revisionMatches = expectedRevision === null || revision === expectedRevision; const healthy = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy"; const diagnostics = result.exitCode === 0 && !healthy ? probeArgoApplicationDiagnostics(state, timeoutSeconds, namespace, applicationName) : null; return { ok: healthy, present: result.exitCode === 0, syncStatus, healthStatus, revision, expectedRevision, revisionMatches, revisionPolicy: "non-blocking-branch-head-drift", warning: healthy && !revisionMatches ? "Argo app is Synced/Healthy but status.sync.revision differs from current GitOps branch HEAD; in multi-sentinel GitOps this can happen when another sentinel path advances the branch. Runtime image/manifest checks remain authoritative for rollout readiness." : null, diagnostics, result: compactCommand(result), }; } function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSeconds: number, namespace: string, applicationName: string): Record { const route = stringAt(state.controlPlaneNode, "kubeRoute"); const appResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsed = parseJsonObject(appResult.stdout); const status = record(parsed?.status); const resourcesRaw = Array.isArray(status.resources) ? status.resources : []; const resources = resourcesRaw .filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) .map((item) => { const health = record(item.health); return { kind: item.kind ?? null, namespace: item.namespace ?? null, name: item.name ?? null, status: item.status ?? null, healthStatus: health.status ?? null, healthMessage: short(health.message), }; }); const problemResources = resources .filter((item) => { const sync = nonEmptyString(item.status); const health = nonEmptyString(item.healthStatus); return (sync !== null && sync !== "Synced") || (health !== null && health !== "Healthy"); }) .slice(0, 12); const conditions = (Array.isArray(status.conditions) ? status.conditions : []) .filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) .slice(-8) .map((item) => ({ type: item.type ?? null, message: short(item.message), lastTransitionTime: item.lastTransitionTime ?? null, })); const operationState = record(status.operationState); const eventResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "events", "--field-selector", `involvedObject.name=${applicationName}`, "--sort-by=.lastTimestamp", "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const eventsJson = parseJsonObject(eventResult.stdout); const events = (Array.isArray(eventsJson?.items) ? eventsJson.items : []) .filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) .slice(-8) .map((item) => ({ type: item.type ?? null, reason: item.reason ?? null, message: short(item.message), count: item.count ?? null, lastTimestamp: item.lastTimestamp ?? item.eventTime ?? null, })); return { ok: appResult.exitCode === 0, resourceCount: resources.length, problemResourceCount: problemResources.length, problemResources, conditions, operationState: { phase: operationState.phase ?? null, message: short(operationState.message), startedAt: operationState.startedAt ?? null, finishedAt: operationState.finishedAt ?? null, }, events, result: compactCommand(appResult), eventsResult: compactCommand(eventResult), drillDown: { application: `trans ${route} kubectl -n ${namespace} get application ${applicationName} -o json`, events: `trans ${route} kubectl -n ${namespace} get events --field-selector involvedObject.name=${applicationName} --sort-by=.lastTimestamp`, }, }; } function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record { const namespace = stringAt(state.cicd, "builder.namespace"); const repository = stringAt(state.controlPlaneTarget, "source.repository"); const branch = stringAt(state.cicd, "argo.targetRevision"); const manifestPath = `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`; const repoPath = `/cache/${repository}.git`; const inner = [ "set -eu", `git --git-dir=${shellQuote(repoPath)} rev-parse ${shellQuote(`refs/heads/${branch}`)}`, `git --git-dir=${shellQuote(repoPath)} show ${shellQuote(`refs/heads/${branch}:${manifestPath}`)}`, ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "exec", "deploy/git-mirror-http", "--", "sh", "-lc", inner], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const [revisionLine, ...manifestLines] = result.stdout.split(/\r?\n/u); const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null; const manifest = manifestLines.join("\n"); const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]); const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`); const compact = compactCommand(result); return { ok: result.exitCode === 0 && revision !== null && imageMatchesRepository, revision, branch, manifestPath, image, imageMatchesRepository, result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` }, valuesRedacted: true, }; } function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, expectedImage: string | null): Record { const namespace = stringAt(state.runtime, "namespace"); const deploymentName = stringAt(state.runtime, "deploymentName"); const serviceName = stringAt(state.runtime, "serviceName"); const pvcName = stringAt(state.runtime, "pvcName"); const configMapName = `${deploymentName}-config`; const serviceAccountName = stringAt(state.runtime, "serviceAccountName"); const script = [ "set +e", `namespace=${shellQuote(namespace)}`, `deployment=${shellQuote(deploymentName)}`, `service=${shellQuote(serviceName)}`, `pvc=${shellQuote(pvcName)}`, `configmap=${shellQuote(configMapName)}`, `serviceaccount=${shellQuote(serviceAccountName)}`, "tmp=$(mktemp -d)", "kubectl -n \"$namespace\" get deploy \"$deployment\" -o json >\"$tmp/deploy.json\" 2>/dev/null; echo $? >\"$tmp/deploy.rc\"", "kubectl -n \"$namespace\" get svc \"$service\" -o json >\"$tmp/svc.json\" 2>/dev/null; echo $? >\"$tmp/svc.rc\"", "kubectl -n \"$namespace\" get pvc \"$pvc\" -o json >\"$tmp/pvc.json\" 2>/dev/null; echo $? >\"$tmp/pvc.rc\"", "kubectl -n \"$namespace\" get cm \"$configmap\" -o json >\"$tmp/cm.json\" 2>/dev/null; echo $? >\"$tmp/cm.rc\"", "kubectl -n \"$namespace\" get sa \"$serviceaccount\" -o json >\"$tmp/sa.json\" 2>/dev/null; echo $? >\"$tmp/sa.rc\"", `expected_image=${shellQuote(expectedImage ?? "")}`, "node - \"$tmp\" \"$expected_image\" <<'NODE'", "const fs = require('node:fs');", "const dir = process.argv[2];", "const expectedImage = process.argv[3] || null;", "function rc(name){ return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); }", "function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }", "const dep = json('deploy');", "const deploymentPresent = rc('deploy') === 0;", "const desired = Number(dep?.spec?.replicas ?? 0);", "const ready = Number(dep?.status?.readyReplicas ?? 0);", "const updated = Number(dep?.status?.updatedReplicas ?? 0);", "const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;", "const imageMatches = expectedImage === null || image === expectedImage;", "const payload = {", " deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },", " service: { present: rc('svc') === 0 },", " pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },", " configMap: { present: rc('cm') === 0 },", " serviceAccount: { present: rc('sa') === 0 },", " valuesRedacted: true", "};", "payload.ok = payload.deployment.present && imageMatches && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;", "console.log(JSON.stringify(payload));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probe = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && probe?.ok === true, probe, result: compactCommand(result) }; } function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record { const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null; if (expected === null) { return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true }; } const metadata = record(expected.metadata); const spec = record(expected.spec); const namespace = stringAt(metadata, "namespace"); const name = stringAt(metadata, "name"); const expectedSchedule = stringAt(spec, "schedule"); const script = [ "set +e", `namespace=${shellQuote(namespace)}`, `cronjob=${shellQuote(name)}`, `sentinel=${shellQuote(state.sentinelId)}`, `expected_schedule=${shellQuote(expectedSchedule)}`, "tmp=$(mktemp -d)", "kubectl -n \"$namespace\" get cronjob \"$cronjob\" -o json >\"$tmp/cronjob.json\" 2>/dev/null; echo $? >\"$tmp/cronjob.rc\"", "kubectl -n \"$namespace\" get jobs -l \"unidesk.ai/web-probe-sentinel-id=$sentinel,app.kubernetes.io/component=cadence-scheduler\" -o json >\"$tmp/jobs.json\" 2>/dev/null; echo $? >\"$tmp/jobs.rc\"", "node - \"$tmp\" \"$namespace\" \"$cronjob\" \"$expected_schedule\" <<'NODE'", "const fs = require('node:fs');", "const [dir, namespace, cronJobName, expectedSchedule] = process.argv.slice(2);", "function rc(name){ try { return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); } catch { return 1; } }", "function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }", "const cron = json('cronjob');", "const jobs = Array.isArray(json('jobs')?.items) ? json('jobs').items : [];", "const present = rc('cronjob') === 0 && !!cron;", "const schedule = cron?.spec?.schedule || null;", "const scheduleMatches = present && schedule === expectedSchedule;", "const suspended = cron?.spec?.suspend === true;", "const active = Array.isArray(cron?.status?.active) ? cron.status.active.length : 0;", "const sortedJobs = jobs.slice().sort((a,b)=>String(b?.metadata?.creationTimestamp||'').localeCompare(String(a?.metadata?.creationTimestamp||''))).slice(0,8);", "let code = null;", "if (!present) code = 'sentinel-cadence-cronjob-missing';", "else if (!scheduleMatches) code = 'sentinel-cadence-cronjob-schedule-mismatch';", "else if (suspended) code = 'sentinel-cadence-cronjob-suspended';", "const latestJob = sortedJobs[0] || null;", "console.log(JSON.stringify({ ok: code === null, code, present, namespace, name: cronJobName, schedule, expectedSchedule, scheduleMatches, suspended, lastScheduleTime: cron?.status?.lastScheduleTime || null, lastSuccessfulTime: cron?.status?.lastSuccessfulTime || null, active, jobCount: jobs.length, latestJobs: sortedJobs.map((job)=>({ name: job?.metadata?.name || null, createdAt: job?.metadata?.creationTimestamp || null, active: Number(job?.status?.active || 0), succeeded: Number(job?.status?.succeeded || 0), failed: Number(job?.status?.failed || 0), completionTime: job?.status?.completionTime || null, valuesRedacted:true })), latestJobName: latestJob?.metadata?.name || null, valuesRedacted: true }));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probe = parseJsonObject(result.stdout); const ok = result.exitCode === 0 && probe?.ok === true; emitWebProbeSentinelSpan({ node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, namespace, runtime: state.runtime, cicd: state.cicd, }, "web_probe_sentinel.cadence.cronjob_observed", { cronJobName: name, namespace, schedule: expectedSchedule, status: ok ? "ok" : text(probe?.code ?? "unknown"), jobName: probe?.latestJobName ?? null, failureKind: probe?.code ?? null, valuesRedacted: true, }, ok); return { ok, probe, result: compactCommand(result), warning: ok ? null : `cadence CronJob is not ready: ${text(probe?.code ?? "probe-failed")}`, valuesRedacted: true, }; } function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record): string | null { const digest = nonEmptyString(record(record(registry).probe).digest); if (digest === null) return null; return `${state.image.repository}@${digest}`; } function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult { const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`; const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName); const namespace = stringAt(state.cicd, "builder.namespace"); sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); if (created.exitCode !== 0) { sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane }); return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "source-mirror"); } const startedAt = Date.now(); const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000; let slowWarningSent = false; let polls = 0; let lastProbe: Record = {}; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probe = parseJsonObject(probeCapture.stdout) ?? {}; lastProbe = { ...probe, capture: compactCommand(probeCapture) }; const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "remote-job", status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running", jobName, polls, elapsedMs: Date.now() - startedAt, pod: probe.pod ?? null, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, }); if (probe.succeeded === true) { const ok = payload.ok === true; return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); } if (probe.failed === true) { return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); } if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { slowWarningSent = true; sentinelProgressEvent("sentinel.source-mirror.warning", { warning: `source mirror sync exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane }); } runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); } function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult { return { ok: false, phase, jobName: "-", payload: { ok: false, status: phase, reason, valuesRedacted: true }, valuesRedacted: true, }; } function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record { const namespace = stringAt(state.cicd, "builder.namespace"); const labels = { "app.kubernetes.io/name": "web-probe-sentinel-source-mirror", "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": state.spec.nodeId, "unidesk.ai/lane": state.spec.lane, }; return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace, labels }, spec: { backoffLimit: 0, activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"), ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"), template: { metadata: { labels }, spec: { restartPolicy: "Never", volumes: [ sentinelGitMirrorCacheVolume(state), { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, ], containers: [{ name: "sync", image: state.image.baseImage, imagePullPolicy: "IfNotPresent", command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)], volumeMounts: [ { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }], }, }, }, }; } function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string { return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit); } function sentinelSourceMirrorSyncShellFromConfig(cicd: Record, controlPlaneNode: Record, jobName: string, selectedCommit: string | null): string { return [ "set -eu", `job_name=${shellQuote(jobName)}`, `source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`, `source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`, `source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`, `source_commit=${shellQuote(selectedCommit ?? "")}`, `source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`, "started_ms=$(node -e 'console.log(Date.now())')", "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }", "trap emit_failed EXIT", ...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode), "repo=\"/cache/${source_repository}.git\"", "mkdir -p \"$(dirname \"$repo\")\"", "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", " git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", "else", " rm -rf \"$repo\"", " git init --bare \"$repo\"", " git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", "fi", "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", "git --git-dir=\"$repo\" config http.uploadpack true", "git --git-dir=\"$repo\" config http.receivepack true", "fetch_ok=0", "for attempt in 1 2 3; do", " if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi", " code=$?", " printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2", " sleep $((attempt * 5))", "done", "test \"$fetch_ok\" = 1", "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")", "if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi", "git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"", "test \"$mirror_commit\" = \"$source_commit\"", "stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"", "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"", "git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"", "git --git-dir=\"$repo\" update-server-info", "finished_ms=$(node -e 'console.log(Date.now())')", "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'", "const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);", "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));", "NODE", "trap - EXIT", ].join("\n"); } function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] { return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode); } function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record): string[] { const proxy = record(valueAtPath(controlPlaneNode, "egressProxy")); const serviceName = nonEmptyString(proxy.serviceName); const namespace = nonEmptyString(proxy.namespace); const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null; const hostRouteProxyUrl = nonEmptyString(proxy.proxyUrl); const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : ""; let proxyHost: string | null = null; let proxyPort: number | null = null; let proxyUrl: string | null = null; if (serviceName !== null && namespace !== null && port !== null) { proxyHost = `${serviceName}.${namespace}.svc.cluster.local`; proxyPort = port; proxyUrl = `http://${proxyHost}:${proxyPort}`; } else if (hostRouteProxyUrl !== null) { try { const parsed = new URL(hostRouteProxyUrl); const parsedPort = Number.parseInt(parsed.port || "80", 10); if ((parsed.protocol === "http:" || parsed.protocol === "https:") && parsed.hostname.length > 0 && Number.isInteger(parsedPort)) { proxyHost = parsed.hostname; proxyPort = parsedPort; proxyUrl = hostRouteProxyUrl; } } catch { proxyHost = null; proxyPort = null; proxyUrl = null; } } const useProxy = proxyHost !== null && proxyPort !== null && proxyUrl !== null; if (!useProxy) { return [ "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", "printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2", "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy", "export NO_PROXY='*'", "export no_proxy='*'", "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", "#!/bin/sh", "exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"", "SH_PROXY", "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", "unset GIT_SSH_COMMAND", ]; } const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; return [ "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", `printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`, `export HTTP_PROXY=${shellQuote(proxyUrl)}`, `export HTTPS_PROXY=${shellQuote(proxyUrl)}`, `export ALL_PROXY=${shellQuote(proxyUrl)}`, `export http_proxy=${shellQuote(proxyUrl)}`, `export https_proxy=${shellQuote(proxyUrl)}`, `export all_proxy=${shellQuote(proxyUrl)}`, `export NO_PROXY=${shellQuote(noProxy)}`, `export no_proxy=${shellQuote(noProxy)}`, "cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'", "#!/usr/bin/env node", "const net = require('node:net');", "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", "const targetPort = Number.parseInt(targetPortRaw || '', 10);", "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {", " console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');", " process.exit(64);", "}", "let settled = false;", "let tunnelEstablished = false;", "function finish(code, message) {", " if (settled) return;", " settled = true;", " if (message) console.error('sentinel source-mirror proxy-connect: ' + message);", " process.exit(code);", "}", "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", "let buffer = Buffer.alloc(0);", "socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", "function onData(chunk) {", " buffer = Buffer.concat([buffer, chunk]);", " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", " if (headerEnd === -1 && buffer.length < 8192) return;", " if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }", " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", " const statusLine = head.split('\\r\\n', 1)[0] || '';", " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {", " const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);", " socket.destroy();", " finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);", " return;", " }", " socket.off('data', onData);", " socket.setTimeout(0);", " tunnelEstablished = true;", " const rest = buffer.slice(headerEnd + 4);", " if (rest.length) process.stdout.write(rest);", " process.stdin.on('error', () => {});", " process.stdout.on('error', () => {});", " process.stdin.pipe(socket);", " socket.pipe(process.stdout);", "}", "socket.on('data', onData);", "NODE_PROXY", "chmod 0700 /tmp/sentinel-github-proxy-connect.cjs", "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", "#!/bin/sh", `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`, "SH_PROXY", "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", "unset GIT_SSH_COMMAND", ]; } function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult { const pipelineRunName = sentinelPipelineRunName(state, rerun); const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops); const namespace = stringAt(state.cicd, "builder.namespace"); sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createTektonPipelineRunScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); if (created.exitCode !== 0) { sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "failed", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-pipelinerun", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish"); } sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "succeeded", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); const startedAt = Date.now(); const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000; let slowWarningSent = false; let polls = 0; let lastProbe: Record = {}; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeTektonPipelineRunScript(namespace, pipelineRunName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const probe = parseJsonObject(probeCapture.stdout) ?? {}; lastProbe = { ...probe, capture: compactCommand(probeCapture) }; const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); sentinelProgressEvent("sentinel.publish.progress", { phase: "remote-pipelinerun", status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running", pipelineRun: pipelineRunName, publishGitops, polls, elapsedMs: Date.now() - startedAt, taskRun: probe.taskRun ?? null, pod: probe.pod ?? null, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, }); if (probe.succeeded === true) { const ok = payload.ok === true; return withSentinelRemoteJobDiagnostics(state, { ok, phase: "pipelinerun-succeeded", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); } if (probe.failed === true) { return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-failed", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); } if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { slowWarningSent = true; sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish PipelineRun exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, pipelineRun: pipelineRunName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane }); } runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-timeout", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); } function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRunName: string, publishGitops: boolean): Record { const namespace = stringAt(state.cicd, "builder.namespace"); const buildkitImage = requireSentinelBuildkitImage(state); const proxyEnv = sentinelImageBuildProxyEnv(state); const labels = { "app.kubernetes.io/name": "web-probe-sentinel-publish", "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "unidesk.ai/spec-ref": "PJ2026-01060508", "unidesk.ai/node": state.spec.nodeId, "unidesk.ai/lane": state.spec.lane, "unidesk.ai/ci-system": "tekton", }; return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRunName, namespace, labels, annotations: { "unidesk.ai/source-commit": state.sourceHead.commit ?? "", "unidesk.ai/source-authority": state.sourceHead.sourceAuthority, "unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "", "unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"), "unidesk.ai/publish-gitops": publishGitops ? "true" : "false", }, }, spec: { timeouts: { pipeline: `${numberAt(state.cicd, "builder.activeDeadlineSeconds")}s` }, taskRunTemplate: { podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, }, }, pipelineSpec: { tasks: [{ name: "publish", taskSpec: { volumes: [ sentinelGitMirrorCacheVolume(state), { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, { name: "workspace", emptyDir: { sizeLimit: "8Gi" } }, sentinelBuildkitStateVolume(state), { name: "tmp", emptyDir: {} }, ], steps: [ { name: "source", image: state.image.baseImage, imagePullPolicy: "IfNotPresent", env: proxyEnv, script: tektonShellScript(sentinelPublishSourceShell(state, pipelineRunName)), volumeMounts: [ { name: "workspace", mountPath: "/workspace" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }, { name: "prepare-buildkit-state", image: state.image.baseImage, imagePullPolicy: "IfNotPresent", script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"), securityContext: { runAsUser: 0, runAsGroup: 0 }, volumeMounts: [ { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, ], }, { name: "image-build", image: buildkitImage, imagePullPolicy: "IfNotPresent", env: [ ...proxyEnv, { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" }, ], script: tektonShellScript(sentinelPublishImageBuildShell(state, pipelineRunName)), securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 }, volumeMounts: [ { name: "workspace", mountPath: "/workspace" }, { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, { name: "tmp", mountPath: "/tmp" }, ], }, { name: "publish", image: state.image.baseImage, imagePullPolicy: "IfNotPresent", env: proxyEnv, script: tektonShellScript(sentinelPublishShell(state, pipelineRunName, publishGitops)), volumeMounts: [ { name: "workspace", mountPath: "/workspace" }, { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }, ], }, }], }, }, }; } function tektonShellScript(body: string): string { return `#!/bin/sh\n${body}`; } function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record { return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget); } function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record): Record { const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath")); if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } }; return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } }; } function sentinelBuildkitStateVolume(state: SentinelCicdState): Record { const buildkitState = monitorWebBuildkitStatePlan(state.cicd); const mode = stringAt(buildkitState, "mode"); if (mode === "hostPath") { return { name: "buildkit-state", hostPath: { path: stringAt(buildkitState, "path"), type: stringAt(buildkitState, "type"), }, }; } if (mode === "persistentVolumeClaim") { return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } }; } if (mode === "emptyDir") { return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } }; } throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); } function requireSentinelBuildkitImage(state: SentinelCicdState): string { const image = state.spec.buildkit?.sidecarImage; if (typeof image !== "string" || image.length === 0) { throw new Error(`config/hwlab-node-lanes.yaml ${state.spec.nodeId}/${state.spec.lane} buildkit.sidecarImage is required for pure k8s web-probe sentinel image publish`); } return image; } function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> { const proxy = state.spec.networkProfile.imageBuildProxy; const noProxy = proxy.noProxy.join(","); return [ { name: "HTTP_PROXY", value: proxy.http }, { name: "http_proxy", value: proxy.http }, { name: "HTTPS_PROXY", value: proxy.https }, { name: "https_proxy", value: proxy.https }, { name: "ALL_PROXY", value: proxy.all }, { name: "all_proxy", value: proxy.all }, { name: "NO_PROXY", value: noProxy }, { name: "no_proxy", value: noProxy }, ]; } function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string { const monitorWeb = record(state.image.monitorWeb); const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => { if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths"); return item; })), "utf8").toString("base64"); const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64"); const envReuseMode = stringAt(monitorWeb, "envReuseMode"); const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath"); return [ "set -eu", `job_name=${shellQuote(jobName)}`, `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`, `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`, `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`, `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, `source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`, `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`, `dockerfile_b64=${shellQuote(dockerfileB64)}`, `env_reuse_mode=${shellQuote(envReuseMode)}`, `env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`, "meta_dir=/workspace/meta", "event_log=/workspace/publish-events.log", "worktree=/workspace/source", "rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json", "mkdir -p \"$meta_dir\"", ": > \"$event_log\"", "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", "started_ms=$(now_ms)", "write_meta started_ms \"$started_ms\"", "write_meta source_commit \"$source_commit\"", "write_meta source_stage_ref \"$source_stage_ref\"", "test -n \"$source_commit\"", "test -n \"$source_stage_ref\"", "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", "git init \"$worktree\"", "cd \"$worktree\"", "git remote add origin \"$source_git_url\"", "git config core.sparseCheckout true", "git config remote.origin.promisor true", "git config remote.origin.partialclonefilter blob:none", "CHECKOUT_PATHS_B64=\"$checkout_paths_b64\" node <<'NODE'", "const fs = require('node:fs');", "const paths = JSON.parse(Buffer.from(process.env.CHECKOUT_PATHS_B64 || '', 'base64').toString('utf8'));", "fs.mkdirSync('.git/info', { recursive: true });", "fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');", "NODE", "source_fetch_started_ms=$(now_ms)", "write_meta source_fetch_started_ms \"$source_fetch_started_ms\"", "emit_stage source-fetch running \"$source_fetch_started_ms\"", "git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"", "git checkout --detach \"$source_commit\"", "mirror_commit=$(git rev-parse HEAD)", "test \"$mirror_commit\" = \"$source_commit\"", "write_meta mirror_commit \"$mirror_commit\"", "source_fetch_finished_ms=$(now_ms)", "write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"", "emit_stage source-fetch succeeded \"$source_fetch_started_ms\"", "env_reuse_node_deps_present=false", "env_reuse_node_deps_entries=0", "if [ -d \"$env_reuse_node_deps_path\" ]; then env_reuse_node_deps_present=true; env_reuse_node_deps_entries=$(find \"$env_reuse_node_deps_path\" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' '); fi", "env_reuse_linked_node_deps=0", "rm -rf node_modules", "if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi", "write_meta env_reuse_mode \"$env_reuse_mode\"", "write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"", "write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"", "write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"", "write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"", "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'", "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", "NODE", "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"", "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", "NODE", "monitor_web_verify_started_ms=$(now_ms)", "write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"", "emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"", "if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi", "cat /tmp/web-probe-sentinel-monitor-web-verify.log", "monitor_web_verify_finished_ms=$(now_ms)", "write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"", "emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"", "mkdir -p .unidesk-sentinel-bin", "cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'", "#!/bin/sh", "exec bun /app/scripts/ssh-cli.ts \"$@\"", "SH_TRANS", "chmod 0755 .unidesk-sentinel-bin/trans", "DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'", "const fs = require('node:fs');", "fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));", "NODE", "cat > .dockerignore <<'EOF_DOCKERIGNORE'", ".git", ".git/**", ".state", ".state/**", "logs", "logs/**", "node_modules", "node_modules/**", "**/node_modules", "**/node_modules/**", "**/dist", "**/dist/**", "**/target", "**/target/**", "**/coverage", "**/coverage/**", "npm-debug.log*", ".env", ".env.*", "EOF_DOCKERIGNORE", "context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')", "write_meta context_ignore_entries \"$context_ignore_entries\"", "chmod -R a+rwX /workspace", "trap - EXIT", ].join("\n"); } function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string { const monitorWeb = record(state.image.monitorWeb); const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode"); const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode"); const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource"); const imageBuildProxy = state.spec.networkProfile.imageBuildProxy; const imageBuildNoProxy = imageBuildProxy.noProxy.join(","); return [ "set -eu", `job_name=${shellQuote(jobName)}`, `image_ref=${shellQuote(state.image.ref)}`, `image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`, `image_build_package_mode=${shellQuote(imageBuildPackageMode)}`, `image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`, `image_build_proxy_source=${shellQuote(imageBuildProxySource)}`, `image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`, `image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`, `image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`, `image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`, "meta_dir=/workspace/meta", "event_log=/workspace/publish-events.log", "build_log=/workspace/image-build.log", "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", "cd /workspace/source", "image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi", "image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi", "image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi", "image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi", "write_meta image_build_builder \"$image_build_builder\"", "write_meta image_build_package_mode \"$image_build_package_mode\"", "write_meta image_build_network_mode \"$image_build_network_mode\"", "write_meta image_build_proxy_source \"$image_build_proxy_source\"", "write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"", "write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"", "write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"", "write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"", "image_build_started_ms=$(now_ms)", "write_meta image_build_started_ms \"$image_build_started_ms\"", "emit_stage image-build running \"$image_build_started_ms\"", "if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi", "cat \"$build_log\"", "image_build_finished_ms=$(now_ms)", "write_meta image_build_finished_ms \"$image_build_finished_ms\"", "metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)", "digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)", "test -n \"$digest\"", "repo_no_tag=${image_ref%:*}", "digest_ref=\"$repo_no_tag@$digest\"", "write_meta digest \"$digest\"", "write_meta digest_ref \"$digest_ref\"", "emit_stage image-build succeeded \"$image_build_started_ms\"", "trap - EXIT", ].join("\n"); } function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string { const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : []; const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({ path: file.path, contentBase64: Buffer.from(file.content, "utf8").toString("base64"), }))), "utf8").toString("base64"); return [ "set -eu", `job_name=${shellQuote(jobName)}`, `image_ref=${shellQuote(state.image.ref)}`, `image_repository=${shellQuote(state.image.repository)}`, `gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`, `gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`, `files_b64=${shellQuote(filesB64)}`, "meta_dir=/workspace/meta", "event_log=/workspace/publish-events.log", "build_log=/workspace/image-build.log", "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", "read_meta() { cat \"$meta_dir/$1\"; }", "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }", "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", "if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi", "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", "started_ms=$(read_meta started_ms)", "source_commit=$(read_meta source_commit)", "source_stage_ref=$(read_meta source_stage_ref)", "mirror_commit=$(read_meta mirror_commit)", "source_fetch_started_ms=$(read_meta source_fetch_started_ms)", "source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)", "monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)", "monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)", "image_build_started_ms=$(read_meta image_build_started_ms)", "image_build_finished_ms=$(read_meta image_build_finished_ms)", "digest_ref=$(read_meta digest_ref)", "env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')", "env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')", "env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)", "env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)", "env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)", "image_build_builder=$(read_meta image_build_builder)", "image_build_package_mode=$(read_meta image_build_package_mode)", "image_build_network_mode=$(read_meta image_build_network_mode)", "image_build_proxy_source=$(read_meta image_build_proxy_source)", "image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)", "image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)", "image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)", "image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)", "context_ignore_entries=$(read_meta context_ignore_entries)", "image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)", "image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)", "image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')", "gitops_commit=''", "changed=false", "file_count=0", "gitops_started_ms=$(now_ms)", "if [ \"$files_b64\" != \"W10=\" ]; then", " emit_stage gitops running \"$gitops_started_ms\"", " gitops_cache=\"/cache/${gitops_repository}.git\"", " gitops_worktree=\"/tmp/$job_name/gitops\"", " git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"", " cd \"$gitops_worktree\"", " git fetch origin \"$gitops_branch\" || true", " if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"; else git checkout --orphan \"$gitops_branch\"; git rm -rf . >/dev/null 2>&1 || true; fi", " FILES_B64=\"$files_b64\" IMAGE_REF=\"$image_ref\" DIGEST_REF=\"$digest_ref\" node <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));", "for (const file of files) {", " const target = path.resolve(process.cwd(), file.path);", " if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);", " fs.mkdirSync(path.dirname(target), { recursive: true });", " const text = Buffer.from(file.contentBase64, 'base64').toString('utf8').split(process.env.IMAGE_REF).join(process.env.DIGEST_REF);", " fs.writeFileSync(target, text);", "}", "console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));", "NODE", " git add .", " file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')", " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi", " git push origin \"HEAD:refs/heads/$gitops_branch\"", " gitops_commit=$(git rev-parse HEAD)", " emit_stage gitops succeeded \"$gitops_started_ms\"", "else", " emit_stage gitops skipped \"$gitops_started_ms\"", "fi", "gitops_finished_ms=$(now_ms)", "finished_ms=$(now_ms)", "node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'", "const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);", "const elapsed = (start, finish) => Number(finish) - Number(start);", "const cacheHits = Number(imageBuildCacheHits || 0);", "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", "NODE", "trap - EXIT", ].join("\n"); } function sentinelGitopsFiles(state: SentinelCicdState): readonly { path: string; content: string }[] { const runtimeManifests = state.manifests.filter((item) => item.kind !== "Application"); return [{ path: `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`, content: `${runtimeManifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`, }]; } function applySentinelArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record { const app = state.manifests.find((item) => item.kind === "Application"); if (app === undefined) return { ok: false, reason: "application-manifest-missing", valuesRedacted: true }; const yaml = `${Bun.YAML.stringify(app).trim()}\n`; const namespace = stringAt(state.cicd, "argo.namespace"); const applicationName = stringAt(state.cicd, "argo.applicationName"); const script = [ "set -eu", "tmp=$(mktemp)", `cat >"$tmp" <<'YAML'\n${yaml}YAML`, "kubectl apply -f \"$tmp\"", `kubectl -n ${shellQuote(namespace)} annotate application ${shellQuote(applicationName)} argocd.argoproj.io/refresh=hard --overwrite`, ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); return { ok: result.exitCode === 0, result: compactCommand(result), valuesRedacted: true }; } function createK8sJobScript(namespace: string, manifest: Record): string { const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`; return [ "set -eu", `kubectl -n ${shellQuote(namespace)} delete job ${shellQuote(stringAt(manifest, "metadata.name"))} --ignore-not-found=true >/dev/null 2>&1 || true`, "tmp=$(mktemp)", `cat >"$tmp" <<'YAML'\n${yaml}YAML`, "kubectl apply -f \"$tmp\"", ].join("\n"); } function createTektonPipelineRunScript(namespace: string, manifest: Record): string { const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`; const pipelineRunName = stringAt(manifest, "metadata.name"); return [ "set -eu", `pipeline_run=${shellQuote(pipelineRunName)}`, `namespace=${shellQuote(namespace)}`, "tmp=$(mktemp)", `cat >"$tmp" <<'YAML'\n${yaml}YAML`, "if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then", " printf '%s\\n' \"sentinel publish PipelineRun already exists; reusing $pipeline_run\"", " exit 0", "fi", "if ! kubectl create -f \"$tmp\"; then", " if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then", " printf '%s\\n' \"sentinel publish PipelineRun appeared concurrently; reusing $pipeline_run\"", " exit 0", " fi", " exit 1", "fi", ].join("\n"); } function probeK8sJobScript(namespace: string, jobName: string): string { return [ "set +e", `namespace=${shellQuote(namespace)}`, `job=${shellQuote(jobName)}`, "succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)", "failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)", "active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)", "pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)", "pod_phase=''", "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi", "logs_tail=''", "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi", "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);", "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", "NODE", ].join("\n"); } function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string): string { return [ "set +e", `namespace=${shellQuote(namespace)}`, `pipeline_run=${shellQuote(pipelineRunName)}`, "condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)", "condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)", "condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)", "if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi", "if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi", "if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi", "task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", "pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", "pod_phase=''", "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi", "logs_tail=''", "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi", "node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", "const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);", "const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');", "const active = conditionStatus === 'Unknown' || (!conditionStatus && (podPhase === 'Pending' || podPhase === 'Running'));", "console.log(JSON.stringify({ succeeded: conditionStatus === 'True', failed: conditionStatus === 'False', active, conditionStatus: conditionStatus || null, conditionReason: conditionReason || null, conditionMessage: message || null, taskRun: taskRun || null, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", "NODE", ].join("\n"); } function sentinelPayloadFromLogs(logsTail: string): Record { const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); for (let index = lines.length - 1; index >= 0; index -= 1) { const line = lines[index]; if (!line.startsWith("{") || !line.endsWith("}")) continue; const parsed = parseJsonObject(line); if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed; } return {}; } function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult { return { ...result, diagnostics: sentinelRemoteJobDiagnostics(state, result, domain), valuesRedacted: true }; } function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): Record { const namespace = stringAt(state.cicd, "builder.namespace"); const probe = record(result.probe); const logsTail = typeof probe.logsTail === "string" ? probe.logsTail : ""; const events = sentinelStageEventsFromLogs(logsTail, domain); const envReuse = sentinelEnvReuseFromLogs(logsTail); const completedStages = sentinelCompletedStages(events, record(result.payload)); const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs); const currentPhase = sentinelCurrentRemotePhase(result, events, domain); const isPipelineRun = result.resourceKind === "PipelineRun"; const commands = { cliStatus: domain === "publish" ? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}` : `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, logs: result.jobName === "-" ? "-" : isPipelineRun ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs -l tekton.dev/pipelineRun=${result.jobName} --all-containers=true --tail=120` : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`, describe: result.jobName === "-" ? "-" : isPipelineRun ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe pipelinerun/${result.jobName}` : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`, gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`, gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`, controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, valuesRedacted: true, }; return { domain, resourceKind: result.resourceKind ?? "Job", pipelineRun: isPipelineRun ? result.jobName : null, taskRun: probe.taskRun ?? null, currentPhase, completedStages, stageTimings, envReuse, pod: probe.pod ?? null, podPhase: probe.podPhase ?? null, active: probe.active ?? null, conditionStatus: probe.conditionStatus ?? null, conditionReason: probe.conditionReason ?? null, recentLogSummary: sentinelRecentLogSummary(logsTail), commands, valuesRedacted: true, }; } function sentinelEnvReuseFromLogs(logsTail: string): Record | null { const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); for (let index = lines.length - 1; index >= 0; index -= 1) { const parsed = parseJsonObject(lines[index]); if (parsed !== null && parsed.event === "sentinel-publish-env-reuse") return { ...parsed, valuesRedacted: true }; } return null; } function sentinelStageEventsFromLogs(logsTail: string, domain: "source-mirror" | "publish"): Record[] { const expectedEvent = domain === "publish" ? "sentinel-publish-stage" : "sentinel-source-mirror-stage"; return logsTail .split(/\r?\n/u) .map((line) => parseJsonObject(line.trim())) .filter((item): item is Record => item !== null && item.event === expectedEvent); } function sentinelCompletedStages(events: readonly Record[], payload: Record): string[] { const completed = events .filter((event) => event.status === "succeeded" || event.status === "skipped") .map((event) => `${text(event.stage)}:${text(event.status)}`); const payloadStages = Array.isArray(payload.completedStages) ? payload.completedStages.map(text) : []; return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-"); } function sentinelStageTimingSummary(events: readonly Record[], payload: Record, fallbackTotalMs: unknown): Record { const payloadTimings = record(payload.stageTimings); const eventElapsed = (stage: string): number | null => { const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed")); return event === undefined ? null : finiteNumberOrNull(event.elapsedMs); }; const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch"); const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify"); const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build"); const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops"); const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null); const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0); const totalMs = finiteNumberOrNull(payloadTimings.totalMs) ?? finiteNumberOrNull(payload.elapsedMs) ?? finiteNumberOrNull(fallbackTotalMs) ?? summedTotalMs; const result = { sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs, totalMs, valuesRedacted: true, }; return Object.values(result).some((item) => item !== null && item !== true) ? result : {}; } function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record[], domain: "source-mirror" | "publish"): string { if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed"; if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase; const reversed = [...events].reverse(); const failed = reversed.find((event) => event.status === "failed"); if (failed !== undefined) return text(failed.stage); const running = reversed.find((event) => event.status === "running"); if (running !== undefined) return text(running.stage); const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage))); const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"]; const next = order.find((stage) => !completed.has(stage)); return next ?? result.phase; } function sentinelRecentLogSummary(logsTail: string): string { const lines = logsTail .split(/\r?\n/u) .map((line) => line.trim()) .filter((line) => line.length > 0 && !line.startsWith("{")) .slice(-5) .map((line) => short(line)); return lines.length === 0 ? "-" : lines.join(" | "); } function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] { const remote = record(job); if (remote.phase !== "job-timeout" && remote.phase !== "pipelinerun-timeout") return []; const diagnostics = record(remote.diagnostics); const commands = record(diagnostics.commands); return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`]; } function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] { if (!observedReady) return []; const warnings: string[] = []; const publishRecord = record(publish); if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) { warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`); } const flushRecord = record(flush); if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) { warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout."); } return warnings; } function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number { return numberAt(state.cicd, "confirmWait.maxSeconds"); } function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] { const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; if (elapsedMs === null || elapsedMs <= budgetMs) return []; return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`]; } function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] { const sync = record(sourceMirrorSync); if (sync.ok === true || state.sourceHead.ok !== true) return []; return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`]; } function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, probe: unknown): Record { return { ok: true, phase: "already-present", jobName: null, probe: record(probe), payload: { ok: true, status: "already-present", sourceCommit: state.sourceHead.commit, mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit, stageRef: state.sourceHead.stageRef, sourceAuthority: state.sourceHead.sourceAuthority, valuesRedacted: true, }, polls: 0, elapsedMs: 0, valuesRedacted: true, }; } function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] { if (applyOnly) return []; const next = sentinelP5Next(state); return [`targetValidation quick verify is outside the CI/CD validation gate; run ${next.quickVerify} only as separate post-deploy evidence if needed.`]; } export function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] { const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; if (elapsedMs === null || elapsedMs <= budgetMs) return []; return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, only Code Agent multi-round business failures should block acceptance.`]; } export function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] { const warnings: string[] = []; for (const item of items) { const values = Array.isArray(item) ? item : [item]; for (const value of values) { if (value === undefined || value === null || value === "") continue; const warning = text(value).trim(); if (warning.length > 0 && warning !== "-" && !warnings.includes(warning)) warnings.push(warning); } } return warnings; } export function withWarnings(payload: Record, warnings: readonly unknown[]): Record { const merged = mergeWarnings(payload.warnings, warnings); return merged.length === 0 ? payload : { ...payload, warnings: merged, valuesRedacted: true }; } function sentinelProgressEvent(event: string, payload: Record): void { console.error(JSON.stringify({ event, at: new Date().toISOString(), ...payload, valuesRedacted: true })); } function installSentinelPublishInterruptHandler(state: SentinelCicdState, context: Record): () => void { let handled = false; const handler = (signal: string) => { if (handled) return; handled = true; const exitCode = signal === "SIGINT" ? 130 : 143; const next = publishCurrentNext(state); sentinelProgressEvent("sentinel.publish.interrupted", { signal, exitCode, phase: context.phase ?? "unknown", pipelineRun: context.pipelineRun ?? sentinelPipelineRunName(state, false), sourceCommit: context.sourceCommit ?? state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, recoveryNext: { status: next.controlPlaneStatus, retry: next.publishCurrent, gitMirrorStatus: next.gitMirrorStatus, gitMirrorSync: next.gitMirrorSync, gitMirrorFlush: next.gitMirrorFlush, valuesRedacted: true, }, valuesRedacted: true, }); process.exit(exitCode); }; process.once("SIGTERM", handler); process.once("SIGINT", handler); return () => { process.off("SIGTERM", handler); process.off("SIGINT", handler); }; } function confirmBlocked(action: string, state: SentinelCicdState): Record { return { code: "sentinel-cicd-confirm-requires-tekton-pipelinerun", action, reason: "Confirmed publish uses a Tekton PipelineRun for image build and GitOps writeback; dry-run refuses to report a deployment mutation before that CI run is submitted.", sourceGitMirrorReadUrl: stringAt(state.cicd, "source.gitMirrorReadUrl"), requiredNextImplementation: [ "clone source from source.gitMirrorReadUrl at selected commit", "build and push digest-pinned image through Tekton PipelineRun on the selected node", "publish manifests to the HWLAB gitops branch/path through git-mirror", "flush/recheck git-mirror and let Argo reconcile the Application", ], valuesRedacted: true, }; } function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelControlPlaneAction): Record { const node = state.spec.nodeId; const lane = state.spec.lane; const suffix = sentinelCliSuffix(state); return { plan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${node} --lane ${lane}${suffix} --dry-run`, status: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`, image: `bun scripts/cli.ts web-probe sentinel image status --node ${node} --lane ${lane}${suffix}`, triggerCurrent: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${node} --lane ${lane}${suffix} --dry-run`, apply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${node} --lane ${lane}${suffix} --confirm --wait`, validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`, quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`, gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`, gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm`, gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`, issue: "https://github.com/pikasTech/unidesk/issues/1285", currentAction: action, }; } function controlPlaneRecoveryNext(state: SentinelCicdState, ok: boolean, publish: unknown, flush: unknown, observed: unknown): Record | null { const payload = record(record(publish).payload); if (ok || nonEmptyString(payload.digestRef) === null) return null; const next = controlPlaneNext(state, "apply"); const flushRecord = record(flush); const observedRecord = record(observed); return { reason: "publish produced an image digest, but GitOps/git-mirror/Argo/runtime alignment is not complete yet", pipelineRun: record(publish).jobName ?? null, digestRef: payload.digestRef, gitopsCommit: payload.gitopsCommit ?? null, flushMode: flushRecord.mode ?? null, observedReady: sentinelObservedReady(observedRecord), publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, nextStatus: next.status, gitMirrorStatus: next.gitMirrorStatus, gitMirrorSync: next.gitMirrorSync, gitMirrorFlush: next.gitMirrorFlush, controlPlaneApply: next.apply, valuesRedacted: true, }; } function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record { const sourcesByPurpose = new Map>(); for (const source of arrayAt(state.secrets, "sources").map(record)) { const purpose = stringAtNullable(source, "purpose"); if (purpose !== null) sourcesByPurpose.set(purpose, source); } const desired: Array<{ namespace: string; name: string; data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> }> = []; const missing: Record[] = []; const skipped: Record[] = []; for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) { const name = stringAt(runtimeSecret, "name"); const namespace = stringAt(runtimeSecret, "namespace"); const data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> = []; for (const item of arrayAt(runtimeSecret, "data").map(record)) { const sourcePurpose = stringAt(item, "sourcePurpose"); const targetKey = stringAt(item, "targetKey"); if (sourcePurpose === "frp-token") { skipped.push({ name, namespace, targetKey, sourcePurpose, reason: "managed-by-publicExposure-frpc", valuesRedacted: true }); continue; } const source = sourcesByPurpose.get(sourcePurpose); if (source === undefined) { missing.push({ name, namespace, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true }); continue; } const sourceRef = stringAt(source, "sourceRef"); const sourceKey = stringAt(source, "sourceKey"); const material = readSentinelSecretSourceValue(source); if (!material.ok) { missing.push({ name, namespace, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true }); continue; } const value = stringAt(material, "value"); data.push({ key: targetKey, value, sourcePurpose, sourceRef, sourceKey, fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`, }); } if (data.length > 0) desired.push({ namespace, name, data }); } if (missing.length > 0) return { ok: false, phase: "local-source", missing, skipped, valuesRedacted: true }; if (desired.length === 0) return { ok: true, phase: "skipped-no-runtime-secrets", secretCount: 0, keyCount: 0, skippedKeyCount: skipped.length, skipped, valuesRedacted: true }; const manifests = desired.map((secret) => ({ apiVersion: "v1", kind: "Secret", metadata: { name: secret.name, namespace: secret.namespace, labels: { "app.kubernetes.io/managed-by": "unidesk", "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", "unidesk.ai/node": state.spec.nodeId, "unidesk.ai/lane": state.spec.lane, "unidesk.ai/web-probe-sentinel-id": state.sentinelId, }, }, type: "Opaque", data: Object.fromEntries(secret.data.map((item) => [item.key, Buffer.from(item.value, "utf8").toString("base64")])), })); const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; const summary = desired.map((secret) => ({ name: secret.name, namespace: secret.namespace, keys: secret.data.map((item) => item.key).sort(), sources: secret.data.map((item) => ({ key: item.key, sourcePurpose: item.sourcePurpose, sourceRef: item.sourceRef, sourceKey: item.sourceKey, fingerprint: item.fingerprint, valuesRedacted: true })), valuesRedacted: true, })); const summaryB64 = Buffer.from(JSON.stringify(summary), "utf8").toString("base64"); const script = [ "set +e", `summary_b64=${shellQuote(summaryB64)}`, "tmp=$(mktemp -d)", "trap 'rm -rf \"$tmp\"' EXIT", "manifest=\"$tmp/runtime-secrets.yaml\"", "cat >\"$manifest\"", "kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-runtime-secrets -f \"$manifest\" >/tmp/web-probe-sentinel-runtime-secrets.out 2>/tmp/web-probe-sentinel-runtime-secrets.err", "apply_rc=$?", "python3 - \"$summary_b64\" \"$apply_rc\" <<'PY'", "import base64, json, subprocess, sys", "summary = json.loads(base64.b64decode(sys.argv[1]).decode('utf-8'))", "apply_rc = int(sys.argv[2])", "items = []", "ok = apply_rc == 0", "for secret in summary:", " proc = subprocess.run(['kubectl', '-n', secret['namespace'], 'get', 'secret', secret['name'], '-o', 'json'], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)", " data = json.loads(proc.stdout).get('data', {}) if proc.returncode == 0 and proc.stdout else {}", " keys = {key: key in data for key in secret['keys']}", " present = proc.returncode == 0 and all(keys.values())", " ok = ok and present", " items.append({'name': secret['name'], 'namespace': secret['namespace'], 'present': present, 'keys': keys, 'sources': secret['sources'], 'valuesRedacted': True})", "print(json.dumps({'ok': ok, 'applyExitCode': apply_rc, 'secretCount': len(summary), 'keyCount': sum(len(item['keys']) for item in summary), 'items': items, 'valuesRedacted': True}, ensure_ascii=False))", "PY", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: manifestYaml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, result: compactCommand(result), valuesRedacted: true }; } function readSentinelSecretSourceValue(source: Record): Record { const sourceRef = stringAt(source, "sourceRef"); const sourceKey = stringAt(source, "sourceKey"); const sourceLine = numberAtNullable(source, "sourceLine"); const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, error: "secret-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const textValue = readFileSync(sourcePath, "utf8"); const value = sourceLine === null ? parseEnvFile(textValue)[sourceKey] : textValue.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, ""); if (value === undefined || value.length === 0) return { ok: false, error: sourceLine === null ? "secret-source-key-missing" : "secret-source-line-missing", sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const format = stringAtNullable(source, "format"); if (format === null) return { ok: true, value, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; if (format === "web-account-json") { const username = readSentinelWebAccountUsername(source); if (!username.ok) return { ok: false, error: username.error, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; return { ok: true, value: JSON.stringify({ username: username.value, password: value }), sourceRef, sourceKey, sourceLine, format, sourcePath: displayPath(sourcePath), valuesRedacted: true }; } return { ok: false, error: "unsupported-secret-source-format", sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true }; } function readSentinelWebAccountUsername(source: Record): { ok: true; value: string } | { ok: false; error: string } { const username = stringAtNullable(source, "username"); if (username !== null) return { ok: true, value: username }; const sourceRef = stringAtNullable(source, "usernameSourceRef"); const sourceLine = numberAtNullable(source, "usernameSourceLine"); if (sourceRef === null || sourceLine === null) return { ok: false, error: "web-account-json-username-missing" }; const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, error: "web-account-json-username-source-missing" }; const value = readFileSync(sourcePath, "utf8").split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? ""; return value.length === 0 ? { ok: false, error: "web-account-json-username-line-missing" } : { ok: true, value }; } function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { const material = readSentinelFrpcMaterial(state); if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true }; const secret = applySentinelFrpcSecret(state, stringAt(material, "frpcToml"), timeoutSeconds); const caddy = applySentinelCaddyBlock(state, timeoutSeconds); return { ok: secret.ok === true && caddy.ok === true, hostname: stringAt(state.publicExposure, "hostname"), publicBaseUrl: stringAt(state.publicExposure, "publicBaseUrl"), material: { ok: true, sourceRef: material.sourceRef, sourcePath: material.sourcePath, fingerprint: material.fingerprint, valuesRedacted: true, }, secret, caddy, valuesRedacted: true, }; } function readSentinelFrpcMaterial(state: SentinelCicdState): Record { const sourceRef = stringAt(state.publicExposure, "frpc.tokenSourceRef"); const sourceKey = stringAt(state.publicExposure, "frpc.tokenSourceKey"); const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, error: "frp-token-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const token = values[sourceKey]; if (token === undefined || token.length === 0) return { ok: false, error: "frp-token-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const proxy = record(valueAtPath(state.publicExposure, "frpc.httpProxy")); const frpcToml = [ `serverAddr = "${tomlEscape(stringAt(state.publicExposure, "frpc.serverAddr"))}"`, `serverPort = ${numberAt(state.publicExposure, "frpc.serverPort")}`, "loginFailExit = true", `auth.token = "${tomlEscape(token)}"`, "", "[[proxies]]", `name = "${tomlEscape(stringAt(proxy, "name"))}"`, 'type = "tcp"', `localIP = "${tomlEscape(stringAt(proxy, "localIP"))}"`, `localPort = ${numberAt(proxy, "localPort")}`, `remotePort = ${numberAt(proxy, "remotePort")}`, "", ].join("\n"); return { ok: true, sourceRef, sourceKey, sourcePath: displayPath(sourcePath), frpcToml, fingerprint: `sha256:${createHash("sha256").update(`${token}\n${frpcToml}`).digest("hex").slice(0, 16)}`, valuesRedacted: true, }; } function applySentinelFrpcSecret(state: SentinelCicdState, frpcToml: string, timeoutSeconds: number): Record { const namespace = stringAt(state.runtime, "namespace"); const secretName = stringAt(state.publicExposure, "frpc.secretName"); const secretKey = stringAt(state.publicExposure, "frpc.secretKey"); const script = [ "set +e", `namespace=${shellQuote(namespace)}`, `secret=${shellQuote(secretName)}`, `key=${shellQuote(secretKey)}`, "tmp=$(mktemp -d)", "trap 'rm -rf \"$tmp\"' EXIT", "cat >\"$tmp/frpc.toml\"", "kubectl -n \"$namespace\" create secret generic \"$secret\" --from-file=\"$key=$tmp/frpc.toml\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-public-exposure -f - >/tmp/web-probe-sentinel-frpc-secret.out 2>/tmp/web-probe-sentinel-frpc-secret.err", "rc=$?", "present=no", "bytes=0", "if kubectl -n \"$namespace\" get secret \"$secret\" -o jsonpath=\"{.data.$key}\" >/tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null; then present=yes; bytes=$(base64 -d /dev/null | wc -c | tr -d ' '); fi", "node - \"$rc\" \"$namespace\" \"$secret\" \"$key\" \"$present\" \"$bytes\" <<'NODE'", "const [rc, namespace, secret, key, present, bytes] = process.argv.slice(2);", "console.log(JSON.stringify({ok:Number(rc)===0&&present==='yes',namespace,secret,key,present:present==='yes',bytes:Number(bytes||0),applyExitCode:Number(rc),valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: frpcToml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: number): Record { const hostname = stringAt(state.publicExposure, "hostname"); const owner = stringAt(state.publicExposure, "caddy.managedBlockOwner"); const configPath = stringAt(state.publicExposure, "caddy.configPath"); const serviceName = stringAt(state.publicExposure, "caddy.serviceName"); const responseHeaderTimeoutSeconds = numberAt(state.publicExposure, "caddy.responseHeaderTimeoutSeconds"); const remotePort = numberAt(state.publicExposure, "frpc.httpProxy.remotePort"); const routePrefix = normalizeRoutePrefix(stringAtNullable(state.publicExposure, "routePrefix")); const rootOrder = stringAtNullable(state.publicExposure, "caddy.rootOrder") ?? "normal"; const monitorRoot = record(state.publicExposure.monitorRoot); const cleanupOwner = monitorRoot.enabled === false ? stringAtNullable(monitorRoot, "caddyManagedBlockOwner") : null; const proxyLines = [ `reverse_proxy 127.0.0.1:${remotePort} {`, " transport http {", ` response_header_timeout ${responseHeaderTimeoutSeconds}s`, " }", "}", ]; const block = [ routePrefix === "/" ? "handle {" : `handle_path ${routePrefix}* {`, ...proxyLines.map((line) => ` ${line}`), "}", "", ].join("\n"); const blockB64 = Buffer.from(block, "utf8").toString("base64"); const script = [ "set +e", `hostname=${shellQuote(hostname)}`, `owner=${shellQuote(owner)}`, `config_path=${shellQuote(configPath)}`, `service=${shellQuote(serviceName)}`, `route_prefix=${shellQuote(routePrefix)}`, `root_order=${shellQuote(rootOrder)}`, `cleanup_owner=${shellQuote(cleanupOwner ?? "")}`, `block_b64=${shellQuote(blockB64)}`, "marker=\"unidesk managed $owner\"", "tmp=$(mktemp -d)", "trap 'rm -rf \"$tmp\"' EXIT", "block=\"$tmp/block\"", "next=\"$tmp/Caddyfile\"", "printf '%s' \"$block_b64\" | base64 -d >\"$block\"", "if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi", "python3 - \"$next\" \"$block\" \"$marker\" \"$hostname\" \"$route_prefix\" \"$root_order\" \"$cleanup_owner\" <<'PY' >/tmp/web-probe-sentinel-caddy-python.out 2>/tmp/web-probe-sentinel-caddy-python.err", "import pathlib, re, sys", "config = pathlib.Path(sys.argv[1])", "block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')", "marker = sys.argv[3]", "hostname = sys.argv[4]", "route_prefix = sys.argv[5]", "root_order = sys.argv[6]", "cleanup_owner = sys.argv[7]", "text = config.read_text(encoding='utf-8') if config.exists() else ''", "begin = f'# BEGIN {marker}'", "end = f'# END {marker}'", "def managed_pattern(marker_text):", " return re.compile(rf'(?ms)^[ \\t]*# BEGIN {re.escape(marker_text)}\\n.*?^[ \\t]*# END {re.escape(marker_text)}\\n*')", "pattern = managed_pattern(marker)", "def collect_nested_managed(segment):", " preserved = []", " lines = segment.splitlines()", " index = 0", " while index < len(lines):", " stripped = lines[index].strip()", " if stripped.startswith('# BEGIN ') and stripped != begin:", " owner_text = stripped[len('# BEGIN '):]", " end_line = '# END ' + owner_text", " block_lines = [lines[index]]", " index += 1", " while index < len(lines):", " block_lines.append(lines[index])", " if lines[index].strip() == end_line:", " break", " index += 1", " preserved.append('\\n'.join(block_lines).rstrip() + '\\n')", " index += 1", " return preserved", "preserved_blocks = []", "for match in pattern.finditer(text):", " preserved_blocks.extend(collect_nested_managed(match.group(0)))", "text = pattern.sub('', text)", "if cleanup_owner:", " cleanup_marker = f'unidesk managed {cleanup_owner}'", " if cleanup_marker != marker:", " text = managed_pattern(cleanup_marker).sub('', text)", "def site_span(src, host):", " match = re.search(rf'(?m)^([ \\t]*){re.escape(host)}[ \\t]*\\{{[ \\t]*\\n', src)", " if not match:", " return None", " depth = 1", " index = match.end()", " while index < len(src):", " char = src[index]", " if char == '{':", " depth += 1", " elif char == '}':", " depth -= 1", " if depth == 0:", " end_index = index + 1", " if end_index < len(src) and src[end_index] == '\\n':", " end_index += 1", " return match.start(), end_index, index, match.end()", " index += 1", " raise ValueError(f'unclosed Caddy site block for {host}')", "handler = '\\n'.join((' ' + line) if line else '' for line in block.rstrip().splitlines())", "managed = f' {begin}\\n{handler}\\n {end}\\n'", "def fallback_insert_pos(site, relative_open, close_rel):", " for match in re.finditer(r'(?m)^[ \\t]*handle[ \\t]*\\{[ \\t]*\\n', site[relative_open:close_rel]):", " pos = relative_open + match.start()", " prev_end = pos - 1", " if prev_end >= 0 and site[prev_end] == '\\n':", " prev_end -= 1", " if prev_end >= 0:", " prev_start = site.rfind('\\n', 0, prev_end + 1) + 1", " if site[prev_start:prev_end + 1].strip().startswith('# BEGIN '):", " return prev_start", " return pos", " return relative_open", "def append_before_close(site, close_rel, addition):", " prefix = site[:close_rel]", " suffix = site[close_rel:]", " if prefix and not prefix.endswith('\\n'):", " prefix += '\\n'", " return prefix + addition + suffix", "span = site_span(text, hostname)", "if span is None:", " body = ''.join(preserved_blocks) + managed", " text = text.rstrip() + '\\n\\n' + f'{hostname} {{\\n{body}}}\\n'", "else:", " start, stop, close_index, open_end = span", " site = text[start:stop]", " relative_open = open_end - start", " close_rel = close_index - start", " additions = ''.join(preserved_blocks) + managed", " if route_prefix == '/' and root_order == 'active':", " replacement = site[:relative_open] + additions + site[relative_open:]", " elif route_prefix == '/':", " replacement = append_before_close(site, close_rel, additions)", " else:", " insert_at = fallback_insert_pos(site, relative_open, close_rel)", " replacement = site[:insert_at] + additions + site[insert_at:]", " text = text[:start] + replacement + text[stop:]", "config.write_text(text, encoding='utf-8')", "PY", "python_rc=$?", "validate_rc=1", "reload_rc=", "if [ \"$python_rc\" = 0 ]; then sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/web-probe-sentinel-caddy-validate.out 2>/tmp/web-probe-sentinel-caddy-validate.err; validate_rc=$?; fi", "if [ \"$validate_rc\" = 0 ]; then sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/web-probe-sentinel-caddy-install.out 2>/tmp/web-probe-sentinel-caddy-install.err && (sudo systemctl reload \"$service\" >/tmp/web-probe-sentinel-caddy-reload.out 2>/tmp/web-probe-sentinel-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/web-probe-sentinel-caddy-reload.out 2>>/tmp/web-probe-sentinel-caddy-reload.err); reload_rc=$?; fi", "probe_rc=1", "probe_status=", "if [ \"$reload_rc\" = 0 ]; then", " probe_path=\"$route_prefix\"", " if [ \"$probe_path\" = \"/\" ]; then probe_url=\"https://$hostname/\"; else probe_url=\"https://$hostname$probe_path/\"; fi", " probe_status=$(curl -k -sS -o /tmp/web-probe-sentinel-caddy-probe.out -w '%{http_code}' --max-time 10 --resolve \"$hostname:443:127.0.0.1\" \"$probe_url\" 2>/tmp/web-probe-sentinel-caddy-probe.err)", " probe_rc=$?", "fi", "after_present=no", "grep -Fq \"# BEGIN $marker\" \"$config_path\" 2>/dev/null && after_present=yes", "active=$(systemctl is-active \"$service\" 2>/dev/null || true)", "err=$(cat /tmp/web-probe-sentinel-caddy-python.err /tmp/web-probe-sentinel-caddy-validate.err /tmp/web-probe-sentinel-caddy-install.err /tmp/web-probe-sentinel-caddy-reload.err /tmp/web-probe-sentinel-caddy-probe.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)", "python3 - \"$python_rc\" \"$validate_rc\" \"$reload_rc\" \"$probe_rc\" \"$probe_status\" \"$after_present\" \"$active\" \"$hostname\" \"$config_path\" \"$err\" <<'PY'", "import json, sys", "python_rc, validate_rc, reload_rc, probe_rc, probe_status, after_present, active, hostname, config_path, error_preview = sys.argv[1:11]", "def num(value):", " if value == '':", " return None", " try:", " return int(value)", " except ValueError:", " return None", "http_status = num(probe_status)", "probe_ok = num(probe_rc) == 0 and http_status is not None and 200 <= http_status < 400", "payload = {", " 'ok': num(python_rc) == 0 and num(validate_rc) == 0 and num(reload_rc) == 0 and probe_ok and after_present == 'yes',", " 'hostname': hostname,", " 'configPath': config_path,", " 'pythonExitCode': num(python_rc),", " 'validateExitCode': num(validate_rc),", " 'reloadExitCode': num(reload_rc),", " 'routeProbeExitCode': num(probe_rc),", " 'routeProbeHttpStatus': http_status,", " 'routeProbeOk': probe_ok,", " 'afterBlockPresent': after_present == 'yes',", " 'active': active,", " 'errorPreview': error_preview,", " 'valuesRedacted': True,", "}", "print(json.dumps(payload, ensure_ascii=False))", "PY", ].join("\n"); const result = runCommand(["trans", stringAt(state.publicExposure, "caddy.route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, rootOrder, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } export function secretSourcePaths(sourceRef: string): string[] { if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef); const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef)); return [...new Set(paths)]; } function ownerFileSourcePaths(sourceRef: string): string[] { if (sourceRef.includes("..") || sourceRef.includes("\0")) return []; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot]; return [...new Set(roots.map((root) => join(root, sourceRef)))]; } export function parseEnvFile(textValue: string): Record { const values: Record = {}; for (const rawLine of textValue.split(/\r?\n/u)) { const line = rawLine.trim(); if (line.length === 0 || line.startsWith("#")) continue; const index = line.indexOf("="); if (index <= 0) continue; const key = line.slice(0, index).trim(); let value = line.slice(index + 1).trim(); if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1); values[key] = value; } return values; } export function stringAtNullable(value: unknown, path: string): string | null { const found = valueAtPath(value, path); return typeof found === "string" && found.length > 0 ? found : null; } export function numberAtNullable(value: unknown, path: string): number | null { const found = valueAtPath(value, path); return typeof found === "number" && Number.isFinite(found) ? found : null; } function booleanAtNullable(value: unknown, path: string): boolean | null { const found = valueAtPath(value, path); return typeof found === "boolean" ? found : null; } export function displayPath(pathValue: string): string { if (pathValue.startsWith(`${repoRoot}/`)) return pathValue.slice(repoRoot.length + 1); const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); if (index >= 0) { const mainRoot = repoRoot.slice(0, index); if (pathValue.startsWith(`${mainRoot}/`)) return pathValue.slice(mainRoot.length + 1); } return pathValue; } function sentinelPipelineRunName(state: SentinelCicdState, rerun = false): string { const commit = state.sourceHead.commit ?? "source"; const base = `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`; if (!rerun) return base; const suffix = `-r${Date.now().toString(36)}`; return `${base.slice(0, Math.max(1, 63 - suffix.length)).replace(/-+$/u, "")}${suffix}`; } export function sentinelCliSuffix(state: SentinelCicdState): string { return ` --sentinel ${state.sentinelId}`; } export function safeJobSegment(value: string): string { return value.replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "").slice(0, 48) || "sentinel"; } function safeKubernetesSegment(value: string, maxLength: number): string { const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, ""); return (normalized || "sentinel").slice(0, Math.max(1, maxLength)).replace(/-+$/u, "") || "sentinel"; } function renderPublishResult(publish: Record): string { const payload = record(publish.payload); const diagnostics = record(publish.diagnostics); const diagnosticEnvReuse = record(diagnostics.envReuse); const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse; const imageBuild = record(payload.imageBuild); const imageBuildProxy = record(imageBuild.proxy); const payloadStageTimings = record(payload.stageTimings); const diagnosticStageTimings = record(diagnostics.stageTimings); const timings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings; const commands = record(diagnostics.commands); const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none"; const runColumn = diagnostics.resourceKind === "PipelineRun" || publish.resourceKind === "PipelineRun" ? "PIPELINERUN" : "JOB"; const lines = [ "PUBLISH", table(["OK", "PHASE", runColumn, "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[ publish.ok, publish.phase, publish.jobName, publish.elapsedMs ?? "-", diagnostics.pod ?? "-", diagnostics.currentPhase ?? "-", short(payload.digestRef), short(payload.gitopsCommit), ]]), ]; if (Object.keys(envReuse).length > 0) { lines.push( "", "PUBLISH_ENV_REUSE", table(["MODE", "NODE_DEPS", "PRESENT", "ENTRIES", "LINKED", "DEPENDENCY"], [[ envReuse.mode, envReuse.nodeDepsPath, envReuse.nodeDepsPresent, envReuse.nodeDepsEntries, envReuse.linkedNodeDeps ?? "-", envReuse.dependencyReuse, ]]), ); } if (Object.keys(imageBuild).length > 0 || Object.keys(timings).length > 0) { lines.push( "", "PUBLISH_BUILD", table(["BUILDER", "PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "TOTAL_MS"], [[ imageBuild.builder ?? "-", imageBuild.packageMode ?? "-", imageBuild.networkMode ?? "-", proxySummary, imageBuild.contextIgnoreEntries ?? "-", imageBuild.layerCache ?? "-", imageBuild.cacheHitLines ?? "-", imageBuild.stepLines ?? "-", timings.sourceFetchMs ?? "-", timings.monitorWebVerifyMs ?? "-", timings.imageBuildMs ?? "-", timings.gitopsMs ?? "-", timings.totalMs ?? payload.elapsedMs ?? "-", ]]), ); } if (Object.keys(diagnostics).length > 0) { lines.push( "", "PUBLISH_DIAGNOSTICS", table(["TASKRUN", "POD_PHASE", "ACTIVE", "CONDITION", "COMPLETED", "RECENT_LOG"], [[ diagnostics.taskRun ?? "-", diagnostics.podPhase ?? "-", diagnostics.active ?? "-", diagnostics.conditionReason ?? diagnostics.conditionStatus ?? "-", Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "-", diagnostics.recentLogSummary ?? "-", ]]), ); } if (publish.ok !== true && Object.keys(commands).length > 0) { lines.push( "", "PUBLISH_DRILLDOWN", ` status: ${commands.cliStatus ?? "-"}`, ` logs: ${commands.logs ?? "-"}`, ` describe: ${commands.describe ?? "-"}`, ` publish-current: ${commands.publishCurrent ?? "-"}`, ` git-mirror: ${commands.gitMirrorStatus ?? "-"}`, ` sync: ${commands.gitMirrorSync ?? "-"}`, ` flush: ${commands.gitMirrorFlush ?? "-"}`, ` apply: ${commands.controlPlaneApply ?? "-"}`, ); } return lines.join("\n"); } function renderPublishCurrentResult(result: Record): string { const source = record(result.source); const image = record(result.image); const controlPlane = record(result.controlPlane); const publish = record(controlPlane.publish); const publishPayload = record(publish.payload); const observed = record(controlPlane.observed); const gitops = record(observed.gitops); const argo = record(observed.argo); const runtime = record(observed.runtime); const runtimeDeployment = record(record(runtime.probe).deployment); const health = record(result.health); const healthBody = record(record(health.health).bodyJson); const timings = record(result.timings); const budget = record(result.budget); const stageBudgets = record(result.stageBudgets); const validationPlan = record(result.validationPlan); const blocker = record(result.blocker); const recoveryNext = record(controlPlane.recoveryNext); const next = record(result.next); const warnings = Array.isArray(result.warnings) ? result.warnings : []; const slowStages = Array.isArray(result.slowStages) ? result.slowStages.map(record) : []; const lines = [ String(result.command), "", table(["NODE", "LANE", "SENTINEL", "STATUS", "MODE", "BUDGET_S", "ELAPSED_S"], [[ result.node, result.lane, result.sentinelId, result.ok === true ? "ok" : "blocked", result.mode, budget.maxSeconds ?? "-", finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000), ]]), "", table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[ `${source.repository ?? "-"}@${source.branch ?? "-"}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref ?? "-", short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest), result.pipelineRun ?? publish.jobName ?? "-", ]]), "", table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "HEALTH"], [[ short(gitops.revision), short(argo.revision), `${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`, short(runtimeDeployment.image), `${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`, health.ok === true ? "pass" : health.skipped === true ? `skipped:${text(health.reason)}` : Object.keys(health).length === 0 ? "planned" : "blocked", ]]), "", table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "VALIDATION_MS", "TOTAL_MS"], [[ timings.sourceSyncMs ?? "-", timings.sourceFetchMs ?? "-", timings.monitorWebVerifyMs ?? "-", timings.imageBuildMs ?? "-", timings.gitopsMs ?? "-", timings.argoRuntimeMs ?? "-", timings.healthValidationMs ?? "-", timings.totalMs ?? "-", ]]), "", table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "VALIDATION"], [[ "YAML publishCurrent", stageBudgets.sourceSyncSeconds ?? "-", stageBudgets.sourceFetchSeconds ?? "-", stageBudgets.monitorWebVerifySeconds ?? "-", stageBudgets.imageBuildSeconds ?? "-", stageBudgets.gitopsSeconds ?? "-", stageBudgets.argoRuntimeSeconds ?? "-", stageBudgets.dashboardVerifySeconds ?? "-", ]]), ]; if (Object.keys(publish).length > 0) { const payloadImageBuild = record(publishPayload.imageBuild); const payloadEnvReuse = record(publishPayload.envReuse); lines.push( "", table(["ENV_REUSE", "NODE_DEPS", "BUILD_PACKAGE", "BUILD_NETWORK", "CACHE", "CACHE_LINES"], [[ payloadEnvReuse.dependencyReuse ?? "-", payloadEnvReuse.nodeDepsPresent ?? "-", payloadImageBuild.packageMode ?? "-", payloadImageBuild.networkMode ?? "-", payloadImageBuild.layerCache ?? "-", payloadImageBuild.cacheHitLines ?? "-", ]]), ); } lines.push( "", Object.keys(health).length === 0 ? "HEALTH_VALIDATION\n-" : table(["ENDPOINT", "HTTP", "OK", "STATUS", "PUBLIC_URL", "INTERNAL_URL"], [[ health.endpoint ?? validationPlan.endpoint ?? "-", health.httpStatus ?? "-", healthBody.ok ?? "-", healthBody.status ?? "-", health.publicUrl ?? "-", health.internalUrl ?? "-", ]]), "", slowStages.length === 0 ? "SLOW_STAGES\n-" : [ "SLOW_STAGES", table(["STAGE", "ELAPSED_MS", "BUDGET_S", "SUGGESTION"], slowStages.map((stage) => [stage.stage, stage.elapsedMs, stage.budgetSeconds, stage.suggestion])), ].join("\n"), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), "", Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [ "RECOVERY_NEXT", table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]), ` publish-current: ${recoveryNext.publishCurrent ?? "-"}`, ` status: ${recoveryNext.nextStatus ?? "-"}`, ` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`, ` sync: ${recoveryNext.gitMirrorSync ?? "-"}`, ` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`, ` apply: ${recoveryNext.controlPlaneApply ?? "-"}`, ].join("\n"), "", "NEXT", ` publish-current: ${next.publishCurrent ?? "-"}`, ` status: ${next.controlPlaneStatus ?? "-"}`, ` post-deploy-dashboard: ${next.dashboardVerify ?? "-"}`, ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, ` sync: ${next.gitMirrorSync ?? "-"}`, ` flush: ${next.gitMirrorFlush ?? "-"}`, "", "DISCLOSURE", ` end-to-end and stage budgets are read from ${Object.keys(validationPlan).length > 0 ? "publishCurrent YAML and runtime.healthPath" : "YAML-required publishCurrent fields"}.`, " CI/CD validation only checks the configured health endpoint; web-probe, Playwright and browser dashboard checks are post-deploy evidence, not this gate.", " image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.", ); return lines.join("\n"); } function renderImageResult(result: Record): string { const source = record(result.source); const sourceMirror = record(result.sourceMirror); const sourceMirrorSync = record(result.sourceMirrorSync); const image = record(result.image); const monitorWeb = record(image.monitorWeb); const registry = record(result.registry); const publish = record(result.publish); const blocker = record(result.blocker); const next = record(result.next); const warnings = Array.isArray(result.warnings) ? result.warnings : []; return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]), "", table(["SOURCE_REPO", "BRANCH", "COMMIT", "AUTHORITY", "STAGE_REF", "MIRROR"], [[source.repository, source.branch, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), short(source.mirrorCommit)]]), "", Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]), "", table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]), "", Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "BUILD_STATE", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, `${record(monitorWeb.imageBuildState).mode ?? "-"}:${record(monitorWeb.imageBuildState).path ?? record(monitorWeb.imageBuildState).claimName ?? record(monitorWeb.imageBuildState).sizeLimit ?? "-"}`, monitorWeb.imageBuildContextIgnore]]), "", Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]), "", Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), "", Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), "", "NEXT", ` status: ${next.status ?? "-"}`, ` dry-run: ${next.dryRun ?? "-"}`, ` confirm: ${next.confirm ?? "-"}`, ` trigger: ${next.controlPlaneTrigger ?? "-"}`, ` control-plane: ${next.controlPlanePlan ?? "-"}`, "", "DISCLOSURE", " valuesRedacted=true; image status shows refs, hashes and object names only.", ].join("\n"); } function renderControlPlaneResult(result: Record): string { const source = record(result.source); const image = record(result.image); const gitops = record(result.gitops); const argo = record(result.argo); const validation = record(result.validation); const observability = record(result.observability); const observed = record(result.observed); const sourceMirrorSync = record(result.sourceMirrorSync); const publish = record(result.publish); const flush = record(result.flush); const runtimeSecretsApply = record(result.runtimeSecretsApply); const publicExposureApply = record(result.publicExposureApply); const publicExposureCaddy = record(publicExposureApply.caddy); const argoApply = record(result.argoApply); const blocker = record(result.blocker); const statusDiagnosis = record(result.statusDiagnosis); const targetValidation = record(result.targetValidation); const targetValidationBusiness = record(targetValidation.businessStatus); const recoveryNext = record(result.recoveryNext); const next = record(result.next); const warnings = Array.isArray(result.warnings) ? result.warnings : []; return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]), "", table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref, short(gitops.manifestSha256)]]), "", table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]), "", table(["SCENARIO", "MAX_SECONDS", "CI_WAIT", "QVERIFY", "SECOND_PATH"], [[validation.scenarioId, validation.maxSeconds, validation.controlPlaneWaitMaxSeconds ?? "-", validation.quickVerifyMode ?? "-", validation.automaticSecondPath]]), "", Object.keys(observability).length === 0 ? "OTEL\n-" : table(["ENABLED", "ENDPOINT", "SERVICE", "COVERAGE"], [[observability.enabled, observability.endpointConfigured, observability.serviceName, observability.coverage]]), "", renderObservedStatus(observed), "", Object.keys(statusDiagnosis).length === 0 ? "STATUS_DIAGNOSIS\n-" : [ "STATUS_DIAGNOSIS", table(["CODE", "PHASE", "PIPELINERUN", "SOURCE", "REGISTRY", "GIT_MIRROR", "GITOPS", "ARGO", "RUNTIME"], [[ statusDiagnosis.code, statusDiagnosis.phase, statusDiagnosis.pipelineRun, statusDiagnosis.sourceMirror, statusDiagnosis.registry, statusDiagnosis.gitMirror, statusDiagnosis.gitops, statusDiagnosis.argo, statusDiagnosis.runtime, ]]), ].join("\n"), "", Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), "", Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "ERROR_TITLE", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ targetValidation.ok, targetValidation.status, targetValidationBusiness.status ?? "-", targetValidation.errorTitleZh ?? targetValidation.failureTitleZh ?? targetValidationBusiness.errorTitleZh ?? "-", targetValidation.scenarioId, targetValidation.runId, targetValidation.observerId, short(targetValidation.reportJsonSha256), targetValidation.findingCount, targetValidation.artifactCount, ]]), "", Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), "", Object.keys(flush).length === 0 ? "FLUSH\n-" : flush.mode === "async-job" ? table(["OK", "MODE", "JOB", "STATUS"], [[flush.ok, flush.mode, record(flush.job).id, record(flush.next).status]]) : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]), "", Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]), "", Object.keys(publicExposureApply).length === 0 ? "PUBLIC_EXPOSURE_APPLY\n-" : table(["OK", "SECRET", "CADDY", "HOST", "ROUTE_HTTP"], [[publicExposureApply.ok, record(publicExposureApply.secret).ok, record(publicExposureApply.caddy).ok, publicExposureApply.hostname, record(publicExposureApply.caddy).routeProbeHttpStatus ?? "-"]]), "", Object.keys(publicExposureCaddy).length === 0 || publicExposureCaddy.ok === true ? "CADDY_APPLY_DETAIL\n-" : table(["PY", "VALIDATE", "RELOAD", "PROBE", "HTTP", "BLOCK", "ACTIVE", "ERROR", "STDOUT", "STDERR"], [[publicExposureCaddy.pythonExitCode, publicExposureCaddy.validateExitCode, publicExposureCaddy.reloadExitCode, publicExposureCaddy.routeProbeExitCode, publicExposureCaddy.routeProbeHttpStatus, publicExposureCaddy.afterBlockPresent, publicExposureCaddy.active, short(publicExposureCaddy.errorPreview), short(record(publicExposureCaddy.result).stdoutPreview), short(record(publicExposureCaddy.result).stderrPreview)]]), "", Object.keys(argoApply).length === 0 ? "ARGO_APPLY\n-" : table(["OK", "EXIT", "PREVIEW"], [[argoApply.ok, record(argoApply.result).exitCode, record(argoApply.result).stdoutPreview]]), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), "", Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [ "RECOVERY_NEXT", table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]), ` publish-current: ${recoveryNext.publishCurrent ?? "-"}`, ` status: ${recoveryNext.nextStatus ?? "-"}`, ` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`, ` sync: ${recoveryNext.gitMirrorSync ?? "-"}`, ` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`, ` apply: ${recoveryNext.controlPlaneApply ?? "-"}`, ].join("\n"), "", "NEXT", ` plan: ${next.plan ?? "-"}`, ` status: ${next.status ?? "-"}`, ` image: ${next.image ?? "-"}`, ` trigger-current: ${next.triggerCurrent ?? "-"}`, ` apply: ${next.apply ?? "-"}`, ` validate: ${next.validate ?? "-"}`, ` quick-verify: ${next.quickVerify ?? "-"}`, ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, ` sync: ${next.gitMirrorSync ?? "-"}`, ` flush: ${next.gitMirrorFlush ?? "-"}`, "", "DISCLOSURE", " default view is a bounded CI/CD summary; full manifest content is represented by object counts and sha256.", " sentinel unavailable policy is structured-failure; no automatic second execution path is rendered.", ].join("\n"); } function renderObservedStatus(observed: Record): string { const rows = [ observedStatusRow("source", observed.sourceMirror), observedStatusRow("registry", observed.registry), observedStatusRow("git-mirror", observed.gitMirror), observedStatusRow("gitops", observed.gitops), observedStatusRow("argo", observed.argo), observedStatusRow("runtime", observed.runtime), observedStatusRow("cadence", observed.cadence), ].filter((row) => row !== null); if (rows.length === 0) return "OBSERVED\n-"; return table(["CHECK", "OK", "DETAIL", "EXIT", "TIMED_OUT", "PREVIEW"], rows); } function observedStatusRow(name: string, value: unknown): unknown[] | null { const item = record(value); if (Object.keys(item).length === 0) return null; const result = record(item.result); return [name, item.ok, observedDetail(name, item), result.exitCode, result.timedOut, result.stdoutPreview]; } function observedDetail(name: string, item: Record): string { if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`; if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`; if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`; if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`; if (name === "argo") { const diagnostics = record(item.diagnostics); const problems = Array.isArray(diagnostics.problemResources) ? diagnostics.problemResources : []; const first = problems.find((entry) => typeof entry === "object" && entry !== null && !Array.isArray(entry)) as Record | undefined; const problemText = Number(diagnostics.problemResourceCount ?? 0) > 0 ? ` degraded=${diagnostics.problemResourceCount}:${first?.kind ?? "-"} ${first?.namespace ?? "-"}/${first?.name ?? "-"} ${first?.healthStatus ?? first?.status ?? "-"}` : ""; return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}${problemText}`; } if (name === "runtime") { const probe = record(item.probe); const deployment = record(probe.deployment); return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`; } if (name === "cadence") { if (item.skipped === true) return `${item.reason ?? "skipped"}`; const probe = record(item.probe); return `${probe.code ?? "ok"} schedule=${probe.schedule ?? "-"}/${probe.expectedSchedule ?? "-"} last=${probe.lastScheduleTime ?? "-"} jobs=${probe.jobCount ?? "-"}`; } return "-"; } export function renderAsyncJobResult(result: Record): string { const job = record(result.job); const next = record(result.next); return [ String(result.command), "", table(["NODE", "LANE", "MODE", "MUTATION", "JOB"], [[result.node, result.lane, result.mode, result.mutation, job.id]]), "", table(["STATUS", "NAME", "CREATED"], [[job.status, job.name, job.createdAt]]), "", "NEXT", ` status: ${next.status ?? "-"}`, ` wait: ${next.wait ?? "-"}`, "", "DISCLOSURE", " confirmed operation is delegated to UniDesk job status to keep interactive calls bounded.", ].join("\n"); } export function rendered(ok: boolean, command: string, text: string): RenderedCliResult { return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" }; } function readConfigFile(file: string): unknown { if (file.startsWith("/") || file.includes("..") || !file.startsWith("config/")) throw new Error(`unsafe configRef file: ${file}`); const abs = rootPath(file); if (!existsSync(abs)) throw new Error(`${file} does not exist`); return Bun.YAML.parse(readFileSync(abs, "utf8")) as unknown; } function configRefFile(ref: string): string { const [file, path, extra] = ref.split("#"); if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) throw new Error(`invalid configRef: ${ref}`); return file; } function valueAtPath(value: unknown, path: string): unknown { let current: unknown = value; for (const segment of path.split(".")) { const match = /^([A-Za-z0-9_-]+)?(?:\[(\d+)\])?$/u.exec(segment); if (match === null) return undefined; if (match[1] !== undefined) { if (!isRecord(current)) return undefined; current = current[match[1]]; } if (match[2] !== undefined) { if (!Array.isArray(current)) return undefined; current = current[Number(match[2])]; } } return current; } export function stringAt(value: unknown, path: string): string { const found = valueAtPath(value, path); if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`); return found; } export function nonEmptyString(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function stringField(value: Record, path: string): string { return stringAt(value, path); } function stringTarget(value: unknown, label: string): string { if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must resolve to a non-empty string`); return value; } export function numberAt(value: unknown, path: string): number { const found = valueAtPath(value, path); if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`); return found; } function booleanAt(value: unknown, path: string): boolean { const found = valueAtPath(value, path); if (typeof found !== "boolean") throw new Error(`${path} must be a boolean`); return found; } function finiteNumberOrNull(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } export function arrayAt(value: unknown, path: string): unknown[] { const found = valueAtPath(value, path); if (!Array.isArray(found)) throw new Error(`${path} must be an array`); return found; } function arrayAtNullable(value: unknown, path: string): Record[] { const found = valueAtPath(value, path); return Array.isArray(found) ? found.map(record) : []; } export function recordTarget(value: unknown, label: string): Record { if (!isRecord(value)) throw new Error(`${label} must resolve to an object`); return value; } export function record(value: unknown): Record { return isRecord(value) ? value : {}; } export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function manifestObjectSummary(items: readonly Record[]): readonly Record[] { return items.map((item) => ({ kind: item.kind ?? null, name: record(item.metadata).name ?? null, namespace: record(item.metadata).namespace ?? null, })); } export function compactCommand(result: CommandResult): CompactCommandResult { return { exitCode: result.exitCode, timedOut: result.timedOut, stdoutBytes: Buffer.byteLength(result.stdout), stderrBytes: Buffer.byteLength(result.stderr), stdoutPreview: result.stdout.trim().slice(0, 500), stderrPreview: result.stderr.trim().slice(0, 500), }; } export function parseJsonObject(text: string): Record | null { const trimmed = text.trim(); if (trimmed.length === 0) return null; try { const parsed = JSON.parse(trimmed) as unknown; return isRecord(parsed) ? parsed : null; } catch { return null; } } export function table(headers: string[], rows: unknown[][]): string { const normalized = [headers, ...rows.map((row) => row.map(text))]; const widths = headers.map((_, index) => Math.max(...normalized.map((row) => text(row[index] ?? "").length))); return normalized.map((row) => row.map((cell, index) => text(cell).padEnd(widths[index])).join(" ").trimEnd()).join("\n"); } export function text(value: unknown): string { if (value === undefined || value === null || value === "") return "-"; if (typeof value === "boolean") return value ? "true" : "false"; return String(value).replace(/\s+/gu, " ").trim(); } export function short(value: unknown): string { const raw = text(value); if (raw === "-") return raw; if (/^sha256:[0-9a-f]{64}$/iu.test(raw)) return `${raw.slice(0, 19)}...`; if (/^[0-9a-f]{40}$/iu.test(raw)) return raw.slice(0, 12); return raw.length > 42 ? `${raw.slice(0, 39)}...` : raw; } function sha256(textValue: string): string { return `sha256:${createHash("sha256").update(textValue).digest("hex")}`; } export function shellQuote(value: string): string { return `'${value.replace(/'/gu, "'\\''")}'`; } function tomlEscape(value: string): string { return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"'); }