Codex
4670 lines
246 KiB
TypeScript
4670 lines
246 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p8-web-probe-sentinel-recovery.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p10-monitor-web-aggregation.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot.
|
|
// Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { repoRoot, rootPath } from "./config";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
import { startJob } from "./jobs";
|
|
import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered } from "./hwlab-node-web-sentinel-config";
|
|
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
|
|
import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver";
|
|
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
|
import type { RenderedCliResult } from "./output";
|
|
import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
|
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
|
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
|
|
|
|
export type WebProbeSentinelConfigAction = "plan" | "status";
|
|
export type WebProbeSentinelImageAction = "status" | "build";
|
|
export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current";
|
|
export type WebProbeSentinelPublishAction = "publish-current";
|
|
export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop";
|
|
export type WebProbeSentinelDashboardAction = "verify" | "screenshot" | "trigger";
|
|
export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary";
|
|
|
|
export type WebProbeSentinelOptions =
|
|
| {
|
|
readonly kind: "config";
|
|
readonly action: WebProbeSentinelConfigAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "image";
|
|
readonly action: WebProbeSentinelImageAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
}
|
|
| {
|
|
readonly kind: "control-plane";
|
|
readonly action: WebProbeSentinelControlPlaneAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
readonly rerun: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "publish";
|
|
readonly action: WebProbeSentinelPublishAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
readonly rerun: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "maintenance";
|
|
readonly action: WebProbeSentinelMaintenanceAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
readonly releaseId: string | null;
|
|
readonly reason: string | null;
|
|
readonly quickVerify: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "validate";
|
|
readonly action: "validate";
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
readonly quickVerify: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "report";
|
|
readonly action: "report";
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly view: WebProbeSentinelReportView;
|
|
readonly runId: string | null;
|
|
readonly latest: boolean;
|
|
readonly traceId: string | null;
|
|
readonly sampleSeq: number | null;
|
|
readonly raw: boolean;
|
|
readonly full: boolean;
|
|
readonly timeoutSeconds: number;
|
|
}
|
|
| {
|
|
readonly kind: "dashboard";
|
|
readonly action: WebProbeSentinelDashboardAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly viewport: string;
|
|
readonly localDir: string;
|
|
readonly name: string | null;
|
|
readonly runId: string | null;
|
|
readonly timeoutMs: number;
|
|
readonly waitTimeoutMs: number;
|
|
readonly timeoutSeconds: number;
|
|
readonly commandTimeoutSeconds: number;
|
|
readonly fullPage: boolean;
|
|
readonly raw: boolean;
|
|
};
|
|
|
|
export interface SentinelCicdState {
|
|
readonly spec: HwlabRuntimeLaneSpec;
|
|
readonly sentinelId: string;
|
|
readonly configRefs: Record<string, string>;
|
|
readonly configReady: boolean;
|
|
readonly runtime: Record<string, unknown>;
|
|
readonly cicd: Record<string, unknown>;
|
|
readonly scenarios: unknown;
|
|
readonly publicExposure: Record<string, unknown>;
|
|
readonly secrets: Record<string, unknown>;
|
|
readonly controlPlaneTarget: Record<string, unknown>;
|
|
readonly controlPlaneNode: Record<string, unknown>;
|
|
readonly sourceHead: SourceHead;
|
|
readonly image: SentinelImagePlan;
|
|
readonly manifests: readonly Record<string, unknown>[];
|
|
readonly manifestSha256: string;
|
|
readonly valuesRedacted: true;
|
|
}
|
|
|
|
interface SourceHead {
|
|
readonly ok: boolean;
|
|
readonly repository: string;
|
|
readonly branch: string;
|
|
readonly commit: string | null;
|
|
readonly stageRef: string | null;
|
|
readonly mirrorCommit: string | null;
|
|
readonly sourceAuthority: "git-mirror-cache" | "git-mirror-snapshot";
|
|
readonly latestDrift: boolean;
|
|
readonly result: CompactCommandResult;
|
|
}
|
|
|
|
interface SentinelImagePlan {
|
|
readonly repository: string;
|
|
readonly tag: string;
|
|
readonly ref: string;
|
|
readonly digestRef: string | null;
|
|
readonly baseImage: string;
|
|
readonly buildContext: string;
|
|
readonly entrypoint: string;
|
|
readonly dockerfileSha256: string;
|
|
readonly dockerfilePreview: string;
|
|
readonly monitorWeb: Record<string, unknown>;
|
|
}
|
|
|
|
interface SentinelObservedStatus {
|
|
readonly sourceMirror: Record<string, unknown>;
|
|
readonly registry: Record<string, unknown>;
|
|
readonly gitMirror: Record<string, unknown>;
|
|
readonly gitops: Record<string, unknown>;
|
|
readonly argo: Record<string, unknown>;
|
|
readonly runtime: Record<string, unknown>;
|
|
readonly cadence: Record<string, unknown>;
|
|
readonly wait?: Record<string, unknown>;
|
|
}
|
|
|
|
interface SentinelObservedExpectation {
|
|
readonly gitopsRevision: string | null;
|
|
readonly runtimeImage: string | null;
|
|
}
|
|
|
|
interface SentinelRemoteJobResult {
|
|
readonly ok: boolean;
|
|
readonly phase: string;
|
|
readonly resourceKind?: "Job" | "PipelineRun";
|
|
readonly jobName: string;
|
|
readonly payload: Record<string, unknown>;
|
|
readonly polls?: number;
|
|
readonly elapsedMs?: number;
|
|
readonly create?: Record<string, unknown>;
|
|
readonly probe?: Record<string, unknown>;
|
|
readonly diagnostics?: Record<string, unknown>;
|
|
readonly valuesRedacted: true;
|
|
}
|
|
|
|
export interface CompactCommandResult {
|
|
readonly exitCode: number | null;
|
|
readonly timedOut: boolean;
|
|
readonly stdoutBytes: number;
|
|
readonly stderrBytes: number;
|
|
readonly stdoutPreview: string;
|
|
readonly stderrPreview: string;
|
|
}
|
|
|
|
export interface ChildCliResult {
|
|
readonly ok: boolean;
|
|
readonly parsed: Record<string, unknown> | null;
|
|
readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string };
|
|
}
|
|
|
|
const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot";
|
|
|
|
type SourceResolveMode = "cached" | "sync";
|
|
|
|
export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult {
|
|
if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId));
|
|
requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
|
|
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options));
|
|
if (options.kind === "image") return runSentinelImage(state, options);
|
|
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
|
|
if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
|
|
if (options.kind === "maintenance") return runSentinelMaintenance(state, options);
|
|
if (options.kind === "validate") return runSentinelValidate(state, options);
|
|
if (options.kind === "dashboard") return runSentinelDashboard(state, options);
|
|
return runSentinelReport(state, options);
|
|
}
|
|
|
|
function sentinelSourceResolveMode(options: WebProbeSentinelOptions): SourceResolveMode {
|
|
if (options.kind === "image" && options.action === "build" && options.confirm && options.wait) return "sync";
|
|
if (options.kind === "control-plane" && options.action === "trigger-current" && options.confirm && options.wait) return "sync";
|
|
if (options.kind === "publish" && options.confirm && options.wait) return "sync";
|
|
return "cached";
|
|
}
|
|
|
|
function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel image ${options.action}`;
|
|
if (options.action === "build" && options.confirm) {
|
|
if (!options.wait) return renderAsyncSentinelJob(state, "image", "build", options.timeoutSeconds);
|
|
return runSentinelImageBuildConfirmed(state, options);
|
|
}
|
|
const sourceMirror = options.action === "status" ? probeSourceMirror(state, options.timeoutSeconds) : null;
|
|
const registry = options.action === "status" ? probeImageRegistry(state, options.timeoutSeconds) : null;
|
|
const sourceMirrorReady = options.action !== "status" || record(sourceMirror).ok === true;
|
|
const registryReady = options.action !== "status" || record(registry?.probe).present === true;
|
|
const result = {
|
|
ok: state.configReady && state.sourceHead.ok && sourceMirrorReady && registryReady,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
|
|
mutation: false,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
sourceMirror,
|
|
image: state.image,
|
|
registry,
|
|
blocker: sourceMirrorReady
|
|
? registryReady ? null : { code: "sentinel-image-missing", reason: "expected sentinel image tag is not present in the node-local registry" }
|
|
: { code: "sentinel-source-mirror-not-ready", reason: "source.gitMirrorReadUrl does not expose the selected source commit yet" },
|
|
next: {
|
|
status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
dryRun: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`,
|
|
confirm: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`,
|
|
controlPlanePlan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderImageResult(result));
|
|
}
|
|
|
|
function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel control-plane ${options.action}`;
|
|
const mutationAction = options.action === "apply" || options.action === "trigger-current";
|
|
if (options.confirm && mutationAction) {
|
|
if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds);
|
|
return runSentinelControlPlaneConfirmed(state, options);
|
|
}
|
|
const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null;
|
|
const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
|
|
const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
|
|
const pipelineRun = sentinelPipelineRunName(state, options.rerun);
|
|
const statusDiagnosis = options.action === "status" && !observedReady ? sentinelObservedStatusDiagnosis(state, observed, pipelineRun) : null;
|
|
const result = {
|
|
ok: state.configReady && state.sourceHead.ok && observedReady,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
|
|
mutation: false,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
pipelineRun,
|
|
gitops: {
|
|
path: stringField(state.cicd, "gitopsPath"),
|
|
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
|
manifestObjects: state.manifests.length,
|
|
manifestSha256: state.manifestSha256,
|
|
},
|
|
argo: {
|
|
namespace: stringAt(state.cicd, "argo.namespace"),
|
|
projectName: stringAt(state.cicd, "argo.projectName"),
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
},
|
|
maintenance: {
|
|
startCommand: stringAt(state.cicd, "maintenance.startCommand"),
|
|
stopCommand: stringAt(state.cicd, "maintenance.stopCommand"),
|
|
serviceUnavailablePolicy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"),
|
|
},
|
|
validation: {
|
|
scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
|
|
maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
|
|
controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state),
|
|
quickVerifyMode: "manual-validate",
|
|
automaticSecondPath: false,
|
|
},
|
|
manifests: {
|
|
objects: manifestObjectSummary(state.manifests),
|
|
sha256: state.manifestSha256,
|
|
},
|
|
observability: webProbeSentinelOtelSummary({
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
namespace: stringAt(state.runtime, "namespace"),
|
|
runtime: state.runtime,
|
|
cicd: state.cicd,
|
|
}),
|
|
observed,
|
|
statusDiagnosis,
|
|
warnings: mergeWarnings(observedWarnings, record(statusDiagnosis).warning),
|
|
blocker: observedReady
|
|
? null
|
|
: record(statusDiagnosis).blocker ?? { code: "sentinel-control-plane-observed-not-ready", reason: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass", valuesRedacted: true },
|
|
recoveryNext: record(statusDiagnosis).recoveryNext ?? null,
|
|
next: controlPlaneNext(state, options.action),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderControlPlaneResult(result));
|
|
}
|
|
|
|
function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
|
|
const command = "web-probe sentinel publish-current";
|
|
if (options.confirm) {
|
|
if (!options.wait) return renderAsyncSentinelJob(state, "publish", options.action, options.timeoutSeconds);
|
|
return runSentinelPublishCurrentConfirmed(state, options);
|
|
}
|
|
const result = {
|
|
ok: state.configReady && state.sourceHead.ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
pipelineRun: sentinelPipelineRunName(state, options.rerun),
|
|
gitops: {
|
|
path: stringAt(state.cicd, "gitopsPath"),
|
|
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
|
manifestSha256: state.manifestSha256,
|
|
},
|
|
argo: {
|
|
namespace: stringAt(state.cicd, "argo.namespace"),
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
},
|
|
budget: publishCurrentBudget(state),
|
|
validationPlan: publishCurrentHealthValidationPlan(state),
|
|
stageBudgets: publishCurrentStageBudgets(state),
|
|
blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" },
|
|
next: publishCurrentNext(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok === true, command, renderPublishCurrentResult(result));
|
|
}
|
|
|
|
function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
|
|
const interruptContext: Record<string, unknown> = {
|
|
phase: "starting",
|
|
pipelineRun: sentinelPipelineRunName(state, options.rerun),
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
valuesRedacted: true,
|
|
};
|
|
const uninstallInterruptHandler = installSentinelPublishInterruptHandler(state, interruptContext);
|
|
try {
|
|
return runSentinelPublishCurrentConfirmedInner(state, options, interruptContext);
|
|
} finally {
|
|
uninstallInterruptHandler();
|
|
}
|
|
}
|
|
|
|
function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>, interruptContext: Record<string, unknown>): RenderedCliResult {
|
|
const startedAt = Date.now();
|
|
const command = "web-probe sentinel publish-current";
|
|
const budget = publishCurrentBudget(state);
|
|
const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds"));
|
|
const deadline = startedAt + budgetSeconds * 1000;
|
|
const remainingBudgetSeconds = () => strictRemainingSeconds(deadline, budgetSeconds);
|
|
let controlResult: Record<string, unknown> | null = null;
|
|
sentinelProgressEvent("sentinel.publish.progress", {
|
|
phase: "publish-current-start",
|
|
status: "running",
|
|
pipelineRun: interruptContext.pipelineRun,
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
});
|
|
if (state.configReady && state.sourceHead.ok && remainingBudgetSeconds() >= 5) {
|
|
interruptContext.phase = "already-current-registry-probe";
|
|
const registryProbe = probeImageRegistry(state, Math.max(1, Math.min(remainingBudgetSeconds(), 5)));
|
|
if (record(record(registryProbe).probe).present === true && remainingBudgetSeconds() >= 5) {
|
|
const preflightStartedAt = Date.now();
|
|
const preflightTimeoutSeconds = Math.max(1, Math.min(remainingBudgetSeconds(), 10));
|
|
interruptContext.phase = "already-current-observed-probe";
|
|
const preflightObserved = withObservedWait(
|
|
collectSentinelObservedStatus(state, preflightTimeoutSeconds, undefined, true),
|
|
preflightStartedAt,
|
|
preflightTimeoutSeconds,
|
|
true,
|
|
);
|
|
if (sentinelObservedReady(preflightObserved)) {
|
|
controlResult = sentinelAlreadyCurrentControlResult(state, preflightObserved, Date.now() - preflightStartedAt);
|
|
}
|
|
}
|
|
}
|
|
interruptContext.phase = "control-plane-trigger-current";
|
|
controlResult ??= sentinelControlPlaneConfirmedResult(state, {
|
|
kind: "control-plane",
|
|
action: "trigger-current",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
sentinelId: options.sentinelId,
|
|
dryRun: false,
|
|
confirm: true,
|
|
wait: true,
|
|
timeoutSeconds: Math.max(1, remainingBudgetSeconds()),
|
|
rerun: options.rerun,
|
|
});
|
|
let health: Record<string, unknown>;
|
|
let healthElapsedMs: number | null = null;
|
|
if (controlResult.ok !== true) {
|
|
health = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true };
|
|
} else if (remainingBudgetSeconds() < 2) {
|
|
health = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-health", valuesRedacted: true };
|
|
} else {
|
|
interruptContext.phase = "health-endpoint-validation";
|
|
const healthStartedAt = Date.now();
|
|
health = probeSentinelRuntimeHealthEndpoint(state, remainingBudgetSeconds());
|
|
healthElapsedMs = Date.now() - healthStartedAt;
|
|
health = { ...health, elapsedMs: healthElapsedMs, valuesRedacted: true };
|
|
}
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const timings = publishCurrentStageTimings(controlResult, health, elapsedMs);
|
|
const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds);
|
|
const withinBudget = elapsedMs <= budgetSeconds * 1000;
|
|
const healthOk = health.ok === true;
|
|
const ok = controlResult.ok === true && healthOk && withinBudget;
|
|
const blocker = ok ? null : publishCurrentBlocker(controlResult, health, withinBudget);
|
|
const result = {
|
|
ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
mode: controlResult.mode === "already-current" ? "already-current" : "confirm-wait",
|
|
mutation: controlResult.mutation !== false,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state, options.rerun),
|
|
controlPlane: controlResult,
|
|
health,
|
|
budget,
|
|
validationPlan: publishCurrentHealthValidationPlan(state),
|
|
stageBudgets: publishCurrentStageBudgets(state),
|
|
elapsedMs,
|
|
withinBudget,
|
|
timings,
|
|
slowStages,
|
|
warnings: mergeWarnings(controlResult.warnings, publishCurrentBudgetWarnings(slowStages, withinBudget, budgetSeconds, elapsedMs)),
|
|
blocker,
|
|
next: publishCurrentNext(state),
|
|
valuesRedacted: true,
|
|
};
|
|
interruptContext.phase = "completed";
|
|
return rendered(ok, command, renderPublishCurrentResult(result));
|
|
}
|
|
|
|
function strictRemainingSeconds(deadline: number, cap: number): number {
|
|
return Math.max(0, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000)));
|
|
}
|
|
|
|
function withObservedWait(observed: SentinelObservedStatus, startedAt: number, timeoutSeconds: number, includeGitMirror: boolean): SentinelObservedStatus {
|
|
const elapsedMs = Date.now() - startedAt;
|
|
return {
|
|
...observed,
|
|
wait: {
|
|
polls: 1,
|
|
elapsedMs,
|
|
timeoutMs: Math.max(1, timeoutSeconds) * 1000,
|
|
ready: sentinelObservedReady(observed),
|
|
includeGitMirror,
|
|
valuesRedacted: true,
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed: SentinelObservedStatus, elapsedMs: number): Record<string, unknown> {
|
|
const registryProbe = record(record(observed.registry).probe);
|
|
const gitops = record(observed.gitops);
|
|
const argo = record(observed.argo);
|
|
const digest = text(registryProbe.digest);
|
|
const digestRef = digest === "-" ? null : `${state.image.repository}@${digest}`;
|
|
const stageTimings = {
|
|
sourceFetchMs: 0,
|
|
monitorWebVerifyMs: 0,
|
|
imageBuildMs: 0,
|
|
gitopsMs: 0,
|
|
totalMs: 0,
|
|
valuesRedacted: true,
|
|
};
|
|
return {
|
|
ok: true,
|
|
command: "web-probe sentinel control-plane trigger-current",
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "already-current",
|
|
mutation: false,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
pipelineRun: "already-current",
|
|
gitops: {
|
|
path: stringAt(state.cicd, "gitopsPath"),
|
|
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
|
manifestObjects: state.manifests.length,
|
|
manifestSha256: state.manifestSha256,
|
|
},
|
|
argo: {
|
|
namespace: stringAt(state.cicd, "argo.namespace"),
|
|
projectName: stringAt(state.cicd, "argo.projectName"),
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
},
|
|
validation: {
|
|
scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
|
|
maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
|
|
controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state),
|
|
quickVerifyMode: "manual-validate",
|
|
automaticSecondPath: false,
|
|
},
|
|
manifests: {
|
|
objects: manifestObjectSummary(state.manifests),
|
|
sha256: state.manifestSha256,
|
|
},
|
|
sourceMirrorSync: { ok: true, phase: "already-current", jobName: "-", elapsedMs: 0, valuesRedacted: true },
|
|
publish: {
|
|
ok: true,
|
|
phase: "already-current",
|
|
resourceKind: "PipelineRun",
|
|
jobName: "already-current",
|
|
elapsedMs: 0,
|
|
payload: {
|
|
ok: true,
|
|
status: "already-current",
|
|
sourceCommit: state.sourceHead.commit,
|
|
imageRef: state.image.ref,
|
|
digestRef,
|
|
gitopsCommit: gitops.revision ?? argo.revision ?? null,
|
|
stageTimings,
|
|
completedStages: ["already-current"],
|
|
valuesRedacted: true,
|
|
},
|
|
diagnostics: {
|
|
domain: "publish",
|
|
resourceKind: "PipelineRun",
|
|
pipelineRun: "already-current",
|
|
currentPhase: "already-current",
|
|
completedStages: ["already-current:skipped"],
|
|
stageTimings,
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
},
|
|
flush: { ok: true, skipped: true, reason: "already-current", valuesRedacted: true },
|
|
runtimeSecretsApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
|
|
publicExposureApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
|
|
argoApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
|
|
observed,
|
|
targetValidation: null,
|
|
elapsedMs,
|
|
warnings: [
|
|
"publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used health endpoint validation only.",
|
|
...sentinelObservedWarnings(observed),
|
|
...targetValidationDeferredWarnings(state, false, controlPlaneWaitWarningSeconds(state)),
|
|
],
|
|
blocker: null,
|
|
recoveryNext: controlPlaneRecoveryNext(state, true, {}, { ok: true }, observed),
|
|
next: controlPlaneNext(state, "trigger-current"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number, sourceResolveMode: SourceResolveMode): SentinelCicdState {
|
|
const sentinel = resolveWebProbeSentinel(spec, sentinelId);
|
|
const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
|
|
const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime);
|
|
const cicd = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.cicd), sentinel.configRefs.cicd);
|
|
const scenarios = readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.scenarios);
|
|
const rawPublicExposure = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
|
|
const publicExposure = effectiveWebProbeSentinelPublicExposure(spec, sentinel.id, rawPublicExposure);
|
|
const secrets = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.secrets), sentinel.configRefs.secrets);
|
|
const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef");
|
|
const controlPlaneTarget = recordTarget(readWebProbeSentinelConfigRefTarget(spec, controlPlaneRef), controlPlaneRef);
|
|
const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef));
|
|
const nodeId = stringField(controlPlaneTarget, "node");
|
|
const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
|
|
validateSentinelSourceAuthority(cicd);
|
|
const sourceHead = resolveSourceHead(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds, sourceResolveMode);
|
|
const image = sentinelImagePlan(spec, cicd, sourceHead);
|
|
const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image);
|
|
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
|
return {
|
|
spec,
|
|
sentinelId: sentinel.id,
|
|
configRefs: sentinel.configRefs,
|
|
configReady: configPlan.ok,
|
|
runtime,
|
|
cicd,
|
|
scenarios,
|
|
publicExposure,
|
|
secrets,
|
|
controlPlaneTarget,
|
|
controlPlaneNode,
|
|
sourceHead,
|
|
image,
|
|
manifests,
|
|
manifestSha256: sha256(manifestYaml),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function resolveSourceHead(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
cicd: Record<string, unknown>,
|
|
controlPlaneTarget: Record<string, unknown>,
|
|
controlPlaneNode: Record<string, unknown>,
|
|
timeoutSeconds: number,
|
|
mode: SourceResolveMode,
|
|
): SourceHead {
|
|
const repository = stringAt(cicd, "source.repository");
|
|
const branch = stringAt(cicd, "source.branch");
|
|
const resolved = mode === "sync"
|
|
? resolveSourceHeadWithK8sSnapshot(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds)
|
|
: probeSourceMirrorCache(cicd, controlPlaneNode, timeoutSeconds, null);
|
|
const probe = record(resolved.probe);
|
|
const commit = nonEmptyString(probe.sourceCommit) ?? nonEmptyString(probe.commit) ?? nonEmptyString(probe.mirrorCommit);
|
|
const stageRef = nonEmptyString(probe.stageRef) ?? (commit === null ? null : sentinelSourceSnapshotRef(cicd, commit));
|
|
const mirrorCommit = nonEmptyString(probe.mirrorCommit) ?? nonEmptyString(probe.commit);
|
|
return {
|
|
ok: resolved.ok === true && commit !== null,
|
|
repository,
|
|
branch,
|
|
commit,
|
|
stageRef,
|
|
mirrorCommit,
|
|
sourceAuthority: mode === "sync" ? "git-mirror-snapshot" : "git-mirror-cache",
|
|
latestDrift: commit !== null && mirrorCommit !== null && commit !== mirrorCommit,
|
|
result: compactCommand(resolved.result),
|
|
};
|
|
}
|
|
|
|
function resolveSourceHeadWithK8sSnapshot(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
cicd: Record<string, unknown>,
|
|
controlPlaneTarget: Record<string, unknown>,
|
|
controlPlaneNode: Record<string, unknown>,
|
|
timeoutSeconds: number,
|
|
): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
|
|
const namespace = stringAt(cicd, "builder.namespace");
|
|
const prefix = `${stringAt(cicd, "builder.jobPrefix")}-source-resolve`;
|
|
const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
|
const manifest = sentinelSourceMirrorResolveJobManifest(spec, cicd, controlPlaneTarget, controlPlaneNode, jobName);
|
|
const created = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
if (created.exitCode !== 0) return { ok: false, probe: { ok: false, status: "create-failed", jobName, valuesRedacted: true }, result: created };
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, 120_000));
|
|
let lastCapture = created;
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
lastCapture = probeCapture;
|
|
const probe = parseJsonObject(probeCapture.stdout) ?? {};
|
|
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
|
if (probe.succeeded === true) return { ok: payload.ok === true, probe: payload, result: probeCapture };
|
|
if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, valuesRedacted: true } : payload, result: probeCapture };
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture };
|
|
}
|
|
|
|
function sentinelSourceMirrorResolveJobManifest(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
cicd: Record<string, unknown>,
|
|
controlPlaneTarget: Record<string, unknown>,
|
|
controlPlaneNode: Record<string, unknown>,
|
|
jobName: string,
|
|
): Record<string, unknown> {
|
|
const namespace = stringAt(cicd, "builder.namespace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": "web-probe-sentinel-source-resolve",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
};
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: { name: jobName, namespace, labels },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: numberAt(cicd, "builder.activeDeadlineSeconds"),
|
|
ttlSecondsAfterFinished: numberAt(cicd, "builder.ttlSecondsAfterFinished"),
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
volumes: [
|
|
sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget),
|
|
{ name: "git-ssh", secret: { secretName: stringAt(cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
],
|
|
containers: [{
|
|
name: "resolve",
|
|
image: sentinelSourceResolverImage(spec, cicd),
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShellFromConfig(cicd, controlPlaneNode, jobName, null)],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelSourceResolverImage(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>): string {
|
|
const baseImageRef = stringAt(cicd, "image.baseImageRef");
|
|
return stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef);
|
|
}
|
|
|
|
function validateSentinelSourceAuthority(cicd: Record<string, unknown>): void {
|
|
const mode = stringAt(cicd, "sourceAuthority.mode");
|
|
const resolver = stringAt(cicd, "sourceAuthority.resolver");
|
|
const allowHostGit = booleanAt(cicd, "sourceAuthority.allowHostGit");
|
|
const allowGithubDirectInPipeline = booleanAt(cicd, "sourceAuthority.allowGithubDirectInPipeline");
|
|
const missingObjectPolicy = stringAt(cicd, "sourceSnapshot.missingObjectPolicy");
|
|
if (mode !== "gitMirrorSnapshot") throw new Error("sourceAuthority.mode must be gitMirrorSnapshot");
|
|
if (resolver !== "k8s-git-mirror") throw new Error("sourceAuthority.resolver must be k8s-git-mirror");
|
|
if (allowHostGit !== false) throw new Error("sourceAuthority.allowHostGit must be false");
|
|
if (allowGithubDirectInPipeline !== false) throw new Error("sourceAuthority.allowGithubDirectInPipeline must be false");
|
|
if (missingObjectPolicy !== "fail-fast") throw new Error("sourceSnapshot.missingObjectPolicy must be fail-fast");
|
|
sentinelSourceSnapshotStageRefPrefix(cicd);
|
|
}
|
|
|
|
function sentinelSourceSnapshotStageRefPrefix(cicd: Record<string, unknown>): string {
|
|
const branch = stringAt(cicd, "source.branch");
|
|
const repository = stringAt(cicd, "source.repository");
|
|
const prefix = stringAt(cicd, "sourceSnapshot.stageRefPrefix")
|
|
.replaceAll("{branch}", branch)
|
|
.replaceAll("{repository}", repository)
|
|
.replace(/\/+$/u, "");
|
|
if (!prefix.startsWith("refs/")) throw new Error("sourceSnapshot.stageRefPrefix must resolve to a git ref prefix");
|
|
return prefix;
|
|
}
|
|
|
|
function sentinelSourceSnapshotRef(cicd: Record<string, unknown>, commit: string): string {
|
|
return `${sentinelSourceSnapshotStageRefPrefix(cicd)}/${commit}`;
|
|
}
|
|
|
|
function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>, sourceHead: SourceHead): SentinelImagePlan {
|
|
const repository = stringAt(cicd, "image.repository");
|
|
const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12);
|
|
const baseImageRef = stringAt(cicd, "image.baseImageRef");
|
|
const baseImage = stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef);
|
|
const entrypoint = stringAt(cicd, "source.entrypoint");
|
|
const monitorWeb = monitorWebCicdPlan(spec, cicd);
|
|
const dockerfile = sentinelDockerfile(baseImage, entrypoint);
|
|
return {
|
|
repository,
|
|
tag,
|
|
ref: `${repository}:${tag}`,
|
|
digestRef: null,
|
|
baseImage,
|
|
buildContext: stringAt(cicd, "source.buildContext"),
|
|
entrypoint,
|
|
dockerfileSha256: sha256(dockerfile),
|
|
dockerfilePreview: dockerfile,
|
|
monitorWeb,
|
|
};
|
|
}
|
|
|
|
function sentinelDockerfile(baseImage: string, entrypoint: string): string {
|
|
return [
|
|
`FROM ${baseImage}`,
|
|
"WORKDIR /app",
|
|
"ENV NODE_ENV=production",
|
|
"COPY .unidesk-sentinel-bin/trans /usr/local/bin/trans",
|
|
"COPY . /app",
|
|
`ENTRYPOINT ["bun", "${entrypoint}"]`,
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
stack: stringAtNullable(cicd, "monitorWeb.frontendStack") ?? "vue3-vendored-browser-build",
|
|
runtimeMode: stringAtNullable(cicd, "monitorWeb.runtimeMode") ?? "runner-served-bridge",
|
|
assetRoot: stringAtNullable(cicd, "monitorWeb.assetRoot") ?? "scripts/assets/web-probe-sentinel-monitor-web",
|
|
verifyCommand: "bun scripts/verify-web-probe-sentinel-monitor-web.ts",
|
|
gitMirrorReadUrl: stringAt(cicd, "source.gitMirrorReadUrl"),
|
|
sourceMode: stringAt(cicd, "builder.sourceMode"),
|
|
envReuseMode: stringAtNullable(cicd, "monitorWeb.envReuse.mode") ?? "k8s-buildkit-and-ci-node-deps",
|
|
envReuseNodeDepsPath: stringAtNullable(cicd, "monitorWeb.envReuse.nodeDepsPath") ?? "/opt/hwlab-ci-node-deps/node_modules",
|
|
verifyPhase: stringAtNullable(cicd, "monitorWeb.imageBuild.verifyPhase") ?? "pre-image-build",
|
|
imageBuildBuilder: spec.buildkit?.sidecarImage ?? null,
|
|
imageBuildPackageMode: stringAtNullable(cicd, "monitorWeb.imageBuild.packageMode") ?? "copy-only-containerfile",
|
|
imageBuildNetworkMode: monitorWebImageBuildNetworkMode(cicd),
|
|
imageBuildProxySource: stringAtNullable(cicd, "monitorWeb.imageBuild.proxySource") ?? "node.networkProfile.imageBuildProxy",
|
|
imageBuildContextIgnore: stringAtNullable(cicd, "monitorWeb.imageBuild.contextIgnore") ?? "generated",
|
|
imageBuildState: monitorWebBuildkitStatePlan(cicd),
|
|
ciBudgetSeconds: numberAtNullable(cicd, "monitorWeb.ciBudget.maxSeconds") ?? numberAt(cicd, "confirmWait.maxSeconds"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function monitorWebImageBuildNetworkMode(cicd: Record<string, unknown>): "default" | "host" {
|
|
const value = stringAtNullable(cicd, "monitorWeb.imageBuild.networkMode") ?? "host";
|
|
if (value !== "default" && value !== "host") throw new Error(`monitorWeb.imageBuild.networkMode must be default or host, got ${value}`);
|
|
return value;
|
|
}
|
|
|
|
function monitorWebBuildkitStatePlan(cicd: Record<string, unknown>): Record<string, unknown> {
|
|
const state = recordTarget(valueAtPath(cicd, "monitorWeb.imageBuild.buildkitState"), "monitorWeb.imageBuild.buildkitState");
|
|
const mode = stringAt(state, "mode");
|
|
if (mode === "hostPath") {
|
|
return {
|
|
mode,
|
|
path: stringAt(state, "path"),
|
|
type: stringAt(state, "type"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
if (mode === "persistentVolumeClaim") {
|
|
return {
|
|
mode,
|
|
claimName: stringAt(state, "claimName"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
if (mode === "emptyDir") {
|
|
return {
|
|
mode,
|
|
sizeLimit: stringAt(state, "sizeLimit"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`);
|
|
}
|
|
|
|
function publishCurrentBudget(state: SentinelCicdState): Record<string, unknown> {
|
|
const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget");
|
|
return {
|
|
maxSeconds: numberAt(budget, "maxSeconds"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publishCurrentStageBudgets(state: SentinelCicdState): Record<string, unknown> {
|
|
const budgets = recordTarget(valueAtPath(state.cicd, "publishCurrent.stageBudgets"), "publishCurrent.stageBudgets");
|
|
return {
|
|
sourceSyncSeconds: numberAt(budgets, "sourceSyncSeconds"),
|
|
sourceFetchSeconds: numberAt(budgets, "sourceFetchSeconds"),
|
|
monitorWebVerifySeconds: numberAt(budgets, "monitorWebVerifySeconds"),
|
|
imageBuildSeconds: numberAt(budgets, "imageBuildSeconds"),
|
|
gitopsSeconds: numberAt(budgets, "gitopsSeconds"),
|
|
argoRuntimeSeconds: numberAt(budgets, "argoRuntimeSeconds"),
|
|
dashboardVerifySeconds: numberAt(budgets, "dashboardVerifySeconds"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publishCurrentDashboardPlan(state: SentinelCicdState): Record<string, unknown> {
|
|
const dashboard = recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard");
|
|
const viewport = stringAt(dashboard, "viewport");
|
|
if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(viewport)) throw new Error(`publishCurrent.dashboard.viewport must look like 1440x900, got ${viewport}`);
|
|
return {
|
|
enabled: booleanAt(dashboard, "enabled"),
|
|
required: booleanAt(dashboard, "required"),
|
|
viewport,
|
|
timeoutMs: numberAt(dashboard, "timeoutMs"),
|
|
waitTimeoutMs: numberAt(dashboard, "waitTimeoutMs"),
|
|
commandTimeoutSeconds: numberAt(dashboard, "commandTimeoutSeconds"),
|
|
fullPage: booleanAt(dashboard, "fullPage"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publishCurrentDashboardEnabled(state: SentinelCicdState): boolean {
|
|
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "enabled");
|
|
}
|
|
|
|
function publishCurrentDashboardRequired(state: SentinelCicdState): boolean {
|
|
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required");
|
|
}
|
|
|
|
function publishCurrentHealthValidationPlan(state: SentinelCicdState): Record<string, unknown> {
|
|
return {
|
|
enabled: true,
|
|
required: true,
|
|
endpoint: stringAt(state.runtime, "healthPath"),
|
|
source: "runtime.healthPath",
|
|
browser: false,
|
|
playwright: false,
|
|
webProbe: false,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publishCurrentDashboardReserveSeconds(state: SentinelCicdState): number {
|
|
if (!publishCurrentDashboardEnabled(state)) return 0;
|
|
const dashboard = publishCurrentDashboardPlan(state);
|
|
const budgets = publishCurrentStageBudgets(state);
|
|
return Math.max(0, Math.min(
|
|
numberAt(budgets, "dashboardVerifySeconds"),
|
|
numberAt(dashboard, "commandTimeoutSeconds"),
|
|
Math.ceil(numberAt(dashboard, "waitTimeoutMs") / 1000),
|
|
));
|
|
}
|
|
|
|
function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract<WebProbeSentinelOptions, { kind: "dashboard" }> {
|
|
const dashboard = publishCurrentDashboardPlan(state);
|
|
const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000));
|
|
const waitTimeoutMs = Math.min(numberAt(dashboard, "waitTimeoutMs"), remainingMs);
|
|
const timeoutMs = Math.min(numberAt(dashboard, "timeoutMs"), waitTimeoutMs);
|
|
return {
|
|
kind: "dashboard",
|
|
action: "verify",
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
viewport: stringAt(dashboard, "viewport"),
|
|
localDir: "/tmp",
|
|
name: null,
|
|
runId: null,
|
|
timeoutMs,
|
|
waitTimeoutMs,
|
|
timeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
|
|
commandTimeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
|
|
fullPage: booleanAt(dashboard, "fullPage"),
|
|
raw: false,
|
|
};
|
|
}
|
|
|
|
function publishCurrentStageTimings(controlResult: Record<string, unknown>, health: Record<string, unknown>, elapsedMs: number): Record<string, unknown> {
|
|
const publish = record(controlResult.publish);
|
|
const payload = record(publish.payload);
|
|
const payloadStageTimings = record(payload.stageTimings);
|
|
const diagnosticStageTimings = record(record(publish.diagnostics).stageTimings);
|
|
const stageTimings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings;
|
|
const observedWait = record(record(controlResult.observed).wait);
|
|
return {
|
|
sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs),
|
|
sourceFetchMs: finiteNumberOrNull(stageTimings.sourceFetchMs),
|
|
monitorWebVerifyMs: finiteNumberOrNull(stageTimings.monitorWebVerifyMs),
|
|
imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs),
|
|
gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs),
|
|
argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs),
|
|
healthValidationMs: finiteNumberOrNull(health.elapsedMs),
|
|
totalMs: elapsedMs,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publishCurrentSlowStages(state: SentinelCicdState, timings: Record<string, unknown>, budgetSeconds: number): Record<string, unknown>[] {
|
|
const budgets = publishCurrentStageBudgets(state);
|
|
const stageMap: Array<[string, string, string, string]> = [
|
|
["source-sync", "sourceSyncMs", "sourceSyncSeconds", "check git-mirror pre-sync, node-local mirror health and SSH-over-proxy latency"],
|
|
["source-fetch", "sourceFetchMs", "sourceFetchSeconds", "keep sparse checkout paths narrow and verify node-local git mirror object availability"],
|
|
["monitor-web-verify", "monitorWebVerifyMs", "monitorWebVerifySeconds", "keep monitor-web verification copy-only and avoid rebuilding frontend assets during publish"],
|
|
["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"],
|
|
["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"],
|
|
["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"],
|
|
["health-validation", "healthValidationMs", "dashboardVerifySeconds", "inspect runtime health endpoint latency and service routing"],
|
|
];
|
|
const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => {
|
|
const elapsed = finiteNumberOrNull(timings[timingKey]);
|
|
const stageBudgetSeconds = finiteNumberOrNull(budgets[budgetKey]);
|
|
if (elapsed === null || stageBudgetSeconds === null || elapsed <= stageBudgetSeconds * 1000) return [];
|
|
return [{ stage, elapsedMs: elapsed, budgetSeconds: stageBudgetSeconds, suggestion, valuesRedacted: true }];
|
|
});
|
|
const total = finiteNumberOrNull(timings.totalMs);
|
|
if (total !== null && total > budgetSeconds * 1000) {
|
|
slow.push({
|
|
stage: "total",
|
|
elapsedMs: total,
|
|
budgetSeconds,
|
|
suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or health endpoint segment before rerun",
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
return slow;
|
|
}
|
|
|
|
function publishCurrentBudgetWarnings(slowStages: readonly Record<string, unknown>[], withinBudget: boolean, budgetSeconds: number, elapsedMs: number): string[] {
|
|
const warnings = slowStages.map((stage) => `${text(stage.stage)} exceeded configured ${text(stage.budgetSeconds)}s budget (${Math.round((finiteNumberOrNull(stage.elapsedMs) ?? 0) / 1000)}s): ${text(stage.suggestion)}`);
|
|
if (!withinBudget) warnings.unshift(`publish-current exceeded configured ${budgetSeconds}s end-to-end budget (${Math.round(elapsedMs / 1000)}s); no further blind wait was attempted.`);
|
|
return warnings;
|
|
}
|
|
|
|
function publishCurrentBlocker(controlResult: Record<string, unknown>, health: Record<string, unknown>, withinBudget: boolean): Record<string, unknown> {
|
|
if (controlResult.ok !== true) {
|
|
const blocker = record(controlResult.blocker);
|
|
return {
|
|
code: blocker.code ?? "sentinel-publish-current-control-plane-blocked",
|
|
reason: blocker.reason ?? "control-plane publish did not converge",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
if (health.ok !== true) {
|
|
const degradedReason = text(health.degradedReason);
|
|
return {
|
|
code: health.skipped === true ? text(health.reason) : "sentinel-publish-current-health-endpoint-failed",
|
|
reason: health.skipped === true
|
|
? "health endpoint validation did not run"
|
|
: degradedReason === "-"
|
|
? "health endpoint validation did not pass"
|
|
: `health endpoint validation did not pass: ${degradedReason}`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
if (!withinBudget) {
|
|
return {
|
|
code: "sentinel-publish-current-over-budget",
|
|
reason: "runtime and health endpoint converged, but the one-click CI/CD path exceeded the YAML end-to-end budget",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
return { code: "sentinel-publish-current-blocked", reason: "publish-current did not satisfy all checks", valuesRedacted: true };
|
|
}
|
|
|
|
function publishCurrentNext(state: SentinelCicdState): Record<string, string> {
|
|
const node = state.spec.nodeId;
|
|
const lane = state.spec.lane;
|
|
const suffix = sentinelCliSuffix(state);
|
|
return {
|
|
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait`,
|
|
controlPlaneStatus: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
|
|
dashboardVerify: `bun scripts/cli.ts web-probe sentinel dashboard verify --node ${node} --lane ${lane}${suffix}`,
|
|
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`,
|
|
gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm`,
|
|
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`,
|
|
};
|
|
}
|
|
|
|
function renderSentinelManifests(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sentinelId: string,
|
|
runtime: Record<string, unknown>,
|
|
cicd: Record<string, unknown>,
|
|
scenarios: unknown,
|
|
publicExposure: Record<string, unknown>,
|
|
secrets: Record<string, unknown>,
|
|
image: SentinelImagePlan,
|
|
): readonly Record<string, unknown>[] {
|
|
const namespace = stringAt(runtime, "namespace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": stringAt(runtime, "deploymentName"),
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
"unidesk.ai/web-probe-sentinel-id": sentinelId,
|
|
};
|
|
const deploymentName = stringAt(runtime, "deploymentName");
|
|
const serviceName = stringAt(runtime, "serviceName");
|
|
const servicePort = numberAt(runtime, "servicePort");
|
|
const pvcStorage = stringAt(runtime, "pvcStorage");
|
|
const stateRoot = stringAt(runtime, "stateRoot");
|
|
const sentinelEnv = sentinelContainerEnv(sentinelId, runtime, cicd, secrets);
|
|
const kubernetesApiEgress = sentinelKubernetesApiEgress(runtime);
|
|
const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv);
|
|
if (cadenceJob !== null) {
|
|
emitWebProbeSentinelSpan({
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sentinelId,
|
|
namespace,
|
|
runtime,
|
|
cicd,
|
|
}, "web_probe_sentinel.cadence.cronjob_rendered", {
|
|
cronJobName: record(cadenceJob.metadata).name ?? null,
|
|
namespace,
|
|
cadence: record(cadenceJob.metadata).annotations === undefined ? null : record(record(cadenceJob.metadata).annotations)["unidesk.ai/cadence"],
|
|
schedule: record(cadenceJob.spec).schedule ?? null,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
return [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ServiceAccount",
|
|
metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels },
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "Role",
|
|
metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels },
|
|
rules: [
|
|
{ apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get"] },
|
|
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "list"] },
|
|
],
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "RoleBinding",
|
|
metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels },
|
|
subjects: [{ kind: "ServiceAccount", name: stringAt(runtime, "serviceAccountName"), namespace }],
|
|
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${deploymentName}-manual-trigger` },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "PersistentVolumeClaim",
|
|
metadata: { name: stringAt(runtime, "pvcName"), namespace, labels },
|
|
spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: pvcStorage } } },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ConfigMap",
|
|
metadata: { name: `${deploymentName}-config`, namespace, labels },
|
|
data: {
|
|
"config-summary.json": JSON.stringify({
|
|
specRef: SPEC_REF,
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sentinelId,
|
|
publicBaseUrl: stringAt(publicExposure, "publicBaseUrl"),
|
|
routePrefix: stringAtNullable(publicExposure, "routePrefix") ?? "/",
|
|
gitopsPath: stringAt(cicd, "gitopsPath"),
|
|
valuesRedacted: true,
|
|
}, null, 2),
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name: deploymentName, namespace, labels },
|
|
spec: {
|
|
replicas: numberAt(runtime, "replicas"),
|
|
selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
serviceAccountName: stringAt(runtime, "serviceAccountName"),
|
|
containers: [{
|
|
name: "sentinel",
|
|
image: image.ref,
|
|
imagePullPolicy: "IfNotPresent",
|
|
args: [
|
|
"--node",
|
|
spec.nodeId,
|
|
"--lane",
|
|
spec.lane,
|
|
"--sentinel",
|
|
sentinelId,
|
|
"--state-root",
|
|
stateRoot,
|
|
"--host",
|
|
stringAt(runtime, "listenHost"),
|
|
"--port",
|
|
String(servicePort),
|
|
],
|
|
env: sentinelEnv,
|
|
ports: [{ name: "http", containerPort: servicePort }],
|
|
readinessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
|
|
livenessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
|
|
volumeMounts: [{ name: "state", mountPath: stateRoot }],
|
|
}],
|
|
volumes: [{ name: "state", persistentVolumeClaim: { claimName: stringAt(runtime, "pvcName") } }],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
metadata: { name: serviceName, namespace, labels },
|
|
spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] },
|
|
},
|
|
...(cadenceJob === null ? [] : [cadenceJob]),
|
|
{
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name: stringAt(publicExposure, "frpc.deploymentName"), namespace, labels: { ...labels, "app.kubernetes.io/component": "tunnel" } },
|
|
spec: {
|
|
replicas: 1,
|
|
selector: { matchLabels: { "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName") } },
|
|
template: {
|
|
metadata: {
|
|
labels: { ...labels, "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName"), "app.kubernetes.io/component": "tunnel" },
|
|
annotations: {
|
|
"unidesk.ai/public-base-url": stringAt(publicExposure, "publicBaseUrl"),
|
|
"unidesk.ai/frp-server": `${stringAt(publicExposure, "frpc.serverAddr")}:${numberAt(publicExposure, "frpc.serverPort")}`,
|
|
"unidesk.ai/frp-remote-port": String(numberAt(publicExposure, "frpc.httpProxy.remotePort")),
|
|
},
|
|
},
|
|
spec: {
|
|
containers: [{
|
|
name: "frpc",
|
|
image: stringAt(publicExposure, "frpc.image"),
|
|
imagePullPolicy: "IfNotPresent",
|
|
args: ["-c", "/etc/frp/frpc.toml"],
|
|
volumeMounts: [{ name: "frpc-config", mountPath: "/etc/frp/frpc.toml", subPath: stringAt(publicExposure, "frpc.secretKey"), readOnly: true }],
|
|
}],
|
|
volumes: [{ name: "frpc-config", secret: { secretName: stringAt(publicExposure, "frpc.secretName") } }],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "networking.k8s.io/v1",
|
|
kind: "NetworkPolicy",
|
|
metadata: { name: `${deploymentName}-egress`, namespace, labels },
|
|
spec: {
|
|
podSelector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
|
|
policyTypes: ["Ingress", "Egress"],
|
|
ingress: [{ from: [{ namespaceSelector: {} }], ports: [{ protocol: "TCP", port: servicePort }] }],
|
|
egress: [
|
|
{ to: [{ namespaceSelector: {} }] },
|
|
...kubernetesApiEgress,
|
|
],
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "argoproj.io/v1alpha1",
|
|
kind: "Application",
|
|
metadata: { name: stringAt(cicd, "argo.applicationName"), namespace: stringAt(cicd, "argo.namespace"), labels },
|
|
spec: {
|
|
project: stringAt(cicd, "argo.projectName"),
|
|
source: {
|
|
repoURL: stringAt(cicd, "argo.repoURL"),
|
|
targetRevision: stringAt(cicd, "argo.targetRevision"),
|
|
path: stringAt(cicd, "gitopsPath"),
|
|
},
|
|
destination: { server: "https://kubernetes.default.svc", namespace },
|
|
syncPolicy: { automated: { prune: true, selfHeal: true } },
|
|
},
|
|
},
|
|
];
|
|
}
|
|
|
|
function sentinelKubernetesApiEgress(runtime: Record<string, unknown>): readonly Record<string, unknown>[] {
|
|
if (booleanAtNullable(runtime, "kubernetesApi.egress.enabled") !== true) return [];
|
|
const rules = arrayAtNullable(runtime, "kubernetesApi.egress.rules");
|
|
const rows = rules.length > 0 ? rules : [{ cidr: stringAt(runtime, "kubernetesApi.egress.cidr"), port: numberAt(runtime, "kubernetesApi.egress.port") }];
|
|
return rows.map((rule) => ({
|
|
to: [{ ipBlock: { cidr: stringAt(rule, "cidr") } }],
|
|
ports: [{ protocol: "TCP", port: numberAt(rule, "port") }],
|
|
}));
|
|
}
|
|
|
|
function sentinelContainerEnv(sentinelId: string, runtime: Record<string, unknown>, cicd: Record<string, unknown>, secrets: Record<string, unknown>): readonly Record<string, unknown>[] {
|
|
const env: Record<string, unknown>[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }];
|
|
const otelEnabled = booleanAtNullable(runtime, "observability.otel.enabled") ?? booleanAtNullable(cicd, "observability.otel.enabled") ?? false;
|
|
const otelEndpoint = stringAtNullable(runtime, "observability.otel.tracesEndpoint")
|
|
?? stringAtNullable(runtime, "observability.otel.endpoint")
|
|
?? stringAtNullable(cicd, "observability.otel.tracesEndpoint")
|
|
?? stringAtNullable(cicd, "observability.otel.endpoint");
|
|
const otelServiceName = stringAtNullable(runtime, "observability.otel.serviceName") ?? stringAtNullable(cicd, "observability.otel.serviceName");
|
|
const otelSampler = stringAtNullable(runtime, "observability.otel.sampler") ?? stringAtNullable(cicd, "observability.otel.sampler");
|
|
const otelSamplerArg = stringAtNullable(runtime, "observability.otel.samplerArg") ?? stringAtNullable(cicd, "observability.otel.samplerArg");
|
|
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
|
for (const source of arrayAt(secrets, "sources").map(record)) {
|
|
const purpose = stringAtNullable(source, "purpose");
|
|
if (purpose !== null) sourcesByPurpose.set(purpose, source);
|
|
}
|
|
const used = new Set(env.map((item) => String(item.name ?? "")));
|
|
const pushEnv = (item: Record<string, unknown>): void => {
|
|
const name = String(item.name ?? "");
|
|
if (name.length === 0 || used.has(name)) return;
|
|
used.add(name);
|
|
env.push(item);
|
|
};
|
|
if (otelEnabled) {
|
|
if (otelEndpoint !== null) pushEnv({ name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: otelEndpoint });
|
|
if (otelServiceName !== null) pushEnv({ name: "OTEL_SERVICE_NAME", value: otelServiceName });
|
|
if (otelSampler !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER", value: otelSampler });
|
|
if (otelSamplerArg !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER_ARG", value: otelSamplerArg });
|
|
}
|
|
for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) {
|
|
const secretName = stringAtNullable(runtimeSecret, "name");
|
|
if (secretName === null) continue;
|
|
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
|
|
const targetKey = stringAtNullable(item, "targetKey");
|
|
const sourcePurpose = stringAtNullable(item, "sourcePurpose");
|
|
const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey");
|
|
const sourceKeyEnvName = sourcePurpose === "bootstrap-admin" || sourcePurpose === "prompt-set" ? sourceKey : null;
|
|
if (targetKey !== null && sourceKeyEnvName !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKeyEnvName)) {
|
|
pushEnv({ name: sourceKeyEnvName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
|
|
}
|
|
const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
|
|
if (envName === null) continue;
|
|
pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
|
|
}
|
|
}
|
|
return env;
|
|
}
|
|
|
|
function sentinelCadenceCronJobPlan(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sentinelId: string,
|
|
runtime: Record<string, unknown>,
|
|
cicd: Record<string, unknown>,
|
|
scenarios: unknown,
|
|
imageRef: string,
|
|
sentinelEnv: readonly Record<string, unknown>[],
|
|
): Record<string, unknown> | null {
|
|
const scheduler = record(valueAtPath(cicd, "targetValidation.cadenceScheduler"));
|
|
const cadenceSchedulerEnabled = booleanAtNullable(cicd, "targetValidation.cadenceScheduler.enabled") === true;
|
|
if (!cadenceSchedulerEnabled) return null;
|
|
const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId");
|
|
if (scenarioId === null) return null;
|
|
const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null;
|
|
if (scenario === null) return null;
|
|
const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null;
|
|
const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds);
|
|
if (schedule === null) return null;
|
|
const namespace = stringAt(runtime, "namespace");
|
|
const deploymentName = stringAt(runtime, "deploymentName");
|
|
const serviceAccountName = stringAt(runtime, "serviceAccountName");
|
|
const timeoutSeconds = numberAt(cicd, "targetValidation.maxSeconds");
|
|
const activeDeadlineSlackSeconds = numberAt(scheduler, "activeDeadlineSlackSeconds");
|
|
const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost");
|
|
const name = sentinelCadenceCronJobName(deploymentName);
|
|
const concurrencyPolicy = stringAt(scheduler, "concurrencyPolicy");
|
|
if (!["Allow", "Forbid", "Replace"].includes(concurrencyPolicy)) throw new Error("targetValidation.cadenceScheduler.concurrencyPolicy must be Allow, Forbid or Replace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": name,
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"app.kubernetes.io/component": "cadence-scheduler",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
"unidesk.ai/web-probe-sentinel-id": sentinelId,
|
|
};
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "CronJob",
|
|
metadata: {
|
|
name,
|
|
namespace,
|
|
labels,
|
|
annotations: {
|
|
"unidesk.ai/cadence": String(scenario.cadence),
|
|
"unidesk.ai/target-validation-max-seconds": String(timeoutSeconds),
|
|
"unidesk.ai/source": "targetValidation.cadenceScheduler",
|
|
},
|
|
},
|
|
spec: {
|
|
schedule,
|
|
concurrencyPolicy,
|
|
successfulJobsHistoryLimit: numberAt(scheduler, "successfulJobsHistoryLimit"),
|
|
failedJobsHistoryLimit: numberAt(scheduler, "failedJobsHistoryLimit"),
|
|
startingDeadlineSeconds: numberAt(scheduler, "startingDeadlineSeconds"),
|
|
jobTemplate: {
|
|
spec: {
|
|
activeDeadlineSeconds: timeoutSeconds + activeDeadlineSlackSeconds,
|
|
ttlSecondsAfterFinished: numberAt(scheduler, "ttlSecondsAfterFinished"),
|
|
backoffLimit: numberAt(scheduler, "backoffLimit"),
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
serviceAccountName,
|
|
containers: [{
|
|
name: "quick-verify",
|
|
image: imageRef,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["bun", "scripts/cli.ts"],
|
|
args: [
|
|
"web-probe",
|
|
"sentinel",
|
|
"validate",
|
|
"--node",
|
|
spec.nodeId,
|
|
"--lane",
|
|
spec.lane,
|
|
"--sentinel",
|
|
sentinelId,
|
|
"--quick-verify",
|
|
"--confirm",
|
|
"--wait",
|
|
"--timeout-seconds",
|
|
String(timeoutSeconds),
|
|
],
|
|
env: [
|
|
...sentinelEnv,
|
|
{ name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" },
|
|
...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]),
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelCadenceCronJobName(deploymentName: string): string {
|
|
return safeKubernetesSegment(`${deploymentName}-quick-verify`, 52);
|
|
}
|
|
|
|
function scenarioRows(value: unknown): Record<string, unknown>[] {
|
|
if (Array.isArray(value)) return value.map(record);
|
|
if (!isRecord(value)) return [];
|
|
if (Array.isArray(value.scenarios)) return value.scenarios.map(record);
|
|
if (isRecord(value.workflow)) return [value.workflow];
|
|
return [value];
|
|
}
|
|
|
|
function parseDurationSeconds(value: string): number | null {
|
|
const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim());
|
|
if (match === null) return null;
|
|
const amount = Number(match[1]);
|
|
const unit = match[2];
|
|
if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000));
|
|
if (unit === "s") return Math.max(60, amount);
|
|
if (unit === "m") return amount * 60;
|
|
if (unit === "h") return amount * 3600;
|
|
return null;
|
|
}
|
|
|
|
function cronScheduleForCadenceSeconds(seconds: number): string | null {
|
|
if (!Number.isFinite(seconds) || seconds <= 0) return null;
|
|
if (seconds <= 60) return "* * * * *";
|
|
if (seconds % 60 === 0) {
|
|
const minutes = Math.trunc(seconds / 60);
|
|
if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`;
|
|
if (minutes % 60 === 0) {
|
|
const hours = Math.trunc(minutes / 60);
|
|
if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`;
|
|
if (hours === 24) return "0 0 * * *";
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null {
|
|
if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null;
|
|
const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, "");
|
|
return segment.length === 0 ? null : `HWLAB_WEB_${segment}_JSON`;
|
|
}
|
|
|
|
export function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record<string, unknown> } {
|
|
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
|
for (const source of arrayAt(state.secrets, "sources").map(record)) {
|
|
const purpose = stringAtNullable(source, "purpose");
|
|
if (purpose !== null) sourcesByPurpose.set(purpose, source);
|
|
}
|
|
const env: NodeJS.ProcessEnv = {};
|
|
const items: Record<string, unknown>[] = [];
|
|
const missing: Record<string, unknown>[] = [];
|
|
for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) {
|
|
const secretName = stringAtNullable(runtimeSecret, "name");
|
|
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
|
|
const targetKey = stringAtNullable(item, "targetKey");
|
|
const sourcePurpose = stringAtNullable(item, "sourcePurpose");
|
|
const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
|
|
if (envName === null || sourcePurpose === null || targetKey === null) continue;
|
|
const source = sourcesByPurpose.get(sourcePurpose);
|
|
const runtimeValue = process.env[envName];
|
|
if (source === undefined) {
|
|
if (runtimeValue !== undefined && runtimeValue.length > 0) {
|
|
env[envName] = runtimeValue;
|
|
items.push({
|
|
envName,
|
|
secretName,
|
|
targetKey,
|
|
sourcePurpose,
|
|
sourceMode: "runtime-env",
|
|
fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
});
|
|
continue;
|
|
}
|
|
missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const sourceRef = stringAt(source, "sourceRef");
|
|
const sourceKey = stringAt(source, "sourceKey");
|
|
const material = readSentinelSecretSourceValue(source);
|
|
if (!material.ok) {
|
|
if (runtimeValue !== undefined && runtimeValue.length > 0) {
|
|
env[envName] = runtimeValue;
|
|
items.push({
|
|
envName,
|
|
secretName,
|
|
targetKey,
|
|
sourcePurpose,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourceMode: "runtime-env",
|
|
fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
});
|
|
continue;
|
|
}
|
|
missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const value = stringAt(material, "value");
|
|
env[envName] = value;
|
|
items.push({
|
|
envName,
|
|
secretName,
|
|
targetKey,
|
|
sourcePurpose,
|
|
sourceRef,
|
|
sourceKey,
|
|
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
}
|
|
const summary = {
|
|
ok: missing.length === 0,
|
|
envCount: items.length,
|
|
items,
|
|
missing,
|
|
valuesRedacted: true,
|
|
};
|
|
return { ok: missing.length === 0, env, summary };
|
|
}
|
|
|
|
function normalizeRoutePrefix(value: string | null): string {
|
|
if (value === null || value.trim() === "" || value.trim() === "/") return "/";
|
|
const prefixed = value.trim().startsWith("/") ? value.trim() : `/${value.trim()}`;
|
|
return prefixed.replace(/\/+$/u, "") || "/";
|
|
}
|
|
|
|
function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const endpoint = stringAt(state.controlPlaneNode, "registry.endpoint");
|
|
const repoTag = state.image.ref.replace(`${endpoint}/`, "");
|
|
const repo = repoTag.slice(0, repoTag.lastIndexOf(":"));
|
|
const tag = repoTag.slice(repoTag.lastIndexOf(":") + 1);
|
|
const registryMode = stringAtNullable(state.controlPlaneNode, "registry.mode") ?? "host-docker";
|
|
const endpointIsLoopback = endpoint === "127.0.0.1:5000" || endpoint.startsWith("127.0.0.1:");
|
|
const probeMode = registryMode === "k8s-workload" && !endpointIsLoopback ? "k8s-service" : "node-loopback";
|
|
const url = probeMode === "k8s-service"
|
|
? `http://${stringAt(state.controlPlaneNode, "registry.serviceName")}.${stringAt(state.controlPlaneNode, "registry.namespace")}.svc.cluster.local:${numberAtNullable(state.controlPlaneNode, "registry.containerPort") ?? 5000}/v2/${repo}/manifests/${tag}`
|
|
: `http://${endpoint}/v2/${repo}/manifests/${tag}`;
|
|
const script = [
|
|
"set +e",
|
|
`url=${shellQuote(url)}`,
|
|
`probe_mode=${shellQuote(probeMode)}`,
|
|
"headers=$(mktemp)",
|
|
"accept='application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json'",
|
|
"if command -v curl >/dev/null 2>&1 && curl -fsSI -H \"Accept: $accept\" --max-time 5 \"$url\" >\"$headers\" 2>/tmp/web-probe-sentinel-image.err; then present=true; else present=false; fi",
|
|
"digest=$(awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}' \"$headers\" 2>/dev/null)",
|
|
"python3 - \"$present\" \"$digest\" \"$url\" \"$probe_mode\" <<'PY'",
|
|
"import json, sys",
|
|
"print(json.dumps({'present': sys.argv[1] == 'true', 'digest': sys.argv[2] or None, 'url': sys.argv[3], 'mode': sys.argv[4], 'valuesRedacted': True}))",
|
|
"PY",
|
|
].join("\n");
|
|
const result = probeMode === "k8s-service"
|
|
? runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", stringAt(state.cicd, "builder.namespace"), "exec", "deploy/git-mirror-http", "--", "sh", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 })
|
|
: runCommand(["trans", stringAt(state.controlPlaneNode, "route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
probe: parseJsonObject(result.stdout),
|
|
result: compactCommand(result),
|
|
};
|
|
}
|
|
|
|
function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
|
|
const startedAt = Date.now();
|
|
const command = "web-probe sentinel image build";
|
|
const sourceMirrorProbe = probeSourceMirror(state, Math.min(options.timeoutSeconds, 20));
|
|
const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds);
|
|
const sourceMirrorReady = sourceMirrorSync.ok === true;
|
|
const publish = sourceMirrorReady
|
|
? runSentinelPublishJob(state, false, options.timeoutSeconds, false)
|
|
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
|
|
const registry = probeImageRegistry(state, options.timeoutSeconds);
|
|
const registryReady = record(registry.probe).present === true;
|
|
const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publish.ok === true && registryReady;
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state);
|
|
const result = {
|
|
ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: true,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
registry,
|
|
sourceMirrorSync,
|
|
publish,
|
|
elapsedMs,
|
|
warnings: [
|
|
...sentinelCicdElapsedWarnings(elapsedMs, "sentinel image build confirm-wait", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds),
|
|
...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"),
|
|
...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
|
|
...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
|
|
],
|
|
blocker: ok
|
|
? null
|
|
: !sourceMirrorReady
|
|
? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" }
|
|
: publish.ok !== true
|
|
? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" }
|
|
: { code: "sentinel-image-registry-missing", reason: "image publish completed but expected registry tag is not visible" },
|
|
next: {
|
|
status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
controlPlaneTrigger: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderImageResult(result));
|
|
}
|
|
|
|
function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
|
|
const result = sentinelControlPlaneConfirmedResult(state, options);
|
|
return rendered(result.ok === true, String(result.command), renderControlPlaneResult(result));
|
|
}
|
|
|
|
function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): Record<string, unknown> {
|
|
const startedAt = Date.now();
|
|
const command = `web-probe sentinel control-plane ${options.action}`;
|
|
const applyOnly = options.action === "apply";
|
|
const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state);
|
|
const waitBudgetSeconds = Math.max(1, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds));
|
|
const deadline = startedAt + waitBudgetSeconds * 1000;
|
|
const remainingCicdWaitSeconds = () => strictRemainingSeconds(deadline, waitBudgetSeconds);
|
|
const remainingCommandSeconds = () => Math.max(1, remainingCicdWaitSeconds());
|
|
const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCommandSeconds(), 20));
|
|
const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCommandSeconds());
|
|
const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true;
|
|
const publish = applyOnly
|
|
? null
|
|
: sourceMirrorReady
|
|
? runSentinelPublishJob(state, true, remainingCommandSeconds(), options.rerun)
|
|
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
|
|
const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8;
|
|
const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true
|
|
? startSentinelGitMirrorFlushAsync(state)
|
|
: null;
|
|
const runtimeSecretsApply = publishWaitBudgetExhausted
|
|
? sentinelSkippedControlStep("publish-wait-budget-exhausted")
|
|
: applySentinelRuntimeSecrets(state, remainingCommandSeconds());
|
|
const publicExposureApply = publishWaitBudgetExhausted
|
|
? sentinelSkippedControlStep("publish-wait-budget-exhausted")
|
|
: applySentinelPublicExposure(state, remainingCommandSeconds());
|
|
const argoApply = publishWaitBudgetExhausted
|
|
? sentinelSkippedControlStep("publish-wait-budget-exhausted")
|
|
: applySentinelArgoApplication(state, remainingCommandSeconds());
|
|
const observed = publishWaitBudgetExhausted
|
|
? sentinelSkippedObservedStatus("publish-wait-budget-exhausted")
|
|
: waitForSentinelObservedStatus(state, remainingCommandSeconds(), undefined, false);
|
|
const observedReady = sentinelObservedReady(observed);
|
|
const publishReady = applyOnly || record(publish).ok === true || observedReady;
|
|
const flushReady = applyOnly || record(flush).ok === true || observedReady;
|
|
const targetValidation = null;
|
|
const targetValidationBlocked = false;
|
|
const ok = state.configReady
|
|
&& state.sourceHead.ok
|
|
&& sourceMirrorReady
|
|
&& publishReady
|
|
&& flushReady
|
|
&& record(runtimeSecretsApply).ok === true
|
|
&& record(publicExposureApply).ok === true
|
|
&& record(argoApply).ok === true
|
|
&& observedReady;
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const blocker = ok ? null : {
|
|
code: !sourceMirrorReady
|
|
? "sentinel-source-mirror-sync-failed"
|
|
: !publishReady
|
|
? "sentinel-image-gitops-publish-not-ready"
|
|
: record(runtimeSecretsApply).ok === false
|
|
? "sentinel-runtime-secret-sync-failed"
|
|
: "sentinel-control-plane-not-ready",
|
|
reason: !sourceMirrorReady
|
|
? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish"
|
|
: !publishReady
|
|
? "Tekton publish did not finish and runtime observation has not proven the selected source/image/GitOps state yet"
|
|
: record(runtimeSecretsApply).ok === false
|
|
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
|
|
: "one or more publicExposure, Argo or runtime observation checks did not pass",
|
|
};
|
|
const publishPipelineRun = applyOnly ? sentinelPipelineRunName(state, options.rerun) : record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun);
|
|
const result = {
|
|
ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: true,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
pipelineRun: publishPipelineRun,
|
|
gitops: {
|
|
path: stringAt(state.cicd, "gitopsPath"),
|
|
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
|
manifestObjects: state.manifests.length,
|
|
manifestSha256: state.manifestSha256,
|
|
},
|
|
argo: {
|
|
namespace: stringAt(state.cicd, "argo.namespace"),
|
|
projectName: stringAt(state.cicd, "argo.projectName"),
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
},
|
|
validation: {
|
|
scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
|
|
maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
|
|
controlPlaneWaitMaxSeconds: cicdWaitWarningSeconds,
|
|
quickVerifyMode: applyOnly ? "not-applicable" : "manual-validate",
|
|
automaticSecondPath: false,
|
|
},
|
|
manifests: {
|
|
objects: manifestObjectSummary(state.manifests),
|
|
sha256: state.manifestSha256,
|
|
},
|
|
sourceMirrorSync,
|
|
publish,
|
|
flush,
|
|
runtimeSecretsApply,
|
|
publicExposureApply,
|
|
argoApply,
|
|
observed,
|
|
targetValidation,
|
|
elapsedMs,
|
|
warnings: Array.from(new Set([
|
|
...sentinelCicdElapsedWarnings(elapsedMs, "sentinel control-plane confirm-wait", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds),
|
|
...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"),
|
|
...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
|
|
...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds),
|
|
...asyncGitMirrorFlushWarnings(flush, cicdWaitWarningSeconds),
|
|
...publishSatisfiedByObservedWarnings(publish, flush, observedReady),
|
|
...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
|
|
...sentinelObservedWarnings(observed),
|
|
...(publishWaitBudgetExhausted ? [`sentinel publish consumed the configured ${waitBudgetSeconds}s confirm-wait budget; skipped runtime Secret, public exposure, Argo apply and observed wait to avoid blind over-budget waiting. Use the reported status/log drill-down after the PipelineRun advances.`] : []),
|
|
...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds),
|
|
...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []),
|
|
...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []),
|
|
])),
|
|
blocker,
|
|
recoveryNext: controlPlaneRecoveryNext(state, ok, publish, flush, observed),
|
|
next: controlPlaneNext(state, options.action),
|
|
valuesRedacted: true,
|
|
};
|
|
return result;
|
|
}
|
|
|
|
function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane" | "publish", action: string, timeoutSeconds: number): RenderedCliResult {
|
|
const args = domain === "image"
|
|
? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
|
|
: domain === "control-plane"
|
|
? ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
|
|
: ["web-probe", "sentinel", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
|
|
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`);
|
|
const command = domain === "publish" ? `web-probe sentinel ${action}` : `web-probe sentinel ${domain} ${action}`;
|
|
const result = {
|
|
ok: true,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "async-job",
|
|
mutation: true,
|
|
reason: "confirmed sentinel publish/build can exceed the short interactive window; use job status for bounded progress.",
|
|
job,
|
|
next: {
|
|
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
|
|
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(true, command, renderAsyncJobResult(result));
|
|
}
|
|
|
|
function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record<string, unknown> {
|
|
const args = ["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"];
|
|
const job = startJob(
|
|
`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_git_mirror_flush`,
|
|
["bun", "scripts/cli.ts", ...args],
|
|
`Flush HWLAB ${state.spec.lane} git mirror after web-probe sentinel ${state.sentinelId} GitOps publish for node ${state.spec.nodeId}`,
|
|
);
|
|
return {
|
|
ok: true,
|
|
mode: "async-job",
|
|
job,
|
|
next: {
|
|
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
|
|
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
|
|
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): string[] {
|
|
const item = record(flush);
|
|
if (item.mode !== "async-job") return [];
|
|
const next = record(item.next);
|
|
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
|
|
}
|
|
|
|
function sentinelSkippedControlStep(reason: string): Record<string, unknown> {
|
|
return { ok: false, skipped: true, reason, valuesRedacted: true };
|
|
}
|
|
|
|
function sentinelSkippedObservedStatus(reason: string): SentinelObservedStatus {
|
|
const skipped = { ok: false, skipped: true, reason, valuesRedacted: true };
|
|
return {
|
|
sourceMirror: skipped,
|
|
registry: skipped,
|
|
gitMirror: { skipped: true, reason, valuesRedacted: true },
|
|
gitops: skipped,
|
|
argo: skipped,
|
|
runtime: skipped,
|
|
cadence: skipped,
|
|
wait: {
|
|
polls: 0,
|
|
elapsedMs: 0,
|
|
timeoutMs: 0,
|
|
ready: false,
|
|
includeGitMirror: false,
|
|
reason,
|
|
valuesRedacted: true,
|
|
},
|
|
};
|
|
}
|
|
|
|
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
|
|
const registry = probeImageRegistry(state, timeoutSeconds);
|
|
const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds);
|
|
const effectiveExpectation = {
|
|
gitopsRevision: expectation?.gitopsRevision ?? nonEmptyString(gitops.revision),
|
|
runtimeImage: expectation?.runtimeImage ?? nonEmptyString(gitops.image) ?? expectedRuntimeImageFromRegistry(state, registry),
|
|
};
|
|
return {
|
|
sourceMirror: probeSourceMirror(state, timeoutSeconds),
|
|
registry,
|
|
gitMirror: includeGitMirror
|
|
? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds)
|
|
: { ok: true, skipped: true, reason: "deferred-to-async-flush", valuesRedacted: true },
|
|
gitops,
|
|
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
|
|
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
|
|
cadence: probeCadenceCronJob(state, timeoutSeconds),
|
|
};
|
|
}
|
|
|
|
function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
|
let polls = 1;
|
|
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
|
polls += 1;
|
|
}
|
|
return {
|
|
...observed,
|
|
wait: {
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
timeoutMs,
|
|
ready: sentinelObservedReady(observed),
|
|
includeGitMirror,
|
|
valuesRedacted: true,
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelObservedReady(value: Record<string, unknown> | SentinelObservedStatus): boolean {
|
|
const observed = record(value);
|
|
const gitMirror = record(observed.gitMirror);
|
|
const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true;
|
|
return record(observed.sourceMirror).ok === true
|
|
&& record(record(observed.registry).probe).present === true
|
|
&& gitMirrorReady
|
|
&& record(observed.gitops).ok === true
|
|
&& record(observed.argo).ok === true
|
|
&& record(observed.runtime).ok === true
|
|
&& record(observed.cadence).ok === true;
|
|
}
|
|
|
|
function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObservedStatus | null): string[] {
|
|
const observed = record(value);
|
|
const argo = record(observed.argo);
|
|
const cadence = record(observed.cadence);
|
|
return mergeWarnings(argo.warning, cadence.warning);
|
|
}
|
|
|
|
function sentinelObservedStatusDiagnosis(state: SentinelCicdState, value: unknown, pipelineRun: string): Record<string, unknown> | null {
|
|
const observed = record(value);
|
|
if (sentinelObservedReady(observed)) return null;
|
|
const sourceMirror = record(observed.sourceMirror);
|
|
const registryProbe = record(record(observed.registry).probe);
|
|
const gitMirror = record(observed.gitMirror);
|
|
const gitops = record(observed.gitops);
|
|
const argo = record(observed.argo);
|
|
const runtimeDeployment = record(record(record(observed.runtime).probe).deployment);
|
|
const cadence = record(observed.cadence);
|
|
const sourceReady = sourceMirror.ok === true;
|
|
const registryPresent = registryProbe.present === true;
|
|
const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true;
|
|
const gitopsReady = gitops.ok === true;
|
|
const argoReady = argo.ok === true;
|
|
const runtimeReady = record(observed.runtime).ok === true;
|
|
const cadenceReady = cadence.ok === true;
|
|
const code = sourceReady && !registryPresent
|
|
? "sentinel-publish-half-state-registry-missing"
|
|
: registryPresent && !gitopsReady
|
|
? "sentinel-gitops-manifest-not-updated"
|
|
: gitopsReady && (!argoReady || !runtimeReady)
|
|
? "sentinel-runtime-not-aligned"
|
|
: !gitMirrorReady
|
|
? "sentinel-git-mirror-not-in-sync"
|
|
: !cadenceReady
|
|
? "sentinel-cadence-not-ready"
|
|
: "sentinel-control-plane-observed-not-ready";
|
|
const reason = code === "sentinel-publish-half-state-registry-missing"
|
|
? "source mirror contains the selected commit, but the expected registry tag is missing; retry publish-current so Tekton builds/pushes the image and advances GitOps/runtime"
|
|
: code === "sentinel-gitops-manifest-not-updated"
|
|
? "registry contains the selected image, but GitOps manifest is not yet updated to a digest-pinned runtime image"
|
|
: code === "sentinel-runtime-not-aligned"
|
|
? "GitOps manifest is present, but Argo or runtime objects have not converged to the selected image yet"
|
|
: code === "sentinel-git-mirror-not-in-sync"
|
|
? "runtime is aligned, but git-mirror status is not in sync; run the controlled git-mirror sync/status path before treating the control plane as fully healthy"
|
|
: code === "sentinel-cadence-not-ready"
|
|
? "runtime is aligned, but cadence CronJob validation did not pass"
|
|
: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass";
|
|
const next = publishCurrentNext(state);
|
|
const controlNext = controlPlaneNext(state, "trigger-current");
|
|
const shouldRetryPublish = code === "sentinel-publish-half-state-registry-missing" || code === "sentinel-gitops-manifest-not-updated";
|
|
return {
|
|
code,
|
|
phase: code === "sentinel-publish-half-state-registry-missing"
|
|
? "source-ready-registry-missing"
|
|
: code === "sentinel-gitops-manifest-not-updated"
|
|
? "registry-ready-gitops-pending"
|
|
: code === "sentinel-runtime-not-aligned"
|
|
? "gitops-ready-runtime-pending"
|
|
: code === "sentinel-git-mirror-not-in-sync"
|
|
? "runtime-ready-git-mirror-pending"
|
|
: code === "sentinel-cadence-not-ready"
|
|
? "runtime-ready-cadence-pending"
|
|
: "observed-not-ready",
|
|
reason,
|
|
sourceMirror: sourceReady ? `ready ${short(record(sourceMirror.probe).commit ?? record(sourceMirror.probe).expectedCommit)}` : `blocked ${short(record(sourceMirror.probe).commit)}/${short(record(sourceMirror.probe).expectedCommit)}`,
|
|
registry: registryPresent ? `present ${short(registryProbe.digest)}` : "missing -",
|
|
gitMirror: gitMirrorReady ? "ready" : "pending",
|
|
gitops: gitopsReady ? `ready ${short(gitops.image)}` : `pending ${short(gitops.image)}`,
|
|
argo: `${argo.syncStatus ?? "-"} ${argo.healthStatus ?? "-"} ${short(argo.revision)}/${short(argo.expectedRevision)}`,
|
|
runtime: `ready=${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"} image=${short(runtimeDeployment.image)} expected=${short(runtimeDeployment.expectedImage)}`,
|
|
pipelineRun,
|
|
warning: code === "sentinel-publish-half-state-registry-missing"
|
|
? `source mirror already exposes ${short(state.sourceHead.commit)} but registry tag ${state.image.tag} is missing; rerun ${next.publishCurrent} and then recheck ${next.controlPlaneStatus}.`
|
|
: code === "sentinel-git-mirror-not-in-sync"
|
|
? `runtime is aligned but git-mirror is not in sync; run ${next.gitMirrorSync} and then recheck ${next.controlPlaneStatus}.`
|
|
: null,
|
|
blocker: { code, reason, valuesRedacted: true },
|
|
recoveryNext: {
|
|
reason,
|
|
pipelineRun,
|
|
digestRef: registryPresent ? expectedRuntimeImageFromRegistry(state, record(observed.registry)) : null,
|
|
gitopsCommit: gitops.revision ?? null,
|
|
publishCurrent: shouldRetryPublish ? next.publishCurrent : null,
|
|
nextStatus: next.controlPlaneStatus,
|
|
gitMirrorStatus: next.gitMirrorStatus,
|
|
gitMirrorSync: !gitMirrorReady ? next.gitMirrorSync : null,
|
|
gitMirrorFlush: null,
|
|
controlPlaneApply: shouldRetryPublish || code === "sentinel-runtime-not-aligned" ? controlNext.apply : null,
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const result = probeSourceMirrorCache(state.cicd, state.controlPlaneNode, timeoutSeconds, state.sourceHead.commit);
|
|
return { ...result, result: compactCommand(result.result) };
|
|
}
|
|
|
|
function probeSourceMirrorCache(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, timeoutSeconds: number, expectedCommit: string | null): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
|
|
const namespace = stringAt(cicd, "builder.namespace");
|
|
const repository = stringAt(cicd, "source.repository");
|
|
const branch = stringAt(cicd, "source.branch");
|
|
const stageRef = expectedCommit === null ? "" : sentinelSourceSnapshotRef(cicd, expectedCommit);
|
|
const script = [
|
|
"set +e",
|
|
`repo_path=${shellQuote(`/cache/${repository}.git`)}`,
|
|
`branch=${shellQuote(branch)}`,
|
|
`expected=${shellQuote(expectedCommit ?? "")}`,
|
|
`stage_ref=${shellQuote(stageRef)}`,
|
|
"commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)",
|
|
"rc=$?",
|
|
"object_rc=1",
|
|
"expected_object_rc=1",
|
|
"stage_object_rc=1",
|
|
"contains_rc=1",
|
|
"if [ \"$rc\" -eq 0 ]; then",
|
|
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
|
|
" object_rc=$?",
|
|
"fi",
|
|
"if [ -n \"$expected\" ]; then",
|
|
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$expected^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
|
|
" expected_object_rc=$?",
|
|
"fi",
|
|
"if [ -n \"$stage_ref\" ]; then",
|
|
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse --verify \\\"$stage_ref^{commit}\\\" >/dev/null 2>&1\" >/dev/null 2>&1",
|
|
" stage_object_rc=$?",
|
|
"fi",
|
|
"if [ \"$rc\" -eq 0 ] && [ \"$expected_object_rc\" -eq 0 ]; then",
|
|
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" merge-base --is-ancestor \\\"$expected\\\" \\\"$commit\\\" 2>/dev/null\" >/dev/null 2>&1",
|
|
" contains_rc=$?",
|
|
"fi",
|
|
"node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$stage_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$stage_ref\" \"$repo_path\" \"$branch\" <<'NODE'",
|
|
"const [rc, objectRc, expectedObjectRc, stageObjectRc, containsRc, commit, expected, stageRef, repoPath, branch] = process.argv.slice(2);",
|
|
"const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');",
|
|
"const objectPresent = present && Number(objectRc) === 0;",
|
|
"const expectedObjectPresent = !expected || Number(expectedObjectRc) === 0;",
|
|
"const stageObjectPresent = !stageRef || Number(stageObjectRc) === 0;",
|
|
"const containsExpected = !expected || commit === expected || Number(containsRc) === 0;",
|
|
"const relation = !expected ? 'unconstrained' : commit === expected ? 'equal' : containsExpected ? 'mirror-ahead' : expectedObjectPresent ? 'diverged-or-behind' : 'expected-object-missing';",
|
|
"console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror-cache', present, objectPresent, expectedObjectPresent, stageObjectPresent, containsExpected, relation, commit: present ? commit : null, sourceCommit: expected || (present ? commit : null), mirrorCommit: present ? commit : null, expectedCommit: expected || null, stageRef: stageRef || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout) ?? {}, result };
|
|
}
|
|
|
|
function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "argo.namespace");
|
|
const applicationName = stringAt(state.cicd, "argo.applicationName");
|
|
const jsonpath = "{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.sync.revision}{\"\\n\"}";
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", `jsonpath=${jsonpath}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const [syncStatusRaw, healthStatusRaw, revisionRaw] = result.stdout.trim().split(/\r?\n/u);
|
|
const syncStatus = nonEmptyString(syncStatusRaw);
|
|
const healthStatus = nonEmptyString(healthStatusRaw);
|
|
const revision = nonEmptyString(revisionRaw);
|
|
const revisionMatches = expectedRevision === null || revision === expectedRevision;
|
|
const healthy = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy";
|
|
const diagnostics = result.exitCode === 0 && !healthy
|
|
? probeArgoApplicationDiagnostics(state, timeoutSeconds, namespace, applicationName)
|
|
: null;
|
|
return {
|
|
ok: healthy,
|
|
present: result.exitCode === 0,
|
|
syncStatus,
|
|
healthStatus,
|
|
revision,
|
|
expectedRevision,
|
|
revisionMatches,
|
|
revisionPolicy: "non-blocking-branch-head-drift",
|
|
warning: healthy && !revisionMatches
|
|
? "Argo app is Synced/Healthy but status.sync.revision differs from current GitOps branch HEAD; in multi-sentinel GitOps this can happen when another sentinel path advances the branch. Runtime image/manifest checks remain authoritative for rollout readiness."
|
|
: null,
|
|
diagnostics,
|
|
result: compactCommand(result),
|
|
};
|
|
}
|
|
|
|
function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSeconds: number, namespace: string, applicationName: string): Record<string, unknown> {
|
|
const route = stringAt(state.controlPlaneNode, "kubeRoute");
|
|
const appResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(appResult.stdout);
|
|
const status = record(parsed?.status);
|
|
const resourcesRaw = Array.isArray(status.resources) ? status.resources : [];
|
|
const resources = resourcesRaw
|
|
.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
|
|
.map((item) => {
|
|
const health = record(item.health);
|
|
return {
|
|
kind: item.kind ?? null,
|
|
namespace: item.namespace ?? null,
|
|
name: item.name ?? null,
|
|
status: item.status ?? null,
|
|
healthStatus: health.status ?? null,
|
|
healthMessage: short(health.message),
|
|
};
|
|
});
|
|
const problemResources = resources
|
|
.filter((item) => {
|
|
const sync = nonEmptyString(item.status);
|
|
const health = nonEmptyString(item.healthStatus);
|
|
return (sync !== null && sync !== "Synced") || (health !== null && health !== "Healthy");
|
|
})
|
|
.slice(0, 12);
|
|
const conditions = (Array.isArray(status.conditions) ? status.conditions : [])
|
|
.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
|
|
.slice(-8)
|
|
.map((item) => ({
|
|
type: item.type ?? null,
|
|
message: short(item.message),
|
|
lastTransitionTime: item.lastTransitionTime ?? null,
|
|
}));
|
|
const operationState = record(status.operationState);
|
|
const eventResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "events", "--field-selector", `involvedObject.name=${applicationName}`, "--sort-by=.lastTimestamp", "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const eventsJson = parseJsonObject(eventResult.stdout);
|
|
const events = (Array.isArray(eventsJson?.items) ? eventsJson.items : [])
|
|
.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
|
|
.slice(-8)
|
|
.map((item) => ({
|
|
type: item.type ?? null,
|
|
reason: item.reason ?? null,
|
|
message: short(item.message),
|
|
count: item.count ?? null,
|
|
lastTimestamp: item.lastTimestamp ?? item.eventTime ?? null,
|
|
}));
|
|
return {
|
|
ok: appResult.exitCode === 0,
|
|
resourceCount: resources.length,
|
|
problemResourceCount: problemResources.length,
|
|
problemResources,
|
|
conditions,
|
|
operationState: {
|
|
phase: operationState.phase ?? null,
|
|
message: short(operationState.message),
|
|
startedAt: operationState.startedAt ?? null,
|
|
finishedAt: operationState.finishedAt ?? null,
|
|
},
|
|
events,
|
|
result: compactCommand(appResult),
|
|
eventsResult: compactCommand(eventResult),
|
|
drillDown: {
|
|
application: `trans ${route} kubectl -n ${namespace} get application ${applicationName} -o json`,
|
|
events: `trans ${route} kubectl -n ${namespace} get events --field-selector involvedObject.name=${applicationName} --sort-by=.lastTimestamp`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const repository = stringAt(state.controlPlaneTarget, "source.repository");
|
|
const branch = stringAt(state.cicd, "argo.targetRevision");
|
|
const manifestPath = `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`;
|
|
const repoPath = `/cache/${repository}.git`;
|
|
const inner = [
|
|
"set -eu",
|
|
`git --git-dir=${shellQuote(repoPath)} rev-parse ${shellQuote(`refs/heads/${branch}`)}`,
|
|
`git --git-dir=${shellQuote(repoPath)} show ${shellQuote(`refs/heads/${branch}:${manifestPath}`)}`,
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "exec", "deploy/git-mirror-http", "--", "sh", "-lc", inner], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const [revisionLine, ...manifestLines] = result.stdout.split(/\r?\n/u);
|
|
const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null;
|
|
const manifest = manifestLines.join("\n");
|
|
const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]);
|
|
const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`);
|
|
const compact = compactCommand(result);
|
|
return {
|
|
ok: result.exitCode === 0 && revision !== null && imageMatchesRepository,
|
|
revision,
|
|
branch,
|
|
manifestPath,
|
|
image,
|
|
imageMatchesRepository,
|
|
result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, expectedImage: string | null): Record<string, unknown> {
|
|
const namespace = stringAt(state.runtime, "namespace");
|
|
const deploymentName = stringAt(state.runtime, "deploymentName");
|
|
const serviceName = stringAt(state.runtime, "serviceName");
|
|
const pvcName = stringAt(state.runtime, "pvcName");
|
|
const configMapName = `${deploymentName}-config`;
|
|
const serviceAccountName = stringAt(state.runtime, "serviceAccountName");
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deploymentName)}`,
|
|
`service=${shellQuote(serviceName)}`,
|
|
`pvc=${shellQuote(pvcName)}`,
|
|
`configmap=${shellQuote(configMapName)}`,
|
|
`serviceaccount=${shellQuote(serviceAccountName)}`,
|
|
"tmp=$(mktemp -d)",
|
|
"kubectl -n \"$namespace\" get deploy \"$deployment\" -o json >\"$tmp/deploy.json\" 2>/dev/null; echo $? >\"$tmp/deploy.rc\"",
|
|
"kubectl -n \"$namespace\" get svc \"$service\" -o json >\"$tmp/svc.json\" 2>/dev/null; echo $? >\"$tmp/svc.rc\"",
|
|
"kubectl -n \"$namespace\" get pvc \"$pvc\" -o json >\"$tmp/pvc.json\" 2>/dev/null; echo $? >\"$tmp/pvc.rc\"",
|
|
"kubectl -n \"$namespace\" get cm \"$configmap\" -o json >\"$tmp/cm.json\" 2>/dev/null; echo $? >\"$tmp/cm.rc\"",
|
|
"kubectl -n \"$namespace\" get sa \"$serviceaccount\" -o json >\"$tmp/sa.json\" 2>/dev/null; echo $? >\"$tmp/sa.rc\"",
|
|
`expected_image=${shellQuote(expectedImage ?? "")}`,
|
|
"node - \"$tmp\" \"$expected_image\" <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const dir = process.argv[2];",
|
|
"const expectedImage = process.argv[3] || null;",
|
|
"function rc(name){ return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); }",
|
|
"function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }",
|
|
"const dep = json('deploy');",
|
|
"const deploymentPresent = rc('deploy') === 0;",
|
|
"const desired = Number(dep?.spec?.replicas ?? 0);",
|
|
"const ready = Number(dep?.status?.readyReplicas ?? 0);",
|
|
"const updated = Number(dep?.status?.updatedReplicas ?? 0);",
|
|
"const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;",
|
|
"const imageMatches = expectedImage === null || image === expectedImage;",
|
|
"const payload = {",
|
|
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },",
|
|
" service: { present: rc('svc') === 0 },",
|
|
" pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },",
|
|
" configMap: { present: rc('cm') === 0 },",
|
|
" serviceAccount: { present: rc('sa') === 0 },",
|
|
" valuesRedacted: true",
|
|
"};",
|
|
"payload.ok = payload.deployment.present && imageMatches && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;",
|
|
"console.log(JSON.stringify(payload));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probe = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && probe?.ok === true, probe, result: compactCommand(result) };
|
|
}
|
|
|
|
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null;
|
|
if (expected === null) {
|
|
return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true };
|
|
}
|
|
const metadata = record(expected.metadata);
|
|
const spec = record(expected.spec);
|
|
const namespace = stringAt(metadata, "namespace");
|
|
const name = stringAt(metadata, "name");
|
|
const expectedSchedule = stringAt(spec, "schedule");
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`cronjob=${shellQuote(name)}`,
|
|
`sentinel=${shellQuote(state.sentinelId)}`,
|
|
`expected_schedule=${shellQuote(expectedSchedule)}`,
|
|
"tmp=$(mktemp -d)",
|
|
"kubectl -n \"$namespace\" get cronjob \"$cronjob\" -o json >\"$tmp/cronjob.json\" 2>/dev/null; echo $? >\"$tmp/cronjob.rc\"",
|
|
"kubectl -n \"$namespace\" get jobs -l \"unidesk.ai/web-probe-sentinel-id=$sentinel,app.kubernetes.io/component=cadence-scheduler\" -o json >\"$tmp/jobs.json\" 2>/dev/null; echo $? >\"$tmp/jobs.rc\"",
|
|
"node - \"$tmp\" \"$namespace\" \"$cronjob\" \"$expected_schedule\" <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const [dir, namespace, cronJobName, expectedSchedule] = process.argv.slice(2);",
|
|
"function rc(name){ try { return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); } catch { return 1; } }",
|
|
"function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }",
|
|
"const cron = json('cronjob');",
|
|
"const jobs = Array.isArray(json('jobs')?.items) ? json('jobs').items : [];",
|
|
"const present = rc('cronjob') === 0 && !!cron;",
|
|
"const schedule = cron?.spec?.schedule || null;",
|
|
"const scheduleMatches = present && schedule === expectedSchedule;",
|
|
"const suspended = cron?.spec?.suspend === true;",
|
|
"const active = Array.isArray(cron?.status?.active) ? cron.status.active.length : 0;",
|
|
"const sortedJobs = jobs.slice().sort((a,b)=>String(b?.metadata?.creationTimestamp||'').localeCompare(String(a?.metadata?.creationTimestamp||''))).slice(0,8);",
|
|
"let code = null;",
|
|
"if (!present) code = 'sentinel-cadence-cronjob-missing';",
|
|
"else if (!scheduleMatches) code = 'sentinel-cadence-cronjob-schedule-mismatch';",
|
|
"else if (suspended) code = 'sentinel-cadence-cronjob-suspended';",
|
|
"const latestJob = sortedJobs[0] || null;",
|
|
"console.log(JSON.stringify({ ok: code === null, code, present, namespace, name: cronJobName, schedule, expectedSchedule, scheduleMatches, suspended, lastScheduleTime: cron?.status?.lastScheduleTime || null, lastSuccessfulTime: cron?.status?.lastSuccessfulTime || null, active, jobCount: jobs.length, latestJobs: sortedJobs.map((job)=>({ name: job?.metadata?.name || null, createdAt: job?.metadata?.creationTimestamp || null, active: Number(job?.status?.active || 0), succeeded: Number(job?.status?.succeeded || 0), failed: Number(job?.status?.failed || 0), completionTime: job?.status?.completionTime || null, valuesRedacted:true })), latestJobName: latestJob?.metadata?.name || null, valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probe = parseJsonObject(result.stdout);
|
|
const ok = result.exitCode === 0 && probe?.ok === true;
|
|
emitWebProbeSentinelSpan({
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
namespace,
|
|
runtime: state.runtime,
|
|
cicd: state.cicd,
|
|
}, "web_probe_sentinel.cadence.cronjob_observed", {
|
|
cronJobName: name,
|
|
namespace,
|
|
schedule: expectedSchedule,
|
|
status: ok ? "ok" : text(probe?.code ?? "unknown"),
|
|
jobName: probe?.latestJobName ?? null,
|
|
failureKind: probe?.code ?? null,
|
|
valuesRedacted: true,
|
|
}, ok);
|
|
return {
|
|
ok,
|
|
probe,
|
|
result: compactCommand(result),
|
|
warning: ok ? null : `cadence CronJob is not ready: ${text(probe?.code ?? "probe-failed")}`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record<string, unknown>): string | null {
|
|
const digest = nonEmptyString(record(record(registry).probe).digest);
|
|
if (digest === null) return null;
|
|
return `${state.image.repository}@${digest}`;
|
|
}
|
|
|
|
function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult {
|
|
const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`;
|
|
const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
|
const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName);
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
if (created.exitCode !== 0) {
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane });
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "source-mirror");
|
|
}
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
|
|
let slowWarningSent = false;
|
|
let polls = 0;
|
|
let lastProbe: Record<string, unknown> = {};
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probe = parseJsonObject(probeCapture.stdout) ?? {};
|
|
lastProbe = { ...probe, capture: compactCommand(probeCapture) };
|
|
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", {
|
|
phase: "remote-job",
|
|
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
pod: probe.pod ?? null,
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
});
|
|
if (probe.succeeded === true) {
|
|
const ok = payload.ok === true;
|
|
return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
|
|
}
|
|
if (probe.failed === true) {
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
|
|
}
|
|
if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
|
|
slowWarningSent = true;
|
|
sentinelProgressEvent("sentinel.source-mirror.warning", { warning: `source mirror sync exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
|
|
}
|
|
|
|
function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult {
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
jobName: "-",
|
|
payload: { ok: false, status: phase, reason, valuesRedacted: true },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": "web-probe-sentinel-source-mirror",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
};
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: { name: jobName, namespace, labels },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"),
|
|
ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"),
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
volumes: [
|
|
sentinelGitMirrorCacheVolume(state),
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
],
|
|
containers: [{
|
|
name: "sync",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string {
|
|
return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit);
|
|
}
|
|
|
|
function sentinelSourceMirrorSyncShellFromConfig(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, jobName: string, selectedCommit: string | null): string {
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`,
|
|
`source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`,
|
|
`source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`,
|
|
`source_commit=${shellQuote(selectedCommit ?? "")}`,
|
|
`source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`,
|
|
"started_ms=$(node -e 'console.log(Date.now())')",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode),
|
|
"repo=\"/cache/${source_repository}.git\"",
|
|
"mkdir -p \"$(dirname \"$repo\")\"",
|
|
"if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
|
|
" git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
|
"else",
|
|
" rm -rf \"$repo\"",
|
|
" git init --bare \"$repo\"",
|
|
" git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
|
"fi",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
|
|
"git --git-dir=\"$repo\" config http.uploadpack true",
|
|
"git --git-dir=\"$repo\" config http.receivepack true",
|
|
"fetch_ok=0",
|
|
"for attempt in 1 2 3; do",
|
|
" if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi",
|
|
" code=$?",
|
|
" printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2",
|
|
" sleep $((attempt * 5))",
|
|
"done",
|
|
"test \"$fetch_ok\" = 1",
|
|
"mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")",
|
|
"if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi",
|
|
"git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"",
|
|
"test \"$mirror_commit\" = \"$source_commit\"",
|
|
"stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"",
|
|
"git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"",
|
|
"git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"",
|
|
"git --git-dir=\"$repo\" update-server-info",
|
|
"finished_ms=$(node -e 'console.log(Date.now())')",
|
|
"node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
|
|
"const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] {
|
|
return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode);
|
|
}
|
|
|
|
function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record<string, unknown>): string[] {
|
|
const proxy = record(valueAtPath(controlPlaneNode, "egressProxy"));
|
|
const serviceName = nonEmptyString(proxy.serviceName);
|
|
const namespace = nonEmptyString(proxy.namespace);
|
|
const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null;
|
|
const hostRouteProxyUrl = nonEmptyString(proxy.proxyUrl);
|
|
const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : "";
|
|
let proxyHost: string | null = null;
|
|
let proxyPort: number | null = null;
|
|
let proxyUrl: string | null = null;
|
|
if (serviceName !== null && namespace !== null && port !== null) {
|
|
proxyHost = `${serviceName}.${namespace}.svc.cluster.local`;
|
|
proxyPort = port;
|
|
proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
|
} else if (hostRouteProxyUrl !== null) {
|
|
try {
|
|
const parsed = new URL(hostRouteProxyUrl);
|
|
const parsedPort = Number.parseInt(parsed.port || "80", 10);
|
|
if ((parsed.protocol === "http:" || parsed.protocol === "https:") && parsed.hostname.length > 0 && Number.isInteger(parsedPort)) {
|
|
proxyHost = parsed.hostname;
|
|
proxyPort = parsedPort;
|
|
proxyUrl = hostRouteProxyUrl;
|
|
}
|
|
} catch {
|
|
proxyHost = null;
|
|
proxyPort = null;
|
|
proxyUrl = null;
|
|
}
|
|
}
|
|
const useProxy = proxyHost !== null && proxyPort !== null && proxyUrl !== null;
|
|
if (!useProxy) {
|
|
return [
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2",
|
|
"unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy",
|
|
"export NO_PROXY='*'",
|
|
"export no_proxy='*'",
|
|
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
|
"#!/bin/sh",
|
|
"exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"",
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
|
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
];
|
|
}
|
|
const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
|
|
return [
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
`printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`,
|
|
`export HTTP_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export HTTPS_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export ALL_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export http_proxy=${shellQuote(proxyUrl)}`,
|
|
`export https_proxy=${shellQuote(proxyUrl)}`,
|
|
`export all_proxy=${shellQuote(proxyUrl)}`,
|
|
`export NO_PROXY=${shellQuote(noProxy)}`,
|
|
`export no_proxy=${shellQuote(noProxy)}`,
|
|
"cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'",
|
|
"#!/usr/bin/env node",
|
|
"const net = require('node:net');",
|
|
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
|
|
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
|
|
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
|
|
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
|
|
" console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');",
|
|
" process.exit(64);",
|
|
"}",
|
|
"let settled = false;",
|
|
"let tunnelEstablished = false;",
|
|
"function finish(code, message) {",
|
|
" if (settled) return;",
|
|
" settled = true;",
|
|
" if (message) console.error('sentinel source-mirror proxy-connect: ' + message);",
|
|
" process.exit(code);",
|
|
"}",
|
|
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
|
|
"let buffer = Buffer.alloc(0);",
|
|
"socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
|
|
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
|
|
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
|
|
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
|
|
"function onData(chunk) {",
|
|
" buffer = Buffer.concat([buffer, chunk]);",
|
|
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
|
|
" if (headerEnd === -1 && buffer.length < 8192) return;",
|
|
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
|
|
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
|
|
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
|
|
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
|
|
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
|
|
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
|
|
" socket.destroy();",
|
|
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
|
|
" return;",
|
|
" }",
|
|
" socket.off('data', onData);",
|
|
" socket.setTimeout(0);",
|
|
" tunnelEstablished = true;",
|
|
" const rest = buffer.slice(headerEnd + 4);",
|
|
" if (rest.length) process.stdout.write(rest);",
|
|
" process.stdin.on('error', () => {});",
|
|
" process.stdout.on('error', () => {});",
|
|
" process.stdin.pipe(socket);",
|
|
" socket.pipe(process.stdout);",
|
|
"}",
|
|
"socket.on('data', onData);",
|
|
"NODE_PROXY",
|
|
"chmod 0700 /tmp/sentinel-github-proxy-connect.cjs",
|
|
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
|
"#!/bin/sh",
|
|
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`,
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
|
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
];
|
|
}
|
|
|
|
function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult {
|
|
const pipelineRunName = sentinelPipelineRunName(state, rerun);
|
|
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops);
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createTektonPipelineRunScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
if (created.exitCode !== 0) {
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "failed", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-pipelinerun", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish");
|
|
}
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "succeeded", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
|
|
let slowWarningSent = false;
|
|
let polls = 0;
|
|
let lastProbe: Record<string, unknown> = {};
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeTektonPipelineRunScript(namespace, pipelineRunName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probe = parseJsonObject(probeCapture.stdout) ?? {};
|
|
lastProbe = { ...probe, capture: compactCommand(probeCapture) };
|
|
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
|
sentinelProgressEvent("sentinel.publish.progress", {
|
|
phase: "remote-pipelinerun",
|
|
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
|
pipelineRun: pipelineRunName,
|
|
publishGitops,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
taskRun: probe.taskRun ?? null,
|
|
pod: probe.pod ?? null,
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
});
|
|
if (probe.succeeded === true) {
|
|
const ok = payload.ok === true;
|
|
return withSentinelRemoteJobDiagnostics(state, { ok, phase: "pipelinerun-succeeded", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
|
}
|
|
if (probe.failed === true) {
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-failed", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
|
}
|
|
if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
|
|
slowWarningSent = true;
|
|
sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish PipelineRun exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, pipelineRun: pipelineRunName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-timeout", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
|
}
|
|
|
|
function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRunName: string, publishGitops: boolean): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const buildkitImage = requireSentinelBuildkitImage(state);
|
|
const proxyEnv = sentinelImageBuildProxyEnv(state);
|
|
const labels = {
|
|
"app.kubernetes.io/name": "web-probe-sentinel-publish",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
"unidesk.ai/ci-system": "tekton",
|
|
};
|
|
return {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "PipelineRun",
|
|
metadata: {
|
|
name: pipelineRunName,
|
|
namespace,
|
|
labels,
|
|
annotations: {
|
|
"unidesk.ai/source-commit": state.sourceHead.commit ?? "",
|
|
"unidesk.ai/source-authority": state.sourceHead.sourceAuthority,
|
|
"unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "",
|
|
"unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"),
|
|
"unidesk.ai/publish-gitops": publishGitops ? "true" : "false",
|
|
},
|
|
},
|
|
spec: {
|
|
timeouts: { pipeline: `${numberAt(state.cicd, "builder.activeDeadlineSeconds")}s` },
|
|
taskRunTemplate: {
|
|
podTemplate: {
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
securityContext: { fsGroup: 1000 },
|
|
},
|
|
},
|
|
pipelineSpec: {
|
|
tasks: [{
|
|
name: "publish",
|
|
taskSpec: {
|
|
volumes: [
|
|
sentinelGitMirrorCacheVolume(state),
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
{ name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
|
|
sentinelBuildkitStateVolume(state),
|
|
{ name: "tmp", emptyDir: {} },
|
|
],
|
|
steps: [
|
|
{
|
|
name: "source",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
env: proxyEnv,
|
|
script: tektonShellScript(sentinelPublishSourceShell(state, pipelineRunName)),
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
},
|
|
{
|
|
name: "prepare-buildkit-state",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"),
|
|
securityContext: { runAsUser: 0, runAsGroup: 0 },
|
|
volumeMounts: [
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
],
|
|
},
|
|
{
|
|
name: "image-build",
|
|
image: buildkitImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
env: [
|
|
...proxyEnv,
|
|
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
|
],
|
|
script: tektonShellScript(sentinelPublishImageBuildShell(state, pipelineRunName)),
|
|
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
{ name: "tmp", mountPath: "/tmp" },
|
|
],
|
|
},
|
|
{
|
|
name: "publish",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
env: proxyEnv,
|
|
script: tektonShellScript(sentinelPublishShell(state, pipelineRunName, publishGitops)),
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
}],
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function tektonShellScript(body: string): string {
|
|
return `#!/bin/sh\n${body}`;
|
|
}
|
|
|
|
function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string, unknown> {
|
|
return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget);
|
|
}
|
|
|
|
function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record<string, unknown>): Record<string, unknown> {
|
|
const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath"));
|
|
if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } };
|
|
return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } };
|
|
}
|
|
|
|
function sentinelBuildkitStateVolume(state: SentinelCicdState): Record<string, unknown> {
|
|
const buildkitState = monitorWebBuildkitStatePlan(state.cicd);
|
|
const mode = stringAt(buildkitState, "mode");
|
|
if (mode === "hostPath") {
|
|
return {
|
|
name: "buildkit-state",
|
|
hostPath: {
|
|
path: stringAt(buildkitState, "path"),
|
|
type: stringAt(buildkitState, "type"),
|
|
},
|
|
};
|
|
}
|
|
if (mode === "persistentVolumeClaim") {
|
|
return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } };
|
|
}
|
|
if (mode === "emptyDir") {
|
|
return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } };
|
|
}
|
|
throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`);
|
|
}
|
|
|
|
function requireSentinelBuildkitImage(state: SentinelCicdState): string {
|
|
const image = state.spec.buildkit?.sidecarImage;
|
|
if (typeof image !== "string" || image.length === 0) {
|
|
throw new Error(`config/hwlab-node-lanes.yaml ${state.spec.nodeId}/${state.spec.lane} buildkit.sidecarImage is required for pure k8s web-probe sentinel image publish`);
|
|
}
|
|
return image;
|
|
}
|
|
|
|
function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> {
|
|
const proxy = state.spec.networkProfile.imageBuildProxy;
|
|
const noProxy = proxy.noProxy.join(",");
|
|
return [
|
|
{ name: "HTTP_PROXY", value: proxy.http },
|
|
{ name: "http_proxy", value: proxy.http },
|
|
{ name: "HTTPS_PROXY", value: proxy.https },
|
|
{ name: "https_proxy", value: proxy.https },
|
|
{ name: "ALL_PROXY", value: proxy.all },
|
|
{ name: "all_proxy", value: proxy.all },
|
|
{ name: "NO_PROXY", value: noProxy },
|
|
{ name: "no_proxy", value: noProxy },
|
|
];
|
|
}
|
|
|
|
function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string {
|
|
const monitorWeb = record(state.image.monitorWeb);
|
|
const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => {
|
|
if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths");
|
|
return item;
|
|
})), "utf8").toString("base64");
|
|
const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64");
|
|
const envReuseMode = stringAt(monitorWeb, "envReuseMode");
|
|
const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath");
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
|
|
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
|
|
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
|
|
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
|
|
`source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`,
|
|
`checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
|
|
`dockerfile_b64=${shellQuote(dockerfileB64)}`,
|
|
`env_reuse_mode=${shellQuote(envReuseMode)}`,
|
|
`env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`,
|
|
"meta_dir=/workspace/meta",
|
|
"event_log=/workspace/publish-events.log",
|
|
"worktree=/workspace/source",
|
|
"rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json",
|
|
"mkdir -p \"$meta_dir\"",
|
|
": > \"$event_log\"",
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"started_ms=$(now_ms)",
|
|
"write_meta started_ms \"$started_ms\"",
|
|
"write_meta source_commit \"$source_commit\"",
|
|
"write_meta source_stage_ref \"$source_stage_ref\"",
|
|
"test -n \"$source_commit\"",
|
|
"test -n \"$source_stage_ref\"",
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
|
"git init \"$worktree\"",
|
|
"cd \"$worktree\"",
|
|
"git remote add origin \"$source_git_url\"",
|
|
"git config core.sparseCheckout true",
|
|
"git config remote.origin.promisor true",
|
|
"git config remote.origin.partialclonefilter blob:none",
|
|
"CHECKOUT_PATHS_B64=\"$checkout_paths_b64\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const paths = JSON.parse(Buffer.from(process.env.CHECKOUT_PATHS_B64 || '', 'base64').toString('utf8'));",
|
|
"fs.mkdirSync('.git/info', { recursive: true });",
|
|
"fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');",
|
|
"NODE",
|
|
"source_fetch_started_ms=$(now_ms)",
|
|
"write_meta source_fetch_started_ms \"$source_fetch_started_ms\"",
|
|
"emit_stage source-fetch running \"$source_fetch_started_ms\"",
|
|
"git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"",
|
|
"git checkout --detach \"$source_commit\"",
|
|
"mirror_commit=$(git rev-parse HEAD)",
|
|
"test \"$mirror_commit\" = \"$source_commit\"",
|
|
"write_meta mirror_commit \"$mirror_commit\"",
|
|
"source_fetch_finished_ms=$(now_ms)",
|
|
"write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"",
|
|
"emit_stage source-fetch succeeded \"$source_fetch_started_ms\"",
|
|
"env_reuse_node_deps_present=false",
|
|
"env_reuse_node_deps_entries=0",
|
|
"if [ -d \"$env_reuse_node_deps_path\" ]; then env_reuse_node_deps_present=true; env_reuse_node_deps_entries=$(find \"$env_reuse_node_deps_path\" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' '); fi",
|
|
"env_reuse_linked_node_deps=0",
|
|
"rm -rf node_modules",
|
|
"if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi",
|
|
"write_meta env_reuse_mode \"$env_reuse_mode\"",
|
|
"write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"",
|
|
"write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"",
|
|
"write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"",
|
|
"write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"",
|
|
"node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'",
|
|
"const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
|
|
"NODE",
|
|
"node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"",
|
|
"const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
|
|
"NODE",
|
|
"monitor_web_verify_started_ms=$(now_ms)",
|
|
"write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"",
|
|
"emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"",
|
|
"if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi",
|
|
"cat /tmp/web-probe-sentinel-monitor-web-verify.log",
|
|
"monitor_web_verify_finished_ms=$(now_ms)",
|
|
"write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"",
|
|
"emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"",
|
|
"mkdir -p .unidesk-sentinel-bin",
|
|
"cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'",
|
|
"#!/bin/sh",
|
|
"exec bun /app/scripts/ssh-cli.ts \"$@\"",
|
|
"SH_TRANS",
|
|
"chmod 0755 .unidesk-sentinel-bin/trans",
|
|
"DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));",
|
|
"NODE",
|
|
"cat > .dockerignore <<'EOF_DOCKERIGNORE'",
|
|
".git",
|
|
".git/**",
|
|
".state",
|
|
".state/**",
|
|
"logs",
|
|
"logs/**",
|
|
"node_modules",
|
|
"node_modules/**",
|
|
"**/node_modules",
|
|
"**/node_modules/**",
|
|
"**/dist",
|
|
"**/dist/**",
|
|
"**/target",
|
|
"**/target/**",
|
|
"**/coverage",
|
|
"**/coverage/**",
|
|
"npm-debug.log*",
|
|
".env",
|
|
".env.*",
|
|
"EOF_DOCKERIGNORE",
|
|
"context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')",
|
|
"write_meta context_ignore_entries \"$context_ignore_entries\"",
|
|
"chmod -R a+rwX /workspace",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string {
|
|
const monitorWeb = record(state.image.monitorWeb);
|
|
const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode");
|
|
const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode");
|
|
const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource");
|
|
const imageBuildProxy = state.spec.networkProfile.imageBuildProxy;
|
|
const imageBuildNoProxy = imageBuildProxy.noProxy.join(",");
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
`image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`,
|
|
`image_build_package_mode=${shellQuote(imageBuildPackageMode)}`,
|
|
`image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`,
|
|
`image_build_proxy_source=${shellQuote(imageBuildProxySource)}`,
|
|
`image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`,
|
|
`image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`,
|
|
`image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`,
|
|
`image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`,
|
|
"meta_dir=/workspace/meta",
|
|
"event_log=/workspace/publish-events.log",
|
|
"build_log=/workspace/image-build.log",
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"cd /workspace/source",
|
|
"image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi",
|
|
"image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi",
|
|
"image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi",
|
|
"image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi",
|
|
"write_meta image_build_builder \"$image_build_builder\"",
|
|
"write_meta image_build_package_mode \"$image_build_package_mode\"",
|
|
"write_meta image_build_network_mode \"$image_build_network_mode\"",
|
|
"write_meta image_build_proxy_source \"$image_build_proxy_source\"",
|
|
"write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"",
|
|
"write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"",
|
|
"write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"",
|
|
"write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"",
|
|
"image_build_started_ms=$(now_ms)",
|
|
"write_meta image_build_started_ms \"$image_build_started_ms\"",
|
|
"emit_stage image-build running \"$image_build_started_ms\"",
|
|
"if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi",
|
|
"cat \"$build_log\"",
|
|
"image_build_finished_ms=$(now_ms)",
|
|
"write_meta image_build_finished_ms \"$image_build_finished_ms\"",
|
|
"metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)",
|
|
"digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)",
|
|
"test -n \"$digest\"",
|
|
"repo_no_tag=${image_ref%:*}",
|
|
"digest_ref=\"$repo_no_tag@$digest\"",
|
|
"write_meta digest \"$digest\"",
|
|
"write_meta digest_ref \"$digest_ref\"",
|
|
"emit_stage image-build succeeded \"$image_build_started_ms\"",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
|
|
const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : [];
|
|
const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({
|
|
path: file.path,
|
|
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
|
|
}))), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
`image_repository=${shellQuote(state.image.repository)}`,
|
|
`gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`,
|
|
`gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`,
|
|
`files_b64=${shellQuote(filesB64)}`,
|
|
"meta_dir=/workspace/meta",
|
|
"event_log=/workspace/publish-events.log",
|
|
"build_log=/workspace/image-build.log",
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
"read_meta() { cat \"$meta_dir/$1\"; }",
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi",
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
|
"started_ms=$(read_meta started_ms)",
|
|
"source_commit=$(read_meta source_commit)",
|
|
"source_stage_ref=$(read_meta source_stage_ref)",
|
|
"mirror_commit=$(read_meta mirror_commit)",
|
|
"source_fetch_started_ms=$(read_meta source_fetch_started_ms)",
|
|
"source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)",
|
|
"monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)",
|
|
"monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)",
|
|
"image_build_started_ms=$(read_meta image_build_started_ms)",
|
|
"image_build_finished_ms=$(read_meta image_build_finished_ms)",
|
|
"digest_ref=$(read_meta digest_ref)",
|
|
"env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')",
|
|
"env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')",
|
|
"env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)",
|
|
"env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)",
|
|
"env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)",
|
|
"image_build_builder=$(read_meta image_build_builder)",
|
|
"image_build_package_mode=$(read_meta image_build_package_mode)",
|
|
"image_build_network_mode=$(read_meta image_build_network_mode)",
|
|
"image_build_proxy_source=$(read_meta image_build_proxy_source)",
|
|
"image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)",
|
|
"image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)",
|
|
"image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)",
|
|
"image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)",
|
|
"context_ignore_entries=$(read_meta context_ignore_entries)",
|
|
"image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)",
|
|
"image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)",
|
|
"image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')",
|
|
"gitops_commit=''",
|
|
"changed=false",
|
|
"file_count=0",
|
|
"gitops_started_ms=$(now_ms)",
|
|
"if [ \"$files_b64\" != \"W10=\" ]; then",
|
|
" emit_stage gitops running \"$gitops_started_ms\"",
|
|
" gitops_cache=\"/cache/${gitops_repository}.git\"",
|
|
" gitops_worktree=\"/tmp/$job_name/gitops\"",
|
|
" git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"",
|
|
" cd \"$gitops_worktree\"",
|
|
" git fetch origin \"$gitops_branch\" || true",
|
|
" if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"; else git checkout --orphan \"$gitops_branch\"; git rm -rf . >/dev/null 2>&1 || true; fi",
|
|
" FILES_B64=\"$files_b64\" IMAGE_REF=\"$image_ref\" DIGEST_REF=\"$digest_ref\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
|
|
"for (const file of files) {",
|
|
" const target = path.resolve(process.cwd(), file.path);",
|
|
" if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);",
|
|
" fs.mkdirSync(path.dirname(target), { recursive: true });",
|
|
" const text = Buffer.from(file.contentBase64, 'base64').toString('utf8').split(process.env.IMAGE_REF).join(process.env.DIGEST_REF);",
|
|
" fs.writeFileSync(target, text);",
|
|
"}",
|
|
"console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));",
|
|
"NODE",
|
|
" git add .",
|
|
" file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')",
|
|
" if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi",
|
|
" git push origin \"HEAD:refs/heads/$gitops_branch\"",
|
|
" gitops_commit=$(git rev-parse HEAD)",
|
|
" emit_stage gitops succeeded \"$gitops_started_ms\"",
|
|
"else",
|
|
" emit_stage gitops skipped \"$gitops_started_ms\"",
|
|
"fi",
|
|
"gitops_finished_ms=$(now_ms)",
|
|
"finished_ms=$(now_ms)",
|
|
"node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
|
|
"const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
|
|
"const elapsed = (start, finish) => Number(finish) - Number(start);",
|
|
"const cacheHits = Number(imageBuildCacheHits || 0);",
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelGitopsFiles(state: SentinelCicdState): readonly { path: string; content: string }[] {
|
|
const runtimeManifests = state.manifests.filter((item) => item.kind !== "Application");
|
|
return [{
|
|
path: `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`,
|
|
content: `${runtimeManifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`,
|
|
}];
|
|
}
|
|
|
|
function applySentinelArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const app = state.manifests.find((item) => item.kind === "Application");
|
|
if (app === undefined) return { ok: false, reason: "application-manifest-missing", valuesRedacted: true };
|
|
const yaml = `${Bun.YAML.stringify(app).trim()}\n`;
|
|
const namespace = stringAt(state.cicd, "argo.namespace");
|
|
const applicationName = stringAt(state.cicd, "argo.applicationName");
|
|
const script = [
|
|
"set -eu",
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"kubectl apply -f \"$tmp\"",
|
|
`kubectl -n ${shellQuote(namespace)} annotate application ${shellQuote(applicationName)} argocd.argoproj.io/refresh=hard --overwrite`,
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
return { ok: result.exitCode === 0, result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function createK8sJobScript(namespace: string, manifest: Record<string, unknown>): string {
|
|
const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
|
return [
|
|
"set -eu",
|
|
`kubectl -n ${shellQuote(namespace)} delete job ${shellQuote(stringAt(manifest, "metadata.name"))} --ignore-not-found=true >/dev/null 2>&1 || true`,
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"kubectl apply -f \"$tmp\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function createTektonPipelineRunScript(namespace: string, manifest: Record<string, unknown>): string {
|
|
const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
|
const pipelineRunName = stringAt(manifest, "metadata.name");
|
|
return [
|
|
"set -eu",
|
|
`pipeline_run=${shellQuote(pipelineRunName)}`,
|
|
`namespace=${shellQuote(namespace)}`,
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then",
|
|
" printf '%s\\n' \"sentinel publish PipelineRun already exists; reusing $pipeline_run\"",
|
|
" exit 0",
|
|
"fi",
|
|
"if ! kubectl create -f \"$tmp\"; then",
|
|
" if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then",
|
|
" printf '%s\\n' \"sentinel publish PipelineRun appeared concurrently; reusing $pipeline_run\"",
|
|
" exit 0",
|
|
" fi",
|
|
" exit 1",
|
|
"fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function probeK8sJobScript(namespace: string, jobName: string): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
"succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)",
|
|
"failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)",
|
|
"active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)",
|
|
"pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)",
|
|
"pod_phase=''",
|
|
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi",
|
|
"logs_tail=''",
|
|
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
|
|
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
|
|
"const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`pipeline_run=${shellQuote(pipelineRunName)}`,
|
|
"condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)",
|
|
"condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)",
|
|
"condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)",
|
|
"if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi",
|
|
"if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi",
|
|
"if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi",
|
|
"task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
|
|
"pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
|
|
"pod_phase=''",
|
|
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi",
|
|
"logs_tail=''",
|
|
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
|
|
"node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
|
|
"const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);",
|
|
"const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');",
|
|
"const active = conditionStatus === 'Unknown' || (!conditionStatus && (podPhase === 'Pending' || podPhase === 'Running'));",
|
|
"console.log(JSON.stringify({ succeeded: conditionStatus === 'True', failed: conditionStatus === 'False', active, conditionStatus: conditionStatus || null, conditionReason: conditionReason || null, conditionMessage: message || null, taskRun: taskRun || null, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelPayloadFromLogs(logsTail: string): Record<string, unknown> {
|
|
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
const line = lines[index];
|
|
if (!line.startsWith("{") || !line.endsWith("}")) continue;
|
|
const parsed = parseJsonObject(line);
|
|
if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed;
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult {
|
|
return { ...result, diagnostics: sentinelRemoteJobDiagnostics(state, result, domain), valuesRedacted: true };
|
|
}
|
|
|
|
function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const probe = record(result.probe);
|
|
const logsTail = typeof probe.logsTail === "string" ? probe.logsTail : "";
|
|
const events = sentinelStageEventsFromLogs(logsTail, domain);
|
|
const envReuse = sentinelEnvReuseFromLogs(logsTail);
|
|
const completedStages = sentinelCompletedStages(events, record(result.payload));
|
|
const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs);
|
|
const currentPhase = sentinelCurrentRemotePhase(result, events, domain);
|
|
const isPipelineRun = result.resourceKind === "PipelineRun";
|
|
const commands = {
|
|
cliStatus: domain === "publish"
|
|
? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`
|
|
: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
logs: result.jobName === "-"
|
|
? "-"
|
|
: isPipelineRun
|
|
? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs -l tekton.dev/pipelineRun=${result.jobName} --all-containers=true --tail=120`
|
|
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`,
|
|
describe: result.jobName === "-"
|
|
? "-"
|
|
: isPipelineRun
|
|
? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe pipelinerun/${result.jobName}`
|
|
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`,
|
|
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
|
gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`,
|
|
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`,
|
|
controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
|
|
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
|
|
valuesRedacted: true,
|
|
};
|
|
return {
|
|
domain,
|
|
resourceKind: result.resourceKind ?? "Job",
|
|
pipelineRun: isPipelineRun ? result.jobName : null,
|
|
taskRun: probe.taskRun ?? null,
|
|
currentPhase,
|
|
completedStages,
|
|
stageTimings,
|
|
envReuse,
|
|
pod: probe.pod ?? null,
|
|
podPhase: probe.podPhase ?? null,
|
|
active: probe.active ?? null,
|
|
conditionStatus: probe.conditionStatus ?? null,
|
|
conditionReason: probe.conditionReason ?? null,
|
|
recentLogSummary: sentinelRecentLogSummary(logsTail),
|
|
commands,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelEnvReuseFromLogs(logsTail: string): Record<string, unknown> | null {
|
|
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
const parsed = parseJsonObject(lines[index]);
|
|
if (parsed !== null && parsed.event === "sentinel-publish-env-reuse") return { ...parsed, valuesRedacted: true };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function sentinelStageEventsFromLogs(logsTail: string, domain: "source-mirror" | "publish"): Record<string, unknown>[] {
|
|
const expectedEvent = domain === "publish" ? "sentinel-publish-stage" : "sentinel-source-mirror-stage";
|
|
return logsTail
|
|
.split(/\r?\n/u)
|
|
.map((line) => parseJsonObject(line.trim()))
|
|
.filter((item): item is Record<string, unknown> => item !== null && item.event === expectedEvent);
|
|
}
|
|
|
|
function sentinelCompletedStages(events: readonly Record<string, unknown>[], payload: Record<string, unknown>): string[] {
|
|
const completed = events
|
|
.filter((event) => event.status === "succeeded" || event.status === "skipped")
|
|
.map((event) => `${text(event.stage)}:${text(event.status)}`);
|
|
const payloadStages = Array.isArray(payload.completedStages) ? payload.completedStages.map(text) : [];
|
|
return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-");
|
|
}
|
|
|
|
function sentinelStageTimingSummary(events: readonly Record<string, unknown>[], payload: Record<string, unknown>, fallbackTotalMs: unknown): Record<string, unknown> {
|
|
const payloadTimings = record(payload.stageTimings);
|
|
const eventElapsed = (stage: string): number | null => {
|
|
const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed"));
|
|
return event === undefined ? null : finiteNumberOrNull(event.elapsedMs);
|
|
};
|
|
const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch");
|
|
const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify");
|
|
const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build");
|
|
const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops");
|
|
const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null);
|
|
const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0);
|
|
const totalMs = finiteNumberOrNull(payloadTimings.totalMs)
|
|
?? finiteNumberOrNull(payload.elapsedMs)
|
|
?? finiteNumberOrNull(fallbackTotalMs)
|
|
?? summedTotalMs;
|
|
const result = {
|
|
sourceFetchMs,
|
|
monitorWebVerifyMs,
|
|
imageBuildMs,
|
|
gitopsMs,
|
|
totalMs,
|
|
valuesRedacted: true,
|
|
};
|
|
return Object.values(result).some((item) => item !== null && item !== true) ? result : {};
|
|
}
|
|
|
|
function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record<string, unknown>[], domain: "source-mirror" | "publish"): string {
|
|
if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed";
|
|
if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase;
|
|
const reversed = [...events].reverse();
|
|
const failed = reversed.find((event) => event.status === "failed");
|
|
if (failed !== undefined) return text(failed.stage);
|
|
const running = reversed.find((event) => event.status === "running");
|
|
if (running !== undefined) return text(running.stage);
|
|
const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage)));
|
|
const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"];
|
|
const next = order.find((stage) => !completed.has(stage));
|
|
return next ?? result.phase;
|
|
}
|
|
|
|
function sentinelRecentLogSummary(logsTail: string): string {
|
|
const lines = logsTail
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter((line) => line.length > 0 && !line.startsWith("{"))
|
|
.slice(-5)
|
|
.map((line) => short(line));
|
|
return lines.length === 0 ? "-" : lines.join(" | ");
|
|
}
|
|
|
|
function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] {
|
|
const remote = record(job);
|
|
if (remote.phase !== "job-timeout" && remote.phase !== "pipelinerun-timeout") return [];
|
|
const diagnostics = record(remote.diagnostics);
|
|
const commands = record(diagnostics.commands);
|
|
return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`];
|
|
}
|
|
|
|
function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] {
|
|
if (!observedReady) return [];
|
|
const warnings: string[] = [];
|
|
const publishRecord = record(publish);
|
|
if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) {
|
|
warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`);
|
|
}
|
|
const flushRecord = record(flush);
|
|
if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) {
|
|
warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout.");
|
|
}
|
|
return warnings;
|
|
}
|
|
|
|
function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number {
|
|
return numberAt(state.cicd, "confirmWait.maxSeconds");
|
|
}
|
|
|
|
function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
|
|
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
|
|
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
|
|
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`];
|
|
}
|
|
|
|
function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] {
|
|
const sync = record(sourceMirrorSync);
|
|
if (sync.ok === true || state.sourceHead.ok !== true) return [];
|
|
return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`];
|
|
}
|
|
|
|
function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, probe: unknown): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
phase: "already-present",
|
|
jobName: null,
|
|
probe: record(probe),
|
|
payload: {
|
|
ok: true,
|
|
status: "already-present",
|
|
sourceCommit: state.sourceHead.commit,
|
|
mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit,
|
|
stageRef: state.sourceHead.stageRef,
|
|
sourceAuthority: state.sourceHead.sourceAuthority,
|
|
valuesRedacted: true,
|
|
},
|
|
polls: 0,
|
|
elapsedMs: 0,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] {
|
|
if (applyOnly) return [];
|
|
const next = sentinelP5Next(state);
|
|
return [`targetValidation quick verify is outside the CI/CD validation gate; run ${next.quickVerify} only as separate post-deploy evidence if needed.`];
|
|
}
|
|
|
|
export function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
|
|
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
|
|
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
|
|
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, only Code Agent multi-round business failures should block acceptance.`];
|
|
}
|
|
|
|
export function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] {
|
|
const warnings: string[] = [];
|
|
for (const item of items) {
|
|
const values = Array.isArray(item) ? item : [item];
|
|
for (const value of values) {
|
|
if (value === undefined || value === null || value === "") continue;
|
|
const warning = text(value).trim();
|
|
if (warning.length > 0 && warning !== "-" && !warnings.includes(warning)) warnings.push(warning);
|
|
}
|
|
}
|
|
return warnings;
|
|
}
|
|
|
|
export function withWarnings(payload: Record<string, unknown>, warnings: readonly unknown[]): Record<string, unknown> {
|
|
const merged = mergeWarnings(payload.warnings, warnings);
|
|
return merged.length === 0 ? payload : { ...payload, warnings: merged, valuesRedacted: true };
|
|
}
|
|
|
|
function sentinelProgressEvent(event: string, payload: Record<string, unknown>): void {
|
|
console.error(JSON.stringify({ event, at: new Date().toISOString(), ...payload, valuesRedacted: true }));
|
|
}
|
|
|
|
function installSentinelPublishInterruptHandler(state: SentinelCicdState, context: Record<string, unknown>): () => void {
|
|
let handled = false;
|
|
const handler = (signal: string) => {
|
|
if (handled) return;
|
|
handled = true;
|
|
const exitCode = signal === "SIGINT" ? 130 : 143;
|
|
const next = publishCurrentNext(state);
|
|
sentinelProgressEvent("sentinel.publish.interrupted", {
|
|
signal,
|
|
exitCode,
|
|
phase: context.phase ?? "unknown",
|
|
pipelineRun: context.pipelineRun ?? sentinelPipelineRunName(state, false),
|
|
sourceCommit: context.sourceCommit ?? state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
recoveryNext: {
|
|
status: next.controlPlaneStatus,
|
|
retry: next.publishCurrent,
|
|
gitMirrorStatus: next.gitMirrorStatus,
|
|
gitMirrorSync: next.gitMirrorSync,
|
|
gitMirrorFlush: next.gitMirrorFlush,
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
});
|
|
process.exit(exitCode);
|
|
};
|
|
process.once("SIGTERM", handler);
|
|
process.once("SIGINT", handler);
|
|
return () => {
|
|
process.off("SIGTERM", handler);
|
|
process.off("SIGINT", handler);
|
|
};
|
|
}
|
|
|
|
function confirmBlocked(action: string, state: SentinelCicdState): Record<string, unknown> {
|
|
return {
|
|
code: "sentinel-cicd-confirm-requires-tekton-pipelinerun",
|
|
action,
|
|
reason: "Confirmed publish uses a Tekton PipelineRun for image build and GitOps writeback; dry-run refuses to report a deployment mutation before that CI run is submitted.",
|
|
sourceGitMirrorReadUrl: stringAt(state.cicd, "source.gitMirrorReadUrl"),
|
|
requiredNextImplementation: [
|
|
"clone source from source.gitMirrorReadUrl at selected commit",
|
|
"build and push digest-pinned image through Tekton PipelineRun on the selected node",
|
|
"publish manifests to the HWLAB gitops branch/path through git-mirror",
|
|
"flush/recheck git-mirror and let Argo reconcile the Application",
|
|
],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelControlPlaneAction): Record<string, string> {
|
|
const node = state.spec.nodeId;
|
|
const lane = state.spec.lane;
|
|
const suffix = sentinelCliSuffix(state);
|
|
return {
|
|
plan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${node} --lane ${lane}${suffix} --dry-run`,
|
|
status: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
|
|
image: `bun scripts/cli.ts web-probe sentinel image status --node ${node} --lane ${lane}${suffix}`,
|
|
triggerCurrent: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${node} --lane ${lane}${suffix} --dry-run`,
|
|
apply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${node} --lane ${lane}${suffix} --confirm --wait`,
|
|
validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`,
|
|
quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`,
|
|
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`,
|
|
gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm`,
|
|
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`,
|
|
issue: "https://github.com/pikasTech/unidesk/issues/1285",
|
|
currentAction: action,
|
|
};
|
|
}
|
|
|
|
function controlPlaneRecoveryNext(state: SentinelCicdState, ok: boolean, publish: unknown, flush: unknown, observed: unknown): Record<string, unknown> | null {
|
|
const payload = record(record(publish).payload);
|
|
if (ok || nonEmptyString(payload.digestRef) === null) return null;
|
|
const next = controlPlaneNext(state, "apply");
|
|
const flushRecord = record(flush);
|
|
const observedRecord = record(observed);
|
|
return {
|
|
reason: "publish produced an image digest, but GitOps/git-mirror/Argo/runtime alignment is not complete yet",
|
|
pipelineRun: record(publish).jobName ?? null,
|
|
digestRef: payload.digestRef,
|
|
gitopsCommit: payload.gitopsCommit ?? null,
|
|
flushMode: flushRecord.mode ?? null,
|
|
observedReady: sentinelObservedReady(observedRecord),
|
|
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
|
|
nextStatus: next.status,
|
|
gitMirrorStatus: next.gitMirrorStatus,
|
|
gitMirrorSync: next.gitMirrorSync,
|
|
gitMirrorFlush: next.gitMirrorFlush,
|
|
controlPlaneApply: next.apply,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
|
for (const source of arrayAt(state.secrets, "sources").map(record)) {
|
|
const purpose = stringAtNullable(source, "purpose");
|
|
if (purpose !== null) sourcesByPurpose.set(purpose, source);
|
|
}
|
|
const desired: Array<{ namespace: string; name: string; data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> }> = [];
|
|
const missing: Record<string, unknown>[] = [];
|
|
const skipped: Record<string, unknown>[] = [];
|
|
for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) {
|
|
const name = stringAt(runtimeSecret, "name");
|
|
const namespace = stringAt(runtimeSecret, "namespace");
|
|
const data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> = [];
|
|
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
|
|
const sourcePurpose = stringAt(item, "sourcePurpose");
|
|
const targetKey = stringAt(item, "targetKey");
|
|
if (sourcePurpose === "frp-token") {
|
|
skipped.push({ name, namespace, targetKey, sourcePurpose, reason: "managed-by-publicExposure-frpc", valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const source = sourcesByPurpose.get(sourcePurpose);
|
|
if (source === undefined) {
|
|
missing.push({ name, namespace, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const sourceRef = stringAt(source, "sourceRef");
|
|
const sourceKey = stringAt(source, "sourceKey");
|
|
const material = readSentinelSecretSourceValue(source);
|
|
if (!material.ok) {
|
|
missing.push({ name, namespace, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const value = stringAt(material, "value");
|
|
data.push({
|
|
key: targetKey,
|
|
value,
|
|
sourcePurpose,
|
|
sourceRef,
|
|
sourceKey,
|
|
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
|
});
|
|
}
|
|
if (data.length > 0) desired.push({ namespace, name, data });
|
|
}
|
|
if (missing.length > 0) return { ok: false, phase: "local-source", missing, skipped, valuesRedacted: true };
|
|
if (desired.length === 0) return { ok: true, phase: "skipped-no-runtime-secrets", secretCount: 0, keyCount: 0, skippedKeyCount: skipped.length, skipped, valuesRedacted: true };
|
|
const manifests = desired.map((secret) => ({
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: {
|
|
name: secret.name,
|
|
namespace: secret.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
"unidesk.ai/web-probe-sentinel-id": state.sentinelId,
|
|
},
|
|
},
|
|
type: "Opaque",
|
|
data: Object.fromEntries(secret.data.map((item) => [item.key, Buffer.from(item.value, "utf8").toString("base64")])),
|
|
}));
|
|
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
|
const summary = desired.map((secret) => ({
|
|
name: secret.name,
|
|
namespace: secret.namespace,
|
|
keys: secret.data.map((item) => item.key).sort(),
|
|
sources: secret.data.map((item) => ({ key: item.key, sourcePurpose: item.sourcePurpose, sourceRef: item.sourceRef, sourceKey: item.sourceKey, fingerprint: item.fingerprint, valuesRedacted: true })),
|
|
valuesRedacted: true,
|
|
}));
|
|
const summaryB64 = Buffer.from(JSON.stringify(summary), "utf8").toString("base64");
|
|
const script = [
|
|
"set +e",
|
|
`summary_b64=${shellQuote(summaryB64)}`,
|
|
"tmp=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp\"' EXIT",
|
|
"manifest=\"$tmp/runtime-secrets.yaml\"",
|
|
"cat >\"$manifest\"",
|
|
"kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-runtime-secrets -f \"$manifest\" >/tmp/web-probe-sentinel-runtime-secrets.out 2>/tmp/web-probe-sentinel-runtime-secrets.err",
|
|
"apply_rc=$?",
|
|
"python3 - \"$summary_b64\" \"$apply_rc\" <<'PY'",
|
|
"import base64, json, subprocess, sys",
|
|
"summary = json.loads(base64.b64decode(sys.argv[1]).decode('utf-8'))",
|
|
"apply_rc = int(sys.argv[2])",
|
|
"items = []",
|
|
"ok = apply_rc == 0",
|
|
"for secret in summary:",
|
|
" proc = subprocess.run(['kubectl', '-n', secret['namespace'], 'get', 'secret', secret['name'], '-o', 'json'], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)",
|
|
" data = json.loads(proc.stdout).get('data', {}) if proc.returncode == 0 and proc.stdout else {}",
|
|
" keys = {key: key in data for key in secret['keys']}",
|
|
" present = proc.returncode == 0 and all(keys.values())",
|
|
" ok = ok and present",
|
|
" items.append({'name': secret['name'], 'namespace': secret['namespace'], 'present': present, 'keys': keys, 'sources': secret['sources'], 'valuesRedacted': True})",
|
|
"print(json.dumps({'ok': ok, 'applyExitCode': apply_rc, 'secretCount': len(summary), 'keyCount': sum(len(item['keys']) for item in summary), 'items': items, 'valuesRedacted': True}, ensure_ascii=False))",
|
|
"PY",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: manifestYaml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function readSentinelSecretSourceValue(source: Record<string, unknown>): Record<string, unknown> {
|
|
const sourceRef = stringAt(source, "sourceRef");
|
|
const sourceKey = stringAt(source, "sourceKey");
|
|
const sourceLine = numberAtNullable(source, "sourceLine");
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, error: "secret-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const textValue = readFileSync(sourcePath, "utf8");
|
|
const value = sourceLine === null ? parseEnvFile(textValue)[sourceKey] : textValue.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "");
|
|
if (value === undefined || value.length === 0) return { ok: false, error: sourceLine === null ? "secret-source-key-missing" : "secret-source-line-missing", sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const format = stringAtNullable(source, "format");
|
|
if (format === null) return { ok: true, value, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
if (format === "web-account-json") {
|
|
const username = readSentinelWebAccountUsername(source);
|
|
if (!username.ok) return { ok: false, error: username.error, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
return { ok: true, value: JSON.stringify({ username: username.value, password: value }), sourceRef, sourceKey, sourceLine, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
}
|
|
return { ok: false, error: "unsupported-secret-source-format", sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
}
|
|
|
|
function readSentinelWebAccountUsername(source: Record<string, unknown>): { ok: true; value: string } | { ok: false; error: string } {
|
|
const username = stringAtNullable(source, "username");
|
|
if (username !== null) return { ok: true, value: username };
|
|
const sourceRef = stringAtNullable(source, "usernameSourceRef");
|
|
const sourceLine = numberAtNullable(source, "usernameSourceLine");
|
|
if (sourceRef === null || sourceLine === null) return { ok: false, error: "web-account-json-username-missing" };
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, error: "web-account-json-username-source-missing" };
|
|
const value = readFileSync(sourcePath, "utf8").split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? "";
|
|
return value.length === 0 ? { ok: false, error: "web-account-json-username-line-missing" } : { ok: true, value };
|
|
}
|
|
|
|
function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const material = readSentinelFrpcMaterial(state);
|
|
if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true };
|
|
const secret = applySentinelFrpcSecret(state, stringAt(material, "frpcToml"), timeoutSeconds);
|
|
const caddy = applySentinelCaddyBlock(state, timeoutSeconds);
|
|
return {
|
|
ok: secret.ok === true && caddy.ok === true,
|
|
hostname: stringAt(state.publicExposure, "hostname"),
|
|
publicBaseUrl: stringAt(state.publicExposure, "publicBaseUrl"),
|
|
material: {
|
|
ok: true,
|
|
sourceRef: material.sourceRef,
|
|
sourcePath: material.sourcePath,
|
|
fingerprint: material.fingerprint,
|
|
valuesRedacted: true,
|
|
},
|
|
secret,
|
|
caddy,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function readSentinelFrpcMaterial(state: SentinelCicdState): Record<string, unknown> {
|
|
const sourceRef = stringAt(state.publicExposure, "frpc.tokenSourceRef");
|
|
const sourceKey = stringAt(state.publicExposure, "frpc.tokenSourceKey");
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, error: "frp-token-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const token = values[sourceKey];
|
|
if (token === undefined || token.length === 0) return { ok: false, error: "frp-token-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const proxy = record(valueAtPath(state.publicExposure, "frpc.httpProxy"));
|
|
const frpcToml = [
|
|
`serverAddr = "${tomlEscape(stringAt(state.publicExposure, "frpc.serverAddr"))}"`,
|
|
`serverPort = ${numberAt(state.publicExposure, "frpc.serverPort")}`,
|
|
"loginFailExit = true",
|
|
`auth.token = "${tomlEscape(token)}"`,
|
|
"",
|
|
"[[proxies]]",
|
|
`name = "${tomlEscape(stringAt(proxy, "name"))}"`,
|
|
'type = "tcp"',
|
|
`localIP = "${tomlEscape(stringAt(proxy, "localIP"))}"`,
|
|
`localPort = ${numberAt(proxy, "localPort")}`,
|
|
`remotePort = ${numberAt(proxy, "remotePort")}`,
|
|
"",
|
|
].join("\n");
|
|
return {
|
|
ok: true,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath: displayPath(sourcePath),
|
|
frpcToml,
|
|
fingerprint: `sha256:${createHash("sha256").update(`${token}\n${frpcToml}`).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function applySentinelFrpcSecret(state: SentinelCicdState, frpcToml: string, timeoutSeconds: number): Record<string, unknown> {
|
|
const namespace = stringAt(state.runtime, "namespace");
|
|
const secretName = stringAt(state.publicExposure, "frpc.secretName");
|
|
const secretKey = stringAt(state.publicExposure, "frpc.secretKey");
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`secret=${shellQuote(secretName)}`,
|
|
`key=${shellQuote(secretKey)}`,
|
|
"tmp=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp\"' EXIT",
|
|
"cat >\"$tmp/frpc.toml\"",
|
|
"kubectl -n \"$namespace\" create secret generic \"$secret\" --from-file=\"$key=$tmp/frpc.toml\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-public-exposure -f - >/tmp/web-probe-sentinel-frpc-secret.out 2>/tmp/web-probe-sentinel-frpc-secret.err",
|
|
"rc=$?",
|
|
"present=no",
|
|
"bytes=0",
|
|
"if kubectl -n \"$namespace\" get secret \"$secret\" -o jsonpath=\"{.data.$key}\" >/tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null; then present=yes; bytes=$(base64 -d </tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null | wc -c | tr -d ' '); fi",
|
|
"node - \"$rc\" \"$namespace\" \"$secret\" \"$key\" \"$present\" \"$bytes\" <<'NODE'",
|
|
"const [rc, namespace, secret, key, present, bytes] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ok:Number(rc)===0&&present==='yes',namespace,secret,key,present:present==='yes',bytes:Number(bytes||0),applyExitCode:Number(rc),valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: frpcToml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const hostname = stringAt(state.publicExposure, "hostname");
|
|
const owner = stringAt(state.publicExposure, "caddy.managedBlockOwner");
|
|
const configPath = stringAt(state.publicExposure, "caddy.configPath");
|
|
const serviceName = stringAt(state.publicExposure, "caddy.serviceName");
|
|
const responseHeaderTimeoutSeconds = numberAt(state.publicExposure, "caddy.responseHeaderTimeoutSeconds");
|
|
const remotePort = numberAt(state.publicExposure, "frpc.httpProxy.remotePort");
|
|
const routePrefix = normalizeRoutePrefix(stringAtNullable(state.publicExposure, "routePrefix"));
|
|
const rootOrder = stringAtNullable(state.publicExposure, "caddy.rootOrder") ?? "normal";
|
|
const monitorRoot = record(state.publicExposure.monitorRoot);
|
|
const cleanupOwner = monitorRoot.enabled === false ? stringAtNullable(monitorRoot, "caddyManagedBlockOwner") : null;
|
|
const proxyLines = [
|
|
`reverse_proxy 127.0.0.1:${remotePort} {`,
|
|
" transport http {",
|
|
` response_header_timeout ${responseHeaderTimeoutSeconds}s`,
|
|
" }",
|
|
"}",
|
|
];
|
|
const block = [
|
|
routePrefix === "/" ? "handle {" : `handle_path ${routePrefix}* {`,
|
|
...proxyLines.map((line) => ` ${line}`),
|
|
"}",
|
|
"",
|
|
].join("\n");
|
|
const blockB64 = Buffer.from(block, "utf8").toString("base64");
|
|
const script = [
|
|
"set +e",
|
|
`hostname=${shellQuote(hostname)}`,
|
|
`owner=${shellQuote(owner)}`,
|
|
`config_path=${shellQuote(configPath)}`,
|
|
`service=${shellQuote(serviceName)}`,
|
|
`route_prefix=${shellQuote(routePrefix)}`,
|
|
`root_order=${shellQuote(rootOrder)}`,
|
|
`cleanup_owner=${shellQuote(cleanupOwner ?? "")}`,
|
|
`block_b64=${shellQuote(blockB64)}`,
|
|
"marker=\"unidesk managed $owner\"",
|
|
"tmp=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp\"' EXIT",
|
|
"block=\"$tmp/block\"",
|
|
"next=\"$tmp/Caddyfile\"",
|
|
"printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
|
|
"if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
|
|
"python3 - \"$next\" \"$block\" \"$marker\" \"$hostname\" \"$route_prefix\" \"$root_order\" \"$cleanup_owner\" <<'PY' >/tmp/web-probe-sentinel-caddy-python.out 2>/tmp/web-probe-sentinel-caddy-python.err",
|
|
"import pathlib, re, sys",
|
|
"config = pathlib.Path(sys.argv[1])",
|
|
"block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
|
|
"marker = sys.argv[3]",
|
|
"hostname = sys.argv[4]",
|
|
"route_prefix = sys.argv[5]",
|
|
"root_order = sys.argv[6]",
|
|
"cleanup_owner = sys.argv[7]",
|
|
"text = config.read_text(encoding='utf-8') if config.exists() else ''",
|
|
"begin = f'# BEGIN {marker}'",
|
|
"end = f'# END {marker}'",
|
|
"def managed_pattern(marker_text):",
|
|
" return re.compile(rf'(?ms)^[ \\t]*# BEGIN {re.escape(marker_text)}\\n.*?^[ \\t]*# END {re.escape(marker_text)}\\n*')",
|
|
"pattern = managed_pattern(marker)",
|
|
"def collect_nested_managed(segment):",
|
|
" preserved = []",
|
|
" lines = segment.splitlines()",
|
|
" index = 0",
|
|
" while index < len(lines):",
|
|
" stripped = lines[index].strip()",
|
|
" if stripped.startswith('# BEGIN ') and stripped != begin:",
|
|
" owner_text = stripped[len('# BEGIN '):]",
|
|
" end_line = '# END ' + owner_text",
|
|
" block_lines = [lines[index]]",
|
|
" index += 1",
|
|
" while index < len(lines):",
|
|
" block_lines.append(lines[index])",
|
|
" if lines[index].strip() == end_line:",
|
|
" break",
|
|
" index += 1",
|
|
" preserved.append('\\n'.join(block_lines).rstrip() + '\\n')",
|
|
" index += 1",
|
|
" return preserved",
|
|
"preserved_blocks = []",
|
|
"for match in pattern.finditer(text):",
|
|
" preserved_blocks.extend(collect_nested_managed(match.group(0)))",
|
|
"text = pattern.sub('', text)",
|
|
"if cleanup_owner:",
|
|
" cleanup_marker = f'unidesk managed {cleanup_owner}'",
|
|
" if cleanup_marker != marker:",
|
|
" text = managed_pattern(cleanup_marker).sub('', text)",
|
|
"def site_span(src, host):",
|
|
" match = re.search(rf'(?m)^([ \\t]*){re.escape(host)}[ \\t]*\\{{[ \\t]*\\n', src)",
|
|
" if not match:",
|
|
" return None",
|
|
" depth = 1",
|
|
" index = match.end()",
|
|
" while index < len(src):",
|
|
" char = src[index]",
|
|
" if char == '{':",
|
|
" depth += 1",
|
|
" elif char == '}':",
|
|
" depth -= 1",
|
|
" if depth == 0:",
|
|
" end_index = index + 1",
|
|
" if end_index < len(src) and src[end_index] == '\\n':",
|
|
" end_index += 1",
|
|
" return match.start(), end_index, index, match.end()",
|
|
" index += 1",
|
|
" raise ValueError(f'unclosed Caddy site block for {host}')",
|
|
"handler = '\\n'.join((' ' + line) if line else '' for line in block.rstrip().splitlines())",
|
|
"managed = f' {begin}\\n{handler}\\n {end}\\n'",
|
|
"def fallback_insert_pos(site, relative_open, close_rel):",
|
|
" for match in re.finditer(r'(?m)^[ \\t]*handle[ \\t]*\\{[ \\t]*\\n', site[relative_open:close_rel]):",
|
|
" pos = relative_open + match.start()",
|
|
" prev_end = pos - 1",
|
|
" if prev_end >= 0 and site[prev_end] == '\\n':",
|
|
" prev_end -= 1",
|
|
" if prev_end >= 0:",
|
|
" prev_start = site.rfind('\\n', 0, prev_end + 1) + 1",
|
|
" if site[prev_start:prev_end + 1].strip().startswith('# BEGIN '):",
|
|
" return prev_start",
|
|
" return pos",
|
|
" return relative_open",
|
|
"def append_before_close(site, close_rel, addition):",
|
|
" prefix = site[:close_rel]",
|
|
" suffix = site[close_rel:]",
|
|
" if prefix and not prefix.endswith('\\n'):",
|
|
" prefix += '\\n'",
|
|
" return prefix + addition + suffix",
|
|
"span = site_span(text, hostname)",
|
|
"if span is None:",
|
|
" body = ''.join(preserved_blocks) + managed",
|
|
" text = text.rstrip() + '\\n\\n' + f'{hostname} {{\\n{body}}}\\n'",
|
|
"else:",
|
|
" start, stop, close_index, open_end = span",
|
|
" site = text[start:stop]",
|
|
" relative_open = open_end - start",
|
|
" close_rel = close_index - start",
|
|
" additions = ''.join(preserved_blocks) + managed",
|
|
" if route_prefix == '/' and root_order == 'active':",
|
|
" replacement = site[:relative_open] + additions + site[relative_open:]",
|
|
" elif route_prefix == '/':",
|
|
" replacement = append_before_close(site, close_rel, additions)",
|
|
" else:",
|
|
" insert_at = fallback_insert_pos(site, relative_open, close_rel)",
|
|
" replacement = site[:insert_at] + additions + site[insert_at:]",
|
|
" text = text[:start] + replacement + text[stop:]",
|
|
"config.write_text(text, encoding='utf-8')",
|
|
"PY",
|
|
"python_rc=$?",
|
|
"validate_rc=1",
|
|
"reload_rc=",
|
|
"if [ \"$python_rc\" = 0 ]; then sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/web-probe-sentinel-caddy-validate.out 2>/tmp/web-probe-sentinel-caddy-validate.err; validate_rc=$?; fi",
|
|
"if [ \"$validate_rc\" = 0 ]; then sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/web-probe-sentinel-caddy-install.out 2>/tmp/web-probe-sentinel-caddy-install.err && (sudo systemctl reload \"$service\" >/tmp/web-probe-sentinel-caddy-reload.out 2>/tmp/web-probe-sentinel-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/web-probe-sentinel-caddy-reload.out 2>>/tmp/web-probe-sentinel-caddy-reload.err); reload_rc=$?; fi",
|
|
"probe_rc=1",
|
|
"probe_status=",
|
|
"if [ \"$reload_rc\" = 0 ]; then",
|
|
" probe_path=\"$route_prefix\"",
|
|
" if [ \"$probe_path\" = \"/\" ]; then probe_url=\"https://$hostname/\"; else probe_url=\"https://$hostname$probe_path/\"; fi",
|
|
" probe_status=$(curl -k -sS -o /tmp/web-probe-sentinel-caddy-probe.out -w '%{http_code}' --max-time 10 --resolve \"$hostname:443:127.0.0.1\" \"$probe_url\" 2>/tmp/web-probe-sentinel-caddy-probe.err)",
|
|
" probe_rc=$?",
|
|
"fi",
|
|
"after_present=no",
|
|
"grep -Fq \"# BEGIN $marker\" \"$config_path\" 2>/dev/null && after_present=yes",
|
|
"active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
|
|
"err=$(cat /tmp/web-probe-sentinel-caddy-python.err /tmp/web-probe-sentinel-caddy-validate.err /tmp/web-probe-sentinel-caddy-install.err /tmp/web-probe-sentinel-caddy-reload.err /tmp/web-probe-sentinel-caddy-probe.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
|
|
"python3 - \"$python_rc\" \"$validate_rc\" \"$reload_rc\" \"$probe_rc\" \"$probe_status\" \"$after_present\" \"$active\" \"$hostname\" \"$config_path\" \"$err\" <<'PY'",
|
|
"import json, sys",
|
|
"python_rc, validate_rc, reload_rc, probe_rc, probe_status, after_present, active, hostname, config_path, error_preview = sys.argv[1:11]",
|
|
"def num(value):",
|
|
" if value == '':",
|
|
" return None",
|
|
" try:",
|
|
" return int(value)",
|
|
" except ValueError:",
|
|
" return None",
|
|
"http_status = num(probe_status)",
|
|
"probe_ok = num(probe_rc) == 0 and http_status is not None and 200 <= http_status < 400",
|
|
"payload = {",
|
|
" 'ok': num(python_rc) == 0 and num(validate_rc) == 0 and num(reload_rc) == 0 and probe_ok and after_present == 'yes',",
|
|
" 'hostname': hostname,",
|
|
" 'configPath': config_path,",
|
|
" 'pythonExitCode': num(python_rc),",
|
|
" 'validateExitCode': num(validate_rc),",
|
|
" 'reloadExitCode': num(reload_rc),",
|
|
" 'routeProbeExitCode': num(probe_rc),",
|
|
" 'routeProbeHttpStatus': http_status,",
|
|
" 'routeProbeOk': probe_ok,",
|
|
" 'afterBlockPresent': after_present == 'yes',",
|
|
" 'active': active,",
|
|
" 'errorPreview': error_preview,",
|
|
" 'valuesRedacted': True,",
|
|
"}",
|
|
"print(json.dumps(payload, ensure_ascii=False))",
|
|
"PY",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.publicExposure, "caddy.route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, rootOrder, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
export function secretSourcePaths(sourceRef: string): string[] {
|
|
if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef);
|
|
const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef));
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
function ownerFileSourcePaths(sourceRef: string): string[] {
|
|
if (sourceRef.includes("..") || sourceRef.includes("\0")) return [];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot];
|
|
return [...new Set(roots.map((root) => join(root, sourceRef)))];
|
|
}
|
|
|
|
export function parseEnvFile(textValue: string): Record<string, string> {
|
|
const values: Record<string, string> = {};
|
|
for (const rawLine of textValue.split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (line.length === 0 || line.startsWith("#")) continue;
|
|
const index = line.indexOf("=");
|
|
if (index <= 0) continue;
|
|
const key = line.slice(0, index).trim();
|
|
let value = line.slice(index + 1).trim();
|
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1);
|
|
values[key] = value;
|
|
}
|
|
return values;
|
|
}
|
|
|
|
export function stringAtNullable(value: unknown, path: string): string | null {
|
|
const found = valueAtPath(value, path);
|
|
return typeof found === "string" && found.length > 0 ? found : null;
|
|
}
|
|
|
|
export function numberAtNullable(value: unknown, path: string): number | null {
|
|
const found = valueAtPath(value, path);
|
|
return typeof found === "number" && Number.isFinite(found) ? found : null;
|
|
}
|
|
|
|
function booleanAtNullable(value: unknown, path: string): boolean | null {
|
|
const found = valueAtPath(value, path);
|
|
return typeof found === "boolean" ? found : null;
|
|
}
|
|
|
|
export function displayPath(pathValue: string): string {
|
|
if (pathValue.startsWith(`${repoRoot}/`)) return pathValue.slice(repoRoot.length + 1);
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) {
|
|
const mainRoot = repoRoot.slice(0, index);
|
|
if (pathValue.startsWith(`${mainRoot}/`)) return pathValue.slice(mainRoot.length + 1);
|
|
}
|
|
return pathValue;
|
|
}
|
|
|
|
function sentinelPipelineRunName(state: SentinelCicdState, rerun = false): string {
|
|
const commit = state.sourceHead.commit ?? "source";
|
|
const base = `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`;
|
|
if (!rerun) return base;
|
|
const suffix = `-r${Date.now().toString(36)}`;
|
|
return `${base.slice(0, Math.max(1, 63 - suffix.length)).replace(/-+$/u, "")}${suffix}`;
|
|
}
|
|
|
|
export function sentinelCliSuffix(state: SentinelCicdState): string {
|
|
return ` --sentinel ${state.sentinelId}`;
|
|
}
|
|
|
|
export function safeJobSegment(value: string): string {
|
|
return value.replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "").slice(0, 48) || "sentinel";
|
|
}
|
|
|
|
function safeKubernetesSegment(value: string, maxLength: number): string {
|
|
const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
return (normalized || "sentinel").slice(0, Math.max(1, maxLength)).replace(/-+$/u, "") || "sentinel";
|
|
}
|
|
|
|
function renderPublishResult(publish: Record<string, unknown>): string {
|
|
const payload = record(publish.payload);
|
|
const diagnostics = record(publish.diagnostics);
|
|
const diagnosticEnvReuse = record(diagnostics.envReuse);
|
|
const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse;
|
|
const imageBuild = record(payload.imageBuild);
|
|
const imageBuildProxy = record(imageBuild.proxy);
|
|
const payloadStageTimings = record(payload.stageTimings);
|
|
const diagnosticStageTimings = record(diagnostics.stageTimings);
|
|
const timings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings;
|
|
const commands = record(diagnostics.commands);
|
|
const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none";
|
|
const runColumn = diagnostics.resourceKind === "PipelineRun" || publish.resourceKind === "PipelineRun" ? "PIPELINERUN" : "JOB";
|
|
const lines = [
|
|
"PUBLISH",
|
|
table(["OK", "PHASE", runColumn, "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[
|
|
publish.ok,
|
|
publish.phase,
|
|
publish.jobName,
|
|
publish.elapsedMs ?? "-",
|
|
diagnostics.pod ?? "-",
|
|
diagnostics.currentPhase ?? "-",
|
|
short(payload.digestRef),
|
|
short(payload.gitopsCommit),
|
|
]]),
|
|
];
|
|
if (Object.keys(envReuse).length > 0) {
|
|
lines.push(
|
|
"",
|
|
"PUBLISH_ENV_REUSE",
|
|
table(["MODE", "NODE_DEPS", "PRESENT", "ENTRIES", "LINKED", "DEPENDENCY"], [[
|
|
envReuse.mode,
|
|
envReuse.nodeDepsPath,
|
|
envReuse.nodeDepsPresent,
|
|
envReuse.nodeDepsEntries,
|
|
envReuse.linkedNodeDeps ?? "-",
|
|
envReuse.dependencyReuse,
|
|
]]),
|
|
);
|
|
}
|
|
if (Object.keys(imageBuild).length > 0 || Object.keys(timings).length > 0) {
|
|
lines.push(
|
|
"",
|
|
"PUBLISH_BUILD",
|
|
table(["BUILDER", "PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "TOTAL_MS"], [[
|
|
imageBuild.builder ?? "-",
|
|
imageBuild.packageMode ?? "-",
|
|
imageBuild.networkMode ?? "-",
|
|
proxySummary,
|
|
imageBuild.contextIgnoreEntries ?? "-",
|
|
imageBuild.layerCache ?? "-",
|
|
imageBuild.cacheHitLines ?? "-",
|
|
imageBuild.stepLines ?? "-",
|
|
timings.sourceFetchMs ?? "-",
|
|
timings.monitorWebVerifyMs ?? "-",
|
|
timings.imageBuildMs ?? "-",
|
|
timings.gitopsMs ?? "-",
|
|
timings.totalMs ?? payload.elapsedMs ?? "-",
|
|
]]),
|
|
);
|
|
}
|
|
if (Object.keys(diagnostics).length > 0) {
|
|
lines.push(
|
|
"",
|
|
"PUBLISH_DIAGNOSTICS",
|
|
table(["TASKRUN", "POD_PHASE", "ACTIVE", "CONDITION", "COMPLETED", "RECENT_LOG"], [[
|
|
diagnostics.taskRun ?? "-",
|
|
diagnostics.podPhase ?? "-",
|
|
diagnostics.active ?? "-",
|
|
diagnostics.conditionReason ?? diagnostics.conditionStatus ?? "-",
|
|
Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "-",
|
|
diagnostics.recentLogSummary ?? "-",
|
|
]]),
|
|
);
|
|
}
|
|
if (publish.ok !== true && Object.keys(commands).length > 0) {
|
|
lines.push(
|
|
"",
|
|
"PUBLISH_DRILLDOWN",
|
|
` status: ${commands.cliStatus ?? "-"}`,
|
|
` logs: ${commands.logs ?? "-"}`,
|
|
` describe: ${commands.describe ?? "-"}`,
|
|
` publish-current: ${commands.publishCurrent ?? "-"}`,
|
|
` git-mirror: ${commands.gitMirrorStatus ?? "-"}`,
|
|
` sync: ${commands.gitMirrorSync ?? "-"}`,
|
|
` flush: ${commands.gitMirrorFlush ?? "-"}`,
|
|
` apply: ${commands.controlPlaneApply ?? "-"}`,
|
|
);
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function renderPublishCurrentResult(result: Record<string, unknown>): string {
|
|
const source = record(result.source);
|
|
const image = record(result.image);
|
|
const controlPlane = record(result.controlPlane);
|
|
const publish = record(controlPlane.publish);
|
|
const publishPayload = record(publish.payload);
|
|
const observed = record(controlPlane.observed);
|
|
const gitops = record(observed.gitops);
|
|
const argo = record(observed.argo);
|
|
const runtime = record(observed.runtime);
|
|
const runtimeDeployment = record(record(runtime.probe).deployment);
|
|
const health = record(result.health);
|
|
const healthBody = record(record(health.health).bodyJson);
|
|
const timings = record(result.timings);
|
|
const budget = record(result.budget);
|
|
const stageBudgets = record(result.stageBudgets);
|
|
const validationPlan = record(result.validationPlan);
|
|
const blocker = record(result.blocker);
|
|
const recoveryNext = record(controlPlane.recoveryNext);
|
|
const next = record(result.next);
|
|
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
const slowStages = Array.isArray(result.slowStages) ? result.slowStages.map(record) : [];
|
|
const lines = [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "SENTINEL", "STATUS", "MODE", "BUDGET_S", "ELAPSED_S"], [[
|
|
result.node,
|
|
result.lane,
|
|
result.sentinelId,
|
|
result.ok === true ? "ok" : "blocked",
|
|
result.mode,
|
|
budget.maxSeconds ?? "-",
|
|
finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000),
|
|
]]),
|
|
"",
|
|
table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[
|
|
`${source.repository ?? "-"}@${source.branch ?? "-"}`,
|
|
short(source.commit),
|
|
source.sourceAuthority ?? "-",
|
|
short(source.stageRef),
|
|
image.ref ?? "-",
|
|
short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest),
|
|
result.pipelineRun ?? publish.jobName ?? "-",
|
|
]]),
|
|
"",
|
|
table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "HEALTH"], [[
|
|
short(gitops.revision),
|
|
short(argo.revision),
|
|
`${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`,
|
|
short(runtimeDeployment.image),
|
|
`${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`,
|
|
health.ok === true ? "pass" : health.skipped === true ? `skipped:${text(health.reason)}` : Object.keys(health).length === 0 ? "planned" : "blocked",
|
|
]]),
|
|
"",
|
|
table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "VALIDATION_MS", "TOTAL_MS"], [[
|
|
timings.sourceSyncMs ?? "-",
|
|
timings.sourceFetchMs ?? "-",
|
|
timings.monitorWebVerifyMs ?? "-",
|
|
timings.imageBuildMs ?? "-",
|
|
timings.gitopsMs ?? "-",
|
|
timings.argoRuntimeMs ?? "-",
|
|
timings.healthValidationMs ?? "-",
|
|
timings.totalMs ?? "-",
|
|
]]),
|
|
"",
|
|
table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "VALIDATION"], [[
|
|
"YAML publishCurrent",
|
|
stageBudgets.sourceSyncSeconds ?? "-",
|
|
stageBudgets.sourceFetchSeconds ?? "-",
|
|
stageBudgets.monitorWebVerifySeconds ?? "-",
|
|
stageBudgets.imageBuildSeconds ?? "-",
|
|
stageBudgets.gitopsSeconds ?? "-",
|
|
stageBudgets.argoRuntimeSeconds ?? "-",
|
|
stageBudgets.dashboardVerifySeconds ?? "-",
|
|
]]),
|
|
];
|
|
if (Object.keys(publish).length > 0) {
|
|
const payloadImageBuild = record(publishPayload.imageBuild);
|
|
const payloadEnvReuse = record(publishPayload.envReuse);
|
|
lines.push(
|
|
"",
|
|
table(["ENV_REUSE", "NODE_DEPS", "BUILD_PACKAGE", "BUILD_NETWORK", "CACHE", "CACHE_LINES"], [[
|
|
payloadEnvReuse.dependencyReuse ?? "-",
|
|
payloadEnvReuse.nodeDepsPresent ?? "-",
|
|
payloadImageBuild.packageMode ?? "-",
|
|
payloadImageBuild.networkMode ?? "-",
|
|
payloadImageBuild.layerCache ?? "-",
|
|
payloadImageBuild.cacheHitLines ?? "-",
|
|
]]),
|
|
);
|
|
}
|
|
lines.push(
|
|
"",
|
|
Object.keys(health).length === 0
|
|
? "HEALTH_VALIDATION\n-"
|
|
: table(["ENDPOINT", "HTTP", "OK", "STATUS", "PUBLIC_URL", "INTERNAL_URL"], [[
|
|
health.endpoint ?? validationPlan.endpoint ?? "-",
|
|
health.httpStatus ?? "-",
|
|
healthBody.ok ?? "-",
|
|
healthBody.status ?? "-",
|
|
health.publicUrl ?? "-",
|
|
health.internalUrl ?? "-",
|
|
]]),
|
|
"",
|
|
slowStages.length === 0 ? "SLOW_STAGES\n-" : [
|
|
"SLOW_STAGES",
|
|
table(["STAGE", "ELAPSED_MS", "BUDGET_S", "SUGGESTION"], slowStages.map((stage) => [stage.stage, stage.elapsedMs, stage.budgetSeconds, stage.suggestion])),
|
|
].join("\n"),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"),
|
|
"",
|
|
Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [
|
|
"RECOVERY_NEXT",
|
|
table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]),
|
|
` publish-current: ${recoveryNext.publishCurrent ?? "-"}`,
|
|
` status: ${recoveryNext.nextStatus ?? "-"}`,
|
|
` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`,
|
|
` sync: ${recoveryNext.gitMirrorSync ?? "-"}`,
|
|
` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`,
|
|
` apply: ${recoveryNext.controlPlaneApply ?? "-"}`,
|
|
].join("\n"),
|
|
"",
|
|
"NEXT",
|
|
` publish-current: ${next.publishCurrent ?? "-"}`,
|
|
` status: ${next.controlPlaneStatus ?? "-"}`,
|
|
` post-deploy-dashboard: ${next.dashboardVerify ?? "-"}`,
|
|
` git-mirror: ${next.gitMirrorStatus ?? "-"}`,
|
|
` sync: ${next.gitMirrorSync ?? "-"}`,
|
|
` flush: ${next.gitMirrorFlush ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
` end-to-end and stage budgets are read from ${Object.keys(validationPlan).length > 0 ? "publishCurrent YAML and runtime.healthPath" : "YAML-required publishCurrent fields"}.`,
|
|
" CI/CD validation only checks the configured health endpoint; web-probe, Playwright and browser dashboard checks are post-deploy evidence, not this gate.",
|
|
" image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.",
|
|
);
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function renderImageResult(result: Record<string, unknown>): string {
|
|
const source = record(result.source);
|
|
const sourceMirror = record(result.sourceMirror);
|
|
const sourceMirrorSync = record(result.sourceMirrorSync);
|
|
const image = record(result.image);
|
|
const monitorWeb = record(image.monitorWeb);
|
|
const registry = record(result.registry);
|
|
const publish = record(result.publish);
|
|
const blocker = record(result.blocker);
|
|
const next = record(result.next);
|
|
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]),
|
|
"",
|
|
table(["SOURCE_REPO", "BRANCH", "COMMIT", "AUTHORITY", "STAGE_REF", "MIRROR"], [[source.repository, source.branch, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), short(source.mirrorCommit)]]),
|
|
"",
|
|
Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]),
|
|
"",
|
|
table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]),
|
|
"",
|
|
Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "BUILD_STATE", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, `${record(monitorWeb.imageBuildState).mode ?? "-"}:${record(monitorWeb.imageBuildState).path ?? record(monitorWeb.imageBuildState).claimName ?? record(monitorWeb.imageBuildState).sizeLimit ?? "-"}`, monitorWeb.imageBuildContextIgnore]]),
|
|
"",
|
|
Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
|
|
"",
|
|
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]),
|
|
"",
|
|
Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"),
|
|
"",
|
|
"NEXT",
|
|
` status: ${next.status ?? "-"}`,
|
|
` dry-run: ${next.dryRun ?? "-"}`,
|
|
` confirm: ${next.confirm ?? "-"}`,
|
|
` trigger: ${next.controlPlaneTrigger ?? "-"}`,
|
|
` control-plane: ${next.controlPlanePlan ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" valuesRedacted=true; image status shows refs, hashes and object names only.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderControlPlaneResult(result: Record<string, unknown>): string {
|
|
const source = record(result.source);
|
|
const image = record(result.image);
|
|
const gitops = record(result.gitops);
|
|
const argo = record(result.argo);
|
|
const validation = record(result.validation);
|
|
const observability = record(result.observability);
|
|
const observed = record(result.observed);
|
|
const sourceMirrorSync = record(result.sourceMirrorSync);
|
|
const publish = record(result.publish);
|
|
const flush = record(result.flush);
|
|
const runtimeSecretsApply = record(result.runtimeSecretsApply);
|
|
const publicExposureApply = record(result.publicExposureApply);
|
|
const publicExposureCaddy = record(publicExposureApply.caddy);
|
|
const argoApply = record(result.argoApply);
|
|
const blocker = record(result.blocker);
|
|
const statusDiagnosis = record(result.statusDiagnosis);
|
|
const targetValidation = record(result.targetValidation);
|
|
const targetValidationBusiness = record(targetValidation.businessStatus);
|
|
const recoveryNext = record(result.recoveryNext);
|
|
const next = record(result.next);
|
|
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]),
|
|
"",
|
|
table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref, short(gitops.manifestSha256)]]),
|
|
"",
|
|
table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]),
|
|
"",
|
|
table(["SCENARIO", "MAX_SECONDS", "CI_WAIT", "QVERIFY", "SECOND_PATH"], [[validation.scenarioId, validation.maxSeconds, validation.controlPlaneWaitMaxSeconds ?? "-", validation.quickVerifyMode ?? "-", validation.automaticSecondPath]]),
|
|
"",
|
|
Object.keys(observability).length === 0 ? "OTEL\n-" : table(["ENABLED", "ENDPOINT", "SERVICE", "COVERAGE"], [[observability.enabled, observability.endpointConfigured, observability.serviceName, observability.coverage]]),
|
|
"",
|
|
renderObservedStatus(observed),
|
|
"",
|
|
Object.keys(statusDiagnosis).length === 0 ? "STATUS_DIAGNOSIS\n-" : [
|
|
"STATUS_DIAGNOSIS",
|
|
table(["CODE", "PHASE", "PIPELINERUN", "SOURCE", "REGISTRY", "GIT_MIRROR", "GITOPS", "ARGO", "RUNTIME"], [[
|
|
statusDiagnosis.code,
|
|
statusDiagnosis.phase,
|
|
statusDiagnosis.pipelineRun,
|
|
statusDiagnosis.sourceMirror,
|
|
statusDiagnosis.registry,
|
|
statusDiagnosis.gitMirror,
|
|
statusDiagnosis.gitops,
|
|
statusDiagnosis.argo,
|
|
statusDiagnosis.runtime,
|
|
]]),
|
|
].join("\n"),
|
|
"",
|
|
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]),
|
|
"",
|
|
Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "ERROR_TITLE", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[
|
|
targetValidation.ok,
|
|
targetValidation.status,
|
|
targetValidationBusiness.status ?? "-",
|
|
targetValidation.errorTitleZh ?? targetValidation.failureTitleZh ?? targetValidationBusiness.errorTitleZh ?? "-",
|
|
targetValidation.scenarioId,
|
|
targetValidation.runId,
|
|
targetValidation.observerId,
|
|
short(targetValidation.reportJsonSha256),
|
|
targetValidation.findingCount,
|
|
targetValidation.artifactCount,
|
|
]]),
|
|
"",
|
|
Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish),
|
|
"",
|
|
Object.keys(flush).length === 0
|
|
? "FLUSH\n-"
|
|
: flush.mode === "async-job"
|
|
? table(["OK", "MODE", "JOB", "STATUS"], [[flush.ok, flush.mode, record(flush.job).id, record(flush.next).status]])
|
|
: table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]),
|
|
"",
|
|
Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]),
|
|
"",
|
|
Object.keys(publicExposureApply).length === 0 ? "PUBLIC_EXPOSURE_APPLY\n-" : table(["OK", "SECRET", "CADDY", "HOST", "ROUTE_HTTP"], [[publicExposureApply.ok, record(publicExposureApply.secret).ok, record(publicExposureApply.caddy).ok, publicExposureApply.hostname, record(publicExposureApply.caddy).routeProbeHttpStatus ?? "-"]]),
|
|
"",
|
|
Object.keys(publicExposureCaddy).length === 0 || publicExposureCaddy.ok === true
|
|
? "CADDY_APPLY_DETAIL\n-"
|
|
: table(["PY", "VALIDATE", "RELOAD", "PROBE", "HTTP", "BLOCK", "ACTIVE", "ERROR", "STDOUT", "STDERR"], [[publicExposureCaddy.pythonExitCode, publicExposureCaddy.validateExitCode, publicExposureCaddy.reloadExitCode, publicExposureCaddy.routeProbeExitCode, publicExposureCaddy.routeProbeHttpStatus, publicExposureCaddy.afterBlockPresent, publicExposureCaddy.active, short(publicExposureCaddy.errorPreview), short(record(publicExposureCaddy.result).stdoutPreview), short(record(publicExposureCaddy.result).stderrPreview)]]),
|
|
"",
|
|
Object.keys(argoApply).length === 0 ? "ARGO_APPLY\n-" : table(["OK", "EXIT", "PREVIEW"], [[argoApply.ok, record(argoApply.result).exitCode, record(argoApply.result).stdoutPreview]]),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"),
|
|
"",
|
|
Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [
|
|
"RECOVERY_NEXT",
|
|
table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]),
|
|
` publish-current: ${recoveryNext.publishCurrent ?? "-"}`,
|
|
` status: ${recoveryNext.nextStatus ?? "-"}`,
|
|
` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`,
|
|
` sync: ${recoveryNext.gitMirrorSync ?? "-"}`,
|
|
` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`,
|
|
` apply: ${recoveryNext.controlPlaneApply ?? "-"}`,
|
|
].join("\n"),
|
|
"",
|
|
"NEXT",
|
|
` plan: ${next.plan ?? "-"}`,
|
|
` status: ${next.status ?? "-"}`,
|
|
` image: ${next.image ?? "-"}`,
|
|
` trigger-current: ${next.triggerCurrent ?? "-"}`,
|
|
` apply: ${next.apply ?? "-"}`,
|
|
` validate: ${next.validate ?? "-"}`,
|
|
` quick-verify: ${next.quickVerify ?? "-"}`,
|
|
` git-mirror: ${next.gitMirrorStatus ?? "-"}`,
|
|
` sync: ${next.gitMirrorSync ?? "-"}`,
|
|
` flush: ${next.gitMirrorFlush ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" default view is a bounded CI/CD summary; full manifest content is represented by object counts and sha256.",
|
|
" sentinel unavailable policy is structured-failure; no automatic second execution path is rendered.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderObservedStatus(observed: Record<string, unknown>): string {
|
|
const rows = [
|
|
observedStatusRow("source", observed.sourceMirror),
|
|
observedStatusRow("registry", observed.registry),
|
|
observedStatusRow("git-mirror", observed.gitMirror),
|
|
observedStatusRow("gitops", observed.gitops),
|
|
observedStatusRow("argo", observed.argo),
|
|
observedStatusRow("runtime", observed.runtime),
|
|
observedStatusRow("cadence", observed.cadence),
|
|
].filter((row) => row !== null);
|
|
if (rows.length === 0) return "OBSERVED\n-";
|
|
return table(["CHECK", "OK", "DETAIL", "EXIT", "TIMED_OUT", "PREVIEW"], rows);
|
|
}
|
|
|
|
function observedStatusRow(name: string, value: unknown): unknown[] | null {
|
|
const item = record(value);
|
|
if (Object.keys(item).length === 0) return null;
|
|
const result = record(item.result);
|
|
return [name, item.ok, observedDetail(name, item), result.exitCode, result.timedOut, result.stdoutPreview];
|
|
}
|
|
|
|
function observedDetail(name: string, item: Record<string, unknown>): string {
|
|
if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`;
|
|
if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`;
|
|
if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`;
|
|
if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`;
|
|
if (name === "argo") {
|
|
const diagnostics = record(item.diagnostics);
|
|
const problems = Array.isArray(diagnostics.problemResources) ? diagnostics.problemResources : [];
|
|
const first = problems.find((entry) => typeof entry === "object" && entry !== null && !Array.isArray(entry)) as Record<string, unknown> | undefined;
|
|
const problemText = Number(diagnostics.problemResourceCount ?? 0) > 0
|
|
? ` degraded=${diagnostics.problemResourceCount}:${first?.kind ?? "-"} ${first?.namespace ?? "-"}/${first?.name ?? "-"} ${first?.healthStatus ?? first?.status ?? "-"}`
|
|
: "";
|
|
return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}${problemText}`;
|
|
}
|
|
if (name === "runtime") {
|
|
const probe = record(item.probe);
|
|
const deployment = record(probe.deployment);
|
|
return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`;
|
|
}
|
|
if (name === "cadence") {
|
|
if (item.skipped === true) return `${item.reason ?? "skipped"}`;
|
|
const probe = record(item.probe);
|
|
return `${probe.code ?? "ok"} schedule=${probe.schedule ?? "-"}/${probe.expectedSchedule ?? "-"} last=${probe.lastScheduleTime ?? "-"} jobs=${probe.jobCount ?? "-"}`;
|
|
}
|
|
return "-";
|
|
}
|
|
|
|
export function renderAsyncJobResult(result: Record<string, unknown>): string {
|
|
const job = record(result.job);
|
|
const next = record(result.next);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "MODE", "MUTATION", "JOB"], [[result.node, result.lane, result.mode, result.mutation, job.id]]),
|
|
"",
|
|
table(["STATUS", "NAME", "CREATED"], [[job.status, job.name, job.createdAt]]),
|
|
"",
|
|
"NEXT",
|
|
` status: ${next.status ?? "-"}`,
|
|
` wait: ${next.wait ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" confirmed operation is delegated to UniDesk job status to keep interactive calls bounded.",
|
|
].join("\n");
|
|
}
|
|
|
|
export function rendered(ok: boolean, command: string, text: string): RenderedCliResult {
|
|
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" };
|
|
}
|
|
|
|
function readConfigFile(file: string): unknown {
|
|
if (file.startsWith("/") || file.includes("..") || !file.startsWith("config/")) throw new Error(`unsafe configRef file: ${file}`);
|
|
const abs = rootPath(file);
|
|
if (!existsSync(abs)) throw new Error(`${file} does not exist`);
|
|
return Bun.YAML.parse(readFileSync(abs, "utf8")) as unknown;
|
|
}
|
|
|
|
function configRefFile(ref: string): string {
|
|
const [file, path, extra] = ref.split("#");
|
|
if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) throw new Error(`invalid configRef: ${ref}`);
|
|
return file;
|
|
}
|
|
|
|
function valueAtPath(value: unknown, path: string): unknown {
|
|
let current: unknown = value;
|
|
for (const segment of path.split(".")) {
|
|
const match = /^([A-Za-z0-9_-]+)?(?:\[(\d+)\])?$/u.exec(segment);
|
|
if (match === null) return undefined;
|
|
if (match[1] !== undefined) {
|
|
if (!isRecord(current)) return undefined;
|
|
current = current[match[1]];
|
|
}
|
|
if (match[2] !== undefined) {
|
|
if (!Array.isArray(current)) return undefined;
|
|
current = current[Number(match[2])];
|
|
}
|
|
}
|
|
return current;
|
|
}
|
|
|
|
export function stringAt(value: unknown, path: string): string {
|
|
const found = valueAtPath(value, path);
|
|
if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return found;
|
|
}
|
|
|
|
export function nonEmptyString(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function stringField(value: Record<string, unknown>, path: string): string {
|
|
return stringAt(value, path);
|
|
}
|
|
|
|
function stringTarget(value: unknown, label: string): string {
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must resolve to a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
export function numberAt(value: unknown, path: string): number {
|
|
const found = valueAtPath(value, path);
|
|
if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`);
|
|
return found;
|
|
}
|
|
|
|
function booleanAt(value: unknown, path: string): boolean {
|
|
const found = valueAtPath(value, path);
|
|
if (typeof found !== "boolean") throw new Error(`${path} must be a boolean`);
|
|
return found;
|
|
}
|
|
|
|
function finiteNumberOrNull(value: unknown): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
export function arrayAt(value: unknown, path: string): unknown[] {
|
|
const found = valueAtPath(value, path);
|
|
if (!Array.isArray(found)) throw new Error(`${path} must be an array`);
|
|
return found;
|
|
}
|
|
|
|
function arrayAtNullable(value: unknown, path: string): Record<string, unknown>[] {
|
|
const found = valueAtPath(value, path);
|
|
return Array.isArray(found) ? found.map(record) : [];
|
|
}
|
|
|
|
export function recordTarget(value: unknown, label: string): Record<string, unknown> {
|
|
if (!isRecord(value)) throw new Error(`${label} must resolve to an object`);
|
|
return value;
|
|
}
|
|
|
|
export function record(value: unknown): Record<string, unknown> {
|
|
return isRecord(value) ? value : {};
|
|
}
|
|
|
|
export function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function manifestObjectSummary(items: readonly Record<string, unknown>[]): readonly Record<string, unknown>[] {
|
|
return items.map((item) => ({
|
|
kind: item.kind ?? null,
|
|
name: record(item.metadata).name ?? null,
|
|
namespace: record(item.metadata).namespace ?? null,
|
|
}));
|
|
}
|
|
|
|
export function compactCommand(result: CommandResult): CompactCommandResult {
|
|
return {
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: Buffer.byteLength(result.stdout),
|
|
stderrBytes: Buffer.byteLength(result.stderr),
|
|
stdoutPreview: result.stdout.trim().slice(0, 500),
|
|
stderrPreview: result.stderr.trim().slice(0, 500),
|
|
};
|
|
}
|
|
|
|
export function parseJsonObject(text: string): Record<string, unknown> | null {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return isRecord(parsed) ? parsed : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function table(headers: string[], rows: unknown[][]): string {
|
|
const normalized = [headers, ...rows.map((row) => row.map(text))];
|
|
const widths = headers.map((_, index) => Math.max(...normalized.map((row) => text(row[index] ?? "").length)));
|
|
return normalized.map((row) => row.map((cell, index) => text(cell).padEnd(widths[index])).join(" ").trimEnd()).join("\n");
|
|
}
|
|
|
|
export function text(value: unknown): string {
|
|
if (value === undefined || value === null || value === "") return "-";
|
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
return String(value).replace(/\s+/gu, " ").trim();
|
|
}
|
|
|
|
export function short(value: unknown): string {
|
|
const raw = text(value);
|
|
if (raw === "-") return raw;
|
|
if (/^sha256:[0-9a-f]{64}$/iu.test(raw)) return `${raw.slice(0, 19)}...`;
|
|
if (/^[0-9a-f]{40}$/iu.test(raw)) return raw.slice(0, 12);
|
|
return raw.length > 42 ? `${raw.slice(0, 39)}...` : raw;
|
|
}
|
|
|
|
function sha256(textValue: string): string {
|
|
return `sha256:${createHash("sha256").update(textValue).digest("hex")}`;
|
|
}
|
|
|
|
export function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, "'\\''")}'`;
|
|
}
|
|
|
|
function tomlEscape(value: string): string {
|
|
return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"');
|
|
}
|