Codex
4892 lines
260 KiB
TypeScript
4892 lines
260 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p8-web-probe-sentinel-recovery.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p10-monitor-web-aggregation.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web.
|
|
// Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { repoRoot, rootPath } from "./config";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
import { startJob } from "./jobs";
|
|
import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered } from "./hwlab-node-web-sentinel-config";
|
|
import { requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver";
|
|
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
|
import type { RenderedCliResult } from "./output";
|
|
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
|
|
|
|
export type WebProbeSentinelConfigAction = "plan" | "status";
|
|
export type WebProbeSentinelImageAction = "status" | "build";
|
|
export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current";
|
|
export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop";
|
|
export type WebProbeSentinelDashboardAction = "verify" | "screenshot";
|
|
export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary";
|
|
|
|
export type WebProbeSentinelOptions =
|
|
| {
|
|
readonly kind: "config";
|
|
readonly action: WebProbeSentinelConfigAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "image";
|
|
readonly action: WebProbeSentinelImageAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
}
|
|
| {
|
|
readonly kind: "control-plane";
|
|
readonly action: WebProbeSentinelControlPlaneAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
}
|
|
| {
|
|
readonly kind: "maintenance";
|
|
readonly action: WebProbeSentinelMaintenanceAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
readonly releaseId: string | null;
|
|
readonly reason: string | null;
|
|
readonly quickVerify: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "validate";
|
|
readonly action: "validate";
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly confirm: boolean;
|
|
readonly wait: boolean;
|
|
readonly timeoutSeconds: number;
|
|
readonly quickVerify: boolean;
|
|
}
|
|
| {
|
|
readonly kind: "report";
|
|
readonly action: "report";
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly view: WebProbeSentinelReportView;
|
|
readonly runId: string | null;
|
|
readonly latest: boolean;
|
|
readonly traceId: string | null;
|
|
readonly sampleSeq: number | null;
|
|
readonly raw: boolean;
|
|
readonly timeoutSeconds: number;
|
|
}
|
|
| {
|
|
readonly kind: "dashboard";
|
|
readonly action: WebProbeSentinelDashboardAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly viewport: string;
|
|
readonly localDir: string;
|
|
readonly name: string | null;
|
|
readonly timeoutMs: number;
|
|
readonly waitTimeoutMs: number;
|
|
readonly timeoutSeconds: number;
|
|
readonly commandTimeoutSeconds: number;
|
|
readonly fullPage: boolean;
|
|
readonly raw: boolean;
|
|
};
|
|
|
|
interface SentinelCicdState {
|
|
readonly spec: HwlabRuntimeLaneSpec;
|
|
readonly sentinelId: string;
|
|
readonly configRefs: Record<string, string>;
|
|
readonly configReady: boolean;
|
|
readonly runtime: Record<string, unknown>;
|
|
readonly cicd: Record<string, unknown>;
|
|
readonly scenarios: unknown;
|
|
readonly publicExposure: Record<string, unknown>;
|
|
readonly secrets: Record<string, unknown>;
|
|
readonly controlPlaneTarget: Record<string, unknown>;
|
|
readonly controlPlaneNode: Record<string, unknown>;
|
|
readonly sourceHead: SourceHead;
|
|
readonly image: SentinelImagePlan;
|
|
readonly manifests: readonly Record<string, unknown>[];
|
|
readonly manifestSha256: string;
|
|
readonly valuesRedacted: true;
|
|
}
|
|
|
|
interface SourceHead {
|
|
readonly ok: boolean;
|
|
readonly repository: string;
|
|
readonly branch: string;
|
|
readonly commit: string | null;
|
|
readonly localHead: string | null;
|
|
readonly result: CompactCommandResult;
|
|
}
|
|
|
|
interface SentinelImagePlan {
|
|
readonly repository: string;
|
|
readonly tag: string;
|
|
readonly ref: string;
|
|
readonly digestRef: string | null;
|
|
readonly baseImage: string;
|
|
readonly buildContext: string;
|
|
readonly entrypoint: string;
|
|
readonly dockerfileSha256: string;
|
|
readonly dockerfilePreview: string;
|
|
readonly monitorWeb: Record<string, unknown>;
|
|
}
|
|
|
|
interface SentinelObservedStatus {
|
|
readonly sourceMirror: Record<string, unknown>;
|
|
readonly registry: Record<string, unknown>;
|
|
readonly gitMirror: Record<string, unknown>;
|
|
readonly gitops: Record<string, unknown>;
|
|
readonly argo: Record<string, unknown>;
|
|
readonly runtime: Record<string, unknown>;
|
|
}
|
|
|
|
interface SentinelObservedExpectation {
|
|
readonly gitopsRevision: string | null;
|
|
readonly runtimeImage: string | null;
|
|
}
|
|
|
|
interface SentinelRemoteJobResult {
|
|
readonly ok: boolean;
|
|
readonly phase: string;
|
|
readonly jobName: string;
|
|
readonly payload: Record<string, unknown>;
|
|
readonly polls?: number;
|
|
readonly elapsedMs?: number;
|
|
readonly create?: Record<string, unknown>;
|
|
readonly probe?: Record<string, unknown>;
|
|
readonly valuesRedacted: true;
|
|
}
|
|
|
|
interface CompactCommandResult {
|
|
readonly exitCode: number | null;
|
|
readonly timedOut: boolean;
|
|
readonly stdoutBytes: number;
|
|
readonly stderrBytes: number;
|
|
readonly stdoutPreview: string;
|
|
readonly stderrPreview: string;
|
|
}
|
|
|
|
interface ChildCliResult {
|
|
readonly ok: boolean;
|
|
readonly parsed: Record<string, unknown> | null;
|
|
readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string };
|
|
}
|
|
|
|
const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard";
|
|
|
|
export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult {
|
|
if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId));
|
|
requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
|
|
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds);
|
|
if (options.kind === "image") return runSentinelImage(state, options);
|
|
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
|
|
if (options.kind === "maintenance") return runSentinelMaintenance(state, options);
|
|
if (options.kind === "validate") return runSentinelValidate(state, options);
|
|
if (options.kind === "dashboard") return runSentinelDashboard(state, options);
|
|
return runSentinelReport(state, options);
|
|
}
|
|
|
|
function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel image ${options.action}`;
|
|
if (options.action === "build" && options.confirm) {
|
|
if (!options.wait) return renderAsyncSentinelJob(state, "image", "build", options.timeoutSeconds);
|
|
return runSentinelImageBuildConfirmed(state, options);
|
|
}
|
|
const sourceMirror = options.action === "status" ? probeSourceMirror(state, options.timeoutSeconds) : null;
|
|
const registry = options.action === "status" ? probeImageRegistry(state, options.timeoutSeconds) : null;
|
|
const sourceMirrorReady = options.action !== "status" || record(sourceMirror).ok === true;
|
|
const registryReady = options.action !== "status" || record(registry?.probe).present === true;
|
|
const result = {
|
|
ok: state.configReady && state.sourceHead.ok && sourceMirrorReady && registryReady,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
|
|
mutation: false,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
sourceMirror,
|
|
image: state.image,
|
|
registry,
|
|
blocker: sourceMirrorReady
|
|
? registryReady ? null : { code: "sentinel-image-missing", reason: "expected sentinel image tag is not present in the node-local registry" }
|
|
: { code: "sentinel-source-mirror-not-ready", reason: "source.gitMirrorReadUrl does not expose the selected source commit yet" },
|
|
next: {
|
|
status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
dryRun: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`,
|
|
confirm: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`,
|
|
controlPlanePlan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderImageResult(result));
|
|
}
|
|
|
|
function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel control-plane ${options.action}`;
|
|
const mutationAction = options.action === "apply" || options.action === "trigger-current";
|
|
if (options.confirm && mutationAction) {
|
|
if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds);
|
|
return runSentinelControlPlaneConfirmed(state, options);
|
|
}
|
|
const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null;
|
|
const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
|
|
const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
|
|
const pipelineRun = sentinelPipelineRunName(state);
|
|
const result = {
|
|
ok: state.configReady && state.sourceHead.ok && observedReady,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
|
|
mutation: false,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
pipelineRun,
|
|
gitops: {
|
|
path: stringField(state.cicd, "gitopsPath"),
|
|
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
|
manifestObjects: state.manifests.length,
|
|
manifestSha256: state.manifestSha256,
|
|
},
|
|
argo: {
|
|
namespace: stringAt(state.cicd, "argo.namespace"),
|
|
projectName: stringAt(state.cicd, "argo.projectName"),
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
},
|
|
maintenance: {
|
|
startCommand: stringAt(state.cicd, "maintenance.startCommand"),
|
|
stopCommand: stringAt(state.cicd, "maintenance.stopCommand"),
|
|
serviceUnavailablePolicy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"),
|
|
},
|
|
validation: {
|
|
scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
|
|
maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
|
|
controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state),
|
|
quickVerifyMode: "manual-validate",
|
|
automaticSecondPath: false,
|
|
},
|
|
manifests: {
|
|
objects: manifestObjectSummary(state.manifests),
|
|
sha256: state.manifestSha256,
|
|
},
|
|
observed,
|
|
warnings: observedWarnings,
|
|
blocker: null,
|
|
next: controlPlaneNext(state, options.action),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderControlPlaneResult(result));
|
|
}
|
|
|
|
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState {
|
|
const sentinel = resolveWebProbeSentinel(spec, sentinelId);
|
|
const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
|
|
const runtime = recordTarget(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime);
|
|
const cicd = recordTarget(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd);
|
|
const scenarios = readConfigRefTarget(sentinel.configRefs.scenarios);
|
|
const publicExposure = recordTarget(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
|
|
const secrets = recordTarget(readConfigRefTarget(sentinel.configRefs.secrets), sentinel.configRefs.secrets);
|
|
const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef");
|
|
const controlPlaneTarget = recordTarget(readConfigRefTarget(controlPlaneRef), controlPlaneRef);
|
|
const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef));
|
|
const nodeId = stringField(controlPlaneTarget, "node");
|
|
const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
|
|
const sourceHead = resolveSourceHead(cicd, timeoutSeconds);
|
|
const image = sentinelImagePlan(cicd, sourceHead);
|
|
const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image);
|
|
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
|
return {
|
|
spec,
|
|
sentinelId: sentinel.id,
|
|
configRefs: sentinel.configRefs,
|
|
configReady: configPlan.ok,
|
|
runtime,
|
|
cicd,
|
|
scenarios,
|
|
publicExposure,
|
|
secrets,
|
|
controlPlaneTarget,
|
|
controlPlaneNode,
|
|
sourceHead,
|
|
image,
|
|
manifests,
|
|
manifestSha256: sha256(manifestYaml),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function resolveSourceHead(cicd: Record<string, unknown>, timeoutSeconds: number): SourceHead {
|
|
const repository = stringAt(cicd, "source.repository");
|
|
const branch = stringAt(cicd, "source.branch");
|
|
const remote = runCommand(["git", "ls-remote", "origin", `refs/heads/${branch}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const local = runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 10_000 });
|
|
const commit = /^[0-9a-f]{40}\b/iu.exec(remote.stdout.trim())?.[0].toLowerCase() ?? null;
|
|
const localHead = /^[0-9a-f]{40}$/iu.test(local.stdout.trim()) ? local.stdout.trim().toLowerCase() : null;
|
|
return {
|
|
ok: remote.exitCode === 0 && commit !== null,
|
|
repository,
|
|
branch,
|
|
commit,
|
|
localHead,
|
|
result: compactCommand(remote),
|
|
};
|
|
}
|
|
|
|
function sentinelImagePlan(cicd: Record<string, unknown>, sourceHead: SourceHead): SentinelImagePlan {
|
|
const repository = stringAt(cicd, "image.repository");
|
|
const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12);
|
|
const baseImageRef = stringAt(cicd, "image.baseImageRef");
|
|
const baseImage = stringTarget(readConfigRefTarget(baseImageRef), baseImageRef);
|
|
const entrypoint = stringAt(cicd, "source.entrypoint");
|
|
const dockerfile = sentinelDockerfile(baseImage, entrypoint);
|
|
return {
|
|
repository,
|
|
tag,
|
|
ref: `${repository}:${tag}`,
|
|
digestRef: null,
|
|
baseImage,
|
|
buildContext: stringAt(cicd, "source.buildContext"),
|
|
entrypoint,
|
|
dockerfileSha256: sha256(dockerfile),
|
|
dockerfilePreview: dockerfile,
|
|
monitorWeb: monitorWebCicdPlan(cicd),
|
|
};
|
|
}
|
|
|
|
function sentinelDockerfile(baseImage: string, entrypoint: string): string {
|
|
return [
|
|
`FROM ${baseImage}`,
|
|
"WORKDIR /app",
|
|
"COPY . /app",
|
|
"RUN if [ -d /opt/hwlab-ci-node-deps/node_modules ]; then mkdir -p /app/node_modules; for dep in /opt/hwlab-ci-node-deps/node_modules/*; do ln -sf \"$dep\" \"/app/node_modules/$(basename \"$dep\")\"; done; fi",
|
|
"RUN printf '%s\\n' '#!/bin/sh' 'exec bun /app/scripts/ssh-cli.ts \"$@\"' > /usr/local/bin/trans && chmod 0755 /usr/local/bin/trans",
|
|
"RUN bun scripts/verify-web-probe-sentinel-monitor-web.ts",
|
|
"ENV NODE_ENV=production",
|
|
`ENTRYPOINT ["bun", "${entrypoint}"]`,
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
function monitorWebCicdPlan(cicd: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
stack: stringAtNullable(cicd, "monitorWeb.frontendStack") ?? "vue3-vendored-browser-build",
|
|
runtimeMode: stringAtNullable(cicd, "monitorWeb.runtimeMode") ?? "runner-served-bridge",
|
|
assetRoot: stringAtNullable(cicd, "monitorWeb.assetRoot") ?? "scripts/assets/web-probe-sentinel-monitor-web",
|
|
verifyCommand: "bun scripts/verify-web-probe-sentinel-monitor-web.ts",
|
|
gitMirrorReadUrl: stringAt(cicd, "source.gitMirrorReadUrl"),
|
|
sourceMode: stringAt(cicd, "builder.sourceMode"),
|
|
envReuseMode: stringAtNullable(cicd, "monitorWeb.envReuse.mode") ?? "docker-layer-and-ci-node-deps",
|
|
envReuseNodeDepsPath: stringAtNullable(cicd, "monitorWeb.envReuse.nodeDepsPath") ?? "/opt/hwlab-ci-node-deps/node_modules",
|
|
ciBudgetSeconds: numberAtNullable(cicd, "monitorWeb.ciBudget.maxSeconds") ?? numberAt(cicd, "confirmWait.maxSeconds"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function renderSentinelManifests(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sentinelId: string,
|
|
runtime: Record<string, unknown>,
|
|
cicd: Record<string, unknown>,
|
|
scenarios: unknown,
|
|
publicExposure: Record<string, unknown>,
|
|
secrets: Record<string, unknown>,
|
|
image: SentinelImagePlan,
|
|
): readonly Record<string, unknown>[] {
|
|
const namespace = stringAt(runtime, "namespace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": stringAt(runtime, "deploymentName"),
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
"unidesk.ai/web-probe-sentinel-id": sentinelId,
|
|
};
|
|
const deploymentName = stringAt(runtime, "deploymentName");
|
|
const serviceName = stringAt(runtime, "serviceName");
|
|
const servicePort = numberAt(runtime, "servicePort");
|
|
const pvcStorage = stringAt(runtime, "pvcStorage");
|
|
const stateRoot = stringAt(runtime, "stateRoot");
|
|
const sentinelEnv = sentinelContainerEnv(sentinelId, secrets);
|
|
const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv);
|
|
return [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ServiceAccount",
|
|
metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "PersistentVolumeClaim",
|
|
metadata: { name: stringAt(runtime, "pvcName"), namespace, labels },
|
|
spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: pvcStorage } } },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ConfigMap",
|
|
metadata: { name: `${deploymentName}-config`, namespace, labels },
|
|
data: {
|
|
"config-summary.json": JSON.stringify({
|
|
specRef: SPEC_REF,
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sentinelId,
|
|
publicBaseUrl: stringAt(publicExposure, "publicBaseUrl"),
|
|
routePrefix: stringAtNullable(publicExposure, "routePrefix") ?? "/",
|
|
gitopsPath: stringAt(cicd, "gitopsPath"),
|
|
valuesRedacted: true,
|
|
}, null, 2),
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name: deploymentName, namespace, labels },
|
|
spec: {
|
|
replicas: numberAt(runtime, "replicas"),
|
|
selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
serviceAccountName: stringAt(runtime, "serviceAccountName"),
|
|
containers: [{
|
|
name: "sentinel",
|
|
image: image.ref,
|
|
imagePullPolicy: "IfNotPresent",
|
|
args: [
|
|
"--node",
|
|
spec.nodeId,
|
|
"--lane",
|
|
spec.lane,
|
|
"--sentinel",
|
|
sentinelId,
|
|
"--state-root",
|
|
stateRoot,
|
|
"--host",
|
|
stringAt(runtime, "listenHost"),
|
|
"--port",
|
|
String(servicePort),
|
|
],
|
|
env: sentinelEnv,
|
|
ports: [{ name: "http", containerPort: servicePort }],
|
|
readinessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
|
|
livenessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
|
|
volumeMounts: [{ name: "state", mountPath: stateRoot }],
|
|
}],
|
|
volumes: [{ name: "state", persistentVolumeClaim: { claimName: stringAt(runtime, "pvcName") } }],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
metadata: { name: serviceName, namespace, labels },
|
|
spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] },
|
|
},
|
|
...(cadenceJob === null ? [] : [cadenceJob]),
|
|
{
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name: stringAt(publicExposure, "frpc.deploymentName"), namespace, labels: { ...labels, "app.kubernetes.io/component": "tunnel" } },
|
|
spec: {
|
|
replicas: 1,
|
|
selector: { matchLabels: { "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName") } },
|
|
template: {
|
|
metadata: {
|
|
labels: { ...labels, "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName"), "app.kubernetes.io/component": "tunnel" },
|
|
annotations: {
|
|
"unidesk.ai/public-base-url": stringAt(publicExposure, "publicBaseUrl"),
|
|
"unidesk.ai/frp-server": `${stringAt(publicExposure, "frpc.serverAddr")}:${numberAt(publicExposure, "frpc.serverPort")}`,
|
|
"unidesk.ai/frp-remote-port": String(numberAt(publicExposure, "frpc.httpProxy.remotePort")),
|
|
},
|
|
},
|
|
spec: {
|
|
containers: [{
|
|
name: "frpc",
|
|
image: stringAt(publicExposure, "frpc.image"),
|
|
imagePullPolicy: "IfNotPresent",
|
|
args: ["-c", "/etc/frp/frpc.toml"],
|
|
volumeMounts: [{ name: "frpc-config", mountPath: "/etc/frp/frpc.toml", subPath: stringAt(publicExposure, "frpc.secretKey"), readOnly: true }],
|
|
}],
|
|
volumes: [{ name: "frpc-config", secret: { secretName: stringAt(publicExposure, "frpc.secretName") } }],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "networking.k8s.io/v1",
|
|
kind: "NetworkPolicy",
|
|
metadata: { name: `${deploymentName}-egress`, namespace, labels },
|
|
spec: {
|
|
podSelector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
|
|
policyTypes: ["Ingress", "Egress"],
|
|
ingress: [{ from: [{ namespaceSelector: {} }], ports: [{ protocol: "TCP", port: servicePort }] }],
|
|
egress: [{ to: [{ namespaceSelector: {} }] }],
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "argoproj.io/v1alpha1",
|
|
kind: "Application",
|
|
metadata: { name: stringAt(cicd, "argo.applicationName"), namespace: stringAt(cicd, "argo.namespace"), labels },
|
|
spec: {
|
|
project: stringAt(cicd, "argo.projectName"),
|
|
source: {
|
|
repoURL: stringAt(cicd, "argo.repoURL"),
|
|
targetRevision: stringAt(cicd, "argo.targetRevision"),
|
|
path: stringAt(cicd, "gitopsPath"),
|
|
},
|
|
destination: { server: "https://kubernetes.default.svc", namespace },
|
|
syncPolicy: { automated: { prune: true, selfHeal: true } },
|
|
},
|
|
},
|
|
];
|
|
}
|
|
|
|
function sentinelContainerEnv(sentinelId: string, secrets: Record<string, unknown>): readonly Record<string, unknown>[] {
|
|
const env: Record<string, unknown>[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }];
|
|
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
|
for (const source of arrayAt(secrets, "sources").map(record)) {
|
|
const purpose = stringAtNullable(source, "purpose");
|
|
if (purpose !== null) sourcesByPurpose.set(purpose, source);
|
|
}
|
|
const used = new Set(env.map((item) => String(item.name ?? "")));
|
|
const pushEnv = (item: Record<string, unknown>): void => {
|
|
const name = String(item.name ?? "");
|
|
if (name.length === 0 || used.has(name)) return;
|
|
used.add(name);
|
|
env.push(item);
|
|
};
|
|
for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) {
|
|
const secretName = stringAtNullable(runtimeSecret, "name");
|
|
if (secretName === null) continue;
|
|
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
|
|
const targetKey = stringAtNullable(item, "targetKey");
|
|
const sourcePurpose = stringAtNullable(item, "sourcePurpose");
|
|
const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey");
|
|
const sourceKeyEnvName = sourcePurpose === "bootstrap-admin" || sourcePurpose === "prompt-set" ? sourceKey : null;
|
|
if (targetKey !== null && sourceKeyEnvName !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKeyEnvName)) {
|
|
pushEnv({ name: sourceKeyEnvName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
|
|
}
|
|
const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
|
|
if (envName === null) continue;
|
|
pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
|
|
}
|
|
}
|
|
return env;
|
|
}
|
|
|
|
function sentinelCadenceCronJobPlan(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sentinelId: string,
|
|
runtime: Record<string, unknown>,
|
|
cicd: Record<string, unknown>,
|
|
scenarios: unknown,
|
|
imageRef: string,
|
|
sentinelEnv: readonly Record<string, unknown>[],
|
|
): Record<string, unknown> | null {
|
|
const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId");
|
|
if (scenarioId === null) return null;
|
|
const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null;
|
|
if (scenario === null) return null;
|
|
const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null;
|
|
const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds);
|
|
if (schedule === null) return null;
|
|
const namespace = stringAt(runtime, "namespace");
|
|
const deploymentName = stringAt(runtime, "deploymentName");
|
|
const serviceAccountName = stringAt(runtime, "serviceAccountName");
|
|
const timeoutSeconds = numberAtNullable(cicd, "targetValidation.maxSeconds") ?? numberAtNullable(scenario, "maxRunSeconds") ?? 300;
|
|
const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost");
|
|
const name = safeKubernetesSegment(`${deploymentName}-quick-verify`, 52);
|
|
const labels = {
|
|
"app.kubernetes.io/name": name,
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"app.kubernetes.io/component": "cadence-scheduler",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
"unidesk.ai/web-probe-sentinel-id": sentinelId,
|
|
};
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "CronJob",
|
|
metadata: {
|
|
name,
|
|
namespace,
|
|
labels,
|
|
annotations: {
|
|
"unidesk.ai/cadence": String(scenario.cadence),
|
|
"unidesk.ai/target-validation-max-seconds": String(timeoutSeconds),
|
|
},
|
|
},
|
|
spec: {
|
|
schedule,
|
|
concurrencyPolicy: "Forbid",
|
|
successfulJobsHistoryLimit: 3,
|
|
failedJobsHistoryLimit: 5,
|
|
startingDeadlineSeconds: Math.max(60, cadenceSeconds),
|
|
jobTemplate: {
|
|
spec: {
|
|
activeDeadlineSeconds: timeoutSeconds + 60,
|
|
ttlSecondsAfterFinished: 86400,
|
|
backoffLimit: 0,
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
serviceAccountName,
|
|
containers: [{
|
|
name: "quick-verify",
|
|
image: imageRef,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["bun", "scripts/cli.ts"],
|
|
args: [
|
|
"web-probe",
|
|
"sentinel",
|
|
"validate",
|
|
"--node",
|
|
spec.nodeId,
|
|
"--lane",
|
|
spec.lane,
|
|
"--sentinel",
|
|
sentinelId,
|
|
"--quick-verify",
|
|
"--confirm",
|
|
"--wait",
|
|
"--timeout-seconds",
|
|
String(timeoutSeconds),
|
|
],
|
|
env: [
|
|
...sentinelEnv,
|
|
{ name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" },
|
|
...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]),
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function scenarioRows(value: unknown): Record<string, unknown>[] {
|
|
if (Array.isArray(value)) return value.map(record);
|
|
if (!isRecord(value)) return [];
|
|
if (Array.isArray(value.scenarios)) return value.scenarios.map(record);
|
|
if (isRecord(value.workflow)) return [value.workflow];
|
|
return [value];
|
|
}
|
|
|
|
function parseDurationSeconds(value: string): number | null {
|
|
const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim());
|
|
if (match === null) return null;
|
|
const amount = Number(match[1]);
|
|
const unit = match[2];
|
|
if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000));
|
|
if (unit === "s") return Math.max(60, amount);
|
|
if (unit === "m") return amount * 60;
|
|
if (unit === "h") return amount * 3600;
|
|
return null;
|
|
}
|
|
|
|
function cronScheduleForCadenceSeconds(seconds: number): string | null {
|
|
if (!Number.isFinite(seconds) || seconds <= 0) return null;
|
|
if (seconds <= 60) return "* * * * *";
|
|
if (seconds % 60 === 0) {
|
|
const minutes = Math.trunc(seconds / 60);
|
|
if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`;
|
|
if (minutes % 60 === 0) {
|
|
const hours = Math.trunc(minutes / 60);
|
|
if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`;
|
|
if (hours === 24) return "0 0 * * *";
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null {
|
|
if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null;
|
|
const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, "");
|
|
return segment.length === 0 ? null : `HWLAB_WEB_${segment}_JSON`;
|
|
}
|
|
|
|
function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record<string, unknown> } {
|
|
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
|
for (const source of arrayAt(state.secrets, "sources").map(record)) {
|
|
const purpose = stringAtNullable(source, "purpose");
|
|
if (purpose !== null) sourcesByPurpose.set(purpose, source);
|
|
}
|
|
const env: NodeJS.ProcessEnv = {};
|
|
const items: Record<string, unknown>[] = [];
|
|
const missing: Record<string, unknown>[] = [];
|
|
for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) {
|
|
const secretName = stringAtNullable(runtimeSecret, "name");
|
|
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
|
|
const targetKey = stringAtNullable(item, "targetKey");
|
|
const sourcePurpose = stringAtNullable(item, "sourcePurpose");
|
|
const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
|
|
if (envName === null || sourcePurpose === null || targetKey === null) continue;
|
|
const source = sourcesByPurpose.get(sourcePurpose);
|
|
const runtimeValue = process.env[envName];
|
|
if (source === undefined) {
|
|
if (runtimeValue !== undefined && runtimeValue.length > 0) {
|
|
env[envName] = runtimeValue;
|
|
items.push({
|
|
envName,
|
|
secretName,
|
|
targetKey,
|
|
sourcePurpose,
|
|
sourceMode: "runtime-env",
|
|
fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
});
|
|
continue;
|
|
}
|
|
missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const sourceRef = stringAt(source, "sourceRef");
|
|
const sourceKey = stringAt(source, "sourceKey");
|
|
const material = readSentinelSecretSourceValue(source);
|
|
if (!material.ok) {
|
|
if (runtimeValue !== undefined && runtimeValue.length > 0) {
|
|
env[envName] = runtimeValue;
|
|
items.push({
|
|
envName,
|
|
secretName,
|
|
targetKey,
|
|
sourcePurpose,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourceMode: "runtime-env",
|
|
fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
});
|
|
continue;
|
|
}
|
|
missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const value = stringAt(material, "value");
|
|
env[envName] = value;
|
|
items.push({
|
|
envName,
|
|
secretName,
|
|
targetKey,
|
|
sourcePurpose,
|
|
sourceRef,
|
|
sourceKey,
|
|
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
}
|
|
const summary = {
|
|
ok: missing.length === 0,
|
|
envCount: items.length,
|
|
items,
|
|
missing,
|
|
valuesRedacted: true,
|
|
};
|
|
return { ok: missing.length === 0, env, summary };
|
|
}
|
|
|
|
function normalizeRoutePrefix(value: string | null): string {
|
|
if (value === null || value.trim() === "" || value.trim() === "/") return "/";
|
|
const prefixed = value.trim().startsWith("/") ? value.trim() : `/${value.trim()}`;
|
|
return prefixed.replace(/\/+$/u, "") || "/";
|
|
}
|
|
|
|
function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const endpoint = stringAt(state.controlPlaneNode, "registry.endpoint");
|
|
const repoTag = state.image.ref.replace(`${endpoint}/`, "");
|
|
const repo = repoTag.slice(0, repoTag.lastIndexOf(":"));
|
|
const tag = repoTag.slice(repoTag.lastIndexOf(":") + 1);
|
|
const route = stringAt(state.controlPlaneNode, "route");
|
|
const script = [
|
|
"set +e",
|
|
`url=${shellQuote(`http://${endpoint}/v2/${repo}/manifests/${tag}`)}`,
|
|
"headers=$(mktemp)",
|
|
"if command -v curl >/dev/null 2>&1 && curl -fsSI --max-time 5 \"$url\" >\"$headers\" 2>/tmp/web-probe-sentinel-image.err; then present=true; else present=false; fi",
|
|
"digest=$(awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}' \"$headers\" 2>/dev/null)",
|
|
"python3 - \"$present\" \"$digest\" \"$url\" <<'PY'",
|
|
"import json, sys",
|
|
"print(json.dumps({'present': sys.argv[1] == 'true', 'digest': sys.argv[2] or None, 'url': sys.argv[3], 'valuesRedacted': True}))",
|
|
"PY",
|
|
].join("\n");
|
|
const result = runCommand(["trans", route, "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
probe: parseJsonObject(result.stdout),
|
|
result: compactCommand(result),
|
|
};
|
|
}
|
|
|
|
function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
|
|
const startedAt = Date.now();
|
|
const command = "web-probe sentinel image build";
|
|
const sourceMirrorProbe = probeSourceMirror(state, Math.min(options.timeoutSeconds, 20));
|
|
const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds);
|
|
const sourceMirrorReady = sourceMirrorSync.ok === true;
|
|
const publish = sourceMirrorReady
|
|
? runSentinelPublishJob(state, false, options.timeoutSeconds)
|
|
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
|
|
const registry = probeImageRegistry(state, options.timeoutSeconds);
|
|
const registryReady = record(registry.probe).present === true;
|
|
const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publish.ok === true && registryReady;
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state);
|
|
const result = {
|
|
ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: true,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
registry,
|
|
sourceMirrorSync,
|
|
publish,
|
|
elapsedMs,
|
|
warnings: [
|
|
...sentinelCicdElapsedWarnings(elapsedMs, "sentinel image build confirm-wait", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds),
|
|
...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
|
|
],
|
|
blocker: ok
|
|
? null
|
|
: !sourceMirrorReady
|
|
? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" }
|
|
: publish.ok !== true
|
|
? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" }
|
|
: { code: "sentinel-image-registry-missing", reason: "image publish completed but expected registry tag is not visible" },
|
|
next: {
|
|
status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
controlPlaneTrigger: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderImageResult(result));
|
|
}
|
|
|
|
function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
|
|
const startedAt = Date.now();
|
|
const command = `web-probe sentinel control-plane ${options.action}`;
|
|
const applyOnly = options.action === "apply";
|
|
const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state);
|
|
const deadline = startedAt + cicdWaitWarningSeconds * 1000;
|
|
const remainingCicdWaitSeconds = () => remainingSeconds(deadline, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds));
|
|
const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCicdWaitSeconds(), 20));
|
|
const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCicdWaitSeconds());
|
|
const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true;
|
|
const publish = applyOnly
|
|
? null
|
|
: sourceMirrorReady
|
|
? runSentinelPublishJob(state, true, remainingCicdWaitSeconds())
|
|
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
|
|
const flush = !applyOnly && record(publish).ok === true
|
|
? startSentinelGitMirrorFlushAsync(state)
|
|
: null;
|
|
const runtimeSecretsApply = applySentinelRuntimeSecrets(state, remainingCicdWaitSeconds());
|
|
const publicExposureApply = applySentinelPublicExposure(state, remainingCicdWaitSeconds());
|
|
const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds());
|
|
const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false);
|
|
const observedReady = sentinelObservedReady(observed);
|
|
const targetValidation = null;
|
|
const targetValidationBlocked = false;
|
|
const ok = state.configReady
|
|
&& state.sourceHead.ok
|
|
&& sourceMirrorReady
|
|
&& (applyOnly || record(publish).ok === true)
|
|
&& (applyOnly || record(flush).ok === true)
|
|
&& record(runtimeSecretsApply).ok === true
|
|
&& record(publicExposureApply).ok === true
|
|
&& record(argoApply).ok === true
|
|
&& observedReady;
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const blocker = ok ? null : {
|
|
code: !sourceMirrorReady ? "sentinel-source-mirror-sync-failed" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready",
|
|
reason: !sourceMirrorReady
|
|
? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish"
|
|
: record(runtimeSecretsApply).ok === false
|
|
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
|
|
: "one or more publish, publicExposure, Argo or runtime observation checks did not pass",
|
|
};
|
|
const result = {
|
|
ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: true,
|
|
specRef: SPEC_REF,
|
|
source: state.sourceHead,
|
|
image: state.image,
|
|
pipelineRun: sentinelPipelineRunName(state),
|
|
gitops: {
|
|
path: stringAt(state.cicd, "gitopsPath"),
|
|
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
|
manifestObjects: state.manifests.length,
|
|
manifestSha256: state.manifestSha256,
|
|
},
|
|
argo: {
|
|
namespace: stringAt(state.cicd, "argo.namespace"),
|
|
projectName: stringAt(state.cicd, "argo.projectName"),
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
},
|
|
validation: {
|
|
scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
|
|
maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
|
|
controlPlaneWaitMaxSeconds: cicdWaitWarningSeconds,
|
|
quickVerifyMode: applyOnly ? "not-applicable" : "manual-validate",
|
|
automaticSecondPath: false,
|
|
},
|
|
manifests: {
|
|
objects: manifestObjectSummary(state.manifests),
|
|
sha256: state.manifestSha256,
|
|
},
|
|
sourceMirrorSync,
|
|
publish,
|
|
flush,
|
|
runtimeSecretsApply,
|
|
publicExposureApply,
|
|
argoApply,
|
|
observed,
|
|
targetValidation,
|
|
elapsedMs,
|
|
warnings: Array.from(new Set([
|
|
...sentinelCicdElapsedWarnings(elapsedMs, "sentinel control-plane confirm-wait", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds),
|
|
...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds),
|
|
...asyncGitMirrorFlushWarnings(flush),
|
|
...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
|
|
...sentinelObservedWarnings(observed),
|
|
...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds),
|
|
...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []),
|
|
...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []),
|
|
])),
|
|
blocker,
|
|
next: controlPlaneNext(state, options.action),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(ok, command, renderControlPlaneResult(result));
|
|
}
|
|
|
|
function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane", action: string, timeoutSeconds: number): RenderedCliResult {
|
|
const args = domain === "image"
|
|
? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
|
|
: ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
|
|
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`);
|
|
const command = `web-probe sentinel ${domain} ${action}`;
|
|
const result = {
|
|
ok: true,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "async-job",
|
|
mutation: true,
|
|
reason: "confirmed sentinel publish/build can exceed the short interactive window; use job status for bounded progress.",
|
|
job,
|
|
next: {
|
|
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
|
|
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(true, command, renderAsyncJobResult(result));
|
|
}
|
|
|
|
function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record<string, unknown> {
|
|
const args = ["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"];
|
|
const job = startJob(
|
|
`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_git_mirror_flush`,
|
|
["bun", "scripts/cli.ts", ...args],
|
|
`Flush HWLAB ${state.spec.lane} git mirror after web-probe sentinel ${state.sentinelId} GitOps publish for node ${state.spec.nodeId}`,
|
|
);
|
|
return {
|
|
ok: true,
|
|
mode: "async-job",
|
|
job,
|
|
next: {
|
|
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
|
|
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
|
|
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function asyncGitMirrorFlushWarnings(flush: unknown): string[] {
|
|
const item = record(flush);
|
|
if (item.mode !== "async-job") return [];
|
|
const next = record(item.next);
|
|
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under 120s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
|
|
}
|
|
|
|
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
|
|
const registry = probeImageRegistry(state, timeoutSeconds);
|
|
const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds);
|
|
const effectiveExpectation = {
|
|
gitopsRevision: expectation?.gitopsRevision ?? nonEmptyString(gitops.revision),
|
|
runtimeImage: expectation?.runtimeImage ?? nonEmptyString(gitops.image) ?? expectedRuntimeImageFromRegistry(state, registry),
|
|
};
|
|
return {
|
|
sourceMirror: probeSourceMirror(state, timeoutSeconds),
|
|
registry,
|
|
gitMirror: includeGitMirror
|
|
? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds)
|
|
: { ok: true, skipped: true, reason: "deferred-to-async-flush", valuesRedacted: true },
|
|
gitops,
|
|
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
|
|
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
|
|
};
|
|
}
|
|
|
|
function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
|
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
|
}
|
|
return observed;
|
|
}
|
|
|
|
function sentinelObservedReady(value: Record<string, unknown> | SentinelObservedStatus): boolean {
|
|
const observed = record(value);
|
|
const gitMirror = record(observed.gitMirror);
|
|
const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true;
|
|
return record(observed.sourceMirror).ok === true
|
|
&& record(record(observed.registry).probe).present === true
|
|
&& gitMirrorReady
|
|
&& record(observed.gitops).ok === true
|
|
&& record(observed.argo).ok === true
|
|
&& record(observed.runtime).ok === true;
|
|
}
|
|
|
|
function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObservedStatus | null): string[] {
|
|
const observed = record(value);
|
|
const argo = record(observed.argo);
|
|
return mergeWarnings(argo.warning);
|
|
}
|
|
|
|
function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const repository = stringAt(state.cicd, "source.repository");
|
|
const branch = stringAt(state.cicd, "source.branch");
|
|
const expectedCommit = state.sourceHead.commit;
|
|
const script = [
|
|
"set +e",
|
|
`repo_path=${shellQuote(`/cache/${repository}.git`)}`,
|
|
`branch=${shellQuote(branch)}`,
|
|
`expected=${shellQuote(expectedCommit ?? "")}`,
|
|
"commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)",
|
|
"rc=$?",
|
|
"object_rc=1",
|
|
"if [ \"$rc\" -eq 0 ]; then",
|
|
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
|
|
" object_rc=$?",
|
|
"fi",
|
|
"node - \"$rc\" \"$object_rc\" \"$commit\" \"$expected\" \"$repo_path\" \"$branch\" <<'NODE'",
|
|
"const [rc, objectRc, commit, expected, repoPath, branch] = process.argv.slice(2);",
|
|
"const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');",
|
|
"const objectPresent = present && Number(objectRc) === 0;",
|
|
"console.log(JSON.stringify({ ok: objectPresent && (!expected || commit === expected), mode: 'internal-git-mirror', present, objectPresent, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, persistentMirrorPresent: objectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) };
|
|
}
|
|
|
|
function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "argo.namespace");
|
|
const applicationName = stringAt(state.cicd, "argo.applicationName");
|
|
const jsonpath = "{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.sync.revision}{\"\\n\"}";
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", `jsonpath=${jsonpath}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const [syncStatusRaw, healthStatusRaw, revisionRaw] = result.stdout.trim().split(/\r?\n/u);
|
|
const syncStatus = nonEmptyString(syncStatusRaw);
|
|
const healthStatus = nonEmptyString(healthStatusRaw);
|
|
const revision = nonEmptyString(revisionRaw);
|
|
const revisionMatches = expectedRevision === null || revision === expectedRevision;
|
|
const healthy = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy";
|
|
return {
|
|
ok: healthy,
|
|
present: result.exitCode === 0,
|
|
syncStatus,
|
|
healthStatus,
|
|
revision,
|
|
expectedRevision,
|
|
revisionMatches,
|
|
revisionPolicy: "non-blocking-branch-head-drift",
|
|
warning: healthy && !revisionMatches
|
|
? "Argo app is Synced/Healthy but status.sync.revision differs from current GitOps branch HEAD; in multi-sentinel GitOps this can happen when another sentinel path advances the branch. Runtime image/manifest checks remain authoritative for rollout readiness."
|
|
: null,
|
|
result: compactCommand(result),
|
|
};
|
|
}
|
|
|
|
function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const repository = stringAt(state.controlPlaneTarget, "source.repository");
|
|
const branch = stringAt(state.cicd, "argo.targetRevision");
|
|
const manifestPath = `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`;
|
|
const repoPath = `/cache/${repository}.git`;
|
|
const inner = [
|
|
"set -eu",
|
|
`git --git-dir=${shellQuote(repoPath)} rev-parse ${shellQuote(`refs/heads/${branch}`)}`,
|
|
`git --git-dir=${shellQuote(repoPath)} show ${shellQuote(`refs/heads/${branch}:${manifestPath}`)}`,
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "exec", "deploy/git-mirror-http", "--", "sh", "-lc", inner], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const [revisionLine, ...manifestLines] = result.stdout.split(/\r?\n/u);
|
|
const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null;
|
|
const manifest = manifestLines.join("\n");
|
|
const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]);
|
|
const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`);
|
|
const compact = compactCommand(result);
|
|
return {
|
|
ok: result.exitCode === 0 && revision !== null && imageMatchesRepository,
|
|
revision,
|
|
branch,
|
|
manifestPath,
|
|
image,
|
|
imageMatchesRepository,
|
|
result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, expectedImage: string | null): Record<string, unknown> {
|
|
const namespace = stringAt(state.runtime, "namespace");
|
|
const deploymentName = stringAt(state.runtime, "deploymentName");
|
|
const serviceName = stringAt(state.runtime, "serviceName");
|
|
const pvcName = stringAt(state.runtime, "pvcName");
|
|
const configMapName = `${deploymentName}-config`;
|
|
const serviceAccountName = stringAt(state.runtime, "serviceAccountName");
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deploymentName)}`,
|
|
`service=${shellQuote(serviceName)}`,
|
|
`pvc=${shellQuote(pvcName)}`,
|
|
`configmap=${shellQuote(configMapName)}`,
|
|
`serviceaccount=${shellQuote(serviceAccountName)}`,
|
|
"tmp=$(mktemp -d)",
|
|
"kubectl -n \"$namespace\" get deploy \"$deployment\" -o json >\"$tmp/deploy.json\" 2>/dev/null; echo $? >\"$tmp/deploy.rc\"",
|
|
"kubectl -n \"$namespace\" get svc \"$service\" -o json >\"$tmp/svc.json\" 2>/dev/null; echo $? >\"$tmp/svc.rc\"",
|
|
"kubectl -n \"$namespace\" get pvc \"$pvc\" -o json >\"$tmp/pvc.json\" 2>/dev/null; echo $? >\"$tmp/pvc.rc\"",
|
|
"kubectl -n \"$namespace\" get cm \"$configmap\" -o json >\"$tmp/cm.json\" 2>/dev/null; echo $? >\"$tmp/cm.rc\"",
|
|
"kubectl -n \"$namespace\" get sa \"$serviceaccount\" -o json >\"$tmp/sa.json\" 2>/dev/null; echo $? >\"$tmp/sa.rc\"",
|
|
`expected_image=${shellQuote(expectedImage ?? "")}`,
|
|
"node - \"$tmp\" \"$expected_image\" <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const dir = process.argv[2];",
|
|
"const expectedImage = process.argv[3] || null;",
|
|
"function rc(name){ return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); }",
|
|
"function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }",
|
|
"const dep = json('deploy');",
|
|
"const deploymentPresent = rc('deploy') === 0;",
|
|
"const desired = Number(dep?.spec?.replicas ?? 0);",
|
|
"const ready = Number(dep?.status?.readyReplicas ?? 0);",
|
|
"const updated = Number(dep?.status?.updatedReplicas ?? 0);",
|
|
"const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;",
|
|
"const imageMatches = expectedImage === null || image === expectedImage;",
|
|
"const payload = {",
|
|
" deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches },",
|
|
" service: { present: rc('svc') === 0 },",
|
|
" pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },",
|
|
" configMap: { present: rc('cm') === 0 },",
|
|
" serviceAccount: { present: rc('sa') === 0 },",
|
|
" valuesRedacted: true",
|
|
"};",
|
|
"payload.ok = payload.deployment.present && imageMatches && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;",
|
|
"console.log(JSON.stringify(payload));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probe = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && probe?.ok === true, probe, result: compactCommand(result) };
|
|
}
|
|
|
|
function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record<string, unknown>): string | null {
|
|
const digest = nonEmptyString(record(record(registry).probe).digest);
|
|
if (digest === null) return null;
|
|
return `${state.image.repository}@${digest}`;
|
|
}
|
|
|
|
function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult {
|
|
const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`;
|
|
const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
|
const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName);
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
if (created.exitCode !== 0) {
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane });
|
|
return { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true };
|
|
}
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
|
|
let slowWarningSent = false;
|
|
let polls = 0;
|
|
let lastProbe: Record<string, unknown> = {};
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probe = parseJsonObject(probeCapture.stdout) ?? {};
|
|
lastProbe = { ...probe, capture: compactCommand(probeCapture) };
|
|
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", {
|
|
phase: "remote-job",
|
|
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
pod: probe.pod ?? null,
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
});
|
|
if (probe.succeeded === true) {
|
|
const ok = payload.ok === true;
|
|
return { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
|
}
|
|
if (probe.failed === true) {
|
|
return { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
|
}
|
|
if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
|
|
slowWarningSent = true;
|
|
sentinelProgressEvent("sentinel.source-mirror.warning", { warning: `source mirror sync exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
return { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
|
}
|
|
|
|
function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult {
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
jobName: "-",
|
|
payload: { ok: false, status: phase, reason, valuesRedacted: true },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": "web-probe-sentinel-source-mirror",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
};
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: { name: jobName, namespace, labels },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"),
|
|
ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"),
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
volumes: [
|
|
{ name: "cache", hostPath: { path: stringAt(state.controlPlaneTarget, "gitMirror.cacheHostPath"), type: "DirectoryOrCreate" } },
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
],
|
|
containers: [{
|
|
name: "sync",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string {
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
|
|
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
|
|
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`,
|
|
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
|
|
"started_ms=$(node -e 'console.log(Date.now())')",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"test -n \"$source_commit\"",
|
|
...sentinelSourceMirrorSshSetupShellLines(state),
|
|
"repo=\"/cache/${source_repository}.git\"",
|
|
"mkdir -p \"$(dirname \"$repo\")\"",
|
|
"if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
|
|
" git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
|
"else",
|
|
" rm -rf \"$repo\"",
|
|
" git init --bare \"$repo\"",
|
|
" git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
|
"fi",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
|
|
"git --git-dir=\"$repo\" config http.uploadpack true",
|
|
"git --git-dir=\"$repo\" config http.receivepack true",
|
|
"fetch_ok=0",
|
|
"for attempt in 1 2 3; do",
|
|
" if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi",
|
|
" code=$?",
|
|
" printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2",
|
|
" sleep $((attempt * 5))",
|
|
"done",
|
|
"test \"$fetch_ok\" = 1",
|
|
"mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")",
|
|
"test \"$mirror_commit\" = \"$source_commit\"",
|
|
"git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"",
|
|
"git --git-dir=\"$repo\" update-server-info",
|
|
"finished_ms=$(node -e 'console.log(Date.now())')",
|
|
"node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
|
|
"const [jobName, repository, branch, sourceCommit, mirrorCommit, startedMs, finishedMs] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] {
|
|
const proxy = record(valueAtPath(state.controlPlaneNode, "egressProxy"));
|
|
const serviceName = nonEmptyString(proxy.serviceName);
|
|
const namespace = nonEmptyString(proxy.namespace);
|
|
const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null;
|
|
const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : "";
|
|
const useProxy = serviceName !== null && namespace !== null && port !== null;
|
|
if (!useProxy) {
|
|
return [
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2",
|
|
"unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy",
|
|
"export NO_PROXY='*'",
|
|
"export no_proxy='*'",
|
|
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
|
"#!/bin/sh",
|
|
"exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"",
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
|
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
];
|
|
}
|
|
const proxyHost = `${serviceName}.${namespace}.svc.cluster.local`;
|
|
const proxyUrl = `http://${proxyHost}:${port}`;
|
|
const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${port} %h %p`;
|
|
return [
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
`printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${port} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`,
|
|
`export HTTP_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export HTTPS_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export ALL_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export http_proxy=${shellQuote(proxyUrl)}`,
|
|
`export https_proxy=${shellQuote(proxyUrl)}`,
|
|
`export all_proxy=${shellQuote(proxyUrl)}`,
|
|
`export NO_PROXY=${shellQuote(noProxy)}`,
|
|
`export no_proxy=${shellQuote(noProxy)}`,
|
|
"cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'",
|
|
"#!/usr/bin/env node",
|
|
"const net = require('node:net');",
|
|
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
|
|
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
|
|
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
|
|
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
|
|
" console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');",
|
|
" process.exit(64);",
|
|
"}",
|
|
"let settled = false;",
|
|
"let tunnelEstablished = false;",
|
|
"function finish(code, message) {",
|
|
" if (settled) return;",
|
|
" settled = true;",
|
|
" if (message) console.error('sentinel source-mirror proxy-connect: ' + message);",
|
|
" process.exit(code);",
|
|
"}",
|
|
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
|
|
"let buffer = Buffer.alloc(0);",
|
|
"socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
|
|
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
|
|
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
|
|
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
|
|
"function onData(chunk) {",
|
|
" buffer = Buffer.concat([buffer, chunk]);",
|
|
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
|
|
" if (headerEnd === -1 && buffer.length < 8192) return;",
|
|
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
|
|
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
|
|
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
|
|
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
|
|
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
|
|
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
|
|
" socket.destroy();",
|
|
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
|
|
" return;",
|
|
" }",
|
|
" socket.off('data', onData);",
|
|
" socket.setTimeout(0);",
|
|
" tunnelEstablished = true;",
|
|
" const rest = buffer.slice(headerEnd + 4);",
|
|
" if (rest.length) process.stdout.write(rest);",
|
|
" process.stdin.on('error', () => {});",
|
|
" process.stdout.on('error', () => {});",
|
|
" process.stdin.pipe(socket);",
|
|
" socket.pipe(process.stdout);",
|
|
"}",
|
|
"socket.on('data', onData);",
|
|
"NODE_PROXY",
|
|
"chmod 0700 /tmp/sentinel-github-proxy-connect.cjs",
|
|
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
|
"#!/bin/sh",
|
|
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`,
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
|
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
];
|
|
}
|
|
|
|
function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult {
|
|
const jobName = `${stringAt(state.cicd, "builder.jobPrefix")}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
|
const manifest = sentinelPublishJobManifest(state, jobName, publishGitops);
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "submitting", jobName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
if (created.exitCode !== 0) {
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "failed", jobName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
|
return { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true };
|
|
}
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "succeeded", jobName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
|
|
let slowWarningSent = false;
|
|
let polls = 0;
|
|
let lastProbe: Record<string, unknown> = {};
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probe = parseJsonObject(probeCapture.stdout) ?? {};
|
|
lastProbe = { ...probe, capture: compactCommand(probeCapture) };
|
|
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
|
sentinelProgressEvent("sentinel.publish.progress", {
|
|
phase: "remote-job",
|
|
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
|
jobName,
|
|
publishGitops,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
pod: probe.pod ?? null,
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
});
|
|
if (probe.succeeded === true) {
|
|
const ok = payload.ok === true;
|
|
return { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
|
}
|
|
if (probe.failed === true) {
|
|
return { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
|
}
|
|
if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
|
|
slowWarningSent = true;
|
|
sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish job exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
return { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
|
}
|
|
|
|
function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, publishGitops: boolean): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": "web-probe-sentinel-publish",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
};
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: { name: jobName, namespace, labels },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"),
|
|
ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"),
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
volumes: [
|
|
{ name: "cache", hostPath: { path: stringAt(state.controlPlaneTarget, "gitMirror.cacheHostPath"), type: "DirectoryOrCreate" } },
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
{ name: "docker-sock", hostPath: { path: stringAt(state.cicd, "builder.dockerSocketPath"), type: "Socket" } },
|
|
],
|
|
containers: [{
|
|
name: "publish",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-ec", sentinelPublishShell(state, jobName, publishGitops)],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
{ name: "docker-sock", mountPath: stringAt(state.cicd, "builder.dockerSocketPath") },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
|
|
const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : [];
|
|
const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({
|
|
path: file.path,
|
|
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
|
|
}))), "utf8").toString("base64");
|
|
const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => {
|
|
if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths");
|
|
return item;
|
|
})), "utf8").toString("base64");
|
|
const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
|
|
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
|
|
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
|
|
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
|
|
`checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
`image_repository=${shellQuote(state.image.repository)}`,
|
|
`dockerfile_b64=${shellQuote(dockerfileB64)}`,
|
|
`gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`,
|
|
`gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`,
|
|
`files_b64=${shellQuote(filesB64)}`,
|
|
"started_ms=$(node -e 'console.log(Date.now())')",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
|
"worktree=\"/tmp/$job_name/source\"",
|
|
"rm -rf \"/tmp/$job_name\"",
|
|
"mkdir -p \"/tmp/$job_name\"",
|
|
"git init \"$worktree\"",
|
|
"cd \"$worktree\"",
|
|
"git remote add origin \"$source_git_url\"",
|
|
"git config core.sparseCheckout true",
|
|
"git config remote.origin.promisor true",
|
|
"git config remote.origin.partialclonefilter blob:none",
|
|
"CHECKOUT_PATHS_B64=\"$checkout_paths_b64\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const paths = JSON.parse(Buffer.from(process.env.CHECKOUT_PATHS_B64 || '', 'base64').toString('utf8'));",
|
|
"fs.mkdirSync('.git/info', { recursive: true });",
|
|
"fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');",
|
|
"NODE",
|
|
"git fetch --depth=1 --filter=blob:none origin \"+refs/heads/$source_branch:refs/remotes/origin/$source_branch\"",
|
|
"git checkout --detach \"$source_commit\"",
|
|
"mirror_commit=$(git rev-parse HEAD)",
|
|
"test \"$mirror_commit\" = \"$source_commit\"",
|
|
"DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"fs.writeFileSync('Dockerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));",
|
|
"NODE",
|
|
"docker build -f Dockerfile.web-probe-sentinel -t \"$image_ref\" .",
|
|
"docker push \"$image_ref\" > /tmp/web-probe-sentinel-docker-push.log 2>&1",
|
|
"cat /tmp/web-probe-sentinel-docker-push.log",
|
|
"tag=${image_ref##*:}",
|
|
"repo_no_tag=${image_ref%:*}",
|
|
"registry_path=${repo_no_tag#127.0.0.1:5000/}",
|
|
"digest=$(awk '/digest: sha256:/ {print $3; exit}' /tmp/web-probe-sentinel-docker-push.log)",
|
|
"if [ -z \"$digest\" ]; then digest=$(curl -fsSI --max-time 10 \"http://127.0.0.1:5000/v2/$registry_path/manifests/$tag\" 2>/dev/null | awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}'); fi",
|
|
"test -n \"$digest\"",
|
|
"digest_ref=\"$repo_no_tag@$digest\"",
|
|
"gitops_commit=''",
|
|
"changed=false",
|
|
"file_count=0",
|
|
"if [ \"$files_b64\" != \"W10=\" ]; then",
|
|
" gitops_cache=\"/cache/${gitops_repository}.git\"",
|
|
" gitops_worktree=\"/tmp/$job_name/gitops\"",
|
|
" git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"",
|
|
" cd \"$gitops_worktree\"",
|
|
" git fetch origin \"$gitops_branch\" || true",
|
|
" if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"; else git checkout --orphan \"$gitops_branch\"; git rm -rf . >/dev/null 2>&1 || true; fi",
|
|
" FILES_B64=\"$files_b64\" IMAGE_REF=\"$image_ref\" DIGEST_REF=\"$digest_ref\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
|
|
"for (const file of files) {",
|
|
" const target = path.resolve(process.cwd(), file.path);",
|
|
" if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);",
|
|
" fs.mkdirSync(path.dirname(target), { recursive: true });",
|
|
" const text = Buffer.from(file.contentBase64, 'base64').toString('utf8').split(process.env.IMAGE_REF).join(process.env.DIGEST_REF);",
|
|
" fs.writeFileSync(target, text);",
|
|
"}",
|
|
"console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));",
|
|
"NODE",
|
|
" git add .",
|
|
" file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')",
|
|
" if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi",
|
|
" git push origin \"HEAD:refs/heads/$gitops_branch\"",
|
|
" gitops_commit=$(git rev-parse HEAD)",
|
|
"fi",
|
|
"finished_ms=$(node -e 'console.log(Date.now())')",
|
|
"node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
|
|
"const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: Number(finishedMs) - Number(startedMs), valuesRedacted:true }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelGitopsFiles(state: SentinelCicdState): readonly { path: string; content: string }[] {
|
|
const runtimeManifests = state.manifests.filter((item) => item.kind !== "Application");
|
|
return [{
|
|
path: `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`,
|
|
content: `${runtimeManifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`,
|
|
}];
|
|
}
|
|
|
|
function applySentinelArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const app = state.manifests.find((item) => item.kind === "Application");
|
|
if (app === undefined) return { ok: false, reason: "application-manifest-missing", valuesRedacted: true };
|
|
const yaml = `${Bun.YAML.stringify(app).trim()}\n`;
|
|
const namespace = stringAt(state.cicd, "argo.namespace");
|
|
const applicationName = stringAt(state.cicd, "argo.applicationName");
|
|
const script = [
|
|
"set -eu",
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"kubectl apply -f \"$tmp\"",
|
|
`kubectl -n ${shellQuote(namespace)} annotate application ${shellQuote(applicationName)} argocd.argoproj.io/refresh=hard --overwrite`,
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
return { ok: result.exitCode === 0, result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function createK8sJobScript(namespace: string, manifest: Record<string, unknown>): string {
|
|
const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
|
return [
|
|
"set -eu",
|
|
`kubectl -n ${shellQuote(namespace)} delete job ${shellQuote(stringAt(manifest, "metadata.name"))} --ignore-not-found=true >/dev/null 2>&1 || true`,
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"kubectl apply -f \"$tmp\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function probeK8sJobScript(namespace: string, jobName: string): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
"succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)",
|
|
"failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)",
|
|
"pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)",
|
|
"logs_tail=''",
|
|
"if [ -n \"$pod\" ]; then logs_tail=$(kubectl -n \"$namespace\" logs \"$pod\" --tail=120 2>/dev/null | tail -c 12000 | base64 | tr -d '\\n'); fi",
|
|
"node - \"$succeeded\" \"$failed\" \"$pod\" \"$logs_tail\" <<'NODE'",
|
|
"const [succeeded, failed, pod, logsB64] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, pod: pod || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelPayloadFromLogs(logsTail: string): Record<string, unknown> {
|
|
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
const line = lines[index];
|
|
if (!line.startsWith("{") || !line.endsWith("}")) continue;
|
|
const parsed = parseJsonObject(line);
|
|
if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed;
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function sentinelElapsedWarnings(value: unknown, subject = "sentinel confirmed operation", budgetSeconds = 120): string[] {
|
|
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
|
|
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
|
|
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s timing budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, investigate wait-stage latency without treating timing alone as HWLAB business blockage.`];
|
|
}
|
|
|
|
function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number {
|
|
return numberAt(state.cicd, "confirmWait.maxSeconds");
|
|
}
|
|
|
|
function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
|
|
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
|
|
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
|
|
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`];
|
|
}
|
|
|
|
function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] {
|
|
const sync = record(sourceMirrorSync);
|
|
if (sync.ok === true || state.sourceHead.ok !== true) return [];
|
|
return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`];
|
|
}
|
|
|
|
function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, probe: unknown): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
phase: "already-present",
|
|
jobName: null,
|
|
probe: record(probe),
|
|
payload: {
|
|
ok: true,
|
|
status: "already-present",
|
|
sourceCommit: state.sourceHead.commit,
|
|
mirrorCommit: state.sourceHead.commit,
|
|
valuesRedacted: true,
|
|
},
|
|
polls: 0,
|
|
elapsedMs: 0,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] {
|
|
if (applyOnly) return [];
|
|
const next = sentinelP5Next(state);
|
|
return [`targetValidation quick verify is deferred from control-plane confirm-wait to keep CI/CD wait under ${Math.round(budgetSeconds)}s; run ${next.quickVerify}.`];
|
|
}
|
|
|
|
function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
|
|
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
|
|
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
|
|
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, only Code Agent multi-round business failures should block acceptance.`];
|
|
}
|
|
|
|
function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] {
|
|
const warnings: string[] = [];
|
|
for (const item of items) {
|
|
const values = Array.isArray(item) ? item : [item];
|
|
for (const value of values) {
|
|
if (value === undefined || value === null || value === "") continue;
|
|
const warning = text(value).trim();
|
|
if (warning.length > 0 && warning !== "-" && !warnings.includes(warning)) warnings.push(warning);
|
|
}
|
|
}
|
|
return warnings;
|
|
}
|
|
|
|
function withWarnings(payload: Record<string, unknown>, warnings: readonly unknown[]): Record<string, unknown> {
|
|
const merged = mergeWarnings(payload.warnings, warnings);
|
|
return merged.length === 0 ? payload : { ...payload, warnings: merged, valuesRedacted: true };
|
|
}
|
|
|
|
function sentinelProgressEvent(event: string, payload: Record<string, unknown>): void {
|
|
console.error(JSON.stringify({ event, at: new Date().toISOString(), ...payload, valuesRedacted: true }));
|
|
}
|
|
|
|
function confirmBlocked(action: string, state: SentinelCicdState): Record<string, unknown> {
|
|
return {
|
|
code: "sentinel-cicd-confirm-requires-remote-publish-job",
|
|
action,
|
|
reason: "P4 currently provides YAML-first render/status/trigger dry-run and refuses to report a deployment mutation before the remote publish job is wired to the node-local git mirror.",
|
|
sourceGitMirrorReadUrl: stringAt(state.cicd, "source.gitMirrorReadUrl"),
|
|
requiredNextImplementation: [
|
|
"clone source from source.gitMirrorReadUrl at selected commit",
|
|
"build and push digest-pinned image on the selected node",
|
|
"publish manifests to the HWLAB gitops branch/path through git-mirror",
|
|
"flush/recheck git-mirror and let Argo reconcile the Application",
|
|
],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelControlPlaneAction): Record<string, string> {
|
|
const node = state.spec.nodeId;
|
|
const lane = state.spec.lane;
|
|
const suffix = sentinelCliSuffix(state);
|
|
return {
|
|
plan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${node} --lane ${lane}${suffix} --dry-run`,
|
|
status: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
|
|
image: `bun scripts/cli.ts web-probe sentinel image status --node ${node} --lane ${lane}${suffix}`,
|
|
triggerCurrent: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${node} --lane ${lane}${suffix} --dry-run`,
|
|
validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`,
|
|
quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`,
|
|
issue: "https://github.com/pikasTech/unidesk/issues/889",
|
|
currentAction: action,
|
|
};
|
|
}
|
|
|
|
function runSentinelMaintenance(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "maintenance" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel maintenance ${options.action}`;
|
|
const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
|
|
if (options.action === "status") {
|
|
const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds);
|
|
const result = {
|
|
ok: serviceHealth.ok && maintenance.ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
serviceHealth,
|
|
maintenance,
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderMaintenanceResult(result));
|
|
}
|
|
if (!options.confirm) {
|
|
const result = {
|
|
ok: serviceHealth.ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "dry-run",
|
|
serviceHealth,
|
|
mutation: false,
|
|
planned: {
|
|
action: options.action,
|
|
releaseId: options.releaseId,
|
|
reason: options.reason,
|
|
quickVerify: options.action === "stop" && options.quickVerify,
|
|
},
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderMaintenanceResult(result));
|
|
}
|
|
if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify);
|
|
if (!serviceHealth.ok) {
|
|
const result = {
|
|
ok: false,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: false,
|
|
serviceHealth,
|
|
blocker: serviceUnavailableBlocker(state),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(false, command, renderMaintenanceResult(result));
|
|
}
|
|
const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true };
|
|
const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds);
|
|
const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok
|
|
? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds)
|
|
: null;
|
|
const result = {
|
|
ok: mutation.ok && (quickVerify === null || quickVerify.ok === true),
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: true,
|
|
serviceHealth,
|
|
maintenance: mutation,
|
|
quickVerify,
|
|
blocker: mutation.ok ? null : serviceUnavailableBlocker(state),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderMaintenanceResult(result));
|
|
}
|
|
|
|
function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
|
|
const command = "web-probe sentinel validate";
|
|
const startedAt = Date.now();
|
|
const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
|
|
let quickVerify: Record<string, unknown> | null = null;
|
|
if (options.quickVerify) {
|
|
if (!options.confirm) {
|
|
const result = {
|
|
ok: initialHealth.ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "dry-run",
|
|
serviceHealth: initialHealth,
|
|
planned: { quickVerify: true, waitRequired: true },
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderValidateResult(result));
|
|
}
|
|
if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true);
|
|
if (!initialHealth.ok) {
|
|
const result = {
|
|
ok: false,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
serviceHealth: initialHealth,
|
|
blocker: serviceUnavailableBlocker(state),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(false, command, renderValidateResult(result));
|
|
}
|
|
quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds);
|
|
}
|
|
const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20);
|
|
const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds);
|
|
const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds);
|
|
const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds);
|
|
const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health");
|
|
const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds);
|
|
const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds);
|
|
const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds);
|
|
const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health;
|
|
const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics;
|
|
const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report;
|
|
const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds);
|
|
const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds);
|
|
if (quickVerify !== null) {
|
|
quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds"))));
|
|
}
|
|
const publicFallbackWarnings = [
|
|
...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []),
|
|
...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []),
|
|
...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []),
|
|
];
|
|
const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health");
|
|
const ok = effectiveHealth.ok
|
|
&& record(effectiveHealth.bodyJson).ok === true
|
|
&& effectiveMetricsOk
|
|
&& effectiveReport.ok
|
|
&& publicExposure.ok === true
|
|
&& publicDashboard.ok === true
|
|
&& (quickVerify === null || quickVerify.ok === true);
|
|
const result = {
|
|
ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: options.quickVerify ? "confirm-wait" : "status",
|
|
serviceHealth: effectiveHealth,
|
|
metrics: effectiveMetrics,
|
|
report: effectiveReport,
|
|
internalServiceHealth: health,
|
|
internalMetrics: metrics,
|
|
internalReport: report,
|
|
publicServiceHealth: publicHealth,
|
|
publicMetrics,
|
|
publicReport,
|
|
publicExposure,
|
|
publicDashboard,
|
|
quickVerify,
|
|
warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []),
|
|
blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(ok, command, renderValidateResult(result));
|
|
}
|
|
|
|
function runSentinelReport(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "report" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`;
|
|
const query = new URLSearchParams({ view: options.view });
|
|
if (options.runId !== null) query.set("run", options.runId);
|
|
if (options.traceId !== null) query.set("traceId", options.traceId);
|
|
if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq));
|
|
const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds);
|
|
const body = record(report.bodyJson);
|
|
const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true });
|
|
const rawPayload = Object.keys(body).length > 0 ? body : report;
|
|
return rendered(report.ok && body.ok !== false, command, options.raw ? JSON.stringify(rawPayload, null, 2) : renderedText);
|
|
}
|
|
|
|
function runSentinelDashboard(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel dashboard ${options.action}`;
|
|
const result = probeSentinelDashboardBrowser(state, options);
|
|
return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result));
|
|
}
|
|
|
|
function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
|
const [widthRaw, heightRaw] = options.viewport.split("x");
|
|
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
|
|
const script = [
|
|
"set -eu",
|
|
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(`${publicBaseUrl}/`)}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`,
|
|
"export PLAYWRIGHT_BROWSERS_PATH=0",
|
|
"if command -v chromium >/dev/null 2>&1; then",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)",
|
|
"elif command -v chromium-browser >/dev/null 2>&1; then",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)",
|
|
"elif command -v google-chrome >/dev/null 2>&1; then",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)",
|
|
"else",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=",
|
|
"fi",
|
|
"cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'",
|
|
sentinelDashboardBrowserModule(),
|
|
"WEB_PROBE_SENTINEL_DASHBOARD_JS",
|
|
"bun \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"",
|
|
].join("\n");
|
|
const route = `${state.spec.nodeId}:${state.spec.workspace}`;
|
|
const job = runWebProbeRemoteArtifactJob({
|
|
route,
|
|
localDir: options.localDir,
|
|
waitTimeoutMs: options.waitTimeoutMs,
|
|
commandTimeoutMs: options.commandTimeoutSeconds * 1000,
|
|
inactivityTimeoutMs: 30000,
|
|
runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`,
|
|
}, script);
|
|
const result = job.result;
|
|
const transport = record(job.transport);
|
|
const remote = record(transport.remote);
|
|
const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : "");
|
|
const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : [];
|
|
const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null;
|
|
const browserOk = page?.ok === true;
|
|
const screenshotOk = options.action === "verify" || screenshot !== null && screenshot.verified === true;
|
|
const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk;
|
|
return {
|
|
ok,
|
|
status: ok ? "pass" : "blocked",
|
|
command: `web-probe sentinel dashboard ${options.action}`,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
publicUrl: `${publicBaseUrl}/`,
|
|
route,
|
|
viewport: options.viewport,
|
|
page,
|
|
screenshot,
|
|
artifacts,
|
|
artifactCount: artifacts.length,
|
|
remote: {
|
|
exitCode: remote.exitCode ?? null,
|
|
remoteDir: remote.remoteDir ?? null,
|
|
stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "",
|
|
stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "",
|
|
},
|
|
transport: {
|
|
ok: transport.ok ?? null,
|
|
runId: transport.runId ?? null,
|
|
artifactCount: transport.artifactCount ?? null,
|
|
expectedArtifactCount: transport.expectedArtifactCount ?? null,
|
|
downloadFailure: transport.downloadFailure ?? null,
|
|
},
|
|
result: compactCommand(result),
|
|
degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelDashboardBrowserModule(): string {
|
|
return String.raw`import { pathToFileURL } from "node:url";
|
|
|
|
const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || "";
|
|
const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright";
|
|
const { chromium } = await import(playwrightModuleSpecifier);
|
|
|
|
const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL;
|
|
const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || "";
|
|
const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1";
|
|
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
|
|
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
|
|
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
|
|
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0";
|
|
const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || "";
|
|
|
|
if (!url) throw new Error("missing dashboard URL");
|
|
|
|
const consoleMessages = [];
|
|
const pageErrors = [];
|
|
const requestFailures = [];
|
|
const browser = await chromium.launch({
|
|
headless: true,
|
|
args: ["--disable-gpu", "--no-sandbox"],
|
|
...(executablePath ? { executablePath } : {}),
|
|
});
|
|
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
|
|
const page = await context.newPage();
|
|
page.on("console", (message) => {
|
|
if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) });
|
|
});
|
|
page.on("pageerror", (error) => {
|
|
if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) });
|
|
});
|
|
page.on("requestfailed", (request) => {
|
|
if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null });
|
|
});
|
|
|
|
let httpStatus = null;
|
|
let navigationError = null;
|
|
let navigationAttempts = 0;
|
|
const maxNavigationAttempts = 3;
|
|
const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts));
|
|
for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) {
|
|
navigationAttempts = attempt;
|
|
navigationError = null;
|
|
try {
|
|
const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" });
|
|
httpStatus = response?.status() ?? null;
|
|
await page.waitForLoadState("networkidle", { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {});
|
|
await page.waitForFunction(() => {
|
|
const root = document.querySelector("#monitor-web-root");
|
|
if (!root) return false;
|
|
const ready = root.getAttribute("data-monitor-ready") === "true";
|
|
const error = document.querySelector("#monitor-web-error");
|
|
const runs = document.querySelectorAll(".run-list .run-row").length;
|
|
const trend = document.querySelector("[data-monitor-trend-curve]");
|
|
return ready && (error || runs > 0 || trend);
|
|
}, null, { timeout: Math.min(15000, perAttemptTimeout) }).catch(() => {});
|
|
await page.waitForTimeout(500);
|
|
const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false);
|
|
if (appReady || attempt === maxNavigationAttempts) break;
|
|
} catch (error) {
|
|
navigationError = String(error?.message || error).slice(0, 500);
|
|
if (attempt === maxNavigationAttempts) break;
|
|
}
|
|
await page.waitForTimeout(750 * attempt);
|
|
}
|
|
|
|
await page.evaluate(() => {
|
|
const detailPane = document.querySelector(".workspace-grid .pane-detail");
|
|
if (detailPane instanceof HTMLElement) detailPane.scrollTop = Math.min(96, Math.max(0, detailPane.scrollHeight - detailPane.clientHeight));
|
|
}).catch(() => {});
|
|
await page.waitForTimeout(150);
|
|
|
|
const trendHoverPoint = await page.evaluate(() => {
|
|
const target = document.querySelector(".trend-dot-hit .trend-dot-red") || document.querySelector(".trend-dot-hit .trend-dot-warning");
|
|
if (!(target instanceof SVGElement)) return null;
|
|
const rect = target.getBoundingClientRect();
|
|
if (rect.width <= 0 || rect.height <= 0) return null;
|
|
return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
|
|
}).catch(() => null);
|
|
if (trendHoverPoint) {
|
|
await page.mouse.move(trendHoverPoint.x, trendHoverPoint.y);
|
|
await page.waitForTimeout(250);
|
|
}
|
|
|
|
if (captureScreenshot && screenshotPath) {
|
|
await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => {
|
|
pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) });
|
|
});
|
|
}
|
|
|
|
const dom = await page.evaluate(() => {
|
|
const visible = (element) => Boolean(element && !element.hidden);
|
|
const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim();
|
|
const root = document.querySelector("#monitor-web-root");
|
|
const shell = document.querySelector("[data-monitor-shell='true']");
|
|
const error = document.querySelector("#monitor-web-error");
|
|
const trend = document.querySelector("[data-monitor-trend-curve]");
|
|
const trendTooltip = document.querySelector("[data-monitor-trend-tooltip='true']");
|
|
const timeline = document.querySelector("[data-monitor-timeline='true']");
|
|
const workspace = document.querySelector("[data-monitor-independent-scroll='true']");
|
|
const panes = Array.from(document.querySelectorAll(".workspace-grid .pane"));
|
|
const detailPane = document.querySelector(".workspace-grid .pane-detail");
|
|
const detailHeader = document.querySelector("#monitor-web-root > div > section.workspace-grid > main > div.pane-header");
|
|
const doc = document.documentElement;
|
|
const body = document.body;
|
|
const viewport = { width: window.innerWidth, height: window.innerHeight };
|
|
const documentSize = {
|
|
width: Math.max(doc.scrollWidth, body?.scrollWidth || 0),
|
|
height: Math.max(doc.scrollHeight, body?.scrollHeight || 0),
|
|
};
|
|
const overflow = [];
|
|
let overflowCount = 0;
|
|
for (const element of Array.from(document.querySelectorAll("body *"))) {
|
|
const rect = element.getBoundingClientRect();
|
|
const overflowRight = rect.right - viewport.width;
|
|
const overflowLeft = -rect.left;
|
|
if (overflowRight > 1 || overflowLeft > 1) {
|
|
overflowCount += 1;
|
|
if (overflow.length < 5) {
|
|
overflow.push({
|
|
tag: element.tagName.toLowerCase(),
|
|
className: String(element.className || "").slice(0, 80),
|
|
text: String(element.textContent || "").replace(/\s+/g, " ").trim().slice(0, 80),
|
|
x: Math.round(rect.x),
|
|
y: Math.round(rect.y),
|
|
width: Math.round(rect.width),
|
|
height: Math.round(rect.height),
|
|
overflowRight: Math.max(0, Math.round(overflowRight)),
|
|
overflowLeft: Math.max(0, Math.round(overflowLeft)),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return {
|
|
shell: Boolean(root && shell),
|
|
ready: root?.getAttribute("data-monitor-ready") === "true",
|
|
dataset: root ? {
|
|
node: root.getAttribute("data-node"),
|
|
lane: root.getAttribute("data-lane"),
|
|
sentinelId: root.getAttribute("data-sentinel-id"),
|
|
basePath: root.getAttribute("data-base-path"),
|
|
contractVersion: root.getAttribute("data-contract-version"),
|
|
} : {},
|
|
title: document.title,
|
|
finalUrl: window.location.href,
|
|
statusText: text(".topbar .pill"),
|
|
subtitle: text(".subtitle"),
|
|
summaryText: text(".status-strip"),
|
|
runRows: document.querySelectorAll(".run-list .run-row").length,
|
|
findingItems: document.querySelectorAll(".finding-list .finding-card").length,
|
|
trendCurve: Boolean(trend),
|
|
trendDotCount: document.querySelectorAll(".trend-dot-hit").length,
|
|
trendTooltip: tooltipSummary(trendTooltip),
|
|
trendPanelText: text("#trend-heading"),
|
|
timelineItems: document.querySelectorAll(".timeline-list .timeline-item").length,
|
|
timelineVisible: Boolean(timeline),
|
|
errorVisible: visible(error),
|
|
errorText: visible(error) ? text("#monitor-web-error").slice(0, 500) : "",
|
|
scrollModel: {
|
|
workspace: Boolean(workspace),
|
|
paneCount: panes.length,
|
|
panes: panes.map((pane) => {
|
|
const style = window.getComputedStyle(pane);
|
|
const rect = pane.getBoundingClientRect();
|
|
return {
|
|
className: String(pane.className || ""),
|
|
overflowY: style.overflowY,
|
|
scrollHeight: pane.scrollHeight,
|
|
clientHeight: pane.clientHeight,
|
|
x: Math.round(rect.x),
|
|
y: Math.round(rect.y),
|
|
width: Math.round(rect.width),
|
|
height: Math.round(rect.height),
|
|
};
|
|
}),
|
|
independentScroll: panes.length >= 3 && panes.every((pane) => {
|
|
const style = window.getComputedStyle(pane);
|
|
return style.overflowY === "auto" || style.overflowY === "scroll";
|
|
}),
|
|
stickyHeader: stickyHeaderSummary(detailPane, detailHeader),
|
|
},
|
|
layout: {
|
|
viewport,
|
|
documentSize,
|
|
horizontalOverflow: documentSize.width > viewport.width + 1,
|
|
overflowCount,
|
|
overflow,
|
|
},
|
|
};
|
|
|
|
function tooltipSummary(element) {
|
|
const body = String(element?.textContent || "").replace(/\s+/g, " ").trim();
|
|
return {
|
|
visible: Boolean(element && body.length > 0),
|
|
text: body.slice(0, 240),
|
|
hasValues: /红色\s+\d+/u.test(body) && /警告\s+\d+/u.test(body) && /总量\s+\d+/u.test(body),
|
|
hasTime: /UTC/u.test(body) || /\d{4}-\d{2}-\d{2}/u.test(body),
|
|
};
|
|
}
|
|
|
|
function stickyHeaderSummary(pane, header) {
|
|
if (!(pane instanceof HTMLElement) || !(header instanceof HTMLElement)) {
|
|
return { present: false, coversScroll: false, backgroundOpaque: false, detailScrollTop: null };
|
|
}
|
|
const rect = header.getBoundingClientRect();
|
|
const style = window.getComputedStyle(header);
|
|
const sampleX = Math.round(rect.left + Math.min(32, Math.max(2, rect.width / 2)));
|
|
const sampleY = Math.round(rect.top + Math.min(12, Math.max(2, rect.height / 2)));
|
|
const topElement = document.elementFromPoint(sampleX, sampleY);
|
|
return {
|
|
present: true,
|
|
detailScrollTop: pane.scrollTop,
|
|
headerTop: Math.round(rect.top),
|
|
headerBottom: Math.round(rect.bottom),
|
|
zIndex: style.zIndex,
|
|
backgroundColor: style.backgroundColor,
|
|
coversScroll: Boolean(topElement && header.contains(topElement)),
|
|
backgroundOpaque: backgroundIsOpaque(style.backgroundColor),
|
|
topElementClass: String(topElement?.className || "").slice(0, 80),
|
|
};
|
|
}
|
|
|
|
function backgroundIsOpaque(value) {
|
|
const rgba = /rgba?\(([^)]+)\)/u.exec(value);
|
|
if (rgba === null) return value.length > 0 && value !== "transparent";
|
|
const parts = rgba[1].split(",").map((part) => part.trim());
|
|
if (parts.length < 4) return true;
|
|
return Number(parts[3]) >= 0.99;
|
|
}
|
|
});
|
|
|
|
const consoleErrors = consoleMessages.filter((item) => item.type === "error");
|
|
const ok = !navigationError
|
|
&& httpStatus !== null
|
|
&& httpStatus >= 200
|
|
&& httpStatus < 300
|
|
&& dom.shell === true
|
|
&& dom.ready === true
|
|
&& dom.errorVisible !== true
|
|
&& dom.trendCurve === true
|
|
&& (dom.trendDotCount === 0 || (dom.trendTooltip?.visible === true && dom.trendTooltip?.hasValues === true && dom.trendTooltip?.hasTime === true))
|
|
&& dom.timelineVisible === true
|
|
&& dom.scrollModel?.independentScroll === true
|
|
&& dom.scrollModel?.stickyHeader?.present === true
|
|
&& dom.scrollModel?.stickyHeader?.coversScroll === true
|
|
&& dom.scrollModel?.stickyHeader?.backgroundOpaque === true
|
|
&& dom.layout?.horizontalOverflow !== true
|
|
&& pageErrors.length === 0;
|
|
|
|
console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
|
|
ok,
|
|
url,
|
|
httpStatus,
|
|
navigationError,
|
|
navigationAttempts,
|
|
executablePath: executablePath || null,
|
|
viewport: { width, height },
|
|
screenshotPath: captureScreenshot ? screenshotPath : null,
|
|
dom,
|
|
consoleCount: consoleMessages.length,
|
|
consoleErrorCount: consoleErrors.length,
|
|
pageErrorCount: pageErrors.length,
|
|
requestFailureCount: requestFailures.length,
|
|
consoleMessages: consoleMessages.slice(0, 8),
|
|
pageErrors: pageErrors.slice(0, 8),
|
|
requestFailures: requestFailures.slice(0, 8),
|
|
valuesRedacted: true,
|
|
}));
|
|
|
|
await context.close().catch(() => {});
|
|
await browser.close().catch(() => {});
|
|
`;
|
|
}
|
|
|
|
function parseDashboardBrowserPayload(textValue: string): Record<string, unknown> | null {
|
|
const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__";
|
|
const index = textValue.lastIndexOf(marker);
|
|
if (index < 0) return null;
|
|
try {
|
|
return record(JSON.parse(textValue.slice(index + marker.length).trim()));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function dashboardScreenshotName(options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>, state: SentinelCicdState): string {
|
|
const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`;
|
|
const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120);
|
|
return safe.endsWith(".png") ? safe : `${safe}.png`;
|
|
}
|
|
|
|
function compactDashboardArtifact(artifact: Record<string, unknown>): Record<string, unknown> {
|
|
const transfer = record(artifact.transfer);
|
|
return {
|
|
remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null,
|
|
localPath: typeof artifact.localPath === "string" ? artifact.localPath : null,
|
|
bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null,
|
|
sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null,
|
|
verified: artifact.verified === true,
|
|
transfer: Object.keys(transfer).length === 0 ? null : {
|
|
strategy: transfer.strategy ?? null,
|
|
transport: transfer.transport ?? null,
|
|
chunks: transfer.chunks ?? null,
|
|
elapsedMs: transfer.elapsedMs ?? null,
|
|
throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null,
|
|
},
|
|
};
|
|
}
|
|
|
|
function dashboardDegradedReason(result: CommandResult, transport: Record<string, unknown>, page: Record<string, unknown> | null, screenshotOk: boolean): string {
|
|
if (result.timedOut) return "sentinel-dashboard-command-timeout";
|
|
if (transport.ok !== true) return "sentinel-dashboard-transport-failed";
|
|
if (page === null) return "sentinel-dashboard-browser-output-missing";
|
|
if (page.ok !== true) return "sentinel-dashboard-render-failed";
|
|
if (!screenshotOk) return "sentinel-dashboard-screenshot-missing";
|
|
return "sentinel-dashboard-unknown";
|
|
}
|
|
|
|
function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult {
|
|
const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
|
|
if (releaseId !== null) args.push("--release-id", releaseId);
|
|
if (reason !== null) args.push("--reason", reason);
|
|
if (quickVerify) args.push("--quick-verify");
|
|
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`);
|
|
const command = `web-probe sentinel ${subcommand.join(" ")}`;
|
|
return rendered(true, command, renderAsyncJobResult({
|
|
ok: true,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "async-job",
|
|
mutation: true,
|
|
job,
|
|
next: {
|
|
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
|
|
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
|
|
},
|
|
valuesRedacted: true,
|
|
}));
|
|
}
|
|
|
|
function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record<string, unknown> = {}): void {
|
|
const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => {
|
|
if (typeof value === "string") return [key, short(value)];
|
|
if (Array.isArray(value)) return [key, value.slice(0, 8)];
|
|
return [key, value];
|
|
}));
|
|
process.stdout.write(`${JSON.stringify({
|
|
event: "sentinel.quick-verify.progress",
|
|
at: new Date().toISOString(),
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
runId,
|
|
...compactExtra,
|
|
phase,
|
|
status,
|
|
valuesRedacted: true,
|
|
})}\n`);
|
|
}
|
|
|
|
function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record<string, unknown> {
|
|
const startedAt = Date.now();
|
|
const elapsedMs = () => Date.now() - startedAt;
|
|
const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
|
|
const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
|
|
const scenario = findScenario(state, scenarioId);
|
|
if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true };
|
|
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
|
|
const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt");
|
|
const prompts = needsPromptSet
|
|
? readPromptSetForScenario(scenario)
|
|
: { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } };
|
|
if (!prompts.ok) return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true };
|
|
const accountEnv = quickVerifyAccountEnv(state);
|
|
if (!accountEnv.ok) {
|
|
const findings = [{
|
|
id: "quick-verify-account-secret-missing",
|
|
severity: "red",
|
|
count: arrayAt(accountEnv.summary, "missing").length || 1,
|
|
summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.",
|
|
missing: arrayAt(accountEnv.summary, "missing"),
|
|
valuesRedacted: true,
|
|
}];
|
|
return recordQuickVerify(state, {
|
|
ok: false,
|
|
runId: `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`,
|
|
scenarioId,
|
|
reason,
|
|
status: "blocked",
|
|
observerId: null,
|
|
elapsedMs: 0,
|
|
steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }],
|
|
failure: "quick-verify-account-secret-missing",
|
|
findingCount: findings.length,
|
|
findings,
|
|
promptSource: prompts.summary,
|
|
accountEnv: accountEnv.summary,
|
|
views: {
|
|
summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
},
|
|
warnings: [],
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs");
|
|
const warningBudgetSeconds = maxSeconds;
|
|
const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds")));
|
|
const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds);
|
|
const deadline = Date.now() + hardBudgetSeconds * 1000;
|
|
const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
|
|
printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds });
|
|
const steps: Record<string, unknown>[] = [];
|
|
const startArgs = [
|
|
"web-probe", "observe", "start",
|
|
"--node", state.spec.nodeId,
|
|
"--lane", state.spec.lane,
|
|
"--target-path", stringAt(scenario, "observeTargetPath"),
|
|
"--sample-interval-ms", String(sampleIntervalMs),
|
|
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
|
|
"--command-timeout-seconds", "55",
|
|
];
|
|
const viewport = stringAtNullable(scenario, "viewport");
|
|
if (viewport !== null) startArgs.push("--viewport", viewport);
|
|
printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) });
|
|
const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env);
|
|
steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
|
|
const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
|
|
printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() });
|
|
if (!started.ok || observerId === null) {
|
|
const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null);
|
|
return recordQuickVerify(state, {
|
|
ok: false,
|
|
runId,
|
|
scenarioId,
|
|
reason,
|
|
status: "blocked",
|
|
observerId,
|
|
elapsedMs: elapsedMs(),
|
|
steps,
|
|
failure: "observe-start-failed",
|
|
findingCount: findings.length,
|
|
findings,
|
|
warnings: elapsedWarnings(),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) });
|
|
const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds);
|
|
steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady });
|
|
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() });
|
|
if (startupReady.ok !== true) {
|
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
|
runId,
|
|
scenarioId,
|
|
reason,
|
|
observerId,
|
|
promptIndex: 0,
|
|
steps,
|
|
failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"),
|
|
elapsedMs: elapsedMs(),
|
|
warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()),
|
|
promptSource: prompts.summary,
|
|
}));
|
|
}
|
|
let promptIndex = 0;
|
|
const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario);
|
|
const nonBlockingCanaryWarnings: string[] = [];
|
|
for (const item of commandSequence) {
|
|
const type = stringAt(item, "type");
|
|
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
|
|
for (let index = 0; index < repeat; index += 1) {
|
|
if (Date.now() >= deadline) {
|
|
printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds });
|
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
|
runId,
|
|
scenarioId,
|
|
reason,
|
|
observerId,
|
|
promptIndex,
|
|
steps,
|
|
failure: "quick-verify-timeout-over-budget",
|
|
elapsedMs: elapsedMs(),
|
|
warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()),
|
|
promptSource: prompts.summary,
|
|
}));
|
|
}
|
|
const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", "55000", "--command-timeout-seconds", String(remainingSeconds(deadline, 55))];
|
|
if (type === "selectProvider") args.push("--provider", stringAt(item, "provider"));
|
|
if (type === "loginAccount" || type === "listSessions" || type === "logout") {
|
|
const accountId = stringAtNullable(item, "accountId");
|
|
if (accountId !== null) args.push("--account-id", accountId);
|
|
}
|
|
if (type === "switchSessions") {
|
|
const fromAccountId = stringAtNullable(item, "fromAccountId");
|
|
const toAccountId = stringAtNullable(item, "toAccountId");
|
|
if (fromAccountId !== null) args.push("--from-account-id", fromAccountId);
|
|
if (toAccountId !== null) args.push("--to-account-id", toAccountId);
|
|
}
|
|
if (type === "sendPrompt") {
|
|
args.push("--text", prompts.prompts[promptIndex % prompts.prompts.length] ?? "");
|
|
args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000));
|
|
promptIndex += 1;
|
|
}
|
|
appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" });
|
|
printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat });
|
|
const commandResult = runChildCli(args, remainingSeconds(deadline, 60));
|
|
steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result });
|
|
printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() });
|
|
if (!commandResult.ok) {
|
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
|
runId,
|
|
scenarioId,
|
|
reason,
|
|
observerId,
|
|
promptIndex,
|
|
steps,
|
|
failure: `observe-command-${type}-failed`,
|
|
elapsedMs: elapsedMs(),
|
|
warnings: elapsedWarnings(),
|
|
promptSource: prompts.summary,
|
|
}));
|
|
}
|
|
if (type === "sendPrompt") {
|
|
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, 55) });
|
|
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds);
|
|
steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult });
|
|
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() });
|
|
if (waitResult.ok !== true) {
|
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
|
runId,
|
|
scenarioId,
|
|
reason,
|
|
observerId,
|
|
promptIndex,
|
|
steps,
|
|
failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"),
|
|
promptSource: prompts.summary,
|
|
elapsedMs: elapsedMs(),
|
|
warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()),
|
|
}));
|
|
}
|
|
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
|
|
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
|
|
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
|
|
if (invariantResult.ok !== true) {
|
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
|
runId,
|
|
scenarioId,
|
|
reason,
|
|
observerId,
|
|
promptIndex,
|
|
steps,
|
|
failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"),
|
|
promptSource: prompts.summary,
|
|
elapsedMs: elapsedMs(),
|
|
warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()),
|
|
}));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) });
|
|
const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120));
|
|
steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result });
|
|
printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() });
|
|
const indexEntry = readLocalObserveIndex(observerId);
|
|
const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30));
|
|
const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30));
|
|
const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30));
|
|
const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame);
|
|
const artifactSummaryRecord = record(artifactSummary);
|
|
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
|
|
const findings = mergeFindingRecords(artifactFindings, controlFindings);
|
|
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
|
|
const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."];
|
|
const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0;
|
|
printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
|
|
return recordQuickVerify(state, {
|
|
ok,
|
|
runId,
|
|
scenarioId,
|
|
reason,
|
|
status: ok ? "analyzed" : "blocked",
|
|
observerId,
|
|
elapsedMs: elapsedMs(),
|
|
stateDir: indexEntry?.stateDir ?? null,
|
|
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
|
|
findingCount: findings.length,
|
|
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
|
|
failure: controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null,
|
|
promptSource: prompts.summary,
|
|
accountEnv: accountEnv.summary,
|
|
steps,
|
|
analysis: artifactSummary,
|
|
views: {
|
|
summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
|
|
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
|
|
},
|
|
findings,
|
|
screenshot: record(artifactSummary).screenshot,
|
|
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
|
|
warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, elapsedWarnings()),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function sessionInvarianceChecksByRound(scenario: Record<string, unknown>): Map<number, Record<string, unknown>[]> {
|
|
const checks = new Map<number, Record<string, unknown>[]>();
|
|
const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : [];
|
|
for (const item of items) {
|
|
const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null;
|
|
if (afterRound === null || afterRound < 0) continue;
|
|
const list = checks.get(afterRound) ?? [];
|
|
list.push(item);
|
|
checks.set(afterRound, list);
|
|
}
|
|
return checks;
|
|
}
|
|
|
|
function runQuickVerifySessionInvarianceChecks(
|
|
state: SentinelCicdState,
|
|
observerId: string,
|
|
checks: readonly Record<string, unknown>[],
|
|
deadline: number,
|
|
promptIndex: number,
|
|
steps: Record<string, unknown>[],
|
|
): Record<string, unknown> {
|
|
const warnings: string[] = [];
|
|
for (const check of checks) {
|
|
const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`;
|
|
const blocking = check.blocking === true;
|
|
const commands: { readonly type: string; readonly enabled: boolean }[] = [
|
|
{ type: "refreshCurrentSession", enabled: check.refreshCurrent === true },
|
|
{ type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true },
|
|
{ type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false },
|
|
];
|
|
for (const command of commands) {
|
|
if (!command.enabled) continue;
|
|
const args = [
|
|
"web-probe", "observe", "command", observerId,
|
|
"--node", state.spec.nodeId,
|
|
"--lane", state.spec.lane,
|
|
"--type", command.type,
|
|
"--after-round", String(promptIndex),
|
|
"--wait-ms", "55000",
|
|
"--command-timeout-seconds", String(remainingSeconds(deadline, 55)),
|
|
];
|
|
const severity = nonEmptyString(check.severity);
|
|
const findingId = nonEmptyString(check.findingId);
|
|
const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange);
|
|
const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy);
|
|
if (severity !== null) args.push("--severity", severity);
|
|
if (findingId !== null) args.push("--finding-id", findingId);
|
|
if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange);
|
|
if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy);
|
|
if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready");
|
|
args.push(check.blocking === true ? "--blocking" : "--non-blocking");
|
|
const result = runChildCli(args, remainingSeconds(deadline, 60));
|
|
steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result });
|
|
if (!result.ok) {
|
|
if (!blocking) {
|
|
warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`);
|
|
continue;
|
|
}
|
|
return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true };
|
|
}
|
|
}
|
|
}
|
|
return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true };
|
|
}
|
|
|
|
function appendScenarioObserveCommandArgs(args: string[], item: Record<string, unknown>, options: { readonly skipText?: boolean } = {}): void {
|
|
const mappings: readonly (readonly [string, string])[] = [
|
|
["path", "--path"],
|
|
["label", "--label"],
|
|
["sessionId", "--session-id"],
|
|
["provider", "--provider"],
|
|
["accountId", "--account-id"],
|
|
["fromAccountId", "--from-account-id"],
|
|
["toAccountId", "--to-account-id"],
|
|
["sourceId", "--source-id"],
|
|
["fileRef", "--file-ref"],
|
|
["filename", "--filename"],
|
|
["taskRef", "--task-ref"],
|
|
["taskId", "--task-id"],
|
|
["task", "--task"],
|
|
["field", "--field"],
|
|
["link", "--link"],
|
|
["title", "--title"],
|
|
["body", "--body"],
|
|
["status", "--status"],
|
|
["hwpodId", "--hwpod-id"],
|
|
["nodeId", "--node-id"],
|
|
["workspaceRoot", "--workspace-root"],
|
|
["root", "--root"],
|
|
];
|
|
for (const [key, flag] of mappings) {
|
|
if (args.includes(flag)) continue;
|
|
const value = stringAtNullable(item, key);
|
|
if (value !== null) args.push(flag, value);
|
|
}
|
|
if (options.skipText !== true && !args.includes("--text")) {
|
|
const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value");
|
|
if (text !== null) args.push("--text", text);
|
|
}
|
|
if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready");
|
|
}
|
|
|
|
function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
|
readonly runId: string;
|
|
readonly scenarioId: string;
|
|
readonly reason: string;
|
|
readonly observerId: string;
|
|
readonly promptIndex: number;
|
|
readonly steps: readonly Record<string, unknown>[];
|
|
readonly failure: string;
|
|
readonly promptSource?: Record<string, unknown>;
|
|
readonly elapsedMs?: number;
|
|
readonly warnings?: readonly unknown[];
|
|
}): Record<string, unknown> {
|
|
const cleanupSteps: Record<string, unknown>[] = [];
|
|
if (input.promptIndex > 0) {
|
|
const cancel = runChildCli([
|
|
"web-probe", "observe", "command", input.observerId,
|
|
"--node", state.spec.nodeId,
|
|
"--lane", state.spec.lane,
|
|
"--type", "cancel",
|
|
"--wait-ms", "55000",
|
|
"--command-timeout-seconds", "55",
|
|
], 60);
|
|
cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result });
|
|
}
|
|
const stop = runChildCli([
|
|
"web-probe", "observe", "stop", input.observerId,
|
|
"--node", state.spec.nodeId,
|
|
"--lane", state.spec.lane,
|
|
"--force",
|
|
"--command-timeout-seconds", "55",
|
|
], 30);
|
|
cleanupSteps.push({ phase: "observe-stop-after-failure", ok: stop.ok, result: stop.result });
|
|
const analysis = runChildCli([
|
|
"web-probe", "observe", "analyze", input.observerId,
|
|
"--node", state.spec.nodeId,
|
|
"--lane", state.spec.lane,
|
|
"--command-timeout-seconds", "55",
|
|
], 60);
|
|
cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result });
|
|
const indexEntry = readLocalObserveIndex(input.observerId);
|
|
const artifactSummary = indexEntry === null
|
|
? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true }
|
|
: readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, 30);
|
|
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30);
|
|
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30);
|
|
const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame);
|
|
const artifactSummaryRecord = record(artifactSummary);
|
|
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
|
|
const findings = mergeFindingRecords(artifactFindings, controlFindings);
|
|
return {
|
|
ok: false,
|
|
runId: input.runId,
|
|
scenarioId: input.scenarioId,
|
|
reason: input.reason,
|
|
status: "blocked",
|
|
observerId: input.observerId,
|
|
elapsedMs: input.elapsedMs ?? null,
|
|
stateDir: indexEntry?.stateDir ?? null,
|
|
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
|
|
findingCount: findings.length,
|
|
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
|
|
failure: input.failure,
|
|
promptSource: input.promptSource,
|
|
steps: [...input.steps, ...cleanupSteps],
|
|
analysis: artifactSummary,
|
|
views: {
|
|
summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
|
|
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
|
|
},
|
|
findings,
|
|
screenshot: record(artifactSummary).screenshot,
|
|
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
|
|
warnings: mergeWarnings(
|
|
Array.isArray(input.warnings) ? input.warnings : [],
|
|
targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", numberAt(state.cicd, "targetValidation.maxSeconds")),
|
|
),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function recordQuickVerify(state: SentinelCicdState, payload: Record<string, unknown>): Record<string, unknown> {
|
|
const views = compactQuickVerifyRecordViews(record(payload.views));
|
|
const summary = {
|
|
reason: payload.reason,
|
|
status: payload.status,
|
|
elapsedMs: payload.elapsedMs,
|
|
failure: payload.failure,
|
|
warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
|
|
analysis: compactQuickVerifyRecordAnalysis(payload.analysis),
|
|
promptSource: payload.promptSource,
|
|
steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [],
|
|
valuesRedacted: true,
|
|
};
|
|
const recordResult = callSentinelService(state, "POST", "/api/runs/record", {
|
|
runId: payload.runId,
|
|
scenarioId: payload.scenarioId,
|
|
status: payload.status,
|
|
observerId: payload.observerId,
|
|
stateDir: payload.stateDir,
|
|
reportJsonSha256: payload.reportJsonSha256,
|
|
findingCount: payload.findingCount,
|
|
artifactCount: payload.artifactCount,
|
|
summary,
|
|
findings: payload.findings,
|
|
views,
|
|
screenshot: payload.screenshot,
|
|
publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"),
|
|
maintenance: payload.reason === "maintenance-stop",
|
|
valuesRedacted: true,
|
|
}, 60);
|
|
return withWarnings({ ...payload, views, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : ["quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]);
|
|
}
|
|
|
|
function compactQuickVerifyRecordViews(views: Record<string, unknown>): Record<string, unknown> {
|
|
const compacted: Record<string, unknown> = {};
|
|
for (const [key, value] of Object.entries(views)) {
|
|
const item = record(value);
|
|
const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000;
|
|
compacted[key] = {
|
|
...item,
|
|
renderedText: boundQuickVerifyRecordText(item.renderedText, limit),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
return compacted;
|
|
}
|
|
|
|
function compactQuickVerifyRecordAnalysis(value: unknown): Record<string, unknown> | null {
|
|
const item = record(value);
|
|
if (Object.keys(item).length === 0) return null;
|
|
return {
|
|
ok: item.ok === true ? true : item.ok === false ? false : null,
|
|
reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null,
|
|
reason: stringAtNullable(item, "reason"),
|
|
stateDir: stringAtNullable(item, "stateDir"),
|
|
reportJsonSha256: stringAtNullable(item, "reportJsonSha256"),
|
|
reportMdSha256: stringAtNullable(item, "reportMdSha256"),
|
|
findingCount: numberAtNullable(item, "findingCount"),
|
|
artifactCount: numberAtNullable(item, "artifactCount"),
|
|
counts: compactQuickVerifyRecordCounts(record(item.counts)),
|
|
screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)),
|
|
findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [],
|
|
pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactQuickVerifyRecordCounts(value: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
samples: numberAtNullable(value, "samples"),
|
|
control: numberAtNullable(value, "control"),
|
|
network: numberAtNullable(value, "network"),
|
|
console: numberAtNullable(value, "console"),
|
|
errors: numberAtNullable(value, "errors"),
|
|
artifacts: numberAtNullable(value, "artifacts"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactQuickVerifyRecordScreenshot(value: Record<string, unknown>): Record<string, unknown> | null {
|
|
if (Object.keys(value).length === 0) return null;
|
|
return {
|
|
path: stringAtNullable(value, "path"),
|
|
sha256: stringAtNullable(value, "sha256"),
|
|
bytes: numberAtNullable(value, "bytes"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactQuickVerifyRecordFinding(value: unknown): Record<string, unknown> {
|
|
const item = record(value);
|
|
return {
|
|
id: stringAtNullable(item, "id"),
|
|
kind: stringAtNullable(item, "kind"),
|
|
code: stringAtNullable(item, "code"),
|
|
severity: stringAtNullable(item, "severity"),
|
|
level: stringAtNullable(item, "level"),
|
|
count: numberAtNullable(item, "count"),
|
|
summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220),
|
|
rootCause: boundQuickVerifyRecordText(item.rootCause, 140),
|
|
rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90),
|
|
rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40),
|
|
nextAction: boundQuickVerifyRecordText(item.nextAction, 240),
|
|
evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence),
|
|
blocking: item.blocking === true,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactQuickVerifyFindingEvidence(value: unknown): string | null {
|
|
const item = record(value);
|
|
if (Object.keys(item).length === 0) return null;
|
|
const keys = [
|
|
"http404Count",
|
|
"responseErrorCount",
|
|
"requestFailedCount",
|
|
"statuses",
|
|
"afterProjectedSeqs",
|
|
"sinceSeqs",
|
|
"traceIds",
|
|
"maxFallbackRatio",
|
|
"maxFallbackTitleCount",
|
|
"overThresholdSampleCount",
|
|
"majorityFallbackSampleCount",
|
|
];
|
|
const compact: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
const raw = item[key];
|
|
if (raw === null || raw === undefined) continue;
|
|
compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw;
|
|
}
|
|
return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240);
|
|
}
|
|
|
|
function compactQuickVerifyRecordStep(value: unknown): Record<string, unknown> {
|
|
const item = record(value);
|
|
return {
|
|
phase: stringAtNullable(item, "phase"),
|
|
ok: item.ok === true ? true : item.ok === false ? false : null,
|
|
promptIndex: numberAtNullable(item, "promptIndex"),
|
|
checkId: stringAtNullable(item, "checkId"),
|
|
failure: stringAtNullable(item, "failure"),
|
|
result: compactQuickVerifyRecordStepResult(record(item.result)),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactQuickVerifyRecordStepResult(value: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
ok: value.ok === true ? true : value.ok === false ? false : null,
|
|
status: stringAtNullable(value, "status"),
|
|
view: stringAtNullable(value, "view"),
|
|
exitCode: numberAtNullable(value, "exitCode"),
|
|
timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null,
|
|
stdoutBytes: numberAtNullable(value, "stdoutBytes"),
|
|
stderrBytes: numberAtNullable(value, "stderrBytes"),
|
|
stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240),
|
|
stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null {
|
|
if (typeof value !== "string") return null;
|
|
if (value.length <= maxChars) return value;
|
|
return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`;
|
|
}
|
|
|
|
function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number): Record<string, unknown> {
|
|
const namespace = stringAt(state.runtime, "namespace");
|
|
const serviceName = stringAt(state.runtime, "serviceName");
|
|
const servicePort = numberAt(state.runtime, "servicePort");
|
|
const deploymentName = stringAt(state.runtime, "deploymentName");
|
|
const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`;
|
|
if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") {
|
|
return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url);
|
|
}
|
|
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
|
|
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
|
|
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
|
|
const postScript = [
|
|
"const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');",
|
|
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
|
|
`const url = 'http://127.0.0.1:${servicePort}' + path;`,
|
|
"fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {",
|
|
" const text = await response.text();",
|
|
" process.stdout.write(text);",
|
|
" if (!response.ok) process.exit(22);",
|
|
"}).catch((error) => {",
|
|
" console.error(error && error.stack ? error.stack : String(error));",
|
|
" process.exit(23);",
|
|
"});",
|
|
].join(" ");
|
|
const script = method === "GET"
|
|
? `kubectl get --raw ${shellQuote(proxyPath)}`
|
|
: [
|
|
"set -eu",
|
|
`kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`,
|
|
].join("\n");
|
|
const maxAttempts = method === "GET" ? 3 : 1;
|
|
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
|
|
const attempts: Record<string, unknown>[] = [];
|
|
let result: CommandResult | null = null;
|
|
let parsed: Record<string, unknown> | null = null;
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 });
|
|
parsed = parseJsonObject(result.stdout);
|
|
attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, valuesRedacted: true });
|
|
if (result.exitCode === 0) break;
|
|
}
|
|
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
|
|
return {
|
|
ok: result?.exitCode === 0,
|
|
method,
|
|
path: pathWithQuery,
|
|
internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`,
|
|
httpStatus: result?.exitCode === 0 ? 200 : null,
|
|
bodyJson: record(compactBodyJson),
|
|
bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "",
|
|
bodyBytes: Buffer.byteLength(result?.stdout ?? ""),
|
|
error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000),
|
|
proxyPath,
|
|
result: result === null ? null : compactCommand(result),
|
|
attempts,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number, url: string): Record<string, unknown> {
|
|
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
|
|
const fetchScript = [
|
|
"const method = process.env.SENTINEL_METHOD || 'GET';",
|
|
"const url = process.env.SENTINEL_URL || '';",
|
|
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
|
|
"const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;",
|
|
"fetch(url, { method, headers, body: method === 'POST' ? body : undefined }).then(async (response) => {",
|
|
" const text = await response.text();",
|
|
" process.stdout.write(text);",
|
|
" if (!response.ok) process.exit(22);",
|
|
"}).catch((error) => {",
|
|
" console.error(error && error.stack ? error.stack : String(error));",
|
|
" process.exit(23);",
|
|
"});",
|
|
].join(" ");
|
|
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
|
|
const result = runCommand(["node", "-e", fetchScript], repoRoot, {
|
|
timeoutMs: attemptTimeoutSeconds * 1000,
|
|
env: {
|
|
...process.env,
|
|
SENTINEL_METHOD: method,
|
|
SENTINEL_URL: url,
|
|
SENTINEL_BODY_B64: bodyB64,
|
|
},
|
|
});
|
|
const parsed = parseJsonObject(result.stdout);
|
|
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
method,
|
|
path: pathWithQuery,
|
|
internalUrl: url,
|
|
httpStatus: result.exitCode === 0 ? 200 : null,
|
|
bodyJson: record(compactBodyJson),
|
|
bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "",
|
|
bodyBytes: Buffer.byteLength(result.stdout),
|
|
error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000),
|
|
proxyPath: null,
|
|
result: compactCommand(result),
|
|
attempts: [{ attempt: 1, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }],
|
|
transport: "direct-service",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
|
|
if (value === null || typeof value.renderedText !== "string") return value;
|
|
return {
|
|
...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]),
|
|
run: pickFields(record(value.run), ["id", "scenario_id", "status", "node", "lane", "observer_id", "state_dir", "report_json_sha256", "finding_count", "artifact_count", "maintenance", "created_at", "updated_at"]),
|
|
summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]),
|
|
findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [],
|
|
renderedText: value.renderedText,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function pickFields(value: Record<string, unknown>, keys: readonly string[]): Record<string, unknown> {
|
|
const picked: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key];
|
|
}
|
|
return picked;
|
|
}
|
|
|
|
function clipTail(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : value.slice(-maxChars);
|
|
}
|
|
|
|
function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
|
const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`;
|
|
const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000));
|
|
const js = [
|
|
"const url=process.env.REQ_URL||'';",
|
|
"const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);",
|
|
"let out;",
|
|
"try{",
|
|
" const controller=new AbortController();",
|
|
" const timer=setTimeout(()=>controller.abort(), timeoutMs);",
|
|
" const started=Date.now();",
|
|
" const res=await fetch(url,{signal:controller.signal});",
|
|
" const text=await res.text();",
|
|
" clearTimeout(timer);",
|
|
" let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}",
|
|
" out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};",
|
|
"}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}",
|
|
"console.log(JSON.stringify(out));",
|
|
].join("");
|
|
const result = runCommand(["bun", "-e", js], repoRoot, {
|
|
timeoutMs: timeoutMs + 2000,
|
|
env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) },
|
|
});
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return {
|
|
ok: result.exitCode === 0 && parsed?.ok === true,
|
|
method: "GET",
|
|
path: pathWithQuery,
|
|
publicUrl: url,
|
|
httpStatus: parsed?.httpStatus ?? null,
|
|
bodyJson: record(parsed?.bodyJson),
|
|
bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "",
|
|
bodyBytes: parsed?.bodyBytes ?? null,
|
|
error: parsed?.error ?? null,
|
|
result: compactCommand(result),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl");
|
|
const hostname = stringAt(state.publicExposure, "hostname");
|
|
const expectedA = stringAt(state.publicExposure, "expectedA");
|
|
const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`;
|
|
const script = [
|
|
"set +e",
|
|
`host=${shellQuote(hostname)}`,
|
|
`expected=${shellQuote(expectedA)}`,
|
|
`url=${shellQuote(probeUrl)}`,
|
|
"dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)",
|
|
"headers=$(mktemp)",
|
|
"body=$(mktemp)",
|
|
"writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)",
|
|
"curl_rc=$?",
|
|
"body_head=$(head -c 1000 \"$body\" | base64 | tr -d '\\n')",
|
|
"node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'",
|
|
"const fs=require('node:fs');",
|
|
"const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);",
|
|
"const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);",
|
|
"const status=Number(statusRaw||0);",
|
|
"const ssl=Number(sslRaw||-1);",
|
|
"const addrs=dns?dns.split(',').filter(Boolean):[];",
|
|
"const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();",
|
|
"const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');",
|
|
"const authCovered=status===401||status===403||status>=200&&status<300;",
|
|
"const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;",
|
|
"const upstreamOk=status>=200&&status<300&&body.includes('valuesRedacted');",
|
|
"const dnsMatches=addrs.includes(expected);",
|
|
"console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
|
const rootUrl = `${publicBaseUrl}/`;
|
|
const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`;
|
|
const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`;
|
|
const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`;
|
|
const script = [
|
|
"set +e",
|
|
`root_url=${shellQuote(rootUrl)}`,
|
|
`css_url=${shellQuote(cssUrl)}`,
|
|
`js_url=${shellQuote(jsUrl)}`,
|
|
`vue_url=${shellQuote(vueUrl)}`,
|
|
"root_body=$(mktemp)",
|
|
"css_body=$(mktemp)",
|
|
"js_body=$(mktemp)",
|
|
"vue_body=$(mktemp)",
|
|
"root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?",
|
|
"css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?",
|
|
"js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?",
|
|
"vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?",
|
|
"node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'",
|
|
"const fs=require('node:fs');",
|
|
"const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);",
|
|
"function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}",
|
|
"const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);",
|
|
"const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&root.includes('monitor-web-bootstrap');",
|
|
"const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.includes('monitor-shell')&&css.includes('workspace-grid')&&css.includes('trend-stage')&&css.length>1000;",
|
|
"const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.includes('createApp')&&js.includes('/api/overview')&&js.includes('data-monitor-trend-curve')&&js.length>1000;",
|
|
"const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.includes('createApp')&&vue.length>80000;",
|
|
"console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contract:root.includes('draft-2026-06-27-p11-monitor-web-observability-dashboard')},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css),workspaceGrid:css.includes('workspace-grid'),trendStage:css.includes('trend-stage')},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js),vueApp:js.includes('createApp'),apiClient:js.includes('/api/overview'),trend:js.includes('data-monitor-trend-curve')},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue),runtime:vue.includes('createApp')},valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
|
for (const source of arrayAt(state.secrets, "sources").map(record)) {
|
|
const purpose = stringAtNullable(source, "purpose");
|
|
if (purpose !== null) sourcesByPurpose.set(purpose, source);
|
|
}
|
|
const desired: Array<{ namespace: string; name: string; data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> }> = [];
|
|
const missing: Record<string, unknown>[] = [];
|
|
const skipped: Record<string, unknown>[] = [];
|
|
for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) {
|
|
const name = stringAt(runtimeSecret, "name");
|
|
const namespace = stringAt(runtimeSecret, "namespace");
|
|
const data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> = [];
|
|
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
|
|
const sourcePurpose = stringAt(item, "sourcePurpose");
|
|
const targetKey = stringAt(item, "targetKey");
|
|
if (sourcePurpose === "frp-token") {
|
|
skipped.push({ name, namespace, targetKey, sourcePurpose, reason: "managed-by-publicExposure-frpc", valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const source = sourcesByPurpose.get(sourcePurpose);
|
|
if (source === undefined) {
|
|
missing.push({ name, namespace, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const sourceRef = stringAt(source, "sourceRef");
|
|
const sourceKey = stringAt(source, "sourceKey");
|
|
const material = readSentinelSecretSourceValue(source);
|
|
if (!material.ok) {
|
|
missing.push({ name, namespace, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
|
|
continue;
|
|
}
|
|
const value = stringAt(material, "value");
|
|
data.push({
|
|
key: targetKey,
|
|
value,
|
|
sourcePurpose,
|
|
sourceRef,
|
|
sourceKey,
|
|
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
|
});
|
|
}
|
|
if (data.length > 0) desired.push({ namespace, name, data });
|
|
}
|
|
if (missing.length > 0) return { ok: false, phase: "local-source", missing, skipped, valuesRedacted: true };
|
|
if (desired.length === 0) return { ok: true, phase: "skipped-no-runtime-secrets", secretCount: 0, keyCount: 0, skippedKeyCount: skipped.length, skipped, valuesRedacted: true };
|
|
const manifests = desired.map((secret) => ({
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: {
|
|
name: secret.name,
|
|
namespace: secret.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
"unidesk.ai/web-probe-sentinel-id": state.sentinelId,
|
|
},
|
|
},
|
|
type: "Opaque",
|
|
data: Object.fromEntries(secret.data.map((item) => [item.key, Buffer.from(item.value, "utf8").toString("base64")])),
|
|
}));
|
|
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
|
const summary = desired.map((secret) => ({
|
|
name: secret.name,
|
|
namespace: secret.namespace,
|
|
keys: secret.data.map((item) => item.key).sort(),
|
|
sources: secret.data.map((item) => ({ key: item.key, sourcePurpose: item.sourcePurpose, sourceRef: item.sourceRef, sourceKey: item.sourceKey, fingerprint: item.fingerprint, valuesRedacted: true })),
|
|
valuesRedacted: true,
|
|
}));
|
|
const summaryB64 = Buffer.from(JSON.stringify(summary), "utf8").toString("base64");
|
|
const script = [
|
|
"set +e",
|
|
`summary_b64=${shellQuote(summaryB64)}`,
|
|
"tmp=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp\"' EXIT",
|
|
"manifest=\"$tmp/runtime-secrets.yaml\"",
|
|
"cat >\"$manifest\"",
|
|
"kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-runtime-secrets -f \"$manifest\" >/tmp/web-probe-sentinel-runtime-secrets.out 2>/tmp/web-probe-sentinel-runtime-secrets.err",
|
|
"apply_rc=$?",
|
|
"python3 - \"$summary_b64\" \"$apply_rc\" <<'PY'",
|
|
"import base64, json, subprocess, sys",
|
|
"summary = json.loads(base64.b64decode(sys.argv[1]).decode('utf-8'))",
|
|
"apply_rc = int(sys.argv[2])",
|
|
"items = []",
|
|
"ok = apply_rc == 0",
|
|
"for secret in summary:",
|
|
" proc = subprocess.run(['kubectl', '-n', secret['namespace'], 'get', 'secret', secret['name'], '-o', 'json'], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)",
|
|
" data = json.loads(proc.stdout).get('data', {}) if proc.returncode == 0 and proc.stdout else {}",
|
|
" keys = {key: key in data for key in secret['keys']}",
|
|
" present = proc.returncode == 0 and all(keys.values())",
|
|
" ok = ok and present",
|
|
" items.append({'name': secret['name'], 'namespace': secret['namespace'], 'present': present, 'keys': keys, 'sources': secret['sources'], 'valuesRedacted': True})",
|
|
"print(json.dumps({'ok': ok, 'applyExitCode': apply_rc, 'secretCount': len(summary), 'keyCount': sum(len(item['keys']) for item in summary), 'items': items, 'valuesRedacted': True}, ensure_ascii=False))",
|
|
"PY",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: manifestYaml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function readSentinelSecretSourceValue(source: Record<string, unknown>): Record<string, unknown> {
|
|
const sourceRef = stringAt(source, "sourceRef");
|
|
const sourceKey = stringAt(source, "sourceKey");
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, error: "secret-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const value = values[sourceKey];
|
|
if (value === undefined || value.length === 0) return { ok: false, error: "secret-source-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const format = stringAtNullable(source, "format");
|
|
if (format === null) return { ok: true, value, sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
if (format === "web-account-json") {
|
|
const username = stringAtNullable(source, "username");
|
|
if (username === null) return { ok: false, error: "web-account-json-username-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
return { ok: true, value: JSON.stringify({ username, password: value }), sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
}
|
|
return { ok: false, error: "unsupported-secret-source-format", sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
}
|
|
|
|
function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const material = readSentinelFrpcMaterial(state);
|
|
if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true };
|
|
const secret = applySentinelFrpcSecret(state, stringAt(material, "frpcToml"), timeoutSeconds);
|
|
const caddy = applySentinelCaddyBlock(state, timeoutSeconds);
|
|
return {
|
|
ok: secret.ok === true && caddy.ok === true,
|
|
hostname: stringAt(state.publicExposure, "hostname"),
|
|
publicBaseUrl: stringAt(state.publicExposure, "publicBaseUrl"),
|
|
material: {
|
|
ok: true,
|
|
sourceRef: material.sourceRef,
|
|
sourcePath: material.sourcePath,
|
|
fingerprint: material.fingerprint,
|
|
valuesRedacted: true,
|
|
},
|
|
secret,
|
|
caddy,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function readSentinelFrpcMaterial(state: SentinelCicdState): Record<string, unknown> {
|
|
const sourceRef = stringAt(state.publicExposure, "frpc.tokenSourceRef");
|
|
const sourceKey = stringAt(state.publicExposure, "frpc.tokenSourceKey");
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, error: "frp-token-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const token = values[sourceKey];
|
|
if (token === undefined || token.length === 0) return { ok: false, error: "frp-token-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const proxy = record(valueAtPath(state.publicExposure, "frpc.httpProxy"));
|
|
const frpcToml = [
|
|
`serverAddr = "${tomlEscape(stringAt(state.publicExposure, "frpc.serverAddr"))}"`,
|
|
`serverPort = ${numberAt(state.publicExposure, "frpc.serverPort")}`,
|
|
"loginFailExit = true",
|
|
`auth.token = "${tomlEscape(token)}"`,
|
|
"",
|
|
"[[proxies]]",
|
|
`name = "${tomlEscape(stringAt(proxy, "name"))}"`,
|
|
'type = "tcp"',
|
|
`localIP = "${tomlEscape(stringAt(proxy, "localIP"))}"`,
|
|
`localPort = ${numberAt(proxy, "localPort")}`,
|
|
`remotePort = ${numberAt(proxy, "remotePort")}`,
|
|
"",
|
|
].join("\n");
|
|
return {
|
|
ok: true,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath: displayPath(sourcePath),
|
|
frpcToml,
|
|
fingerprint: `sha256:${createHash("sha256").update(`${token}\n${frpcToml}`).digest("hex").slice(0, 16)}`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function applySentinelFrpcSecret(state: SentinelCicdState, frpcToml: string, timeoutSeconds: number): Record<string, unknown> {
|
|
const namespace = stringAt(state.runtime, "namespace");
|
|
const secretName = stringAt(state.publicExposure, "frpc.secretName");
|
|
const secretKey = stringAt(state.publicExposure, "frpc.secretKey");
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`secret=${shellQuote(secretName)}`,
|
|
`key=${shellQuote(secretKey)}`,
|
|
"tmp=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp\"' EXIT",
|
|
"cat >\"$tmp/frpc.toml\"",
|
|
"kubectl -n \"$namespace\" create secret generic \"$secret\" --from-file=\"$key=$tmp/frpc.toml\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-public-exposure -f - >/tmp/web-probe-sentinel-frpc-secret.out 2>/tmp/web-probe-sentinel-frpc-secret.err",
|
|
"rc=$?",
|
|
"present=no",
|
|
"bytes=0",
|
|
"if kubectl -n \"$namespace\" get secret \"$secret\" -o jsonpath=\"{.data.$key}\" >/tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null; then present=yes; bytes=$(base64 -d </tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null | wc -c | tr -d ' '); fi",
|
|
"node - \"$rc\" \"$namespace\" \"$secret\" \"$key\" \"$present\" \"$bytes\" <<'NODE'",
|
|
"const [rc, namespace, secret, key, present, bytes] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ok:Number(rc)===0&&present==='yes',namespace,secret,key,present:present==='yes',bytes:Number(bytes||0),applyExitCode:Number(rc),valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: frpcToml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const hostname = stringAt(state.publicExposure, "hostname");
|
|
const owner = stringAt(state.publicExposure, "caddy.managedBlockOwner");
|
|
const configPath = stringAt(state.publicExposure, "caddy.configPath");
|
|
const serviceName = stringAt(state.publicExposure, "caddy.serviceName");
|
|
const responseHeaderTimeoutSeconds = numberAt(state.publicExposure, "caddy.responseHeaderTimeoutSeconds");
|
|
const remotePort = numberAt(state.publicExposure, "frpc.httpProxy.remotePort");
|
|
const routePrefix = normalizeRoutePrefix(stringAtNullable(state.publicExposure, "routePrefix"));
|
|
const proxyLines = [
|
|
`reverse_proxy 127.0.0.1:${remotePort} {`,
|
|
" transport http {",
|
|
` response_header_timeout ${responseHeaderTimeoutSeconds}s`,
|
|
" }",
|
|
"}",
|
|
];
|
|
const block = [
|
|
routePrefix === "/" ? "handle {" : `handle_path ${routePrefix}* {`,
|
|
...proxyLines.map((line) => ` ${line}`),
|
|
"}",
|
|
"",
|
|
].join("\n");
|
|
const blockB64 = Buffer.from(block, "utf8").toString("base64");
|
|
const script = [
|
|
"set +e",
|
|
`hostname=${shellQuote(hostname)}`,
|
|
`owner=${shellQuote(owner)}`,
|
|
`config_path=${shellQuote(configPath)}`,
|
|
`service=${shellQuote(serviceName)}`,
|
|
`route_prefix=${shellQuote(routePrefix)}`,
|
|
`block_b64=${shellQuote(blockB64)}`,
|
|
"marker=\"unidesk managed $owner\"",
|
|
"tmp=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp\"' EXIT",
|
|
"block=\"$tmp/block\"",
|
|
"next=\"$tmp/Caddyfile\"",
|
|
"printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
|
|
"if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
|
|
"python3 - \"$next\" \"$block\" \"$marker\" \"$hostname\" \"$route_prefix\" <<'PY' >/tmp/web-probe-sentinel-caddy-python.out 2>/tmp/web-probe-sentinel-caddy-python.err",
|
|
"import pathlib, re, sys",
|
|
"config = pathlib.Path(sys.argv[1])",
|
|
"block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
|
|
"marker = sys.argv[3]",
|
|
"hostname = sys.argv[4]",
|
|
"route_prefix = sys.argv[5]",
|
|
"text = config.read_text(encoding='utf-8') if config.exists() else ''",
|
|
"begin = f'# BEGIN {marker}'",
|
|
"end = f'# END {marker}'",
|
|
"pattern = re.compile(rf'(?ms)^[ \\t]*# BEGIN {re.escape(marker)}\\n.*?^[ \\t]*# END {re.escape(marker)}\\n*')",
|
|
"def collect_nested_managed(segment):",
|
|
" preserved = []",
|
|
" lines = segment.splitlines()",
|
|
" index = 0",
|
|
" while index < len(lines):",
|
|
" stripped = lines[index].strip()",
|
|
" if stripped.startswith('# BEGIN ') and stripped != begin:",
|
|
" owner_text = stripped[len('# BEGIN '):]",
|
|
" end_line = '# END ' + owner_text",
|
|
" block_lines = [lines[index]]",
|
|
" index += 1",
|
|
" while index < len(lines):",
|
|
" block_lines.append(lines[index])",
|
|
" if lines[index].strip() == end_line:",
|
|
" break",
|
|
" index += 1",
|
|
" preserved.append('\\n'.join(block_lines).rstrip() + '\\n')",
|
|
" index += 1",
|
|
" return preserved",
|
|
"preserved_blocks = []",
|
|
"for match in pattern.finditer(text):",
|
|
" preserved_blocks.extend(collect_nested_managed(match.group(0)))",
|
|
"text = pattern.sub('', text)",
|
|
"def site_span(src, host):",
|
|
" match = re.search(rf'(?m)^([ \\t]*){re.escape(host)}[ \\t]*\\{{[ \\t]*\\n', src)",
|
|
" if not match:",
|
|
" return None",
|
|
" depth = 1",
|
|
" index = match.end()",
|
|
" while index < len(src):",
|
|
" char = src[index]",
|
|
" if char == '{':",
|
|
" depth += 1",
|
|
" elif char == '}':",
|
|
" depth -= 1",
|
|
" if depth == 0:",
|
|
" end_index = index + 1",
|
|
" if end_index < len(src) and src[end_index] == '\\n':",
|
|
" end_index += 1",
|
|
" return match.start(), end_index, index, match.end()",
|
|
" index += 1",
|
|
" raise ValueError(f'unclosed Caddy site block for {host}')",
|
|
"handler = '\\n'.join((' ' + line) if line else '' for line in block.rstrip().splitlines())",
|
|
"managed = f' {begin}\\n{handler}\\n {end}\\n'",
|
|
"def fallback_insert_pos(site, relative_open, close_rel):",
|
|
" for match in re.finditer(r'(?m)^[ \\t]*handle[ \\t]*\\{[ \\t]*\\n', site[relative_open:close_rel]):",
|
|
" pos = relative_open + match.start()",
|
|
" prev_end = pos - 1",
|
|
" if prev_end >= 0 and site[prev_end] == '\\n':",
|
|
" prev_end -= 1",
|
|
" if prev_end >= 0:",
|
|
" prev_start = site.rfind('\\n', 0, prev_end + 1) + 1",
|
|
" if site[prev_start:prev_end + 1].strip().startswith('# BEGIN '):",
|
|
" return prev_start",
|
|
" return pos",
|
|
" return relative_open",
|
|
"def append_before_close(site, close_rel, addition):",
|
|
" prefix = site[:close_rel]",
|
|
" suffix = site[close_rel:]",
|
|
" if prefix and not prefix.endswith('\\n'):",
|
|
" prefix += '\\n'",
|
|
" return prefix + addition + suffix",
|
|
"span = site_span(text, hostname)",
|
|
"if span is None:",
|
|
" body = ''.join(preserved_blocks) + managed",
|
|
" text = text.rstrip() + '\\n\\n' + f'{hostname} {{\\n{body}}}\\n'",
|
|
"else:",
|
|
" start, stop, close_index, open_end = span",
|
|
" site = text[start:stop]",
|
|
" relative_open = open_end - start",
|
|
" close_rel = close_index - start",
|
|
" additions = ''.join(preserved_blocks) + managed",
|
|
" if route_prefix == '/':",
|
|
" replacement = append_before_close(site, close_rel, additions)",
|
|
" else:",
|
|
" insert_at = fallback_insert_pos(site, relative_open, close_rel)",
|
|
" replacement = site[:insert_at] + additions + site[insert_at:]",
|
|
" text = text[:start] + replacement + text[stop:]",
|
|
"config.write_text(text, encoding='utf-8')",
|
|
"PY",
|
|
"python_rc=$?",
|
|
"validate_rc=1",
|
|
"reload_rc=",
|
|
"if [ \"$python_rc\" = 0 ]; then sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/web-probe-sentinel-caddy-validate.out 2>/tmp/web-probe-sentinel-caddy-validate.err; validate_rc=$?; fi",
|
|
"if [ \"$validate_rc\" = 0 ]; then sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/web-probe-sentinel-caddy-install.out 2>/tmp/web-probe-sentinel-caddy-install.err && (sudo systemctl reload \"$service\" >/tmp/web-probe-sentinel-caddy-reload.out 2>/tmp/web-probe-sentinel-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/web-probe-sentinel-caddy-reload.out 2>>/tmp/web-probe-sentinel-caddy-reload.err); reload_rc=$?; fi",
|
|
"probe_rc=1",
|
|
"probe_status=",
|
|
"if [ \"$reload_rc\" = 0 ]; then",
|
|
" probe_path=\"$route_prefix\"",
|
|
" if [ \"$probe_path\" = \"/\" ]; then probe_url=\"https://$hostname/\"; else probe_url=\"https://$hostname$probe_path/\"; fi",
|
|
" probe_status=$(curl -k -sS -o /tmp/web-probe-sentinel-caddy-probe.out -w '%{http_code}' --max-time 10 --resolve \"$hostname:443:127.0.0.1\" \"$probe_url\" 2>/tmp/web-probe-sentinel-caddy-probe.err)",
|
|
" probe_rc=$?",
|
|
"fi",
|
|
"after_present=no",
|
|
"grep -Fq \"# BEGIN $marker\" \"$config_path\" 2>/dev/null && after_present=yes",
|
|
"active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
|
|
"err=$(cat /tmp/web-probe-sentinel-caddy-python.err /tmp/web-probe-sentinel-caddy-validate.err /tmp/web-probe-sentinel-caddy-install.err /tmp/web-probe-sentinel-caddy-reload.err /tmp/web-probe-sentinel-caddy-probe.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
|
|
"python3 - \"$python_rc\" \"$validate_rc\" \"$reload_rc\" \"$probe_rc\" \"$probe_status\" \"$after_present\" \"$active\" \"$hostname\" \"$config_path\" \"$err\" <<'PY'",
|
|
"import json, sys",
|
|
"python_rc, validate_rc, reload_rc, probe_rc, probe_status, after_present, active, hostname, config_path, error_preview = sys.argv[1:11]",
|
|
"def num(value):",
|
|
" if value == '':",
|
|
" return None",
|
|
" try:",
|
|
" return int(value)",
|
|
" except ValueError:",
|
|
" return None",
|
|
"http_status = num(probe_status)",
|
|
"probe_ok = num(probe_rc) == 0 and http_status is not None and 200 <= http_status < 400",
|
|
"payload = {",
|
|
" 'ok': num(python_rc) == 0 and num(validate_rc) == 0 and num(reload_rc) == 0 and probe_ok and after_present == 'yes',",
|
|
" 'hostname': hostname,",
|
|
" 'configPath': config_path,",
|
|
" 'pythonExitCode': num(python_rc),",
|
|
" 'validateExitCode': num(validate_rc),",
|
|
" 'reloadExitCode': num(reload_rc),",
|
|
" 'routeProbeExitCode': num(probe_rc),",
|
|
" 'routeProbeHttpStatus': http_status,",
|
|
" 'routeProbeOk': probe_ok,",
|
|
" 'afterBlockPresent': after_present == 'yes',",
|
|
" 'active': active,",
|
|
" 'errorPreview': error_preview,",
|
|
" 'valuesRedacted': True,",
|
|
"}",
|
|
"print(json.dumps(payload, ensure_ascii=False))",
|
|
"PY",
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.publicExposure, "caddy.route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record<string, unknown> {
|
|
if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true };
|
|
const script = [
|
|
"set -eu",
|
|
`state_dir=${shellQuote(stateDir)}`,
|
|
"node - \"$state_dir\" <<'NODE'",
|
|
"const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');",
|
|
"const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');",
|
|
"const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);",
|
|
"const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;",
|
|
"const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);",
|
|
"let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}",
|
|
"let artifactCount=0; let screenshot=null;",
|
|
"function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}",
|
|
"walk(stateDir);",
|
|
"const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});",
|
|
"const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});",
|
|
"console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: string, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record<string, unknown> {
|
|
const observations: Record<string, unknown>[] = [];
|
|
const indexEntry = readLocalObserveIndex(observerId);
|
|
if (indexEntry === null) {
|
|
return {
|
|
ok: false,
|
|
failure: "observe-index-entry-missing",
|
|
observerId,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
const pollSleepMs = Math.max(250, Math.min(500, Math.trunc(pollIntervalMs / 2) || 250));
|
|
while (Date.now() < deadline) {
|
|
const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now()));
|
|
const script = quickVerifyObserverStartupWaitScript(indexEntry.stateDir, waitMs, pollSleepMs);
|
|
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 });
|
|
const payload = parseJsonObject(result.stdout);
|
|
if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record));
|
|
const terminalPayload = {
|
|
observerId,
|
|
stateDir: indexEntry.stateDir,
|
|
status: typeof payload?.status === "string" ? payload.status : null,
|
|
heartbeatStatus: typeof payload?.heartbeatStatus === "string" ? payload.heartbeatStatus : null,
|
|
startup: record(payload?.startup),
|
|
observations: observations.slice(-6),
|
|
waitResult: compactCommand(result),
|
|
valuesRedacted: true,
|
|
};
|
|
if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-startup-wait-chunk-timeout") {
|
|
return {
|
|
ok: false,
|
|
failure: text(payload?.failure ?? "quick-verify-startup-artifact-wait-failed"),
|
|
...terminalPayload,
|
|
};
|
|
}
|
|
if (payload.ok === true) return { ok: true, ...terminalPayload };
|
|
}
|
|
return {
|
|
ok: false,
|
|
failure: "quick-verify-timeout-over-budget",
|
|
observerId,
|
|
stateDir: indexEntry.stateDir,
|
|
observations: observations.slice(-6),
|
|
warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for the observe runner startup to finish before sending the first command.`],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function quickVerifyObserverStartupWaitScript(stateDir: string, timeoutMs: number, pollSleepMs: number): string {
|
|
return [
|
|
"set -eu",
|
|
`state_dir=${shellQuote(stateDir)}`,
|
|
`timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`,
|
|
`poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`,
|
|
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }",
|
|
"node - \"$state_dir\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const dir = process.argv[2];",
|
|
"const timeoutMs = Number(process.argv[3]);",
|
|
"const pollMs = Number(process.argv[4]);",
|
|
"const startedAt = Date.now();",
|
|
"const startupIds = ['startup-login', 'startup-goto', 'startup-observer-goto'];",
|
|
"const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };",
|
|
"const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };",
|
|
"const clip = (value, limit = 160) => value == null ? null : String(value).replace(/\\s+/gu, ' ').trim().slice(0, limit);",
|
|
"const norm = (value) => String(value || '').trim().toLowerCase().replace(/_/gu, '-');",
|
|
"const terminal = new Set(['failed', 'force-stopped', 'stopped', 'abandoned', 'completed']);",
|
|
"function commandEvents(control, id) { return control.filter((item) => item && item.commandId === id); }",
|
|
"function lastPhase(control, id) { return commandEvents(control, id).filter((item) => typeof item.phase === 'string').slice(-1)[0]?.phase || null; }",
|
|
"function firstFailedStartup(control) { return control.filter((item) => item && startupIds.includes(item.commandId) && item.phase === 'failed').slice(-1)[0] || null; }",
|
|
"function rowFor() {",
|
|
" const heartbeat = readJson('heartbeat.json') || {};",
|
|
" const manifest = readJson('manifest.json') || {};",
|
|
" const control = readJsonl('control.jsonl');",
|
|
" const phases = Object.fromEntries(startupIds.map((id) => [id, lastPhase(control, id)]));",
|
|
" const failed = firstFailedStartup(control);",
|
|
" const heartbeatStatus = norm(heartbeat.status || manifest.status);",
|
|
" const ready = startupIds.every((id) => phases[id] === 'completed') && heartbeatStatus === 'running';",
|
|
" const terminalBeforeReady = !ready && terminal.has(heartbeatStatus);",
|
|
" const degraded = control.filter((item) => item && item.type === 'observer-startup-degraded').slice(-1)[0] || null;",
|
|
" return {",
|
|
" ok: ready,",
|
|
" status: ready ? 'startup-ready' : terminalBeforeReady ? 'startup-terminal' : 'startup-waiting',",
|
|
" heartbeatStatus,",
|
|
" startup: { phases, failedCommandId: failed?.commandId || null, failedType: failed?.type || null, failedMessage: clip(failed?.detail?.error?.message || failed?.detail?.error || failed?.error?.message), observerStartupDegraded: !!degraded, degradedReason: clip(degraded?.reason || degraded?.result?.failureKind || degraded?.result?.reason), sampleSeq: heartbeat.sampleSeq ?? null, commandSeq: heartbeat.commandSeq ?? null, currentUrl: clip(heartbeat.currentUrl, 180), observerUrl: clip(heartbeat.observerUrl, 180), valuesRedacted: true },",
|
|
" valuesRedacted: true",
|
|
" };",
|
|
"}",
|
|
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
|
|
"(async () => {",
|
|
" const observations = [];",
|
|
" while (Date.now() - startedAt <= timeoutMs) {",
|
|
" const row = rowFor();",
|
|
" observations.push(row);",
|
|
" if (row.ok === true) { console.log(JSON.stringify({ ok: true, ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
|
|
" if (row.startup.failedCommandId) { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-command-failed', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
|
|
" if (row.status === 'startup-terminal') { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-terminal', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
|
|
" await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));",
|
|
" }",
|
|
" const row = rowFor();",
|
|
" observations.push(row);",
|
|
" console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-chunk-timeout', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));",
|
|
"})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function collectObserveView(state: SentinelCicdState, observerId: string, view: "turn-summary" | "trace-frame", turn: number | null, timeoutSeconds: number): Record<string, unknown> {
|
|
const args = ["web-probe", "observe", "collect", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--view", view, "--command-timeout-seconds", String(Math.max(5, Math.min(timeoutSeconds, 55))), "--raw", "--compact-raw"];
|
|
if (turn !== null) args.push("--turn", String(turn));
|
|
const result = runChildCli(args, timeoutSeconds);
|
|
const payload = cliDataPayload(result.parsed);
|
|
const collect = record(payload.collect);
|
|
return {
|
|
ok: result.ok && result.parsed !== null && payload.ok !== false && collect.ok !== false,
|
|
view,
|
|
renderedText: typeof collect.renderedText === "string" ? collect.renderedText : typeof payload.renderedText === "string" ? payload.renderedText : String(record(result.result).stdoutTail ?? record(result.result).stdoutPreview ?? ""),
|
|
collect,
|
|
payload,
|
|
result: result.result,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function runChildCli(args: string[], timeoutSeconds: number, input?: string, env?: NodeJS.ProcessEnv): ChildCliResult {
|
|
const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, {
|
|
input,
|
|
env: env === undefined ? undefined : { ...process.env, ...env },
|
|
timeoutMs: Math.max(5, Math.min(timeoutSeconds, 120)) * 1000,
|
|
});
|
|
return {
|
|
ok: result.exitCode === 0 && !result.timedOut,
|
|
parsed: parseJsonObject(result.stdout),
|
|
result: compactCommandWithTail(result),
|
|
};
|
|
}
|
|
|
|
function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record<string, unknown> {
|
|
const observations: Record<string, unknown>[] = [];
|
|
const indexEntry = readLocalObserveIndex(observerId);
|
|
if (indexEntry === null) {
|
|
return {
|
|
ok: false,
|
|
failure: "observe-index-entry-missing",
|
|
round: promptIndex,
|
|
observerId,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
const pollSleepMs = Math.max(250, Math.min(500, Math.trunc(pollIntervalMs / 2) || 250));
|
|
while (Date.now() < deadline) {
|
|
const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now()));
|
|
const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs);
|
|
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 });
|
|
const payload = parseJsonObject(result.stdout);
|
|
if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record));
|
|
const status = typeof payload?.status === "string" ? payload.status : null;
|
|
const terminalPayload = {
|
|
round: promptIndex,
|
|
status,
|
|
traceId: payload?.traceId ?? null,
|
|
finalResponseEmpty: payload?.finalResponseEmpty === true,
|
|
composerReadyForTurn: payload?.composerReadyForTurn === true,
|
|
composerAction: typeof payload?.composerAction === "string" ? payload.composerAction : null,
|
|
observations: observations.slice(-6),
|
|
waitResult: compactCommand(result),
|
|
valuesRedacted: true,
|
|
};
|
|
if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-wait-chunk-timeout") {
|
|
return {
|
|
ok: false,
|
|
failure: text(payload?.failure ?? "quick-verify-artifact-wait-failed"),
|
|
...terminalPayload,
|
|
};
|
|
}
|
|
if (payload.ok === true) return { ok: true, ...terminalPayload };
|
|
if (isQuickVerifyTurnSuccessful(status)) {
|
|
if (payload?.finalResponseEmpty !== true) return { ok: true, ...terminalPayload };
|
|
continue;
|
|
}
|
|
if (isQuickVerifyTurnTerminal(status)) {
|
|
return {
|
|
ok: false,
|
|
failure: "observe-turn-terminal-non-success",
|
|
...terminalPayload,
|
|
};
|
|
}
|
|
}
|
|
return {
|
|
ok: false,
|
|
failure: "quick-verify-timeout-over-budget",
|
|
round: promptIndex,
|
|
observations: observations.slice(-6),
|
|
warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for a submitted turn to become terminal; investigate Code Agent multi-round continuity before retrying.`],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function quickVerifyPromptWaitScript(stateDir: string, promptIndex: number, timeoutMs: number, pollSleepMs: number): string {
|
|
return [
|
|
"set -eu",
|
|
`state_dir=${shellQuote(stateDir)}`,
|
|
`prompt_index=${shellQuote(String(promptIndex))}`,
|
|
`timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`,
|
|
`poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`,
|
|
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }",
|
|
"node - \"$state_dir\" \"$prompt_index\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const dir = process.argv[2];",
|
|
"const promptIndex = Number(process.argv[3]);",
|
|
"const timeoutMs = Number(process.argv[4]);",
|
|
"const pollMs = Number(process.argv[5]);",
|
|
"const startedAt = Date.now();",
|
|
"const short = (value, limit = 160) => String(value || '').replace(/\\s+/gu, ' ').trim().slice(0, limit);",
|
|
"const textOf = (value) => String(value?.text || value?.textPreview || value?.preview || '');",
|
|
"const arr = (value) => Array.isArray(value) ? value : [];",
|
|
"const unique = (values) => Array.from(new Set(values.filter(Boolean)));",
|
|
"const numOrNull = (value) => { const n = Number(value); return Number.isFinite(n) ? n : null; };",
|
|
"const tsMs = (value) => { const ms = Date.parse(String(value || '')); return Number.isFinite(ms) ? ms : null; };",
|
|
"const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };",
|
|
"const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };",
|
|
"const readJsonlTail = (rel, maxBytes = 2000000) => {",
|
|
" try {",
|
|
" const file = path.join(dir, rel);",
|
|
" const stat = fs.statSync(file);",
|
|
" const start = Math.max(0, stat.size - maxBytes);",
|
|
" const length = stat.size - start;",
|
|
" const fd = fs.openSync(file, 'r');",
|
|
" try {",
|
|
" const buffer = Buffer.alloc(length);",
|
|
" fs.readSync(fd, buffer, 0, length, start);",
|
|
" const lines = buffer.toString('utf8').split(/\\r?\\n/u);",
|
|
" if (start > 0) lines.shift();",
|
|
" return lines.filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);",
|
|
" } finally {",
|
|
" fs.closeSync(fd);",
|
|
" }",
|
|
" } catch {",
|
|
" return [];",
|
|
" }",
|
|
"};",
|
|
"const readDone = (id) => id ? readJson(path.join('commands', 'done', `${id}.json`)) : null;",
|
|
"const readFailed = (id) => id ? readJson(path.join('commands', 'failed', `${id}.json`)) : null;",
|
|
"function sessionIdFromUrl(value) { const match = String(value || '').match(/\\/workbench\\/sessions\\/(ses_[A-Za-z0-9_-]+)/u); return match ? match[1] : null; }",
|
|
"function commandSessionId(item) { const done = readDone(item?.commandId); return item?.sessionId || item?.detail?.sessionId || item?.input?.sessionId || item?.result?.sessionId || done?.result?.sessionId || done?.result?.observer?.sessionId || sessionIdFromUrl(item?.afterUrl) || sessionIdFromUrl(item?.detail?.afterUrl) || sessionIdFromUrl(done?.result?.afterUrl) || null; }",
|
|
"function firstTraceId(value) { const match = String(value || '').match(/\\btrc_[A-Za-z0-9_-]+\\b/u); return match ? match[0] : null; }",
|
|
"function commandTraceId(item) { const done = readDone(item?.commandId); return item?.traceId || item?.detail?.chatSubmit?.traceId || item?.detail?.traceId || item?.input?.traceId || item?.result?.chatSubmit?.traceId || done?.result?.chatSubmit?.traceId || done?.result?.traceId || null; }",
|
|
"function itemTraceId(item) { return item?.traceId || firstTraceId(textOf(item)) || null; }",
|
|
"function completedNewSessionIdsBefore(control, ts) { const limit = tsMs(ts); return control.filter((item) => item.type === 'newSession' && item.phase === 'completed').filter((item) => limit === null || tsMs(item.ts) === null || tsMs(item.ts) <= limit).map(commandSessionId).filter(Boolean); }",
|
|
"function authoritativeSessionIdForPrompts(control, prompts) { const ids = completedNewSessionIdsBefore(control, prompts[0]?.firstTs || null); return ids.slice(-1)[0] || unique(prompts.map((item) => item.sessionId))[0] || null; }",
|
|
"function promptCommands(control) {",
|
|
" const map = new Map();",
|
|
" for (const item of control) {",
|
|
" if (item.type !== 'sendPrompt' || !['started', 'completed', 'failed'].includes(item.phase)) continue;",
|
|
" const id = item.commandId || item.seq || String(map.size + 1);",
|
|
" const existing = map.get(id) || {};",
|
|
" map.set(id, { ...existing, ...item, input: { ...(existing.input || {}), ...(item.input || {}) }, sessionId: existing.sessionId || commandSessionId(item), traceId: existing.traceId || commandTraceId(item), firstTs: existing.firstTs || item.ts, lastTs: item.ts });",
|
|
" }",
|
|
" const prompts = Array.from(map.values()).filter((item) => tsMs(item.firstTs) !== null).sort((a, b) => tsMs(a.firstTs) - tsMs(b.firstTs));",
|
|
" const sessionId = authoritativeSessionIdForPrompts(control, prompts);",
|
|
" if (!sessionId) return prompts;",
|
|
" const scoped = prompts.filter((item) => item.sessionId === sessionId);",
|
|
" return scoped.length > 0 ? scoped : prompts;",
|
|
"}",
|
|
"function segmentFor(samples, prompts, index) { const start = tsMs(prompts[index]?.firstTs); const end = index + 1 < prompts.length ? tsMs(prompts[index + 1].firstTs) : Infinity; return samples.filter((sample) => { const ms = tsMs(sample.ts); return ms !== null && ms >= start && ms < end; }); }",
|
|
"function entryGroups(sample) { return [...arr(sample.turns).map((item) => ({ group: 'turn', item })), ...arr(sample.traceRows).map((item) => ({ group: 'traceRow', item })), ...arr(sample.messages).map((item) => ({ group: 'message', item }))]; }",
|
|
"function traceIdsFromSamples(items) { const ids = []; for (const sample of items) for (const entry of entryGroups(sample)) { const id = itemTraceId(entry.item); if (id) ids.push(id); } return unique(ids); }",
|
|
"function chooseTraceId(segment, prompt) { const promptTraceId = commandTraceId(prompt) || prompt?.traceId || null; const ids = traceIdsFromSamples(segment); if (promptTraceId && (ids.length === 0 || ids.includes(promptTraceId))) return promptTraceId; return ids.slice(-1)[0] || promptTraceId || null; }",
|
|
"function traceEntries(items, traceId) { const entries = []; for (const sample of items) for (const entry of entryGroups(sample)) { const text = textOf(entry.item); const id = itemTraceId(entry.item); if (!traceId || id === traceId || text.includes(traceId)) entries.push({ ...entry, sample, text }); } return entries; }",
|
|
"function normalizeLifecycleStatus(value) { const raw = String(value || '').trim().toLowerCase(); if (/^(canceled|cancelled)$/u.test(raw)) return 'canceled'; if (/^(failed|failure|error)$/u.test(raw)) return 'failed'; if (/^(completed|complete|succeeded|success|terminal)$/u.test(raw)) return 'completed'; if (/^(running|admitting|queued|pending|in_progress|in-progress)$/u.test(raw)) return 'running'; return null; }",
|
|
"function statusFor(items, traceId) { const entries = traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) }))); const lastTurn = entries.filter((entry) => entry.group === 'turn').slice(-1)[0]?.item || null; const turnStatus = normalizeLifecycleStatus(lastTurn?.status); if (turnStatus) return turnStatus; const lastMessage = entries.filter((entry) => entry.group === 'message').slice(-1)[0]?.item || null; return normalizeLifecycleStatus(lastMessage?.status) || 'unknown'; }",
|
|
"function cleanFinalResponseText(value) { const raw = String(value || '').trim(); if (!raw) return ''; if (/^(completed|failed|canceled|cancelled|轮次完成|轮次失败|轮次取消|已记录)$/iu.test(raw.replace(/\\s+/gu, ' '))) return ''; if (/^(admitted|run|ok|error)\\s+/iu.test(raw)) return ''; return raw; }",
|
|
"function finalResponseTextFromEntry(entry) {",
|
|
" const explicit = cleanFinalResponseText(entry.item?.finalResponse?.text || entry.item?.finalResponse?.preview || '');",
|
|
" if (explicit && !/^Code Agent\\s*耗时/iu.test(explicit)) return explicit;",
|
|
" if (entry.group !== 'message') return '';",
|
|
" const role = String(entry.item?.role || entry.item?.dataRole || entry.item?.messageRole || '').toLowerCase();",
|
|
" if (role && !/assistant|agent|system/u.test(role)) return '';",
|
|
" const text = cleanFinalResponseText(entry.text);",
|
|
" return text && !/^Code Agent\\s*耗时/iu.test(text) ? text : '';",
|
|
"}",
|
|
"function finalResponseEmpty(items, traceId) { if (!/^(completed|failed|canceled)$/u.test(statusFor(items, traceId))) return true; const entries = (traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) })))).slice().reverse(); for (const entry of entries) { if (finalResponseTextFromEntry(entry)) return false; } return true; }",
|
|
"function rowFor() {",
|
|
" const control = readJsonl('control.jsonl');",
|
|
" const samples = readJsonlTail('samples.jsonl');",
|
|
" const prompts = promptCommands(control);",
|
|
" const prompt = prompts[promptIndex - 1] || null;",
|
|
" if (!prompt) return { ok: true, round: promptIndex, status: 'command-pending', traceId: null, finalResponseEmpty: true, lastSeq: null, lastTs: null, promptMissing: true, valuesRedacted: true };",
|
|
" const done = readDone(prompt.commandId);",
|
|
" const failed = readFailed(prompt.commandId);",
|
|
" if (failed) return { ok: false, failure: 'observe-command-sendPrompt-failed', round: promptIndex, status: 'command-failed', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandFailure: short(failed.error?.message || failed.failure || failed.status || 'command failed'), valuesRedacted: true };",
|
|
" const promptTraceId = commandTraceId(prompt);",
|
|
" if (!done || !promptTraceId) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: promptTraceId || null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: !promptTraceId, valuesRedacted: true };",
|
|
" const segment = segmentFor(samples, prompts, promptIndex - 1);",
|
|
" const controlSegment = segment.filter((sample) => sample.pageRole === 'control');",
|
|
" const statusSegment = controlSegment.length > 0 ? controlSegment : segment;",
|
|
" const traceId = promptTraceId || chooseTraceId(statusSegment, prompt) || chooseTraceId(segment, prompt);",
|
|
" const status = statusFor(statusSegment, traceId);",
|
|
" const sampleForTrace = traceId ? statusSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;",
|
|
" const lastSample = sampleForTrace || segment.slice(-1)[0] || null;",
|
|
" const composerSample = segment.filter((sample) => sample.pageRole === 'control').slice(-1)[0] || lastSample;",
|
|
" const composer = composerSample?.composer || {};",
|
|
" const composerReadyForTurn = composer.inputPresent === true && composer.inputDisabled !== true && composer.submitPresent === true && composer.warningPresent !== true && composer.submitAction === 'turn';",
|
|
" return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(statusSegment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, sampleScope: controlSegment.length > 0 ? 'control' : 'all', segmentSampleCount: segment.length, statusSampleCount: statusSegment.length, source: 'observe-artifact-wait-script', valuesRedacted: true };",
|
|
"}",
|
|
"function norm(value) { return String(value || '').trim().toLowerCase().replace(/_/gu, '-'); }",
|
|
"function successful(value) { return ['completed', 'succeeded', 'success'].includes(norm(value)); }",
|
|
"function terminal(value) { return ['completed', 'succeeded', 'success', 'failed', 'error', 'blocked', 'timeout', 'canceled', 'cancelled', 'terminal'].includes(norm(value)); }",
|
|
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
|
|
"(async () => {",
|
|
" const observations = [];",
|
|
" while (Date.now() - startedAt <= timeoutMs) {",
|
|
" const row = rowFor();",
|
|
" observations.push(row);",
|
|
" if (row.ok === false) { console.log(JSON.stringify({ ...row, ok: false, failure: row.failure || 'quick-verify-artifact-row-failed', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
|
|
" if (successful(row.status) && row.finalResponseEmpty !== true) { console.log(JSON.stringify({ ...row, ok: true, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
|
|
" if (!successful(row.status) && terminal(row.status)) { console.log(JSON.stringify({ ...row, ok: false, failure: 'observe-turn-terminal-non-success', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
|
|
" await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));",
|
|
" }",
|
|
" const row = rowFor();",
|
|
" observations.push(row);",
|
|
" console.log(JSON.stringify({ ...row, ok: false, failure: 'quick-verify-wait-chunk-timeout', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));",
|
|
"})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-artifact-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function isQuickVerifyTurnSuccessful(value: string | null): boolean {
|
|
const status = normalizeQuickVerifyStatus(value);
|
|
return status === "completed" || status === "succeeded" || status === "success";
|
|
}
|
|
|
|
function isQuickVerifyTurnTerminal(value: string | null): boolean {
|
|
const status = normalizeQuickVerifyStatus(value);
|
|
return status === "completed"
|
|
|| status === "succeeded"
|
|
|| status === "success"
|
|
|| status === "failed"
|
|
|| status === "error"
|
|
|| status === "blocked"
|
|
|| status === "timeout"
|
|
|| status === "canceled"
|
|
|| status === "cancelled"
|
|
|| status === "terminal";
|
|
}
|
|
|
|
function normalizeQuickVerifyStatus(value: string | null): string {
|
|
return String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
|
}
|
|
|
|
function cliDataPayload(parsed: Record<string, unknown> | null): Record<string, unknown> {
|
|
const root = record(parsed);
|
|
return isRecord(root.data) ? root.data : root;
|
|
}
|
|
|
|
function findScenario(state: SentinelCicdState, scenarioId: string): Record<string, unknown> | null {
|
|
const scenarios = readConfigRefTarget(state.configRefs.scenarios);
|
|
const items = Array.isArray(scenarios) ? scenarios : isRecord(scenarios) ? [scenarios] : [];
|
|
return items.map(record).find((item) => item.id === scenarioId) ?? null;
|
|
}
|
|
|
|
function readPromptSetForScenario(scenario: Record<string, unknown>): { ok: true; prompts: string[]; summary: Record<string, unknown> } | { ok: false; error: string; summary: Record<string, unknown> } {
|
|
const promptSet = recordTarget(readConfigRefTarget(stringAt(scenario, "promptSetRef")), stringAt(scenario, "promptSetRef"));
|
|
const sourceRef = stringAt(promptSet, "promptSourceRef");
|
|
const key = stringAt(promptSet, "promptSourceKey");
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true };
|
|
const runtimeRaw = process.env[key];
|
|
const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
|
|
const raw = values[key] ?? runtimeRaw;
|
|
const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null;
|
|
if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary };
|
|
if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary };
|
|
const parsed = parsePromptJson(raw);
|
|
if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary };
|
|
return {
|
|
ok: true,
|
|
prompts: parsed,
|
|
summary: {
|
|
...summary,
|
|
sourceMode,
|
|
promptCount: parsed.length,
|
|
promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))),
|
|
promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`),
|
|
promptTextBytes: parsed.map((item) => Buffer.byteLength(item)),
|
|
valuesRedacted: true,
|
|
},
|
|
};
|
|
}
|
|
|
|
function parsePromptJson(raw: string): string[] {
|
|
try {
|
|
const parsed = JSON.parse(raw) as unknown;
|
|
if (Array.isArray(parsed)) return parsed.filter((item): item is string => typeof item === "string" && item.length > 0);
|
|
const recordValue = record(parsed);
|
|
if (Array.isArray(recordValue.prompts)) return recordValue.prompts.filter((item): item is string => typeof item === "string" && item.length > 0);
|
|
if (typeof recordValue.prompt === "string" && recordValue.prompt.length > 0) return [recordValue.prompt];
|
|
} catch {
|
|
if (raw.trim().length > 0) return [raw];
|
|
}
|
|
return [];
|
|
}
|
|
|
|
function readLocalObserveIndex(observerId: string): { stateDir: string } | null {
|
|
const path = rootPath(".state/web-observe/index.json");
|
|
if (!existsSync(path)) return null;
|
|
const parsed = parseJsonObject(readFileSync(path, "utf8"));
|
|
const entry = record(parsed?.[observerId]);
|
|
const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : null;
|
|
return stateDir === null ? null : { stateDir };
|
|
}
|
|
|
|
function secretSourcePaths(sourceRef: string): string[] {
|
|
const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef));
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
function parseEnvFile(textValue: string): Record<string, string> {
|
|
const values: Record<string, string> = {};
|
|
for (const rawLine of textValue.split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (line.length === 0 || line.startsWith("#")) continue;
|
|
const index = line.indexOf("=");
|
|
if (index <= 0) continue;
|
|
const key = line.slice(0, index).trim();
|
|
let value = line.slice(index + 1).trim();
|
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1);
|
|
values[key] = value;
|
|
}
|
|
return values;
|
|
}
|
|
|
|
function observerIdFromText(textValue: string): string | null {
|
|
return /\bwebobs-[a-z0-9-]+\b/iu.exec(textValue)?.[0] ?? null;
|
|
}
|
|
|
|
function remainingSeconds(deadline: number, cap: number): number {
|
|
return Math.max(5, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000)));
|
|
}
|
|
|
|
function metricNames(textValue: unknown): string[] {
|
|
if (typeof textValue !== "string") return [];
|
|
return textValue.split(/\r?\n/u).map((line) => /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line)?.[1]).filter((item): item is string => typeof item === "string");
|
|
}
|
|
|
|
function validationBlocker(health: Record<string, unknown>, metrics: Record<string, unknown>, report: Record<string, unknown>, publicExposure: Record<string, unknown>, publicDashboard: Record<string, unknown>, quickVerify: Record<string, unknown> | null): Record<string, unknown> {
|
|
const blockers = [];
|
|
if (!health.ok || record(health.bodyJson).ok !== true) blockers.push("health");
|
|
if (!metrics.ok || !metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health")) blockers.push("metrics");
|
|
if (!report.ok) blockers.push("recent-report");
|
|
if (publicExposure.ok !== true) blockers.push("public-exposure");
|
|
if (publicDashboard.ok !== true) blockers.push("public-dashboard");
|
|
if (quickVerify !== null && quickVerify.ok !== true) blockers.push("quick-verify");
|
|
return { code: "sentinel-validation-failed", blockers, valuesRedacted: true };
|
|
}
|
|
|
|
function serviceUnavailableBlocker(state: SentinelCicdState): Record<string, unknown> {
|
|
return {
|
|
code: "sentinel-service-unavailable",
|
|
policy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"),
|
|
reason: "sentinel service must be reachable through k3s internal Service DNS before quick verify can run; no public/fallback path is used.",
|
|
retry: `bun scripts/cli.ts web-probe sentinel validate --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelP5Next(state: SentinelCicdState): Record<string, string> {
|
|
const node = state.spec.nodeId;
|
|
const lane = state.spec.lane;
|
|
const suffix = sentinelCliSuffix(state);
|
|
return {
|
|
validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`,
|
|
quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`,
|
|
maintenanceStart: `bun scripts/cli.ts web-probe sentinel maintenance start --node ${node} --lane ${lane}${suffix} --confirm --wait`,
|
|
maintenanceStop: `bun scripts/cli.ts web-probe sentinel maintenance stop --node ${node} --lane ${lane}${suffix} --confirm --wait`,
|
|
report: `bun scripts/cli.ts web-probe sentinel report --node ${node} --lane ${lane}${suffix} --view summary`,
|
|
};
|
|
}
|
|
|
|
function isSafeRelativeStateDir(value: string): boolean {
|
|
return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes("..");
|
|
}
|
|
|
|
function stringAtNullable(value: unknown, path: string): string | null {
|
|
const found = valueAtPath(value, path);
|
|
return typeof found === "string" && found.length > 0 ? found : null;
|
|
}
|
|
|
|
function numberAtNullable(value: unknown, path: string): number | null {
|
|
const found = valueAtPath(value, path);
|
|
return typeof found === "number" && Number.isFinite(found) ? found : null;
|
|
}
|
|
|
|
function displayPath(pathValue: string): string {
|
|
if (pathValue.startsWith(`${repoRoot}/`)) return pathValue.slice(repoRoot.length + 1);
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) {
|
|
const mainRoot = repoRoot.slice(0, index);
|
|
if (pathValue.startsWith(`${mainRoot}/`)) return pathValue.slice(mainRoot.length + 1);
|
|
}
|
|
return pathValue;
|
|
}
|
|
|
|
function mergeFindingRecords(primary: readonly Record<string, unknown>[], extra: readonly Record<string, unknown>[]): Record<string, unknown>[] {
|
|
const merged: Record<string, unknown>[] = [];
|
|
const seen = new Set<string>();
|
|
for (const item of [...primary, ...extra]) {
|
|
const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? stringAtNullable(item, "finding_id") ?? "finding";
|
|
const severity = stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "unknown";
|
|
const key = `${id}\0${severity}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
merged.push(item);
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
function isQuickVerifyBlockingFinding(item: Record<string, unknown>): boolean {
|
|
const severity = (stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "").toLowerCase();
|
|
if (!["critical", "red", "fatal", "error", "failed", "blocked"].includes(severity)) return false;
|
|
const id = (stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? "").toLowerCase();
|
|
if (id === "observer-command-failed") return observerCommandFailureBlocks(item);
|
|
return [
|
|
"quick-verify-no-business-turn",
|
|
"quick-verify-command-sequence-failed",
|
|
"quick-verify-observer-start-failed",
|
|
"quick-verify-account-secret-missing",
|
|
"prompt-chat-submit-failed",
|
|
"route-active-session-mismatch",
|
|
"final-response-flicker",
|
|
"round-completion-final-response-missing",
|
|
"turn-trace-id-missing",
|
|
"no-samples",
|
|
"jsonl-read-issues",
|
|
].includes(id);
|
|
}
|
|
|
|
function observerCommandFailureBlocks(item: Record<string, unknown>): boolean {
|
|
const commands = Array.isArray(item.commands) ? item.commands.map(record) : [];
|
|
if (commands.length === 0) return true;
|
|
return commands.some((command) => {
|
|
const type = (stringAtNullable(command, "type") ?? "").toLowerCase();
|
|
if (["stop", "cancel", "mark", "screenshot"].includes(type)) return false;
|
|
return true;
|
|
});
|
|
}
|
|
|
|
function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null): Record<string, unknown>[] {
|
|
const rendered = [
|
|
typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "",
|
|
typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "",
|
|
].join("\n");
|
|
const noPromptScenario = promptIndex <= 0;
|
|
if (noPromptScenario && failure === null) return [];
|
|
if (noPromptScenario && failure !== null) {
|
|
const observerStartFailure = failure === "observe-start-failed";
|
|
return [{
|
|
id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed",
|
|
severity: "red",
|
|
count: 1,
|
|
summary: observerStartFailure
|
|
? "quick verify observer failed to start before the no-prompt scenario could run."
|
|
: "quick verify no-prompt command sequence failed before the account/session workflow completed.",
|
|
failure,
|
|
promptIndex,
|
|
valuesRedacted: true,
|
|
}];
|
|
}
|
|
const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered);
|
|
const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered);
|
|
if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return [];
|
|
return [{
|
|
id: "quick-verify-no-business-turn",
|
|
severity: "red",
|
|
count: 1,
|
|
summary: "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
|
|
failure: failure ?? null,
|
|
promptIndex,
|
|
valuesRedacted: true,
|
|
}];
|
|
}
|
|
|
|
function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } {
|
|
return {
|
|
...compactCommand(result),
|
|
stdoutPreview: result.stdout.trim().slice(0, 1200),
|
|
stderrPreview: result.stderr.trim().slice(0, 1200),
|
|
stdoutTail: result.stdout.trim().slice(-4000),
|
|
stderrTail: result.stderr.trim().slice(-4000),
|
|
};
|
|
}
|
|
|
|
function renderQuickVerifySummary(input: Record<string, unknown>): string {
|
|
const artifact = record(input.artifactSummary);
|
|
const findings = Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : [];
|
|
return [
|
|
"Web Probe Sentinel Quick Verify",
|
|
"=======================================================",
|
|
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
|
|
`report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${artifact.findingCount ?? findings.length}`,
|
|
`publicOrigin=${input.publicOrigin ?? "-"}`,
|
|
"",
|
|
"Findings",
|
|
findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"} ${item.summary ?? item.message ?? ""}`).join("\n"),
|
|
].join("\n");
|
|
}
|
|
|
|
function renderAuthSessionSwitchQuickVerifySummary(input: Record<string, unknown>): string {
|
|
const artifact = record(input.artifactSummary);
|
|
const accountEnv = record(input.accountEnv);
|
|
const findingRows = Array.isArray(input.findings)
|
|
? input.findings.map(record).slice(0, 8)
|
|
: Array.isArray(artifact.findings)
|
|
? artifact.findings.map(record).slice(0, 8)
|
|
: [];
|
|
const steps = Array.isArray(input.steps) ? input.steps.map(record) : [];
|
|
const commandSteps = steps
|
|
.filter((step) => {
|
|
const phase = stringAtNullable(step, "phase") ?? "";
|
|
return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env";
|
|
})
|
|
.map((step) => {
|
|
const phase = stringAtNullable(step, "phase") ?? "-";
|
|
const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-";
|
|
const result = record(step.result);
|
|
const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-";
|
|
return `${phase} ${ok} ${status}`;
|
|
});
|
|
return [
|
|
"Auth Session Switch Quick Verify",
|
|
"=======================================================",
|
|
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
|
|
`status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`,
|
|
`accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`,
|
|
"",
|
|
"Command Sequence",
|
|
commandSteps.length === 0 ? "-" : commandSteps.join("\n"),
|
|
"",
|
|
"Findings",
|
|
findingRows.length === 0 ? "-" : findingRows.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"} ${item.summary ?? item.message ?? ""}`).join("\n"),
|
|
].join("\n");
|
|
}
|
|
|
|
function renderMaintenanceResult(result: Record<string, unknown>): string {
|
|
const serviceHealth = record(result.serviceHealth);
|
|
const maintenance = record(result.maintenance);
|
|
const quickVerify = record(result.quickVerify);
|
|
const planned = record(result.planned);
|
|
const blocker = record(result.blocker);
|
|
const next = record(result.next);
|
|
const maintenanceBody = record(maintenance.bodyJson);
|
|
const state = record(maintenanceBody.maintenance);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]),
|
|
"",
|
|
table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]),
|
|
"",
|
|
Object.keys(state).length > 0
|
|
? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]])
|
|
: Object.keys(planned).length > 0
|
|
? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]])
|
|
: "MAINTENANCE\n-",
|
|
"",
|
|
Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
|
|
"",
|
|
"NEXT",
|
|
` validate: ${next.validate ?? "-"}`,
|
|
` report: ${next.report ?? "-"}`,
|
|
` maintenance-stop: ${next.maintenanceStop ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderValidateResult(result: Record<string, unknown>): string {
|
|
const health = record(result.serviceHealth);
|
|
const metrics = record(result.metrics);
|
|
const report = record(result.report);
|
|
const publicExposure = record(result.publicExposure);
|
|
const publicDashboard = record(result.publicDashboard);
|
|
const quickVerify = record(result.quickVerify);
|
|
const blocker = record(result.blocker);
|
|
const next = record(result.next);
|
|
const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]),
|
|
"",
|
|
table(["CHECK", "OK", "DETAIL"], [
|
|
["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`],
|
|
["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`],
|
|
["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`],
|
|
["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`],
|
|
["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`],
|
|
["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`],
|
|
]),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]),
|
|
"",
|
|
"NEXT",
|
|
` quick-verify: ${next.quickVerify ?? "-"}`,
|
|
` report: ${next.report ?? "-"}`,
|
|
` maintenance-start: ${next.maintenanceStart ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderDashboardResult(result: Record<string, unknown>): string {
|
|
const page = record(result.page);
|
|
const dom = record(page.dom);
|
|
const dataset = record(dom.dataset);
|
|
const layout = record(dom.layout);
|
|
const screenshot = record(result.screenshot);
|
|
const remote = record(result.remote);
|
|
const transport = record(result.transport);
|
|
const degradedReason = result.degradedReason ?? null;
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]),
|
|
"",
|
|
table(["HTTP", "SHELL", "RUN_ROWS", "FINDINGS", "TABS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[
|
|
page.httpStatus ?? "-",
|
|
dom.shell,
|
|
dom.runRows,
|
|
dom.findingItems,
|
|
dom.detailTabs,
|
|
page.pageErrorCount,
|
|
page.consoleErrorCount,
|
|
page.requestFailureCount,
|
|
]]),
|
|
"",
|
|
table(["TITLE", "STATUS_TEXT", "CONTRACT", "BASE_PATH"], [[dom.title, dom.statusText, dataset.contractVersion, dataset.basePath ?? "-"]]),
|
|
"",
|
|
table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[
|
|
result.viewport,
|
|
`${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`,
|
|
layout.horizontalOverflow,
|
|
layout.overflowCount,
|
|
]]),
|
|
"",
|
|
Object.keys(screenshot).length === 0
|
|
? "SCREENSHOT\n-"
|
|
: table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]),
|
|
"",
|
|
degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT"], [[degradedReason, remote.exitCode, transport.ok]]),
|
|
"",
|
|
"NEXT",
|
|
` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
|
` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" dashboard verify uses the YAML publicExposure URL and remote browser execution; it does not start a sentinel run or inspect provider payloads.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderReportResult(result: Record<string, unknown>): string {
|
|
const report = record(result.report);
|
|
const body = record(report.bodyJson);
|
|
const run = record(body.run);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]),
|
|
"",
|
|
table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]),
|
|
"",
|
|
"DISCLOSURE",
|
|
" report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelPipelineRunName(state: SentinelCicdState): string {
|
|
const commit = state.sourceHead.commit ?? "source";
|
|
return `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`;
|
|
}
|
|
|
|
function sentinelCliSuffix(state: SentinelCicdState): string {
|
|
return ` --sentinel ${state.sentinelId}`;
|
|
}
|
|
|
|
function safeJobSegment(value: string): string {
|
|
return value.replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "").slice(0, 48) || "sentinel";
|
|
}
|
|
|
|
function safeKubernetesSegment(value: string, maxLength: number): string {
|
|
const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
return (normalized || "sentinel").slice(0, Math.max(1, maxLength)).replace(/-+$/u, "") || "sentinel";
|
|
}
|
|
|
|
function renderImageResult(result: Record<string, unknown>): string {
|
|
const source = record(result.source);
|
|
const sourceMirror = record(result.sourceMirror);
|
|
const sourceMirrorSync = record(result.sourceMirrorSync);
|
|
const image = record(result.image);
|
|
const monitorWeb = record(image.monitorWeb);
|
|
const registry = record(result.registry);
|
|
const publish = record(result.publish);
|
|
const blocker = record(result.blocker);
|
|
const next = record(result.next);
|
|
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]),
|
|
"",
|
|
table(["SOURCE_REPO", "BRANCH", "COMMIT", "LOCAL_HEAD"], [[source.repository, source.branch, short(source.commit), short(source.localHead)]]),
|
|
"",
|
|
Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]),
|
|
"",
|
|
table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]),
|
|
"",
|
|
Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`]]),
|
|
"",
|
|
Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
|
|
"",
|
|
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
|
|
"",
|
|
Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
|
|
"",
|
|
"NEXT",
|
|
` status: ${next.status ?? "-"}`,
|
|
` dry-run: ${next.dryRun ?? "-"}`,
|
|
` confirm: ${next.confirm ?? "-"}`,
|
|
` trigger: ${next.controlPlaneTrigger ?? "-"}`,
|
|
` control-plane: ${next.controlPlanePlan ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" valuesRedacted=true; image status shows refs, hashes and object names only.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderControlPlaneResult(result: Record<string, unknown>): string {
|
|
const source = record(result.source);
|
|
const image = record(result.image);
|
|
const gitops = record(result.gitops);
|
|
const argo = record(result.argo);
|
|
const validation = record(result.validation);
|
|
const observed = record(result.observed);
|
|
const sourceMirrorSync = record(result.sourceMirrorSync);
|
|
const publish = record(result.publish);
|
|
const flush = record(result.flush);
|
|
const runtimeSecretsApply = record(result.runtimeSecretsApply);
|
|
const publicExposureApply = record(result.publicExposureApply);
|
|
const publicExposureCaddy = record(publicExposureApply.caddy);
|
|
const argoApply = record(result.argoApply);
|
|
const blocker = record(result.blocker);
|
|
const targetValidation = record(result.targetValidation);
|
|
const next = record(result.next);
|
|
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]),
|
|
"",
|
|
table(["SOURCE", "COMMIT", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), image.ref, short(gitops.manifestSha256)]]),
|
|
"",
|
|
table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]),
|
|
"",
|
|
table(["SCENARIO", "MAX_SECONDS", "CI_WAIT", "QVERIFY", "SECOND_PATH"], [[validation.scenarioId, validation.maxSeconds, validation.controlPlaneWaitMaxSeconds ?? "-", validation.quickVerifyMode ?? "-", validation.automaticSecondPath]]),
|
|
"",
|
|
renderObservedStatus(observed),
|
|
"",
|
|
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
|
|
"",
|
|
Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[
|
|
targetValidation.ok,
|
|
targetValidation.status,
|
|
targetValidation.scenarioId,
|
|
targetValidation.runId,
|
|
targetValidation.observerId,
|
|
short(targetValidation.reportJsonSha256),
|
|
targetValidation.findingCount,
|
|
targetValidation.artifactCount,
|
|
]]),
|
|
"",
|
|
Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]),
|
|
"",
|
|
Object.keys(flush).length === 0
|
|
? "FLUSH\n-"
|
|
: flush.mode === "async-job"
|
|
? table(["OK", "MODE", "JOB", "STATUS"], [[flush.ok, flush.mode, record(flush.job).id, record(flush.next).status]])
|
|
: table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]),
|
|
"",
|
|
Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]),
|
|
"",
|
|
Object.keys(publicExposureApply).length === 0 ? "PUBLIC_EXPOSURE_APPLY\n-" : table(["OK", "SECRET", "CADDY", "HOST", "ROUTE_HTTP"], [[publicExposureApply.ok, record(publicExposureApply.secret).ok, record(publicExposureApply.caddy).ok, publicExposureApply.hostname, record(publicExposureApply.caddy).routeProbeHttpStatus ?? "-"]]),
|
|
"",
|
|
Object.keys(publicExposureCaddy).length === 0 || publicExposureCaddy.ok === true
|
|
? "CADDY_APPLY_DETAIL\n-"
|
|
: table(["PY", "VALIDATE", "RELOAD", "PROBE", "HTTP", "BLOCK", "ACTIVE", "ERROR", "STDOUT", "STDERR"], [[publicExposureCaddy.pythonExitCode, publicExposureCaddy.validateExitCode, publicExposureCaddy.reloadExitCode, publicExposureCaddy.routeProbeExitCode, publicExposureCaddy.routeProbeHttpStatus, publicExposureCaddy.afterBlockPresent, publicExposureCaddy.active, short(publicExposureCaddy.errorPreview), short(record(publicExposureCaddy.result).stdoutPreview), short(record(publicExposureCaddy.result).stderrPreview)]]),
|
|
"",
|
|
Object.keys(argoApply).length === 0 ? "ARGO_APPLY\n-" : table(["OK", "EXIT", "PREVIEW"], [[argoApply.ok, record(argoApply.result).exitCode, record(argoApply.result).stdoutPreview]]),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
|
|
"",
|
|
"NEXT",
|
|
` plan: ${next.plan ?? "-"}`,
|
|
` status: ${next.status ?? "-"}`,
|
|
` image: ${next.image ?? "-"}`,
|
|
` trigger-current: ${next.triggerCurrent ?? "-"}`,
|
|
` validate: ${next.validate ?? "-"}`,
|
|
` quick-verify: ${next.quickVerify ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" default view is a bounded CI/CD summary; full manifest content is represented by object counts and sha256.",
|
|
" sentinel unavailable policy is structured-failure; no automatic second execution path is rendered.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderObservedStatus(observed: Record<string, unknown>): string {
|
|
const rows = [
|
|
observedStatusRow("source", observed.sourceMirror),
|
|
observedStatusRow("registry", observed.registry),
|
|
observedStatusRow("git-mirror", observed.gitMirror),
|
|
observedStatusRow("gitops", observed.gitops),
|
|
observedStatusRow("argo", observed.argo),
|
|
observedStatusRow("runtime", observed.runtime),
|
|
].filter((row) => row !== null);
|
|
if (rows.length === 0) return "OBSERVED\n-";
|
|
return table(["CHECK", "OK", "DETAIL", "EXIT", "TIMED_OUT", "PREVIEW"], rows);
|
|
}
|
|
|
|
function observedStatusRow(name: string, value: unknown): unknown[] | null {
|
|
const item = record(value);
|
|
if (Object.keys(item).length === 0) return null;
|
|
const result = record(item.result);
|
|
return [name, item.ok, observedDetail(name, item), result.exitCode, result.timedOut, result.stdoutPreview];
|
|
}
|
|
|
|
function observedDetail(name: string, item: Record<string, unknown>): string {
|
|
if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`;
|
|
if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`;
|
|
if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`;
|
|
if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`;
|
|
if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}`;
|
|
if (name === "runtime") {
|
|
const probe = record(item.probe);
|
|
const deployment = record(probe.deployment);
|
|
return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`;
|
|
}
|
|
return "-";
|
|
}
|
|
|
|
function renderAsyncJobResult(result: Record<string, unknown>): string {
|
|
const job = record(result.job);
|
|
const next = record(result.next);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "MODE", "MUTATION", "JOB"], [[result.node, result.lane, result.mode, result.mutation, job.id]]),
|
|
"",
|
|
table(["STATUS", "NAME", "CREATED"], [[job.status, job.name, job.createdAt]]),
|
|
"",
|
|
"NEXT",
|
|
` status: ${next.status ?? "-"}`,
|
|
` wait: ${next.wait ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" confirmed operation is delegated to UniDesk job status to keep interactive calls bounded.",
|
|
].join("\n");
|
|
}
|
|
|
|
function rendered(ok: boolean, command: string, text: string): RenderedCliResult {
|
|
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" };
|
|
}
|
|
|
|
function readConfigRefTarget(ref: string): unknown {
|
|
const file = configRefFile(ref);
|
|
const path = configRefPath(ref);
|
|
return valueAtPath(readConfigFile(file), path);
|
|
}
|
|
|
|
function readConfigFile(file: string): unknown {
|
|
if (file.startsWith("/") || file.includes("..") || !file.startsWith("config/")) throw new Error(`unsafe configRef file: ${file}`);
|
|
const abs = rootPath(file);
|
|
if (!existsSync(abs)) throw new Error(`${file} does not exist`);
|
|
return Bun.YAML.parse(readFileSync(abs, "utf8")) as unknown;
|
|
}
|
|
|
|
function configRefFile(ref: string): string {
|
|
const [file, path, extra] = ref.split("#");
|
|
if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) throw new Error(`invalid configRef: ${ref}`);
|
|
return file;
|
|
}
|
|
|
|
function configRefPath(ref: string): string {
|
|
const [, path] = ref.split("#");
|
|
if (path === undefined || path.length === 0) throw new Error(`invalid configRef: ${ref}`);
|
|
return path;
|
|
}
|
|
|
|
function valueAtPath(value: unknown, path: string): unknown {
|
|
let current: unknown = value;
|
|
for (const segment of path.split(".")) {
|
|
const match = /^([A-Za-z0-9_-]+)?(?:\[(\d+)\])?$/u.exec(segment);
|
|
if (match === null) return undefined;
|
|
if (match[1] !== undefined) {
|
|
if (!isRecord(current)) return undefined;
|
|
current = current[match[1]];
|
|
}
|
|
if (match[2] !== undefined) {
|
|
if (!Array.isArray(current)) return undefined;
|
|
current = current[Number(match[2])];
|
|
}
|
|
}
|
|
return current;
|
|
}
|
|
|
|
function stringAt(value: unknown, path: string): string {
|
|
const found = valueAtPath(value, path);
|
|
if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return found;
|
|
}
|
|
|
|
function nonEmptyString(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function stringField(value: Record<string, unknown>, path: string): string {
|
|
return stringAt(value, path);
|
|
}
|
|
|
|
function stringTarget(value: unknown, label: string): string {
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must resolve to a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
function numberAt(value: unknown, path: string): number {
|
|
const found = valueAtPath(value, path);
|
|
if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`);
|
|
return found;
|
|
}
|
|
|
|
function arrayAt(value: unknown, path: string): unknown[] {
|
|
const found = valueAtPath(value, path);
|
|
if (!Array.isArray(found)) throw new Error(`${path} must be an array`);
|
|
return found;
|
|
}
|
|
|
|
function recordTarget(value: unknown, label: string): Record<string, unknown> {
|
|
if (!isRecord(value)) throw new Error(`${label} must resolve to an object`);
|
|
return value;
|
|
}
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return isRecord(value) ? value : {};
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function manifestObjectSummary(items: readonly Record<string, unknown>[]): readonly Record<string, unknown>[] {
|
|
return items.map((item) => ({
|
|
kind: item.kind ?? null,
|
|
name: record(item.metadata).name ?? null,
|
|
namespace: record(item.metadata).namespace ?? null,
|
|
}));
|
|
}
|
|
|
|
function compactCommand(result: CommandResult): CompactCommandResult {
|
|
return {
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: Buffer.byteLength(result.stdout),
|
|
stderrBytes: Buffer.byteLength(result.stderr),
|
|
stdoutPreview: result.stdout.trim().slice(0, 500),
|
|
stderrPreview: result.stderr.trim().slice(0, 500),
|
|
};
|
|
}
|
|
|
|
function parseJsonObject(text: string): Record<string, unknown> | null {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return isRecord(parsed) ? parsed : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function table(headers: string[], rows: unknown[][]): string {
|
|
const normalized = [headers, ...rows.map((row) => row.map(text))];
|
|
const widths = headers.map((_, index) => Math.max(...normalized.map((row) => text(row[index] ?? "").length)));
|
|
return normalized.map((row) => row.map((cell, index) => text(cell).padEnd(widths[index])).join(" ").trimEnd()).join("\n");
|
|
}
|
|
|
|
function text(value: unknown): string {
|
|
if (value === undefined || value === null || value === "") return "-";
|
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
return String(value).replace(/\s+/gu, " ").trim();
|
|
}
|
|
|
|
function short(value: unknown): string {
|
|
const raw = text(value);
|
|
if (raw === "-") return raw;
|
|
if (/^sha256:[0-9a-f]{64}$/iu.test(raw)) return `${raw.slice(0, 19)}...`;
|
|
if (/^[0-9a-f]{40}$/iu.test(raw)) return raw.slice(0, 12);
|
|
return raw.length > 42 ? `${raw.slice(0, 39)}...` : raw;
|
|
}
|
|
|
|
function sha256(textValue: string): string {
|
|
return `sha256:${createHash("sha256").update(textValue).digest("hex")}`;
|
|
}
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, "'\\''")}'`;
|
|
}
|
|
|
|
function tomlEscape(value: string): string {
|
|
return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"');
|
|
}
|