From db3fed012e28b43430a515722535140dd492768f Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Fri, 26 Jun 2026 00:46:45 +0800 Subject: [PATCH] feat: wire web probe sentinel validation (#912) Co-authored-by: Codex --- .../public-exposure.d601-v03.yaml | 2 + scripts/src/hwlab-node-help.ts | 14 +- scripts/src/hwlab-node-web-sentinel-cicd.ts | 976 +++++++++++++++++- scripts/src/hwlab-node-web-sentinel-config.ts | 2 + .../src/hwlab-node-web-sentinel-service.ts | 153 ++- scripts/src/hwlab-node/web-probe-observe.ts | 71 +- 6 files changed, 1205 insertions(+), 13 deletions(-) diff --git a/config/hwlab-web-probe-sentinel/public-exposure.d601-v03.yaml b/config/hwlab-web-probe-sentinel/public-exposure.d601-v03.yaml index b08fb223..af9ad9be 100644 --- a/config/hwlab-web-probe-sentinel/public-exposure.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/public-exposure.d601-v03.yaml @@ -12,6 +12,8 @@ sentinel: hostname: monitor.pikapython.com expectedA: 82.156.23.220 frpc: + deploymentName: hwlab-web-probe-sentinel-frpc + image: 127.0.0.1:5000/hwlab/frpc:v0.68.1 serverAddr: 82.156.23.220 serverPort: 22000 tokenSourceRef: platform-infra/pk01-frp.env diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index 6ccfdaf5..176b29c0 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -58,6 +58,9 @@ export function hwlabNodeHelp(): Record { "bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane plan --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane trigger-current --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes web-probe sentinel validate --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes web-probe sentinel maintenance stop --node D601 --lane v03 --confirm --wait", + "bun scripts/cli.ts hwlab nodes web-probe sentinel report --node D601 --lane v03 --view turn-summary", "bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run", @@ -70,6 +73,7 @@ export function hwlabNodeHelp(): Record { "`control-plane sync --confirm` syncs the YAML-declared local-k3s postgres bootstrap Secret, terminates a stale running Argo operation, deletes failed Argo hook Jobs, and recreates stale non-ready StatefulSet pods that are still pinned to an old controller revision with pull/backoff errors.", "`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.", "`web-probe sentinel image/control-plane` renders the YAML-first image, GitOps and Argo plan from the owning configRefs; unavailable service validation is a structured failure, not an automatic second execution path.", + "`web-probe sentinel validate|maintenance|report` talks to the sentinel through k3s internal Service DNS, records only analyze summaries/views, and never runs a public/fallback validation path.", "Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not." ], }; @@ -112,13 +116,19 @@ export function hwlabNodeWebProbeHelp(): Record { "bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane plan --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane trigger-current --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes web-probe sentinel validate --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes web-probe sentinel validate --node D601 --lane v03 --quick-verify --confirm --wait", + "bun scripts/cli.ts hwlab nodes web-probe sentinel maintenance start --node D601 --lane v03 --confirm --wait --release-id ", + "bun scripts/cli.ts hwlab nodes web-probe sentinel maintenance stop --node D601 --lane v03 --confirm --wait --release-id ", + "bun scripts/cli.ts hwlab nodes web-probe sentinel report --node D601 --lane v03 --view summary", + "bun scripts/cli.ts hwlab nodes web-probe sentinel report --node D601 --lane v03 --view trace-frame", "bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 <<'JS'\nexport default async ({ waitWorkbenchReady, fetchJson, fetchApiMatrix, recordStep, collectText, safeEvaluate, screenshot }) => {\n const ready = await waitWorkbenchReady();\n const workspace = await fetchJson('/v1/workbench/workspace?projectId=prj_hwpod_workbench');\n const apiMatrix = await fetchApiMatrix(['/v1/workbench/workspace?projectId=prj_hwpod_workbench', '/auth/session']);\n const workspaceText = await collectText('#workspace');\n const evaluated = await safeEvaluate(({ a, b }) => ({ sum: a + b }), { a: 1, b: 2 });\n await screenshot('workbench.png');\n recordStep('workbench-summary', { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, apiMatrixOk: apiMatrix.ok });\n return { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, workspaceText, evaluated };\n};\nJS", ], actions: { run: "Run the repo-owned scripts/web-live-dom-probe.mjs helper.", script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus gotoStable/reloadStable/gotoCurrentStable/safeReload/fetchJson/safeFetchJson/fetchApiMatrix/recordStep/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.", observe: "Start, inspect, control, stop, collect, and analyze a pure-client long-running Workbench observer on the target host. The observer runs a control page plus a passive observer page in a shared-auth browser context, receives commands through stateDir/commands files, writes JSONL artifacts, and does not expose any inbound service API.", - sentinel: "Render the YAML-first service wrapper configRef graph plus image/GitOps/Argo control-plane plan for the production web-probe sentinel. This reads observability.webProbe.sentinel.enabled/configRefs and validates owning YAML presence, shape, redacted hashes, and cross-ref consistency without starting a browser or reading secret values.", + sentinel: "Render the YAML-first service wrapper configRef graph plus image/GitOps/Argo control-plane plan for the production web-probe sentinel, then validate/maintenance/report through the k3s internal sentinel Service DNS. Quick verify still uses web-probe observe/analyze artifacts as truth and records only redacted report summaries/views.", }, notes: [ "The default probe URL, browser proxy mode, observe/analyze alert thresholds, and project-management observe behavior come from config/hwlab-node-lanes.yaml webProbe; pass --url only when intentionally overriding the YAML-selected origin.", @@ -134,7 +144,7 @@ export function hwlabNodeWebProbeHelp(): Record { "observe analyze scans every sampled DOM point, extracts Workbench timing text such as 总耗时/total and 最近 N 秒/分前, and writes a sample point vs turn timing report: each Markdown table row starts with the timestamp, followed by each turn's 总耗时(s) and 最近更新(s). Timing series are reported for post-processing/manual analysis instead of auto-judged from status tail output.", "observe analyze also reports visible “加载中” count, owner attribution, concurrent loading owners, and continuous visible segments; fixes must reduce real loading latency, not reveal incomplete content early to make this metric disappear.", "script/observe support --browser-proxy-mode auto|direct; use direct for A/B evidence when frontend RUM is slow but OTel server spans are absent or fast, instead of falling back to raw Playwright.", - "sentinel plan/status is a configuration visibility command for the service wrapper; sentinel image/control-plane renders image, GitOps and Argo control-plane state from owning YAML and refuses to report deployment mutation until the remote publish job is wired. observe start/status/command/collect/analyze remain the sampling and analysis truth.", + "sentinel plan/status is a configuration visibility command for the service wrapper; sentinel image/control-plane renders image, GitOps and Argo control-plane state from owning YAML. sentinel validate checks /api/health, /metrics, recent analyze report and publicExposure; maintenance stop can run the configured quick verify and index the observe/analyze report. observe start/status/command/collect/analyze remain the sampling and analysis truth.", "Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.", "Use reloadStable(), gotoCurrentStable(), or safeReload() for bounded retries around page reload/current-URL navigation jitter such as ERR_NETWORK_CHANGED.", "Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).", diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 04d8e0c9..2c90f801 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -1,7 +1,8 @@ // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel. -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; import { repoRoot, rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; @@ -12,6 +13,8 @@ import type { RenderedCliResult } from "./output"; export type WebProbeSentinelConfigAction = "plan" | "status"; export type WebProbeSentinelImageAction = "status" | "build"; export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current"; +export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop"; +export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame"; export type WebProbeSentinelOptions = | { @@ -40,6 +43,42 @@ export type WebProbeSentinelOptions = readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; + } + | { + readonly kind: "maintenance"; + readonly action: WebProbeSentinelMaintenanceAction; + readonly node: string; + readonly lane: string; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + readonly releaseId: string | null; + readonly reason: string | null; + readonly quickVerify: boolean; + } + | { + readonly kind: "validate"; + readonly action: "validate"; + readonly node: string; + readonly lane: string; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + readonly quickVerify: boolean; + } + | { + readonly kind: "report"; + readonly action: "report"; + readonly node: string; + readonly lane: string; + readonly view: WebProbeSentinelReportView; + readonly runId: string | null; + readonly traceId: string | null; + readonly sampleSeq: number | null; + readonly raw: boolean; + readonly timeoutSeconds: number; }; interface SentinelCicdState { @@ -119,7 +158,10 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action)); const state = loadSentinelCicdState(spec, options.timeoutSeconds); if (options.kind === "image") return runSentinelImage(state, options); - return runSentinelControlPlane(state, options); + if (options.kind === "control-plane") return runSentinelControlPlane(state, options); + if (options.kind === "maintenance") return runSentinelMaintenance(state, options); + if (options.kind === "validate") return runSentinelValidate(state, options); + return runSentinelReport(state, options); } function runSentinelImage(state: SentinelCicdState, options: Extract): RenderedCliResult { @@ -379,6 +421,35 @@ function renderSentinelManifests( metadata: { name: serviceName, namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] }, }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: stringAt(publicExposure, "frpc.deploymentName"), namespace, labels: { ...labels, "app.kubernetes.io/component": "tunnel" } }, + spec: { + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName") } }, + template: { + metadata: { + labels: { ...labels, "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName"), "app.kubernetes.io/component": "tunnel" }, + annotations: { + "unidesk.ai/public-base-url": stringAt(publicExposure, "publicBaseUrl"), + "unidesk.ai/frp-server": `${stringAt(publicExposure, "frpc.serverAddr")}:${numberAt(publicExposure, "frpc.serverPort")}`, + "unidesk.ai/frp-remote-port": String(numberAt(publicExposure, "frpc.httpProxy.remotePort")), + }, + }, + spec: { + containers: [{ + name: "frpc", + image: stringAt(publicExposure, "frpc.image"), + imagePullPolicy: "IfNotPresent", + args: ["-c", "/etc/frp/frpc.toml"], + volumeMounts: [{ name: "frpc-config", mountPath: "/etc/frp/frpc.toml", subPath: stringAt(publicExposure, "frpc.secretKey"), readOnly: true }], + }], + volumes: [{ name: "frpc-config", secret: { secretName: stringAt(publicExposure, "frpc.secretName") } }], + }, + }, + }, + }, { apiVersion: "networking.k8s.io/v1", kind: "NetworkPolicy", @@ -468,14 +539,36 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext const flush = !applyOnly && record(publish).ok === true ? runChildCli(["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"], options.timeoutSeconds) : null; + const publicExposureApply = applySentinelPublicExposure(state, options.timeoutSeconds); const argoApply = applySentinelArgoApplication(state, options.timeoutSeconds); const observed = waitForSentinelObservedStatus(state, options.timeoutSeconds); + const observedReady = sentinelObservedReady(observed); + const targetValidation = applyOnly + ? null + : observedReady + ? runSentinelQuickVerify(state, "control-plane-target-validation", options.timeoutSeconds) + : { + ok: false, + status: "blocked", + scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), + reason: "runtime-not-ready", + valuesRedacted: true, + }; + const targetValidationOk = applyOnly || record(targetValidation).ok === true; const ok = state.configReady && state.sourceHead.ok && (applyOnly || record(publish).ok === true) && (applyOnly || record(flush).ok === true) + && record(publicExposureApply).ok === true && record(argoApply).ok === true - && sentinelObservedReady(observed); + && observedReady + && targetValidationOk; + const blocker = ok ? null : { + code: targetValidationOk ? "sentinel-control-plane-not-ready" : "sentinel-target-validation-failed", + reason: targetValidationOk + ? "one or more publish, publicExposure, Argo or runtime observation checks did not pass" + : text(record(targetValidation).failure ?? record(targetValidation).reason ?? "quick verify did not pass"), + }; const result = { ok, command, @@ -509,13 +602,16 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext }, publish, flush, + publicExposureApply, argoApply, observed, + targetValidation, warnings: Array.from(new Set([ ...sentinelElapsedWarnings(record(publish).elapsedMs), ...sentinelElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs), + ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []), ])), - blocker: null, + blocker, next: controlPlaneNext(state, options.action), valuesRedacted: true, }; @@ -1027,6 +1123,859 @@ function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelCont }; } +function runSentinelMaintenance(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = `hwlab nodes web-probe sentinel maintenance ${options.action}`; + const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); + if (options.action === "status") { + const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds); + const result = { + ok: serviceHealth.ok && maintenance.ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + serviceHealth, + maintenance, + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderMaintenanceResult(result)); + } + if (!options.confirm) { + const result = { + ok: serviceHealth.ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "dry-run", + serviceHealth, + mutation: false, + planned: { + action: options.action, + releaseId: options.releaseId, + reason: options.reason, + quickVerify: options.action === "stop" && options.quickVerify, + }, + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderMaintenanceResult(result)); + } + if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify); + if (!serviceHealth.ok) { + const result = { + ok: false, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "confirm-wait", + mutation: false, + serviceHealth, + blocker: serviceUnavailableBlocker(state), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(false, command, renderMaintenanceResult(result)); + } + const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true }; + const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds); + const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok + ? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds) + : null; + const result = { + ok: mutation.ok && (quickVerify === null || quickVerify.ok === true), + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "confirm-wait", + mutation: true, + serviceHealth, + maintenance: mutation, + quickVerify, + blocker: mutation.ok ? null : serviceUnavailableBlocker(state), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderMaintenanceResult(result)); +} + +function runSentinelValidate(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = "hwlab nodes web-probe sentinel validate"; + const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); + let quickVerify: Record | null = null; + if (options.quickVerify) { + if (!options.confirm) { + const result = { + ok: initialHealth.ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "dry-run", + serviceHealth: initialHealth, + planned: { quickVerify: true, waitRequired: true }, + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderValidateResult(result)); + } + if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true); + if (!initialHealth.ok) { + const result = { + ok: false, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "confirm-wait", + serviceHealth: initialHealth, + blocker: serviceUnavailableBlocker(state), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(false, command, renderValidateResult(result)); + } + quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds); + } + const health = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); + const metrics = callSentinelService(state, "GET", "/metrics", null, options.timeoutSeconds); + const report = callSentinelService(state, "GET", "/api/report?view=summary", null, options.timeoutSeconds); + const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds); + const ok = health.ok + && record(health.bodyJson).ok === true + && metrics.ok + && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health") + && report.ok + && publicExposure.ok === true + && (quickVerify === null || quickVerify.ok === true); + const result = { + ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: options.quickVerify ? "confirm-wait" : "status", + serviceHealth: health, + metrics, + report, + publicExposure, + quickVerify, + blocker: ok ? null : validationBlocker(health, metrics, report, publicExposure, quickVerify), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(ok, command, renderValidateResult(result)); +} + +function runSentinelReport(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = `hwlab nodes web-probe sentinel report --view ${options.view}`; + const query = new URLSearchParams({ view: options.view }); + if (options.runId !== null) query.set("run", options.runId); + if (options.traceId !== null) query.set("traceId", options.traceId); + if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq)); + const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds); + const body = record(report.bodyJson); + const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true }); + return rendered(report.ok && body.ok !== false, command, options.raw ? JSON.stringify(body, null, 2) : renderedText); +} + +function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult { + const args = ["hwlab", "nodes", "web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; + if (releaseId !== null) args.push("--release-id", releaseId); + if (reason !== null) args.push("--reason", reason); + if (quickVerify) args.push("--quick-verify"); + const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${subcommand.join(" ")} for node ${state.spec.nodeId}`); + const command = `hwlab nodes web-probe sentinel ${subcommand.join(" ")}`; + return rendered(true, command, renderAsyncJobResult({ + ok: true, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "async-job", + mutation: true, + job, + next: { + status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, + wait: ["bun", "scripts/cli.ts", ...args].join(" "), + }, + valuesRedacted: true, + })); +} + +function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record { + const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId"); + const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); + const scenario = findScenario(state, scenarioId); + if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true }; + const prompts = readPromptSetForScenario(scenario); + if (!prompts.ok) return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true }; + const deadline = Date.now() + Math.min(timeoutSeconds, maxSeconds) * 1000; + const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; + const steps: Record[] = []; + const startArgs = [ + "hwlab", "nodes", "web-probe", "observe", "start", + "--node", state.spec.nodeId, + "--lane", state.spec.lane, + "--target-path", stringAt(scenario, "observeTargetPath"), + "--sample-interval-ms", String(numberAt(scenario, "sampleIntervalMs")), + "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), + "--command-timeout-seconds", "55", + ]; + const started = runChildCli(startArgs, remainingSeconds(deadline, 55)); + steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); + const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); + if (!started.ok || observerId === null) { + return recordQuickVerify(state, { + ok: false, + runId, + scenarioId, + reason, + status: "blocked", + observerId, + steps, + failure: "observe-start-failed", + valuesRedacted: true, + }); + } + let promptIndex = 0; + for (const item of arrayAt(scenario, "commandSequence").map(record)) { + const type = stringAt(item, "type"); + const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1); + for (let index = 0; index < repeat; index += 1) { + if (Date.now() >= deadline) { + return recordQuickVerify(state, quickVerifyTimeoutPayload(state, runId, scenarioId, reason, observerId, steps)); + } + const args = ["hwlab", "nodes", "web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", "55000", "--command-timeout-seconds", String(remainingSeconds(deadline, 55))]; + let input: string | undefined; + if (type === "selectProvider") args.push("--provider", stringAt(item, "provider")); + if (type === "sendPrompt") { + args.push("--text-stdin"); + input = prompts.prompts[promptIndex % prompts.prompts.length] ?? ""; + promptIndex += 1; + } + const commandResult = runChildCli(args, remainingSeconds(deadline, 60), input); + steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result }); + if (!commandResult.ok) { + return recordQuickVerify(state, { + ok: false, + runId, + scenarioId, + reason, + status: "blocked", + observerId, + steps, + failure: `observe-command-${type}-failed`, + promptSource: prompts.summary, + valuesRedacted: true, + }); + } + } + } + const analysis = runChildCli(["hwlab", "nodes", "web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120)); + steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); + const indexEntry = readLocalObserveIndex(observerId); + const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30)); + const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); + const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30)); + const ok = analysis.ok && record(artifactSummary).ok === true; + return recordQuickVerify(state, { + ok, + runId, + scenarioId, + reason, + status: ok ? "analyzed" : "blocked", + observerId, + stateDir: indexEntry?.stateDir ?? null, + reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), + findingCount: numberAtNullable(artifactSummary, "findingCount") ?? 0, + artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, + promptSource: prompts.summary, + steps, + analysis: artifactSummary, + views: { + summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, + "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, + }, + findings: Array.isArray(record(artifactSummary).findings) ? record(artifactSummary).findings : [], + screenshot: record(artifactSummary).screenshot, + publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), + valuesRedacted: true, + }); +} + +function quickVerifyTimeoutPayload(state: SentinelCicdState, runId: string, scenarioId: string, reason: string, observerId: string, steps: readonly Record[]): Record { + return { + ok: false, + runId, + scenarioId, + reason, + status: "blocked", + observerId, + steps, + failure: "quick-verify-timeout-over-120s", + warnings: ["quick verify exceeded the configured 120s targetValidation budget; investigate env-reuse/git mirror/source build path before retrying."], + valuesRedacted: true, + }; +} + +function recordQuickVerify(state: SentinelCicdState, payload: Record): Record { + const recordResult = callSentinelService(state, "POST", "/api/runs/record", { + runId: payload.runId, + scenarioId: payload.scenarioId, + status: payload.status, + observerId: payload.observerId, + stateDir: payload.stateDir, + reportJsonSha256: payload.reportJsonSha256, + findingCount: payload.findingCount, + artifactCount: payload.artifactCount, + summary: { + reason: payload.reason, + status: payload.status, + analysis: payload.analysis, + promptSource: payload.promptSource, + steps: payload.steps, + valuesRedacted: true, + }, + findings: payload.findings, + views: payload.views, + screenshot: payload.screenshot, + publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"), + maintenance: payload.reason === "maintenance-stop", + valuesRedacted: true, + }, 60); + return { ...payload, recordResult, valuesRedacted: true }; +} + +function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number): Record { + const namespace = stringAt(state.runtime, "namespace"); + const deploymentName = stringAt(state.runtime, "deploymentName"); + const serviceName = stringAt(state.runtime, "serviceName"); + const servicePort = numberAt(state.runtime, "servicePort"); + const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`; + const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); + const js = [ + "const method=process.env.REQ_METHOD||'GET';", + "const url=process.env.REQ_URL||'';", + "const body=Buffer.from(process.env.REQ_BODY_B64||'', 'base64').toString('utf8');", + "const init={method,headers:{}};", + "if(method!=='GET'&&method!=='HEAD'){init.headers['content-type']='application/json';init.body=body;}", + "let out;", + "try{const res=await fetch(url,init);const text=await res.text();let bodyJson=null;try{bodyJson=JSON.parse(text)}catch{};out={ok:res.ok,httpStatus:res.status,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,12000),bodyBytes:Buffer.byteLength(text),valuesRedacted:true};}", + "catch(error){out={ok:false,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}", + "console.log(JSON.stringify(out));", + ].join(""); + const script = [ + "set +e", + `namespace=${shellQuote(namespace)}`, + `deployment=${shellQuote(deploymentName)}`, + `method=${shellQuote(method)}`, + `url=${shellQuote(url)}`, + `body_b64=${shellQuote(bodyB64)}`, + `js=${shellQuote(js)}`, + "if ! kubectl -n \"$namespace\" get deploy \"$deployment\" >/dev/null 2>&1; then node - \"$namespace\" \"$deployment\" <<'NODE'\nconst [namespace,deployment]=process.argv.slice(2); console.log(JSON.stringify({ok:false,error:'sentinel-deployment-missing',namespace,deployment,valuesRedacted:true}));\nNODE\nexit 0; fi", + "kubectl -n \"$namespace\" exec deploy/\"$deployment\" -- env REQ_METHOD=\"$method\" REQ_URL=\"$url\" REQ_BODY_B64=\"$body_b64\" bun -e \"$js\"", + ].join("\n"); + const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { + ok: result.exitCode === 0 && parsed?.ok === true, + method, + path: pathWithQuery, + internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`, + httpStatus: parsed?.httpStatus ?? null, + bodyJson: record(parsed?.bodyJson), + bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "", + bodyBytes: parsed?.bodyBytes ?? null, + error: parsed?.error ?? null, + result: compactCommand(result), + valuesRedacted: true, + }; +} + +function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { + const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl"); + const hostname = stringAt(state.publicExposure, "hostname"); + const expectedA = stringAt(state.publicExposure, "expectedA"); + const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`; + const script = [ + "set +e", + `host=${shellQuote(hostname)}`, + `expected=${shellQuote(expectedA)}`, + `url=${shellQuote(probeUrl)}`, + "dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)", + "headers=$(mktemp)", + "body=$(mktemp)", + "writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)", + "curl_rc=$?", + "body_head=$(head -c 1000 \"$body\" | base64 | tr -d '\\n')", + "node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'", + "const fs=require('node:fs');", + "const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);", + "const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);", + "const status=Number(statusRaw||0);", + "const ssl=Number(sslRaw||-1);", + "const addrs=dns?dns.split(',').filter(Boolean):[];", + "const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();", + "const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');", + "const authCovered=status===401||status===403||status>=200&&status<300;", + "const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;", + "const upstreamOk=status>=200&&status<300&&body.includes('valuesRedacted');", + "const dnsMatches=addrs.includes(expected);", + "console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));", + "NODE", + ].join("\n"); + const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; +} + +function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { + const material = readSentinelFrpcMaterial(state); + if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true }; + const secret = applySentinelFrpcSecret(state, stringAt(material, "frpcToml"), timeoutSeconds); + const caddy = applySentinelCaddyBlock(state, timeoutSeconds); + return { + ok: secret.ok === true && caddy.ok === true, + hostname: stringAt(state.publicExposure, "hostname"), + publicBaseUrl: stringAt(state.publicExposure, "publicBaseUrl"), + material: { + ok: true, + sourceRef: material.sourceRef, + sourcePath: material.sourcePath, + fingerprint: material.fingerprint, + valuesRedacted: true, + }, + secret, + caddy, + valuesRedacted: true, + }; +} + +function readSentinelFrpcMaterial(state: SentinelCicdState): Record { + const sourceRef = stringAt(state.publicExposure, "frpc.tokenSourceRef"); + const sourceKey = stringAt(state.publicExposure, "frpc.tokenSourceKey"); + const paths = secretSourcePaths(sourceRef); + const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); + if (!existsSync(sourcePath)) return { ok: false, error: "frp-token-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const token = values[sourceKey]; + if (token === undefined || token.length === 0) return { ok: false, error: "frp-token-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + const proxy = record(valueAtPath(state.publicExposure, "frpc.httpProxy")); + const frpcToml = [ + `serverAddr = "${tomlEscape(stringAt(state.publicExposure, "frpc.serverAddr"))}"`, + `serverPort = ${numberAt(state.publicExposure, "frpc.serverPort")}`, + "loginFailExit = true", + `auth.token = "${tomlEscape(token)}"`, + "", + "[[proxies]]", + `name = "${tomlEscape(stringAt(proxy, "name"))}"`, + 'type = "tcp"', + `localIP = "${tomlEscape(stringAt(proxy, "localIP"))}"`, + `localPort = ${numberAt(proxy, "localPort")}`, + `remotePort = ${numberAt(proxy, "remotePort")}`, + "", + ].join("\n"); + return { + ok: true, + sourceRef, + sourceKey, + sourcePath: displayPath(sourcePath), + frpcToml, + fingerprint: `sha256:${createHash("sha256").update(`${token}\n${frpcToml}`).digest("hex").slice(0, 16)}`, + valuesRedacted: true, + }; +} + +function applySentinelFrpcSecret(state: SentinelCicdState, frpcToml: string, timeoutSeconds: number): Record { + const namespace = stringAt(state.runtime, "namespace"); + const secretName = stringAt(state.publicExposure, "frpc.secretName"); + const secretKey = stringAt(state.publicExposure, "frpc.secretKey"); + const script = [ + "set +e", + `namespace=${shellQuote(namespace)}`, + `secret=${shellQuote(secretName)}`, + `key=${shellQuote(secretKey)}`, + "tmp=$(mktemp -d)", + "trap 'rm -rf \"$tmp\"' EXIT", + "cat >\"$tmp/frpc.toml\"", + "kubectl -n \"$namespace\" create secret generic \"$secret\" --from-file=\"$key=$tmp/frpc.toml\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-public-exposure -f - >/tmp/web-probe-sentinel-frpc-secret.out 2>/tmp/web-probe-sentinel-frpc-secret.err", + "rc=$?", + "present=no", + "bytes=0", + "if kubectl -n \"$namespace\" get secret \"$secret\" -o jsonpath=\"{.data.$key}\" >/tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null; then present=yes; bytes=$(base64 -d /dev/null | wc -c | tr -d ' '); fi", + "node - \"$rc\" \"$namespace\" \"$secret\" \"$key\" \"$present\" \"$bytes\" <<'NODE'", + "const [rc, namespace, secret, key, present, bytes] = process.argv.slice(2);", + "console.log(JSON.stringify({ok:Number(rc)===0&&present==='yes',namespace,secret,key,present:present==='yes',bytes:Number(bytes||0),applyExitCode:Number(rc),valuesRedacted:true}));", + "NODE", + ].join("\n"); + const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: frpcToml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; +} + +function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: number): Record { + const hostname = stringAt(state.publicExposure, "hostname"); + const owner = stringAt(state.publicExposure, "caddy.managedBlockOwner"); + const configPath = stringAt(state.publicExposure, "caddy.configPath"); + const serviceName = stringAt(state.publicExposure, "caddy.serviceName"); + const responseHeaderTimeoutSeconds = numberAt(state.publicExposure, "caddy.responseHeaderTimeoutSeconds"); + const remotePort = numberAt(state.publicExposure, "frpc.httpProxy.remotePort"); + const block = [ + `${hostname} {`, + ` reverse_proxy 127.0.0.1:${remotePort} {`, + " transport http {", + ` response_header_timeout ${responseHeaderTimeoutSeconds}s`, + " }", + " }", + "}", + "", + ].join("\n"); + const blockB64 = Buffer.from(block, "utf8").toString("base64"); + const script = [ + "set +e", + `hostname=${shellQuote(hostname)}`, + `owner=${shellQuote(owner)}`, + `config_path=${shellQuote(configPath)}`, + `service=${shellQuote(serviceName)}`, + `block_b64=${shellQuote(blockB64)}`, + "marker=\"unidesk managed $owner\"", + "tmp=$(mktemp -d)", + "trap 'rm -rf \"$tmp\"' EXIT", + "block=\"$tmp/block\"", + "next=\"$tmp/Caddyfile\"", + "printf '%s' \"$block_b64\" | base64 -d >\"$block\"", + "if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi", + "python3 - \"$next\" \"$block\" \"$marker\" <<'PY' >/tmp/web-probe-sentinel-caddy-python.out 2>/tmp/web-probe-sentinel-caddy-python.err", + "import pathlib, re, sys", + "config = pathlib.Path(sys.argv[1])", + "block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')", + "marker = sys.argv[3]", + "text = config.read_text(encoding='utf-8') if config.exists() else ''", + "begin = f'# BEGIN {marker}'", + "end = f'# END {marker}'", + "managed = f'{begin}\\n{block.rstrip()}\\n# END {marker}\\n'", + "pattern = re.compile(rf'(?ms)^# BEGIN {re.escape(marker)}\\n.*?\\n# END {re.escape(marker)}\\n*')", + "if pattern.search(text):", + " text = pattern.sub(managed, text)", + "else:", + " text = text.rstrip() + '\\n\\n' + managed", + "config.write_text(text, encoding='utf-8')", + "PY", + "python_rc=$?", + "validate_rc=1", + "reload_rc=", + "if [ \"$python_rc\" = 0 ]; then sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/web-probe-sentinel-caddy-validate.out 2>/tmp/web-probe-sentinel-caddy-validate.err; validate_rc=$?; fi", + "if [ \"$validate_rc\" = 0 ]; then sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/web-probe-sentinel-caddy-install.out 2>/tmp/web-probe-sentinel-caddy-install.err && (sudo systemctl reload \"$service\" >/tmp/web-probe-sentinel-caddy-reload.out 2>/tmp/web-probe-sentinel-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/web-probe-sentinel-caddy-reload.out 2>>/tmp/web-probe-sentinel-caddy-reload.err); reload_rc=$?; fi", + "after_present=no", + "grep -Fq \"# BEGIN $marker\" \"$config_path\" 2>/dev/null && after_present=yes", + "active=$(systemctl is-active \"$service\" 2>/dev/null || true)", + "err=$(cat /tmp/web-probe-sentinel-caddy-python.err /tmp/web-probe-sentinel-caddy-validate.err /tmp/web-probe-sentinel-caddy-install.err /tmp/web-probe-sentinel-caddy-reload.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)", + "node - \"$python_rc\" \"$validate_rc\" \"$reload_rc\" \"$after_present\" \"$active\" \"$hostname\" \"$config_path\" \"$err\" <<'NODE'", + "const [pythonRc, validateRc, reloadRc, afterPresent, active, hostname, configPath, errorPreview] = process.argv.slice(2);", + "console.log(JSON.stringify({ok:Number(pythonRc)===0&&Number(validateRc)===0&&Number(reloadRc)===0&&afterPresent==='yes',hostname,configPath,pythonExitCode:Number(pythonRc),validateExitCode:Number(validateRc),reloadExitCode:reloadRc===''?null:Number(reloadRc),afterBlockPresent:afterPresent==='yes',active,errorPreview,valuesRedacted:true}));", + "NODE", + ].join("\n"); + const result = runCommand(["trans", stringAt(state.publicExposure, "caddy.route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; +} + +function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record { + if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true }; + const script = [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + "node - \"$state_dir\" <<'NODE'", + "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", + "const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", + "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);", + "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", + "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", + "let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}", + "let artifactCount=0; let screenshot=null;", + "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", + "walk(stateDir);", + "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220)};});", + "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", + "console.log(JSON.stringify({ok:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", + "NODE", + ].join("\n"); + const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; +} + +function collectObserveView(state: SentinelCicdState, observerId: string, view: "turn-summary" | "trace-frame", turn: number | null, timeoutSeconds: number): Record { + const args = ["hwlab", "nodes", "web-probe", "observe", "collect", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--view", view, "--command-timeout-seconds", String(Math.max(5, Math.min(timeoutSeconds, 55)))]; + if (turn !== null) args.push("--turn", String(turn)); + const result = runChildCli(args, timeoutSeconds); + return { ok: result.ok, view, renderedText: String(record(result.result).stdoutTail ?? record(result.result).stdoutPreview ?? ""), result: result.result, valuesRedacted: true }; +} + +function runChildCli(args: string[], timeoutSeconds: number, input?: string): { ok: boolean; result: Record } { + const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, { input, timeoutMs: Math.max(5, timeoutSeconds) * 1000 }); + return { + ok: result.exitCode === 0 && !result.timedOut, + result: compactCommandWithTail(result), + }; +} + +function findScenario(state: SentinelCicdState, scenarioId: string): Record | null { + const sentinel = state.spec.observability.webProbe?.sentinel; + if (sentinel === undefined) return null; + const scenarios = readConfigRefTarget(sentinel.configRefs.scenarios); + if (!Array.isArray(scenarios)) return null; + return scenarios.map(record).find((item) => item.id === scenarioId) ?? null; +} + +function readPromptSetForScenario(scenario: Record): { ok: true; prompts: string[]; summary: Record } | { ok: false; error: string; summary: Record } { + const promptSet = recordTarget(readConfigRefTarget(stringAt(scenario, "promptSetRef")), stringAt(scenario, "promptSetRef")); + const sourceRef = stringAt(promptSet, "promptSourceRef"); + const key = stringAt(promptSet, "promptSourceKey"); + const paths = secretSourcePaths(sourceRef); + const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); + const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + if (!existsSync(sourcePath)) return { ok: false, error: "prompt-source-missing", summary }; + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const raw = values[key]; + if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary }; + const parsed = parsePromptJson(raw); + if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary }; + return { + ok: true, + prompts: parsed, + summary: { + ...summary, + promptCount: parsed.length, + promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`), + promptTextBytes: parsed.map((item) => Buffer.byteLength(item)), + valuesRedacted: true, + }, + }; +} + +function parsePromptJson(raw: string): string[] { + try { + const parsed = JSON.parse(raw) as unknown; + if (Array.isArray(parsed)) return parsed.filter((item): item is string => typeof item === "string" && item.length > 0); + const recordValue = record(parsed); + if (Array.isArray(recordValue.prompts)) return recordValue.prompts.filter((item): item is string => typeof item === "string" && item.length > 0); + if (typeof recordValue.prompt === "string" && recordValue.prompt.length > 0) return [recordValue.prompt]; + } catch { + if (raw.trim().length > 0) return [raw]; + } + return []; +} + +function readLocalObserveIndex(observerId: string): { stateDir: string } | null { + const path = rootPath(".state/web-observe/index.json"); + if (!existsSync(path)) return null; + const parsed = parseJsonObject(readFileSync(path, "utf8")); + const entry = record(parsed?.[observerId]); + const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : null; + return stateDir === null ? null : { stateDir }; +} + +function secretSourcePaths(sourceRef: string): string[] { + const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef)); + return [...new Set(paths)]; +} + +function parseEnvFile(textValue: string): Record { + const values: Record = {}; + for (const rawLine of textValue.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const index = line.indexOf("="); + if (index <= 0) continue; + const key = line.slice(0, index).trim(); + let value = line.slice(index + 1).trim(); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1); + values[key] = value; + } + return values; +} + +function observerIdFromText(textValue: string): string | null { + return /\bwebobs-[a-z0-9-]+\b/iu.exec(textValue)?.[0] ?? null; +} + +function remainingSeconds(deadline: number, cap: number): number { + return Math.max(5, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000))); +} + +function metricNames(textValue: unknown): string[] { + if (typeof textValue !== "string") return []; + return textValue.split(/\r?\n/u).map((line) => /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line)?.[1]).filter((item): item is string => typeof item === "string"); +} + +function validationBlocker(health: Record, metrics: Record, report: Record, publicExposure: Record, quickVerify: Record | null): Record { + const blockers = []; + if (!health.ok || record(health.bodyJson).ok !== true) blockers.push("health"); + if (!metrics.ok || !metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health")) blockers.push("metrics"); + if (!report.ok) blockers.push("recent-report"); + if (publicExposure.ok !== true) blockers.push("public-exposure"); + if (quickVerify !== null && quickVerify.ok !== true) blockers.push("quick-verify"); + return { code: "sentinel-validation-failed", blockers, valuesRedacted: true }; +} + +function serviceUnavailableBlocker(state: SentinelCicdState): Record { + return { + code: "sentinel-service-unavailable", + policy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"), + reason: "sentinel service must be reachable through k3s internal Service DNS before quick verify can run; no public/fallback path is used.", + retry: `bun scripts/cli.ts hwlab nodes web-probe sentinel validate --node ${state.spec.nodeId} --lane ${state.spec.lane}`, + valuesRedacted: true, + }; +} + +function sentinelP5Next(state: SentinelCicdState): Record { + const node = state.spec.nodeId; + const lane = state.spec.lane; + return { + validate: `bun scripts/cli.ts hwlab nodes web-probe sentinel validate --node ${node} --lane ${lane}`, + quickVerify: `bun scripts/cli.ts hwlab nodes web-probe sentinel validate --node ${node} --lane ${lane} --quick-verify --confirm --wait`, + maintenanceStart: `bun scripts/cli.ts hwlab nodes web-probe sentinel maintenance start --node ${node} --lane ${lane} --confirm --wait`, + maintenanceStop: `bun scripts/cli.ts hwlab nodes web-probe sentinel maintenance stop --node ${node} --lane ${lane} --confirm --wait`, + report: `bun scripts/cli.ts hwlab nodes web-probe sentinel report --node ${node} --lane ${lane} --view summary`, + }; +} + +function isSafeRelativeStateDir(value: string): boolean { + return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes(".."); +} + +function stringAtNullable(value: unknown, path: string): string | null { + const found = valueAtPath(value, path); + return typeof found === "string" && found.length > 0 ? found : null; +} + +function numberAtNullable(value: unknown, path: string): number | null { + const found = valueAtPath(value, path); + return typeof found === "number" && Number.isFinite(found) ? found : null; +} + +function displayPath(pathValue: string): string { + if (pathValue.startsWith(`${repoRoot}/`)) return pathValue.slice(repoRoot.length + 1); + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + if (index >= 0) { + const mainRoot = repoRoot.slice(0, index); + if (pathValue.startsWith(`${mainRoot}/`)) return pathValue.slice(mainRoot.length + 1); + } + return pathValue; +} + +function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } { + return { + ...compactCommand(result), + stdoutPreview: result.stdout.trim().slice(0, 1200), + stderrPreview: result.stderr.trim().slice(0, 1200), + stdoutTail: result.stdout.trim().slice(-4000), + stderrTail: result.stderr.trim().slice(-4000), + }; +} + +function renderQuickVerifySummary(input: Record): string { + const artifact = record(input.artifactSummary); + const findings = Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : []; + return [ + "Web Probe Sentinel Quick Verify", + "=======================================================", + `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, + `report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${artifact.findingCount ?? findings.length}`, + `publicOrigin=${input.publicOrigin ?? "-"}`, + "", + "Findings", + findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"} ${item.summary ?? item.message ?? ""}`).join("\n"), + ].join("\n"); +} + +function renderMaintenanceResult(result: Record): string { + const serviceHealth = record(result.serviceHealth); + const maintenance = record(result.maintenance); + const quickVerify = record(result.quickVerify); + const planned = record(result.planned); + const blocker = record(result.blocker); + const next = record(result.next); + const maintenanceBody = record(maintenance.bodyJson); + const state = record(maintenanceBody.maintenance); + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]), + "", + table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]), + "", + Object.keys(state).length > 0 + ? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]]) + : Object.keys(planned).length > 0 + ? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]]) + : "MAINTENANCE\n-", + "", + Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), + "", + "NEXT", + ` validate: ${next.validate ?? "-"}`, + ` report: ${next.report ?? "-"}`, + ` maintenance-stop: ${next.maintenanceStop ?? "-"}`, + "", + "DISCLOSURE", + " maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.", + ].join("\n"); +} + +function renderValidateResult(result: Record): string { + const health = record(result.serviceHealth); + const metrics = record(result.metrics); + const report = record(result.report); + const publicExposure = record(result.publicExposure); + const quickVerify = record(result.quickVerify); + const blocker = record(result.blocker); + const next = record(result.next); + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]), + "", + table(["CHECK", "OK", "DETAIL"], [ + ["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl)}`], + ["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`], + ["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`], + ["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`], + ["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`], + ]), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]), + "", + "NEXT", + ` quick-verify: ${next.quickVerify ?? "-"}`, + ` report: ${next.report ?? "-"}`, + ` maintenance-start: ${next.maintenanceStart ?? "-"}`, + "", + "DISCLOSURE", + " validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.", + ].join("\n"); +} + +function renderReportResult(result: Record): string { + const report = record(result.report); + const body = record(report.bodyJson); + const run = record(body.run); + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]), + "", + table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]), + "", + "DISCLOSURE", + " report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.", + ].join("\n"); +} + function sentinelPipelineRunName(state: SentinelCicdState): string { const commit = state.sourceHead.commit ?? "source"; return `hwlab-web-probe-sentinel-${commit.slice(0, 12)}`; @@ -1087,8 +2036,10 @@ function renderControlPlaneResult(result: Record): string { const observed = record(result.observed); const publish = record(result.publish); const flush = record(result.flush); + const publicExposureApply = record(result.publicExposureApply); const argoApply = record(result.argoApply); const blocker = record(result.blocker); + const targetValidation = record(result.targetValidation); const next = record(result.next); const warnings = Array.isArray(result.warnings) ? result.warnings : []; return [ @@ -1104,10 +2055,23 @@ function renderControlPlaneResult(result: Record): string { "", renderObservedStatus(observed), "", + Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ + targetValidation.ok, + targetValidation.status, + targetValidation.scenarioId, + targetValidation.runId, + targetValidation.observerId, + short(targetValidation.reportJsonSha256), + targetValidation.findingCount, + targetValidation.artifactCount, + ]]), + "", Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]), "", Object.keys(flush).length === 0 ? "FLUSH\n-" : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]), "", + Object.keys(publicExposureApply).length === 0 ? "PUBLIC_EXPOSURE_APPLY\n-" : table(["OK", "SECRET", "CADDY", "HOST"], [[publicExposureApply.ok, record(publicExposureApply.secret).ok, record(publicExposureApply.caddy).ok, publicExposureApply.hostname]]), + "", Object.keys(argoApply).length === 0 ? "ARGO_APPLY\n-" : table(["OK", "EXIT", "PREVIEW"], [[argoApply.ok, record(argoApply.result).exitCode, record(argoApply.result).stdoutPreview]]), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), @@ -1325,3 +2289,7 @@ function sha256(textValue: string): string { function shellQuote(value: string): string { return `'${value.replace(/'/gu, "'\\''")}'`; } + +function tomlEscape(value: string): string { + return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"'); +} diff --git a/scripts/src/hwlab-node-web-sentinel-config.ts b/scripts/src/hwlab-node-web-sentinel-config.ts index 6433203a..a9255f14 100644 --- a/scripts/src/hwlab-node-web-sentinel-config.ts +++ b/scripts/src/hwlab-node-web-sentinel-config.ts @@ -108,6 +108,8 @@ const REQUIRED_TARGET_SHAPES: Record): MaintenanceState; planScenarioRun(scenarioId: string, reason: string): Record; + recordRun(input: Record): Record; + report(view: string, runId: string | null): Record; metrics(): string; dashboardHtml(): string; fetch(request: Request): Promise; @@ -172,6 +175,7 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp startedAt: nowIso(), stoppedAt: current.stoppedAt, quickVerifyPlannedAt: current.quickVerifyPlannedAt, + quickVerifyPlannedRunId: current.quickVerifyPlannedRunId, } : { active: false, @@ -180,11 +184,17 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp startedAt: current.startedAt, stoppedAt: nowIso(), quickVerifyPlannedAt: nowIso(), + quickVerifyPlannedRunId: null, }; writeMetadata(db, "maintenance", next); if (!active) { const scenarioId = firstEnabledScenarioId(config); - if (scenarioId !== null) this.planScenarioRun(scenarioId, "maintenance-stop-quick-verify"); + if (scenarioId !== null) { + const planned = this.planScenarioRun(scenarioId, "maintenance-stop-quick-verify"); + const withPlan = { ...next, quickVerifyPlannedRunId: stringOrNull(planned.runId) }; + writeMetadata(db, "maintenance", withPlan); + return withPlan; + } } return next; }, @@ -198,6 +208,12 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp .run(runId, scenarioId, config.node, config.lane, "planned", this.maintenance().active ? 1 : 0, createdAt, createdAt, JSON.stringify({ reason, commandPlan, valuesRedacted: true })); return { ok: true, runId, scenarioId, status: "planned", commandPlanSha256: sha256Json(commandPlan), valuesRedacted: true }; }, + recordRun(input: Record) { + return recordRunResult(config, db, input); + }, + report(view: string, runId: string | null) { + return reportRunView(config, db, view, runId); + }, metrics() { return renderMetrics(config, db, this.health(), this.maintenance()); }, @@ -239,6 +255,16 @@ async function sentinelFetch(service: WebProbeSentinelService, request: Request) const body = await readJsonBody(request); return jsonResponse(service.planScenarioRun(stringField(body, "scenarioId"), stringOrNull(body.reason) ?? "manual")); } + if (request.method === "POST" && url.pathname === "/api/runs/record") { + const body = await readJsonBody(request); + return jsonResponse(service.recordRun(body)); + } + if (request.method === "GET" && url.pathname === "/api/report") { + const view = url.searchParams.get("view") ?? stringOrNull(service.config.reportViews.defaultView) ?? "summary"; + const runId = url.searchParams.get("run") ?? url.searchParams.get("runId"); + const report = service.report(view, runId); + return jsonResponse(report, report.ok === false ? 404 : 200); + } if (request.method === "GET" && url.pathname === "/metrics") { return new Response(service.metrics(), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } }); } @@ -478,7 +504,124 @@ function jsonResponse(value: unknown, status = 200): Response { } function emptyMaintenance(): MaintenanceState { - return { active: false, reason: null, releaseId: null, startedAt: null, stoppedAt: null, quickVerifyPlannedAt: null }; + return { active: false, reason: null, releaseId: null, startedAt: null, stoppedAt: null, quickVerifyPlannedAt: null, quickVerifyPlannedRunId: null }; +} + +function recordRunResult(config: WebProbeSentinelServiceConfig, db: Database, input: Record): Record { + const now = nowIso(); + const runId = stringOrNull(input.runId) ?? `sentinel-run-${Date.now()}-${randomUUID().slice(0, 8)}`; + const scenarioId = stringOrNull(input.scenarioId) ?? firstEnabledScenarioId(config); + if (scenarioId === null) return { ok: false, error: "scenario-missing", valuesRedacted: true }; + const status = stringOrNull(input.status) ?? "analyzed"; + const observerId = stringOrNull(input.observerId); + const stateDir = stringOrNull(input.stateDir); + const reportJsonSha256 = stringOrNull(input.reportJsonSha256); + const findings = arrayRecords(input.findings).slice(0, 50); + const findingCount = numberOr(input.findingCount, findings.length); + const artifactCount = numberOr(input.artifactCount, 0); + const createdAt = stringOrNull(input.createdAt) ?? now; + db.query(` + INSERT INTO runs (id, scenario_id, node, lane, status, observer_id, state_dir, report_json_sha256, finding_count, artifact_count, maintenance, created_at, updated_at, command_plan_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + status = excluded.status, + observer_id = excluded.observer_id, + state_dir = excluded.state_dir, + report_json_sha256 = excluded.report_json_sha256, + finding_count = excluded.finding_count, + artifact_count = excluded.artifact_count, + updated_at = excluded.updated_at + `).run(runId, scenarioId, config.node, config.lane, status, observerId, stateDir, reportJsonSha256, findingCount, artifactCount, thisMaintenanceFlag(input), createdAt, now, JSON.stringify({ source: "recorded-analyze-summary", valuesRedacted: true })); + db.query("DELETE FROM findings WHERE run_id = ?").run(runId); + for (const item of findings) { + const findingId = stringOrNull(item.id) ?? stringOrNull(item.kind) ?? stringOrNull(item.code) ?? "finding"; + const severity = stringOrNull(item.severity) ?? stringOrNull(item.level) ?? "unknown"; + const summary = stringOrNull(item.summary) ?? stringOrNull(item.message) ?? findingId; + db.query("INSERT INTO findings (run_id, finding_id, severity, count, summary, report_json_sha256, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(runId, findingId.slice(0, 160), severity.slice(0, 40), numberOr(item.count, 1), summary.slice(0, 500), reportJsonSha256, now); + } + writeMetadata(db, `run.report.${runId}`, { + runId, + scenarioId, + observerId, + stateDir, + reportJsonSha256, + summary: record(input.summary), + views: record(input.views), + publicOrigin: stringOrNull(input.publicOrigin), + screenshot: record(input.screenshot), + artifactCount, + findingCount, + valuesRedacted: true, + }); + return { ok: true, runId, scenarioId, status, reportJsonSha256, findingCount, artifactCount, valuesRedacted: true }; +} + +function reportRunView(config: WebProbeSentinelServiceConfig, db: Database, view: string, runId: string | null): Record { + if (!stringArrayAt(config.reportViews, "views").includes(view)) { + return { ok: false, error: "unsupported-report-view", view, valuesRedacted: true }; + } + const row = runId === null + ? db.query("SELECT * FROM runs WHERE report_json_sha256 IS NOT NULL ORDER BY updated_at DESC LIMIT 1").get() as Record | null + : db.query("SELECT * FROM runs WHERE id = ?").get(runId) as Record | null; + if (row === null) return { ok: false, error: "report-run-missing", runId, view, valuesRedacted: true }; + const selectedRunId = stringOrNull(row.id); + if (selectedRunId === null) return { ok: false, error: "report-run-id-missing", view, valuesRedacted: true }; + const stored = readMetadata(db, `run.report.${selectedRunId}`) ?? {}; + const findings = db.query("SELECT finding_id, severity, count, summary, report_json_sha256, created_at FROM findings WHERE run_id = ? ORDER BY created_at DESC LIMIT 50") + .all(selectedRunId) as Record[]; + const views = record(stored.views); + const storedView = record(views[view]); + const renderedText = typeof storedView.renderedText === "string" ? storedView.renderedText : view === "summary" ? renderStoredSummary(row, stored, findings) : view === "findings" ? renderStoredFindings(row, findings) : null; + if (renderedText === null) { + return { ok: false, error: "report-view-not-indexed", runId: selectedRunId, view, availableViews: Object.keys(views), valuesRedacted: true }; + } + return { + ok: true, + view, + run: row, + summary: record(stored.summary), + findings, + renderedText, + valuesRedacted: true, + }; +} + +function renderStoredSummary(row: Record, stored: Record, findings: readonly Record[]): string { + const summary = record(stored.summary); + return [ + "Web Probe Sentinel Report", + "=======================================================", + `run=${stringOrNull(row.id) ?? "-"} scenario=${stringOrNull(row.scenario_id) ?? "-"} status=${stringOrNull(row.status) ?? "-"}`, + `observer=${stringOrNull(row.observer_id) ?? "-"} stateDir=${stringOrNull(row.state_dir) ?? "-"}`, + `report=${stringOrNull(row.report_json_sha256) ?? "-"} artifacts=${String(row.artifact_count ?? 0)} findings=${String(row.finding_count ?? findings.length)}`, + `publicOrigin=${stringOrNull(stored.publicOrigin) ?? "-"}`, + `analysisWindow=${JSON.stringify(record(summary.analysisWindow))}`, + "", + "Findings", + findings.length === 0 ? "-" : findings.slice(0, 12).map((item) => `${item.severity ?? "-"} ${item.finding_id ?? "-"} count=${item.count ?? "-"} ${item.summary ?? ""}`).join("\n"), + ].join("\n"); +} + +function renderStoredFindings(row: Record, findings: readonly Record[]): string { + return [ + "Web Probe Sentinel Findings", + "=======================================================", + `run=${stringOrNull(row.id) ?? "-"} report=${stringOrNull(row.report_json_sha256) ?? "-"}`, + findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? "-"} ${item.finding_id ?? "-"} count=${item.count ?? "-"} ${item.summary ?? ""}`).join("\n"), + ].join("\n"); +} + +function thisMaintenanceFlag(input: Record): number { + return input.maintenance === true ? 1 : 0; +} + +function numberOr(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +function arrayRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.map(record) : []; } function checkPath(value: unknown, path: string): unknown { @@ -511,6 +654,12 @@ function arrayAt(value: unknown, path: string): Record[] { return result.filter(record); } +function stringArrayAt(value: unknown, path: string): string[] { + const result = checkPath(value, path); + if (!Array.isArray(result)) throw new Error(`${path} must be an array`); + return result.filter((item): item is string => typeof item === "string" && item.length > 0); +} + function stringField(value: Record, key: string): string { const found = value[key]; if (typeof found !== "string" || found.length === 0) throw new Error(`${key} must be a non-empty string`); diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index ff8de418..31f63170 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -21,7 +21,7 @@ import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectVie import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; -import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; +import { runWebProbeSentinelCommand, type WebProbeSentinelOptions, type WebProbeSentinelReportView } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; @@ -38,10 +38,29 @@ import { displayRepoPath, readBootstrapAdminPasswordMaterial, sleepSync } from " export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSentinelOptions { const [sentinelActionRaw] = args; - if (sentinelActionRaw !== "plan" && sentinelActionRaw !== "status" && sentinelActionRaw !== "image" && sentinelActionRaw !== "control-plane") { - throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane --node NODE --lane vNN [--dry-run|--confirm]"); + if ( + sentinelActionRaw !== "plan" + && sentinelActionRaw !== "status" + && sentinelActionRaw !== "image" + && sentinelActionRaw !== "control-plane" + && sentinelActionRaw !== "validate" + && sentinelActionRaw !== "maintenance" + && sentinelActionRaw !== "report" + ) { + throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane|validate|maintenance|report --node NODE --lane vNN [--dry-run|--confirm]"); } - assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--wait"])); + assertKnownOptions(args, new Set([ + "--node", + "--lane", + "--timeout-seconds", + "--release-id", + "--reason", + "--view", + "--run", + "--run-id", + "--trace-id", + "--sample-seq", + ]), new Set(["--dry-run", "--confirm", "--wait", "--quick-verify", "--raw"])); const node = requiredOption(args, "--node"); assertNodeId(node); const lane = requiredOption(args, "--lane"); @@ -58,12 +77,49 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe const imageAction = args[1]; if (imageAction !== "status" && imageAction !== "build") throw new Error("web-probe sentinel image usage: image status|build --node NODE --lane vNN [--dry-run|--confirm]"); sentinel = { kind: "image", action: imageAction, node, lane, dryRun: imageAction === "build" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds }; - } else { + } else if (sentinelActionRaw === "control-plane") { const controlPlaneAction = args[1]; if (controlPlaneAction !== "plan" && controlPlaneAction !== "apply" && controlPlaneAction !== "status" && controlPlaneAction !== "trigger-current") { throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]"); } sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds }; + } else if (sentinelActionRaw === "maintenance") { + const maintenanceAction = args[1]; + if (maintenanceAction !== "status" && maintenanceAction !== "start" && maintenanceAction !== "stop") { + throw new Error("web-probe sentinel maintenance usage: maintenance status|start|stop --node NODE --lane vNN [--dry-run|--confirm]"); + } + sentinel = { + kind: "maintenance", + action: maintenanceAction, + node, + lane, + dryRun: maintenanceAction === "status" ? dryRun : dryRun || !confirm, + confirm, + wait: args.includes("--wait"), + timeoutSeconds, + releaseId: optionValue(args, "--release-id") ?? null, + reason: optionValue(args, "--reason") ?? null, + quickVerify: maintenanceAction === "stop" || args.includes("--quick-verify"), + }; + } else if (sentinelActionRaw === "validate") { + sentinel = { kind: "validate", action: "validate", node, lane, dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, quickVerify: args.includes("--quick-verify") }; + } else { + const view = parseWebProbeSentinelReportView(optionValue(args, "--view") ?? "summary"); + const sampleSeqRaw = optionValue(args, "--sample-seq") ?? null; + const sampleSeq = sampleSeqRaw === null ? null : Number(sampleSeqRaw); + if (sampleSeq !== null && (!Number.isInteger(sampleSeq) || sampleSeq < 1)) throw new Error("web-probe sentinel report --sample-seq must be a positive integer"); + sentinel = { + kind: "report", + action: "report", + node, + lane, + view, + runId: optionValue(args, "--run") ?? optionValue(args, "--run-id") ?? null, + traceId: optionValue(args, "--trace-id") ?? null, + sampleSeq, + raw: args.includes("--raw"), + timeoutSeconds, + }; } return { action: "sentinel", @@ -73,6 +129,11 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe }; } +function parseWebProbeSentinelReportView(value: string): WebProbeSentinelReportView { + if (value === "summary" || value === "turn-summary" || value === "findings" || value === "trace-frame") return value; + throw new Error(`web-probe sentinel report --view must be summary, turn-summary, findings, or trace-frame; got ${value}`); +} + export function normalizeNodeWebProbeObserveArgs(args: string[]): { args: string[]; id: string | null } { const [observeActionRaw, maybeId, ...rest] = args; if (observeActionRaw !== "start" && maybeId !== undefined && !maybeId.startsWith("--")) {