From 4830410e6898ca49a22993310575851955a5b76a Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 1 Jul 2026 11:42:24 +0000 Subject: [PATCH] feat(web-sentinel): add dashboard manual trigger --- .agents/skills/unidesk-monitor/SKILL.md | 3 +- .../monitor-web.css | 38 +++ .../monitor-web.js | 88 +++++- scripts/src/hwlab-node-help.ts | 1 + scripts/src/hwlab-node-web-sentinel-cicd.ts | 18 +- scripts/src/hwlab-node-web-sentinel-otel.ts | 1 + scripts/src/hwlab-node-web-sentinel-p5.ts | 60 +++- .../src/hwlab-node-web-sentinel-service.ts | 258 ++++++++++++++++++ scripts/src/hwlab-node/web-probe-observe.ts | 4 +- 9 files changed, 464 insertions(+), 7 deletions(-) diff --git a/.agents/skills/unidesk-monitor/SKILL.md b/.agents/skills/unidesk-monitor/SKILL.md index 4b729e8e..3327bbf1 100644 --- a/.agents/skills/unidesk-monitor/SKILL.md +++ b/.agents/skills/unidesk-monitor/SKILL.md @@ -32,12 +32,13 @@ bun scripts/cli.ts web-probe sentinel dashboard screenshot --node --lane bun scripts/cli.ts web-probe sentinel report --node --lane --sentinel --latest --view summary bun scripts/cli.ts web-probe sentinel report --node --lane --sentinel --latest --view summary --raw bun scripts/cli.ts web-probe sentinel report --node --lane --sentinel --latest --view summary --full +bun scripts/cli.ts web-probe sentinel dashboard trigger --node --lane --sentinel bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node --lane --sentinel --confirm trans :k3s kubectl -n get cronjob -l app.kubernetes.io/component=cadence-scheduler trans :k3s kubectl -n create job --from=cronjob/ ``` -For k3s cadence validation, first use the controlled control-plane status/trigger commands, then inspect the rendered CronJob in the target k3s namespace. Manual `kubectl create job --from=cronjob/...` is validation evidence only; persistent cadence changes must be made through YAML/GitOps and redeployed. +For WebUI manual validation, use `web-probe sentinel dashboard trigger` so the remote browser clicks the monitor-web button. Direct `kubectl create job --from=cronjob/...` is a last-resort diagnostic only, not acceptance evidence. For k3s cadence validation, first use the controlled control-plane status/trigger commands, then inspect the rendered CronJob in the target k3s namespace. Persistent cadence changes must be made through YAML/GitOps and redeployed. For long Workbench/user-path evidence, use the normal Web probe surface: diff --git a/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.css b/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.css index 09e5a0fd..9091b0ed 100644 --- a/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.css +++ b/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.css @@ -152,6 +152,44 @@ select { padding: 0 12px; } +.toolbar-button:disabled { + cursor: progress; + opacity: 0.62; +} + +.trigger-button { + border-color: #9fbbb4; + background: #eff7f4; + font-weight: 700; +} + +.trigger-status { + display: inline-flex; + align-items: center; + max-width: 220px; + min-height: 30px; + padding: 0 9px; + border: 1px solid var(--line); + border-radius: 8px; + background: #ffffff; + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; +} + +.trigger-status[data-trigger-state="triggered"], +.trigger-status[data-trigger-state="already-active"] { + border-color: #b9ddc9; + background: var(--green-soft); + color: #17633f; +} + +.trigger-status[data-trigger-state="failed"] { + border-color: #efb3ad; + background: var(--red-soft); + color: #8b1f1a; +} + select { padding: 0 28px 0 10px; } diff --git a/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.js b/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.js index 52e352dd..14c66057 100644 --- a/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.js +++ b/scripts/assets/web-probe-sentinel-monitor-web/monitor-web.js @@ -20,6 +20,9 @@ createApp({ const initialDeepLink = readMonitorDeepLink(); const loading = ref(true); const error = ref(""); + const manualTriggering = ref(false); + const manualTriggerResult = ref(null); + const manualTriggerError = ref(""); const overview = ref(null); const runs = ref([]); const findings = ref([]); @@ -189,6 +192,20 @@ createApp({ text: `${key} ${value?.ok === false ? "异常" : "ok"}`, })); }); + const manualTriggerState = computed(() => { + if (manualTriggering.value) return "triggering"; + if (manualTriggerError.value) return "failed"; + return String(manualTriggerResult.value?.status || "idle"); + }); + const manualTriggerJobName = computed(() => String(manualTriggerResult.value?.jobName || "")); + const manualTriggerStatusText = computed(() => { + if (manualTriggering.value) return "触发中"; + if (manualTriggerError.value) return manualTriggerError.value; + const result = manualTriggerResult.value || {}; + if (result.status === "triggered") return `已触发 ${shortId(result.jobName)}`; + if (result.status === "already-active") return `运行中 ${shortId(result.jobName)}`; + return ""; + }); async function loadAll(options = {}) { if (!options.silent) loading.value = true; @@ -254,6 +271,27 @@ createApp({ void loadAll(); } + async function triggerQuickVerify() { + if (manualTriggering.value) return; + manualTriggering.value = true; + manualTriggerError.value = ""; + manualTriggerResult.value = null; + try { + const payload = await postJson("/api/quick-verify/trigger", { + source: "monitor-web", + reason: "manual-web-ui", + valuesRedacted: true, + }); + manualTriggerResult.value = payload; + autoRefresh.value = true; + await loadAll({ silent: true }); + } catch (cause) { + manualTriggerError.value = String(cause?.message || cause).slice(0, 160); + } finally { + manualTriggering.value = false; + } + } + function selectCheckRun() { const run = runs.value.find((item) => item.id === checkRunId.value); if (run) void selectRun(run); @@ -429,6 +467,9 @@ createApp({ bootstrap, loading, error, + manualTriggering, + manualTriggerResult, + manualTriggerError, overview, runs, findings, @@ -480,12 +521,16 @@ createApp({ checkScopeText, cadence, healthChecks, + manualTriggerState, + manualTriggerJobName, + manualTriggerStatusText, loadAll, selectRun, selectCheckRun, openCheckDetail, closeCheckDetail, refreshNow, + triggerQuickVerify, currentHref, showTrendTooltip, hideTrendTooltip, @@ -532,7 +577,13 @@ createApp({ }; }, template: ` -
+
@@ -554,6 +605,26 @@ createApp({ + + + {{ manualTriggerStatusText }} +
@@ -1049,6 +1120,21 @@ async function fetchJson(path) { return await response.json(); } +async function postJson(path, body) { + const response = await fetch(apiUrl(path), { + method: "POST", + cache: "no-store", + headers: { "content-type": "application/json", accept: "application/json" }, + body: JSON.stringify(body || {}), + }); + const payload = await response.json().catch(() => ({})); + if (!response.ok || payload.ok === false) { + const reason = payload.reason || payload.error || `${path} HTTP ${response.status}`; + throw new Error(String(reason)); + } + return payload; +} + function apiUrl(path) { const prefix = bootstrap.basePath || ""; const suffix = path.startsWith("/") ? path : `/${path}`; diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index afd389cc..bd41b407 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -77,6 +77,7 @@ export function hwlabNodeWebProbeHelp(): Record { "bun scripts/cli.ts web-probe sentinel publish-current --node JD01 --lane v03 --sentinel jd01-web-probe-sentinel --confirm --wait", "bun scripts/cli.ts web-probe sentinel dashboard verify --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x", "bun scripts/cli.ts web-probe sentinel dashboard screenshot --node D601 --lane v03 --sentinel workbench-auth-session-switch-2users", + "bun scripts/cli.ts web-probe sentinel dashboard trigger --node JD01 --lane v03 --sentinel jd01-web-probe-sentinel", "bun scripts/cli.ts web-probe sentinel report --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --latest --view summary --raw", "bun scripts/cli.ts web-probe sentinel report --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --latest --view summary --full", "bun scripts/cli.ts web-probe sentinel maintenance stop --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --confirm --wait --release-id ", diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index c63feb5b..2dfd3416 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -29,7 +29,7 @@ export type WebProbeSentinelImageAction = "status" | "build"; export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current"; export type WebProbeSentinelPublishAction = "publish-current"; export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop"; -export type WebProbeSentinelDashboardAction = "verify" | "screenshot"; +export type WebProbeSentinelDashboardAction = "verify" | "screenshot" | "trigger"; export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary"; export type WebProbeSentinelOptions = @@ -1098,6 +1098,22 @@ function renderSentinelManifests( kind: "ServiceAccount", metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels }, }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels }, + rules: [ + { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get"] }, + { apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "list"] }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels }, + subjects: [{ kind: "ServiceAccount", name: stringAt(runtime, "serviceAccountName"), namespace }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${deploymentName}-manual-trigger` }, + }, { apiVersion: "v1", kind: "PersistentVolumeClaim", diff --git a/scripts/src/hwlab-node-web-sentinel-otel.ts b/scripts/src/hwlab-node-web-sentinel-otel.ts index 36fd774b..6f8b37b1 100644 --- a/scripts/src/hwlab-node-web-sentinel-otel.ts +++ b/scripts/src/hwlab-node-web-sentinel-otel.ts @@ -70,6 +70,7 @@ export function webProbeSentinelOtelSummary(context: SentinelOtelContext): Recor "web_probe_sentinel.cadence.expected", "web_probe_sentinel.cadence.cronjob_rendered", "web_probe_sentinel.cadence.cronjob_observed", + "web_probe_sentinel.manual_trigger", "web_probe_sentinel.quick_verify.job_start", "web_probe_sentinel.quick_verify.job_finish", "web_probe_sentinel.record_run", diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index a838c139..b6bdb69c 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -537,6 +537,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: `export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`, `export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`, `export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`, + `export UNIDESK_SENTINEL_DASHBOARD_TRIGGER=${shellQuote(options.action === "trigger" ? "1" : "0")}`, `export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`, `export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`, `export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, @@ -633,6 +634,7 @@ const { chromium } = await import(playwrightModuleSpecifier); const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL; const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || ""; const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1"; +const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1"; const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440); const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900); const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000); @@ -709,7 +711,32 @@ if (requestedRunId) { await page.waitForTimeout(150); } -const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection }) => { +let manualTrigger = { requested: triggerManual, ok: !triggerManual, status: triggerManual ? "not-attempted" : "not-requested", jobName: null, statusText: "" }; +if (triggerManual) { + manualTrigger = await page.evaluate(async () => { + const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms)); + const root = document.querySelector("[data-monitor-shell='true']"); + const button = document.querySelector("[data-monitor-manual-trigger='true']"); + if (!(button instanceof HTMLButtonElement)) return { requested: true, ok: false, status: "button-missing", jobName: null, statusText: "" }; + if (button.disabled) return { requested: true, ok: false, status: "button-disabled", jobName: button.getAttribute("data-trigger-job") || null, statusText: "" }; + button.click(); + for (let index = 0; index < 160; index += 1) { + const state = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || ""; + const jobName = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null; + const statusText = String(document.querySelector("[data-monitor-manual-trigger-status='true']")?.textContent || "").replace(/\s+/g, " ").trim(); + if (state === "triggered" || state === "already-active" || state === "failed") { + return { requested: true, ok: state === "triggered" || state === "already-active", status: state, jobName, statusText }; + } + await wait(250); + } + const finalState = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "timeout"; + const finalJob = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null; + return { requested: true, ok: false, status: finalState || "timeout", jobName: finalJob, statusText: "timeout" }; + }).catch((error) => ({ requested: true, ok: false, status: "trigger-error", jobName: null, statusText: String(error?.message || error).slice(0, 300) })); + await page.waitForTimeout(300); +} + +const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger }) => { const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; const objectOrNull = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null; const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim(); @@ -720,6 +747,8 @@ const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId const trendErrorCurve = document.querySelector("[data-monitor-trend-error-curve='true']"); const trendWarningCurve = document.querySelector("[data-monitor-trend-warning-curve='true']"); const memoryChart = document.querySelector("[data-run-memory-chart='true']"); + const manualTriggerButton = document.querySelector("[data-monitor-manual-trigger='true']"); + const manualTriggerStatus = document.querySelector("[data-monitor-manual-trigger-status='true']"); const chartTiming = { hasDurationCurve: Boolean(trend), hasErrorCurve: Boolean(trendErrorCurve), @@ -884,6 +913,16 @@ const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId requestMatched: !requestedRunId || targetRunId === requestedRunId, }, requestedRunSelection, + manualTriggerUi: { + requested: manualTrigger.requested, + ok: manualTrigger.ok, + status: manualTrigger.status, + jobName: manualTrigger.jobName, + buttonPresent: Boolean(manualTriggerButton), + buttonDisabled: manualTriggerButton instanceof HTMLButtonElement ? manualTriggerButton.disabled : null, + buttonState: manualTriggerButton?.getAttribute("data-trigger-state") || null, + statusText: String(manualTriggerStatus?.textContent || "").replace(/\s+/g, " ").trim(), + }, chartTiming, memorySummary, api: { @@ -901,7 +940,7 @@ const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId overflow: overflow.slice(0, 2), }, }; -}, { expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection }); +}, { expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger }); if (captureScreenshot && screenshotPath) { if (requestedRunId && dom.memorySummary?.present === true && dom.memorySummary?.matchesTargetRun === true) { @@ -932,6 +971,7 @@ const ok = navigationOk && dom.sentinelBoundary?.routePrefixMatches === true && dom.errorVisible !== true && dom.requestedRunSelection?.ok === true + && manualTrigger.ok === true && dom.chartTiming?.ok === true && dom.memorySummary?.contractOk === true && dom.layout?.horizontalOverflow !== true @@ -946,6 +986,7 @@ console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({ executablePath: executablePath || null, viewport: { width, height }, screenshotPath: captureScreenshot ? screenshotPath : null, + manualTrigger, dom, consoleCount: consoleMessages.length, consoleErrorCount: consoleErrors.length, @@ -1006,6 +1047,8 @@ function dashboardDegradedReason(result: CommandResult, transport: Record): string { const chartTiming = record(dom.chartTiming); const memorySummary = record(dom.memorySummary); const requestedRunSelection = record(dom.requestedRunSelection); + const manualTrigger = record(page.manualTrigger); + const manualTriggerUi = record(dom.manualTriggerUi); const screenshot = record(result.screenshot); const remote = record(result.remote); const transport = record(result.transport); @@ -1464,6 +1509,16 @@ function renderDashboardResult(result: Record): string { chartTiming.hasWarningCurve ?? "-", ]]), "", + manualTrigger.requested === true + ? table(["TRIGGER", "STATUS", "JOB", "UI_STATE", "UI_TEXT"], [[ + manualTrigger.ok ?? "-", + manualTrigger.status ?? "-", + manualTrigger.jobName ?? "-", + manualTriggerUi.buttonState ?? "-", + manualTriggerUi.statusText ?? "-", + ]]) + : "TRIGGER\n-", + "", table(["MEMORY_CHART", "MEMORY_RUN", "MEMORY_MATCH", "MEMORY_PAGES", "MEMORY_SAMPLES", "API_PAGES", "API_SAMPLES", "CONTRACT", "STATUS", "MEMORY_SOURCE"], [[ memorySummary.present ?? "-", memorySummary.runId ?? "-", @@ -1491,6 +1546,7 @@ function renderDashboardResult(result: Record): string { degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT", "POLL_PHASE"], [[degradedReason, remote.exitCode, transport.ok, pollFailure.phase ?? "-"]]), "", "NEXT", + ` trigger: bun scripts/cli.ts web-probe sentinel dashboard trigger --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, ` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, ` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, "", diff --git a/scripts/src/hwlab-node-web-sentinel-service.ts b/scripts/src/hwlab-node-web-sentinel-service.ts index 5f34adb4..83275910 100644 --- a/scripts/src/hwlab-node-web-sentinel-service.ts +++ b/scripts/src/hwlab-node-web-sentinel-service.ts @@ -9,6 +9,7 @@ import { Buffer } from "node:buffer"; import { createHash, randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { request as httpsRequest } from "node:https"; import { join } from "node:path"; import { Database } from "bun:sqlite"; import { renderWebProbeSentinelDashboardHtml, webProbeSentinelDashboardAssetResponse } from "./hwlab-node-web-sentinel-dashboard-assets"; @@ -82,6 +83,7 @@ export interface WebProbeSentinelService { maintenance(): MaintenanceState; setMaintenance(active: boolean, input: Record): MaintenanceState; planScenarioRun(scenarioId: string, reason: string): Record; + triggerQuickVerify(input: Record): Promise>; recordRun(input: Record): Record; report(view: string, runId: string | null): Record; metrics(): string; @@ -257,6 +259,12 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp .run(runId, config.sentinelId, scenarioId, config.node, config.lane, "planned", this.maintenance().active ? 1 : 0, createdAt, createdAt, JSON.stringify({ reason, commandPlan, valuesRedacted: true })); return { ok: true, runId, scenarioId, status: "planned", commandPlanSha256: sha256Json(commandPlan), valuesRedacted: true }; }, + async triggerQuickVerify(input: Record) { + const result = await triggerQuickVerifyCronJob(config, input); + writeMetadata(db, "manual-trigger.latest", result); + emitManualTriggerSpan(config, result); + return result; + }, recordRun(input: Record) { const result = recordRunResult(config, db, input); emitRecordRunSpan(config, input, result); @@ -334,6 +342,11 @@ async function sentinelFetch(service: WebProbeSentinelService, request: Request) const body = await readJsonBody(request); return jsonResponse(service.planScenarioRun(stringField(body, "scenarioId"), stringOrNull(body.reason) ?? "manual")); } + if (request.method === "POST" && pathname === "/api/quick-verify/trigger") { + const body = await readJsonBody(request); + const result = await service.triggerQuickVerify(body); + return jsonResponse(result, result.ok === false ? 409 : 200); + } if (request.method === "POST" && pathname === "/api/runs/record") { const body = await readJsonBody(request); return jsonResponse(service.recordRun(body)); @@ -691,6 +704,17 @@ function emitRecordRunSpan(config: WebProbeSentinelServiceConfig, input: Record< }, result.ok === true); } +function emitManualTriggerSpan(config: WebProbeSentinelServiceConfig, result: Record): void { + emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.manual_trigger", { + scenarioId: result.scenarioId ?? null, + status: result.status ?? null, + jobName: result.jobName ?? null, + cronJobName: result.cronJobName ?? null, + failureKind: result.ok === true ? null : result.reason ?? "manual-trigger-failed", + valuesRedacted: true, + }, result.ok === true); +} + function sentinelOtelContext(config: WebProbeSentinelServiceConfig): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record; readonly cicd: Record } { return { node: config.node, @@ -2002,6 +2026,230 @@ function firstEnabledScenarioId(config: WebProbeSentinelServiceConfig): string | return scenario === undefined ? null : stringAt(scenario, "id"); } +async function triggerQuickVerifyCronJob(config: WebProbeSentinelServiceConfig, input: Record): Promise> { + const namespace = stringAt(config.runtime, "namespace"); + const cronJobName = sentinelCadenceCronJobName(config); + const scenarioId = stringOrNull(input.scenarioId) + ?? stringAtNullable(config.cicd, "targetValidation.scenarioId") + ?? firstEnabledScenarioId(config); + const reason = stringOrNull(input.reason) ?? "manual-web-ui"; + const source = stringOrNull(input.source) ?? "monitor-web"; + if (scenarioId === null) { + return { ok: false, status: "blocked", reason: "scenario-missing", namespace, cronJobName, valuesRedacted: true }; + } + + const activeJobs = await k8sApiJson("GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs?labelSelector=${encodeURIComponent(`unidesk.ai/web-probe-sentinel-id=${config.sentinelId}`)}`, null); + if (activeJobs.ok === true) { + const active = k8sJobItems(activeJobs.bodyJson).filter((job) => numberAtNullable(job, "status.active") > 0); + if (active.length > 0) { + return { + ok: true, + status: "already-active", + mutation: false, + reason: "sentinel-quick-verify-job-already-active", + namespace, + cronJobName, + jobName: stringAtNullable(active[0], "metadata.name"), + activeJobCount: active.length, + scenarioId, + source, + valuesRedacted: true, + }; + } + } + + const cronJob = await k8sApiJson("GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/cronjobs/${encodeURIComponent(cronJobName)}`, null); + if (cronJob.ok !== true) { + return { + ok: false, + status: "blocked", + reason: "cadence-cronjob-read-failed", + namespace, + cronJobName, + scenarioId, + source, + k8s: compactK8sApiResult(cronJob), + valuesRedacted: true, + }; + } + + const now = nowIso(); + const jobName = manualQuickVerifyJobName(config, now); + const job = jobFromCronJobTemplate(config, record(cronJob.bodyJson), jobName, now, reason, source); + const created = await k8sApiJson("POST", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs`, job); + if (created.ok !== true) { + return { + ok: false, + status: "blocked", + reason: "manual-job-create-failed", + namespace, + cronJobName, + jobName, + scenarioId, + source, + k8s: compactK8sApiResult(created), + valuesRedacted: true, + }; + } + + return { + ok: true, + status: "triggered", + mutation: true, + namespace, + cronJobName, + jobName, + scenarioId, + source, + createdAt: now, + k8s: compactK8sApiResult(created), + valuesRedacted: true, + }; +} + +function jobFromCronJobTemplate( + config: WebProbeSentinelServiceConfig, + cronJob: Record, + jobName: string, + triggeredAt: string, + reason: string, + source: string, +): Record { + const namespace = stringAt(config.runtime, "namespace"); + const jobTemplate = record(valueAtPath(cronJob, "spec.jobTemplate")); + const jobSpec = cloneJson(record(jobTemplate.spec)); + if (Object.keys(jobSpec).length === 0) throw new Error("cadence CronJob jobTemplate.spec missing"); + const templateMetadata = record(valueAtPath(jobSpec, "template.metadata")); + const templateLabels = labelRecord(templateMetadata.labels); + const cronJobLabels = labelRecord(record(cronJob.metadata).labels); + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName, + namespace, + labels: { + ...cronJobLabels, + ...templateLabels, + "unidesk.ai/manual-trigger": "true", + "unidesk.ai/trigger-source": safeKubernetesLabelValue(source), + }, + annotations: { + "unidesk.ai/trigger-source": source, + "unidesk.ai/trigger-reason": reason, + "unidesk.ai/triggered-at": triggeredAt, + "unidesk.ai/cronjob-name": stringAt(cronJob, "metadata.name"), + }, + }, + spec: jobSpec, + }; +} + +async function k8sApiJson(method: "GET" | "POST", path: string, body: Record | null): Promise> { + const host = process.env.KUBERNETES_SERVICE_HOST; + const port = Number(process.env.KUBERNETES_SERVICE_PORT_HTTPS ?? process.env.KUBERNETES_SERVICE_PORT ?? 443); + if (typeof host !== "string" || host.length === 0) return { ok: false, status: "blocked", reason: "kubernetes-service-env-missing", valuesRedacted: true }; + const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"; + const caPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; + if (!existsSync(tokenPath)) return { ok: false, status: "blocked", reason: "serviceaccount-token-missing", valuesRedacted: true }; + const bodyText = body === null ? "" : JSON.stringify(body); + const token = readFileSync(tokenPath, "utf8").trim(); + const ca = existsSync(caPath) ? readFileSync(caPath) : undefined; + return await new Promise((resolve) => { + const req = httpsRequest({ + host, + port, + path, + method, + ca, + timeout: 12_000, + headers: { + accept: "application/json", + authorization: `Bearer ${token}`, + ...(body === null ? {} : { "content-type": "application/json", "content-length": Buffer.byteLength(bodyText) }), + }, + }, (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); + res.on("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + const bodyJson = parseJsonRecord(text); + const httpStatus = Number(res.statusCode ?? 0); + resolve({ + ok: httpStatus >= 200 && httpStatus < 300, + httpStatus, + bodyJson, + bodyTextPreview: bodyJson === null ? text.slice(0, 1000) : "", + valuesRedacted: true, + }); + }); + }); + req.on("timeout", () => req.destroy(new Error("kubernetes-api-timeout"))); + req.on("error", (error) => { + resolve({ ok: false, httpStatus: null, error: String(error?.message || error).slice(0, 500), valuesRedacted: true }); + }); + if (bodyText.length > 0) req.write(bodyText); + req.end(); + }); +} + +function compactK8sApiResult(result: Record): Record { + const body = record(result.bodyJson); + return { + ok: result.ok === true, + httpStatus: result.httpStatus ?? null, + status: body.status ?? null, + reason: body.reason ?? result.reason ?? null, + message: typeof body.message === "string" ? body.message.slice(0, 300) : result.error ?? null, + valuesRedacted: true, + }; +} + +function k8sJobItems(value: unknown): Record[] { + const items = record(value).items; + return Array.isArray(items) ? items.map(record) : []; +} + +function sentinelCadenceCronJobName(config: WebProbeSentinelServiceConfig): string { + return safeKubernetesSegment(`${stringAt(config.runtime, "deploymentName")}-quick-verify`, 52); +} + +function manualQuickVerifyJobName(config: WebProbeSentinelServiceConfig, now: string): string { + const stamp = Date.parse(now).toString(36).replace(/[^a-z0-9]/giu, "").toLowerCase(); + const prefix = `wps-${config.node.toLowerCase()}-qv`; + return safeKubernetesSegment(`${prefix}-${stamp}-${randomUUID().slice(0, 5)}`, 63); +} + +function safeKubernetesSegment(value: string, maxLength: number): string { + const cleaned = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, "").replace(/-{2,}/gu, "-"); + const clipped = cleaned.slice(0, maxLength).replace(/-+$/gu, ""); + return clipped.length > 0 ? clipped : "web-probe-sentinel"; +} + +function safeKubernetesLabelValue(value: string): string { + return safeKubernetesSegment(value, 63); +} + +function labelRecord(value: unknown): Record { + const result: Record = {}; + for (const [key, raw] of Object.entries(record(value))) { + if (typeof raw === "string" && raw.length > 0) result[key] = raw; + } + return result; +} + +function cloneJson(value: Record): Record { + return JSON.parse(JSON.stringify(value)) as Record; +} + +function parseJsonRecord(text: string): Record | null { + try { + return record(JSON.parse(text) as unknown); + } catch { + return null; + } +} + async function readJsonBody(request: Request): Promise> { if ((request.headers.get("content-length") ?? "0") === "0") return {}; const value = await request.json().catch(() => ({})) as unknown; @@ -2226,12 +2474,22 @@ function stringAt(value: unknown, path: string): string { return result; } +function stringAtNullable(value: unknown, path: string): string | null { + const result = valueAtPath(value, path); + return typeof result === "string" && result.length > 0 ? result : null; +} + function numberAt(value: unknown, path: string): number { const result = checkPath(value, path); if (typeof result !== "number" || !Number.isFinite(result)) throw new Error(`${path} must be a number`); return result; } +function numberAtNullable(value: unknown, path: string): number { + const result = valueAtPath(value, path); + return typeof result === "number" && Number.isFinite(result) ? result : 0; +} + function boolAt(value: unknown, path: string): boolean { const result = checkPath(value, path); if (typeof result !== "boolean") throw new Error(`${path} must be a boolean`); diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 1acaebce..ff2d8ca3 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -176,8 +176,8 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe } function parseWebProbeSentinelDashboardAction(value: string | undefined): WebProbeSentinelDashboardAction { - if (value === "verify" || value === "screenshot") return value; - throw new Error("web-probe sentinel dashboard usage: dashboard verify|screenshot --node NODE --lane vNN --sentinel "); + if (value === "verify" || value === "screenshot" || value === "trigger") return value; + throw new Error("web-probe sentinel dashboard usage: dashboard verify|screenshot|trigger --node NODE --lane vNN --sentinel "); } function parseWebProbeSentinelDashboardViewport(value: string): string {