feat(web-sentinel): add dashboard manual trigger
This commit is contained in:
@@ -32,12 +32,13 @@ bun scripts/cli.ts web-probe sentinel dashboard screenshot --node <node> --lane
|
||||
bun scripts/cli.ts web-probe sentinel report --node <node> --lane <lane> --sentinel <id> --latest --view summary
|
||||
bun scripts/cli.ts web-probe sentinel report --node <node> --lane <lane> --sentinel <id> --latest --view summary --raw
|
||||
bun scripts/cli.ts web-probe sentinel report --node <node> --lane <lane> --sentinel <id> --latest --view summary --full
|
||||
bun scripts/cli.ts web-probe sentinel dashboard trigger --node <node> --lane <lane> --sentinel <id>
|
||||
bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node <node> --lane <lane> --sentinel <id> --confirm
|
||||
trans <node>:k3s kubectl -n <namespace> get cronjob -l app.kubernetes.io/component=cadence-scheduler
|
||||
trans <node>:k3s kubectl -n <namespace> create job --from=cronjob/<quick-verify-cronjob> <manual-job-name>
|
||||
```
|
||||
|
||||
For k3s cadence validation, first use the controlled control-plane status/trigger commands, then inspect the rendered CronJob in the target k3s namespace. Manual `kubectl create job --from=cronjob/...` is validation evidence only; persistent cadence changes must be made through YAML/GitOps and redeployed.
|
||||
For WebUI manual validation, use `web-probe sentinel dashboard trigger` so the remote browser clicks the monitor-web button. Direct `kubectl create job --from=cronjob/...` is a last-resort diagnostic only, not acceptance evidence. For k3s cadence validation, first use the controlled control-plane status/trigger commands, then inspect the rendered CronJob in the target k3s namespace. Persistent cadence changes must be made through YAML/GitOps and redeployed.
|
||||
|
||||
For long Workbench/user-path evidence, use the normal Web probe surface:
|
||||
|
||||
|
||||
@@ -152,6 +152,44 @@ select {
|
||||
padding: 0 12px;
|
||||
}
|
||||
|
||||
.toolbar-button:disabled {
|
||||
cursor: progress;
|
||||
opacity: 0.62;
|
||||
}
|
||||
|
||||
.trigger-button {
|
||||
border-color: #9fbbb4;
|
||||
background: #eff7f4;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.trigger-status {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
max-width: 220px;
|
||||
min-height: 30px;
|
||||
padding: 0 9px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: #ffffff;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.trigger-status[data-trigger-state="triggered"],
|
||||
.trigger-status[data-trigger-state="already-active"] {
|
||||
border-color: #b9ddc9;
|
||||
background: var(--green-soft);
|
||||
color: #17633f;
|
||||
}
|
||||
|
||||
.trigger-status[data-trigger-state="failed"] {
|
||||
border-color: #efb3ad;
|
||||
background: var(--red-soft);
|
||||
color: #8b1f1a;
|
||||
}
|
||||
|
||||
select {
|
||||
padding: 0 28px 0 10px;
|
||||
}
|
||||
|
||||
@@ -20,6 +20,9 @@ createApp({
|
||||
const initialDeepLink = readMonitorDeepLink();
|
||||
const loading = ref(true);
|
||||
const error = ref("");
|
||||
const manualTriggering = ref(false);
|
||||
const manualTriggerResult = ref(null);
|
||||
const manualTriggerError = ref("");
|
||||
const overview = ref(null);
|
||||
const runs = ref([]);
|
||||
const findings = ref([]);
|
||||
@@ -189,6 +192,20 @@ createApp({
|
||||
text: `${key} ${value?.ok === false ? "异常" : "ok"}`,
|
||||
}));
|
||||
});
|
||||
const manualTriggerState = computed(() => {
|
||||
if (manualTriggering.value) return "triggering";
|
||||
if (manualTriggerError.value) return "failed";
|
||||
return String(manualTriggerResult.value?.status || "idle");
|
||||
});
|
||||
const manualTriggerJobName = computed(() => String(manualTriggerResult.value?.jobName || ""));
|
||||
const manualTriggerStatusText = computed(() => {
|
||||
if (manualTriggering.value) return "触发中";
|
||||
if (manualTriggerError.value) return manualTriggerError.value;
|
||||
const result = manualTriggerResult.value || {};
|
||||
if (result.status === "triggered") return `已触发 ${shortId(result.jobName)}`;
|
||||
if (result.status === "already-active") return `运行中 ${shortId(result.jobName)}`;
|
||||
return "";
|
||||
});
|
||||
|
||||
async function loadAll(options = {}) {
|
||||
if (!options.silent) loading.value = true;
|
||||
@@ -254,6 +271,27 @@ createApp({
|
||||
void loadAll();
|
||||
}
|
||||
|
||||
async function triggerQuickVerify() {
|
||||
if (manualTriggering.value) return;
|
||||
manualTriggering.value = true;
|
||||
manualTriggerError.value = "";
|
||||
manualTriggerResult.value = null;
|
||||
try {
|
||||
const payload = await postJson("/api/quick-verify/trigger", {
|
||||
source: "monitor-web",
|
||||
reason: "manual-web-ui",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
manualTriggerResult.value = payload;
|
||||
autoRefresh.value = true;
|
||||
await loadAll({ silent: true });
|
||||
} catch (cause) {
|
||||
manualTriggerError.value = String(cause?.message || cause).slice(0, 160);
|
||||
} finally {
|
||||
manualTriggering.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
function selectCheckRun() {
|
||||
const run = runs.value.find((item) => item.id === checkRunId.value);
|
||||
if (run) void selectRun(run);
|
||||
@@ -429,6 +467,9 @@ createApp({
|
||||
bootstrap,
|
||||
loading,
|
||||
error,
|
||||
manualTriggering,
|
||||
manualTriggerResult,
|
||||
manualTriggerError,
|
||||
overview,
|
||||
runs,
|
||||
findings,
|
||||
@@ -480,12 +521,16 @@ createApp({
|
||||
checkScopeText,
|
||||
cadence,
|
||||
healthChecks,
|
||||
manualTriggerState,
|
||||
manualTriggerJobName,
|
||||
manualTriggerStatusText,
|
||||
loadAll,
|
||||
selectRun,
|
||||
selectCheckRun,
|
||||
openCheckDetail,
|
||||
closeCheckDetail,
|
||||
refreshNow,
|
||||
triggerQuickVerify,
|
||||
currentHref,
|
||||
showTrendTooltip,
|
||||
hideTrendTooltip,
|
||||
@@ -532,7 +577,13 @@ createApp({
|
||||
};
|
||||
},
|
||||
template: `
|
||||
<div class="monitor-shell" data-monitor-shell="true" :data-monitor-status="currentStatus">
|
||||
<div
|
||||
class="monitor-shell"
|
||||
data-monitor-shell="true"
|
||||
:data-monitor-status="currentStatus"
|
||||
:data-manual-trigger-state="manualTriggerState"
|
||||
:data-manual-trigger-job="manualTriggerJobName"
|
||||
>
|
||||
<header class="topbar">
|
||||
<div class="title-block">
|
||||
<div class="mark" aria-hidden="true"></div>
|
||||
@@ -554,6 +605,26 @@ createApp({
|
||||
</select>
|
||||
<button class="toolbar-button" type="button" @click="autoRefresh = !autoRefresh">{{ autoRefresh ? "自动" : "手动" }}</button>
|
||||
<button class="toolbar-button" type="button" @click="refreshNow">刷新</button>
|
||||
<button
|
||||
class="toolbar-button trigger-button"
|
||||
data-monitor-manual-trigger="true"
|
||||
type="button"
|
||||
:disabled="manualTriggering"
|
||||
:data-trigger-state="manualTriggerState"
|
||||
:data-trigger-job="manualTriggerJobName"
|
||||
@click="triggerQuickVerify"
|
||||
>
|
||||
{{ manualTriggering ? "触发中" : "触发巡检" }}
|
||||
</button>
|
||||
<span
|
||||
v-if="manualTriggerStatusText"
|
||||
class="trigger-status"
|
||||
data-monitor-manual-trigger-status="true"
|
||||
:data-trigger-state="manualTriggerState"
|
||||
:data-trigger-job="manualTriggerJobName"
|
||||
>
|
||||
{{ manualTriggerStatusText }}
|
||||
</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@@ -1049,6 +1120,21 @@ async function fetchJson(path) {
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
async function postJson(path, body) {
|
||||
const response = await fetch(apiUrl(path), {
|
||||
method: "POST",
|
||||
cache: "no-store",
|
||||
headers: { "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify(body || {}),
|
||||
});
|
||||
const payload = await response.json().catch(() => ({}));
|
||||
if (!response.ok || payload.ok === false) {
|
||||
const reason = payload.reason || payload.error || `${path} HTTP ${response.status}`;
|
||||
throw new Error(String(reason));
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
function apiUrl(path) {
|
||||
const prefix = bootstrap.basePath || "";
|
||||
const suffix = path.startsWith("/") ? path : `/${path}`;
|
||||
|
||||
@@ -77,6 +77,7 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts web-probe sentinel publish-current --node JD01 --lane v03 --sentinel jd01-web-probe-sentinel --confirm --wait",
|
||||
"bun scripts/cli.ts web-probe sentinel dashboard verify --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x",
|
||||
"bun scripts/cli.ts web-probe sentinel dashboard screenshot --node D601 --lane v03 --sentinel workbench-auth-session-switch-2users",
|
||||
"bun scripts/cli.ts web-probe sentinel dashboard trigger --node JD01 --lane v03 --sentinel jd01-web-probe-sentinel",
|
||||
"bun scripts/cli.ts web-probe sentinel report --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --latest --view summary --raw",
|
||||
"bun scripts/cli.ts web-probe sentinel report --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --latest --view summary --full",
|
||||
"bun scripts/cli.ts web-probe sentinel maintenance stop --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --confirm --wait --release-id <id>",
|
||||
|
||||
@@ -29,7 +29,7 @@ export type WebProbeSentinelImageAction = "status" | "build";
|
||||
export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current";
|
||||
export type WebProbeSentinelPublishAction = "publish-current";
|
||||
export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop";
|
||||
export type WebProbeSentinelDashboardAction = "verify" | "screenshot";
|
||||
export type WebProbeSentinelDashboardAction = "verify" | "screenshot" | "trigger";
|
||||
export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary";
|
||||
|
||||
export type WebProbeSentinelOptions =
|
||||
@@ -1098,6 +1098,22 @@ function renderSentinelManifests(
|
||||
kind: "ServiceAccount",
|
||||
metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels },
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels },
|
||||
rules: [
|
||||
{ apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get"] },
|
||||
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "list"] },
|
||||
],
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels },
|
||||
subjects: [{ kind: "ServiceAccount", name: stringAt(runtime, "serviceAccountName"), namespace }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${deploymentName}-manual-trigger` },
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "PersistentVolumeClaim",
|
||||
|
||||
@@ -70,6 +70,7 @@ export function webProbeSentinelOtelSummary(context: SentinelOtelContext): Recor
|
||||
"web_probe_sentinel.cadence.expected",
|
||||
"web_probe_sentinel.cadence.cronjob_rendered",
|
||||
"web_probe_sentinel.cadence.cronjob_observed",
|
||||
"web_probe_sentinel.manual_trigger",
|
||||
"web_probe_sentinel.quick_verify.job_start",
|
||||
"web_probe_sentinel.quick_verify.job_finish",
|
||||
"web_probe_sentinel.record_run",
|
||||
|
||||
@@ -537,6 +537,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options:
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_TRIGGER=${shellQuote(options.action === "trigger" ? "1" : "0")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
|
||||
@@ -633,6 +634,7 @@ const { chromium } = await import(playwrightModuleSpecifier);
|
||||
const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL;
|
||||
const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || "";
|
||||
const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1";
|
||||
const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1";
|
||||
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
|
||||
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
|
||||
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
|
||||
@@ -709,7 +711,32 @@ if (requestedRunId) {
|
||||
await page.waitForTimeout(150);
|
||||
}
|
||||
|
||||
const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection }) => {
|
||||
let manualTrigger = { requested: triggerManual, ok: !triggerManual, status: triggerManual ? "not-attempted" : "not-requested", jobName: null, statusText: "" };
|
||||
if (triggerManual) {
|
||||
manualTrigger = await page.evaluate(async () => {
|
||||
const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms));
|
||||
const root = document.querySelector("[data-monitor-shell='true']");
|
||||
const button = document.querySelector("[data-monitor-manual-trigger='true']");
|
||||
if (!(button instanceof HTMLButtonElement)) return { requested: true, ok: false, status: "button-missing", jobName: null, statusText: "" };
|
||||
if (button.disabled) return { requested: true, ok: false, status: "button-disabled", jobName: button.getAttribute("data-trigger-job") || null, statusText: "" };
|
||||
button.click();
|
||||
for (let index = 0; index < 160; index += 1) {
|
||||
const state = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "";
|
||||
const jobName = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null;
|
||||
const statusText = String(document.querySelector("[data-monitor-manual-trigger-status='true']")?.textContent || "").replace(/\s+/g, " ").trim();
|
||||
if (state === "triggered" || state === "already-active" || state === "failed") {
|
||||
return { requested: true, ok: state === "triggered" || state === "already-active", status: state, jobName, statusText };
|
||||
}
|
||||
await wait(250);
|
||||
}
|
||||
const finalState = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "timeout";
|
||||
const finalJob = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null;
|
||||
return { requested: true, ok: false, status: finalState || "timeout", jobName: finalJob, statusText: "timeout" };
|
||||
}).catch((error) => ({ requested: true, ok: false, status: "trigger-error", jobName: null, statusText: String(error?.message || error).slice(0, 300) }));
|
||||
await page.waitForTimeout(300);
|
||||
}
|
||||
|
||||
const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger }) => {
|
||||
const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0;
|
||||
const objectOrNull = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
||||
const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim();
|
||||
@@ -720,6 +747,8 @@ const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId
|
||||
const trendErrorCurve = document.querySelector("[data-monitor-trend-error-curve='true']");
|
||||
const trendWarningCurve = document.querySelector("[data-monitor-trend-warning-curve='true']");
|
||||
const memoryChart = document.querySelector("[data-run-memory-chart='true']");
|
||||
const manualTriggerButton = document.querySelector("[data-monitor-manual-trigger='true']");
|
||||
const manualTriggerStatus = document.querySelector("[data-monitor-manual-trigger-status='true']");
|
||||
const chartTiming = {
|
||||
hasDurationCurve: Boolean(trend),
|
||||
hasErrorCurve: Boolean(trendErrorCurve),
|
||||
@@ -884,6 +913,16 @@ const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId
|
||||
requestMatched: !requestedRunId || targetRunId === requestedRunId,
|
||||
},
|
||||
requestedRunSelection,
|
||||
manualTriggerUi: {
|
||||
requested: manualTrigger.requested,
|
||||
ok: manualTrigger.ok,
|
||||
status: manualTrigger.status,
|
||||
jobName: manualTrigger.jobName,
|
||||
buttonPresent: Boolean(manualTriggerButton),
|
||||
buttonDisabled: manualTriggerButton instanceof HTMLButtonElement ? manualTriggerButton.disabled : null,
|
||||
buttonState: manualTriggerButton?.getAttribute("data-trigger-state") || null,
|
||||
statusText: String(manualTriggerStatus?.textContent || "").replace(/\s+/g, " ").trim(),
|
||||
},
|
||||
chartTiming,
|
||||
memorySummary,
|
||||
api: {
|
||||
@@ -901,7 +940,7 @@ const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId
|
||||
overflow: overflow.slice(0, 2),
|
||||
},
|
||||
};
|
||||
}, { expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection });
|
||||
}, { expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger });
|
||||
|
||||
if (captureScreenshot && screenshotPath) {
|
||||
if (requestedRunId && dom.memorySummary?.present === true && dom.memorySummary?.matchesTargetRun === true) {
|
||||
@@ -932,6 +971,7 @@ const ok = navigationOk
|
||||
&& dom.sentinelBoundary?.routePrefixMatches === true
|
||||
&& dom.errorVisible !== true
|
||||
&& dom.requestedRunSelection?.ok === true
|
||||
&& manualTrigger.ok === true
|
||||
&& dom.chartTiming?.ok === true
|
||||
&& dom.memorySummary?.contractOk === true
|
||||
&& dom.layout?.horizontalOverflow !== true
|
||||
@@ -946,6 +986,7 @@ console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
|
||||
executablePath: executablePath || null,
|
||||
viewport: { width, height },
|
||||
screenshotPath: captureScreenshot ? screenshotPath : null,
|
||||
manualTrigger,
|
||||
dom,
|
||||
consoleCount: consoleMessages.length,
|
||||
consoleErrorCount: consoleErrors.length,
|
||||
@@ -1006,6 +1047,8 @@ function dashboardDegradedReason(result: CommandResult, transport: Record<string
|
||||
return "sentinel-dashboard-transport-failed";
|
||||
}
|
||||
if (page === null) return "sentinel-dashboard-browser-output-missing";
|
||||
const manualTrigger = record(page.manualTrigger);
|
||||
if (manualTrigger.requested === true && manualTrigger.ok !== true) return "sentinel-dashboard-manual-trigger-failed";
|
||||
if (page.ok !== true) return "sentinel-dashboard-render-failed";
|
||||
if (!screenshotOk) return "sentinel-dashboard-screenshot-missing";
|
||||
return "sentinel-dashboard-unknown";
|
||||
@@ -1415,6 +1458,8 @@ function renderDashboardResult(result: Record<string, unknown>): string {
|
||||
const chartTiming = record(dom.chartTiming);
|
||||
const memorySummary = record(dom.memorySummary);
|
||||
const requestedRunSelection = record(dom.requestedRunSelection);
|
||||
const manualTrigger = record(page.manualTrigger);
|
||||
const manualTriggerUi = record(dom.manualTriggerUi);
|
||||
const screenshot = record(result.screenshot);
|
||||
const remote = record(result.remote);
|
||||
const transport = record(result.transport);
|
||||
@@ -1464,6 +1509,16 @@ function renderDashboardResult(result: Record<string, unknown>): string {
|
||||
chartTiming.hasWarningCurve ?? "-",
|
||||
]]),
|
||||
"",
|
||||
manualTrigger.requested === true
|
||||
? table(["TRIGGER", "STATUS", "JOB", "UI_STATE", "UI_TEXT"], [[
|
||||
manualTrigger.ok ?? "-",
|
||||
manualTrigger.status ?? "-",
|
||||
manualTrigger.jobName ?? "-",
|
||||
manualTriggerUi.buttonState ?? "-",
|
||||
manualTriggerUi.statusText ?? "-",
|
||||
]])
|
||||
: "TRIGGER\n-",
|
||||
"",
|
||||
table(["MEMORY_CHART", "MEMORY_RUN", "MEMORY_MATCH", "MEMORY_PAGES", "MEMORY_SAMPLES", "API_PAGES", "API_SAMPLES", "CONTRACT", "STATUS", "MEMORY_SOURCE"], [[
|
||||
memorySummary.present ?? "-",
|
||||
memorySummary.runId ?? "-",
|
||||
@@ -1491,6 +1546,7 @@ function renderDashboardResult(result: Record<string, unknown>): string {
|
||||
degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT", "POLL_PHASE"], [[degradedReason, remote.exitCode, transport.ok, pollFailure.phase ?? "-"]]),
|
||||
"",
|
||||
"NEXT",
|
||||
` trigger: bun scripts/cli.ts web-probe sentinel dashboard trigger --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
||||
` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
||||
` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
||||
"",
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { request as httpsRequest } from "node:https";
|
||||
import { join } from "node:path";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { renderWebProbeSentinelDashboardHtml, webProbeSentinelDashboardAssetResponse } from "./hwlab-node-web-sentinel-dashboard-assets";
|
||||
@@ -82,6 +83,7 @@ export interface WebProbeSentinelService {
|
||||
maintenance(): MaintenanceState;
|
||||
setMaintenance(active: boolean, input: Record<string, unknown>): MaintenanceState;
|
||||
planScenarioRun(scenarioId: string, reason: string): Record<string, unknown>;
|
||||
triggerQuickVerify(input: Record<string, unknown>): Promise<Record<string, unknown>>;
|
||||
recordRun(input: Record<string, unknown>): Record<string, unknown>;
|
||||
report(view: string, runId: string | null): Record<string, unknown>;
|
||||
metrics(): string;
|
||||
@@ -257,6 +259,12 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp
|
||||
.run(runId, config.sentinelId, scenarioId, config.node, config.lane, "planned", this.maintenance().active ? 1 : 0, createdAt, createdAt, JSON.stringify({ reason, commandPlan, valuesRedacted: true }));
|
||||
return { ok: true, runId, scenarioId, status: "planned", commandPlanSha256: sha256Json(commandPlan), valuesRedacted: true };
|
||||
},
|
||||
async triggerQuickVerify(input: Record<string, unknown>) {
|
||||
const result = await triggerQuickVerifyCronJob(config, input);
|
||||
writeMetadata(db, "manual-trigger.latest", result);
|
||||
emitManualTriggerSpan(config, result);
|
||||
return result;
|
||||
},
|
||||
recordRun(input: Record<string, unknown>) {
|
||||
const result = recordRunResult(config, db, input);
|
||||
emitRecordRunSpan(config, input, result);
|
||||
@@ -334,6 +342,11 @@ async function sentinelFetch(service: WebProbeSentinelService, request: Request)
|
||||
const body = await readJsonBody(request);
|
||||
return jsonResponse(service.planScenarioRun(stringField(body, "scenarioId"), stringOrNull(body.reason) ?? "manual"));
|
||||
}
|
||||
if (request.method === "POST" && pathname === "/api/quick-verify/trigger") {
|
||||
const body = await readJsonBody(request);
|
||||
const result = await service.triggerQuickVerify(body);
|
||||
return jsonResponse(result, result.ok === false ? 409 : 200);
|
||||
}
|
||||
if (request.method === "POST" && pathname === "/api/runs/record") {
|
||||
const body = await readJsonBody(request);
|
||||
return jsonResponse(service.recordRun(body));
|
||||
@@ -691,6 +704,17 @@ function emitRecordRunSpan(config: WebProbeSentinelServiceConfig, input: Record<
|
||||
}, result.ok === true);
|
||||
}
|
||||
|
||||
function emitManualTriggerSpan(config: WebProbeSentinelServiceConfig, result: Record<string, unknown>): void {
|
||||
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.manual_trigger", {
|
||||
scenarioId: result.scenarioId ?? null,
|
||||
status: result.status ?? null,
|
||||
jobName: result.jobName ?? null,
|
||||
cronJobName: result.cronJobName ?? null,
|
||||
failureKind: result.ok === true ? null : result.reason ?? "manual-trigger-failed",
|
||||
valuesRedacted: true,
|
||||
}, result.ok === true);
|
||||
}
|
||||
|
||||
function sentinelOtelContext(config: WebProbeSentinelServiceConfig): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record<string, unknown>; readonly cicd: Record<string, unknown> } {
|
||||
return {
|
||||
node: config.node,
|
||||
@@ -2002,6 +2026,230 @@ function firstEnabledScenarioId(config: WebProbeSentinelServiceConfig): string |
|
||||
return scenario === undefined ? null : stringAt(scenario, "id");
|
||||
}
|
||||
|
||||
async function triggerQuickVerifyCronJob(config: WebProbeSentinelServiceConfig, input: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
const namespace = stringAt(config.runtime, "namespace");
|
||||
const cronJobName = sentinelCadenceCronJobName(config);
|
||||
const scenarioId = stringOrNull(input.scenarioId)
|
||||
?? stringAtNullable(config.cicd, "targetValidation.scenarioId")
|
||||
?? firstEnabledScenarioId(config);
|
||||
const reason = stringOrNull(input.reason) ?? "manual-web-ui";
|
||||
const source = stringOrNull(input.source) ?? "monitor-web";
|
||||
if (scenarioId === null) {
|
||||
return { ok: false, status: "blocked", reason: "scenario-missing", namespace, cronJobName, valuesRedacted: true };
|
||||
}
|
||||
|
||||
const activeJobs = await k8sApiJson("GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs?labelSelector=${encodeURIComponent(`unidesk.ai/web-probe-sentinel-id=${config.sentinelId}`)}`, null);
|
||||
if (activeJobs.ok === true) {
|
||||
const active = k8sJobItems(activeJobs.bodyJson).filter((job) => numberAtNullable(job, "status.active") > 0);
|
||||
if (active.length > 0) {
|
||||
return {
|
||||
ok: true,
|
||||
status: "already-active",
|
||||
mutation: false,
|
||||
reason: "sentinel-quick-verify-job-already-active",
|
||||
namespace,
|
||||
cronJobName,
|
||||
jobName: stringAtNullable(active[0], "metadata.name"),
|
||||
activeJobCount: active.length,
|
||||
scenarioId,
|
||||
source,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const cronJob = await k8sApiJson("GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/cronjobs/${encodeURIComponent(cronJobName)}`, null);
|
||||
if (cronJob.ok !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
reason: "cadence-cronjob-read-failed",
|
||||
namespace,
|
||||
cronJobName,
|
||||
scenarioId,
|
||||
source,
|
||||
k8s: compactK8sApiResult(cronJob),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
const now = nowIso();
|
||||
const jobName = manualQuickVerifyJobName(config, now);
|
||||
const job = jobFromCronJobTemplate(config, record(cronJob.bodyJson), jobName, now, reason, source);
|
||||
const created = await k8sApiJson("POST", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs`, job);
|
||||
if (created.ok !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
reason: "manual-job-create-failed",
|
||||
namespace,
|
||||
cronJobName,
|
||||
jobName,
|
||||
scenarioId,
|
||||
source,
|
||||
k8s: compactK8sApiResult(created),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
status: "triggered",
|
||||
mutation: true,
|
||||
namespace,
|
||||
cronJobName,
|
||||
jobName,
|
||||
scenarioId,
|
||||
source,
|
||||
createdAt: now,
|
||||
k8s: compactK8sApiResult(created),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function jobFromCronJobTemplate(
|
||||
config: WebProbeSentinelServiceConfig,
|
||||
cronJob: Record<string, unknown>,
|
||||
jobName: string,
|
||||
triggeredAt: string,
|
||||
reason: string,
|
||||
source: string,
|
||||
): Record<string, unknown> {
|
||||
const namespace = stringAt(config.runtime, "namespace");
|
||||
const jobTemplate = record(valueAtPath(cronJob, "spec.jobTemplate"));
|
||||
const jobSpec = cloneJson(record(jobTemplate.spec));
|
||||
if (Object.keys(jobSpec).length === 0) throw new Error("cadence CronJob jobTemplate.spec missing");
|
||||
const templateMetadata = record(valueAtPath(jobSpec, "template.metadata"));
|
||||
const templateLabels = labelRecord(templateMetadata.labels);
|
||||
const cronJobLabels = labelRecord(record(cronJob.metadata).labels);
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name: jobName,
|
||||
namespace,
|
||||
labels: {
|
||||
...cronJobLabels,
|
||||
...templateLabels,
|
||||
"unidesk.ai/manual-trigger": "true",
|
||||
"unidesk.ai/trigger-source": safeKubernetesLabelValue(source),
|
||||
},
|
||||
annotations: {
|
||||
"unidesk.ai/trigger-source": source,
|
||||
"unidesk.ai/trigger-reason": reason,
|
||||
"unidesk.ai/triggered-at": triggeredAt,
|
||||
"unidesk.ai/cronjob-name": stringAt(cronJob, "metadata.name"),
|
||||
},
|
||||
},
|
||||
spec: jobSpec,
|
||||
};
|
||||
}
|
||||
|
||||
async function k8sApiJson(method: "GET" | "POST", path: string, body: Record<string, unknown> | null): Promise<Record<string, unknown>> {
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = Number(process.env.KUBERNETES_SERVICE_PORT_HTTPS ?? process.env.KUBERNETES_SERVICE_PORT ?? 443);
|
||||
if (typeof host !== "string" || host.length === 0) return { ok: false, status: "blocked", reason: "kubernetes-service-env-missing", valuesRedacted: true };
|
||||
const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||
const caPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt";
|
||||
if (!existsSync(tokenPath)) return { ok: false, status: "blocked", reason: "serviceaccount-token-missing", valuesRedacted: true };
|
||||
const bodyText = body === null ? "" : JSON.stringify(body);
|
||||
const token = readFileSync(tokenPath, "utf8").trim();
|
||||
const ca = existsSync(caPath) ? readFileSync(caPath) : undefined;
|
||||
return await new Promise((resolve) => {
|
||||
const req = httpsRequest({
|
||||
host,
|
||||
port,
|
||||
path,
|
||||
method,
|
||||
ca,
|
||||
timeout: 12_000,
|
||||
headers: {
|
||||
accept: "application/json",
|
||||
authorization: `Bearer ${token}`,
|
||||
...(body === null ? {} : { "content-type": "application/json", "content-length": Buffer.byteLength(bodyText) }),
|
||||
},
|
||||
}, (res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
res.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
||||
res.on("end", () => {
|
||||
const text = Buffer.concat(chunks).toString("utf8");
|
||||
const bodyJson = parseJsonRecord(text);
|
||||
const httpStatus = Number(res.statusCode ?? 0);
|
||||
resolve({
|
||||
ok: httpStatus >= 200 && httpStatus < 300,
|
||||
httpStatus,
|
||||
bodyJson,
|
||||
bodyTextPreview: bodyJson === null ? text.slice(0, 1000) : "",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
req.on("timeout", () => req.destroy(new Error("kubernetes-api-timeout")));
|
||||
req.on("error", (error) => {
|
||||
resolve({ ok: false, httpStatus: null, error: String(error?.message || error).slice(0, 500), valuesRedacted: true });
|
||||
});
|
||||
if (bodyText.length > 0) req.write(bodyText);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function compactK8sApiResult(result: Record<string, unknown>): Record<string, unknown> {
|
||||
const body = record(result.bodyJson);
|
||||
return {
|
||||
ok: result.ok === true,
|
||||
httpStatus: result.httpStatus ?? null,
|
||||
status: body.status ?? null,
|
||||
reason: body.reason ?? result.reason ?? null,
|
||||
message: typeof body.message === "string" ? body.message.slice(0, 300) : result.error ?? null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function k8sJobItems(value: unknown): Record<string, unknown>[] {
|
||||
const items = record(value).items;
|
||||
return Array.isArray(items) ? items.map(record) : [];
|
||||
}
|
||||
|
||||
function sentinelCadenceCronJobName(config: WebProbeSentinelServiceConfig): string {
|
||||
return safeKubernetesSegment(`${stringAt(config.runtime, "deploymentName")}-quick-verify`, 52);
|
||||
}
|
||||
|
||||
function manualQuickVerifyJobName(config: WebProbeSentinelServiceConfig, now: string): string {
|
||||
const stamp = Date.parse(now).toString(36).replace(/[^a-z0-9]/giu, "").toLowerCase();
|
||||
const prefix = `wps-${config.node.toLowerCase()}-qv`;
|
||||
return safeKubernetesSegment(`${prefix}-${stamp}-${randomUUID().slice(0, 5)}`, 63);
|
||||
}
|
||||
|
||||
function safeKubernetesSegment(value: string, maxLength: number): string {
|
||||
const cleaned = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, "").replace(/-{2,}/gu, "-");
|
||||
const clipped = cleaned.slice(0, maxLength).replace(/-+$/gu, "");
|
||||
return clipped.length > 0 ? clipped : "web-probe-sentinel";
|
||||
}
|
||||
|
||||
function safeKubernetesLabelValue(value: string): string {
|
||||
return safeKubernetesSegment(value, 63);
|
||||
}
|
||||
|
||||
function labelRecord(value: unknown): Record<string, string> {
|
||||
const result: Record<string, string> = {};
|
||||
for (const [key, raw] of Object.entries(record(value))) {
|
||||
if (typeof raw === "string" && raw.length > 0) result[key] = raw;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function cloneJson(value: Record<string, unknown>): Record<string, unknown> {
|
||||
return JSON.parse(JSON.stringify(value)) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function parseJsonRecord(text: string): Record<string, unknown> | null {
|
||||
try {
|
||||
return record(JSON.parse(text) as unknown);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function readJsonBody(request: Request): Promise<Record<string, unknown>> {
|
||||
if ((request.headers.get("content-length") ?? "0") === "0") return {};
|
||||
const value = await request.json().catch(() => ({})) as unknown;
|
||||
@@ -2226,12 +2474,22 @@ function stringAt(value: unknown, path: string): string {
|
||||
return result;
|
||||
}
|
||||
|
||||
function stringAtNullable(value: unknown, path: string): string | null {
|
||||
const result = valueAtPath(value, path);
|
||||
return typeof result === "string" && result.length > 0 ? result : null;
|
||||
}
|
||||
|
||||
function numberAt(value: unknown, path: string): number {
|
||||
const result = checkPath(value, path);
|
||||
if (typeof result !== "number" || !Number.isFinite(result)) throw new Error(`${path} must be a number`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function numberAtNullable(value: unknown, path: string): number {
|
||||
const result = valueAtPath(value, path);
|
||||
return typeof result === "number" && Number.isFinite(result) ? result : 0;
|
||||
}
|
||||
|
||||
function boolAt(value: unknown, path: string): boolean {
|
||||
const result = checkPath(value, path);
|
||||
if (typeof result !== "boolean") throw new Error(`${path} must be a boolean`);
|
||||
|
||||
@@ -176,8 +176,8 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
}
|
||||
|
||||
function parseWebProbeSentinelDashboardAction(value: string | undefined): WebProbeSentinelDashboardAction {
|
||||
if (value === "verify" || value === "screenshot") return value;
|
||||
throw new Error("web-probe sentinel dashboard usage: dashboard verify|screenshot --node NODE --lane vNN --sentinel <id>");
|
||||
if (value === "verify" || value === "screenshot" || value === "trigger") return value;
|
||||
throw new Error("web-probe sentinel dashboard usage: dashboard verify|screenshot|trigger --node NODE --lane vNN --sentinel <id>");
|
||||
}
|
||||
|
||||
function parseWebProbeSentinelDashboardViewport(value: string): string {
|
||||
|
||||
Reference in New Issue
Block a user