refactor: split control-plane cli modules
This commit is contained in:
Codex
@@ -0,0 +1,996 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. observability module for scripts/src/hwlab-node-impl.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-node-impl.ts:568-1599 for #903.
|
||||
|
||||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
||||
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand, type CommandResult } from "../command";
|
||||
import { startJob } from "../jobs";
|
||||
import { classifySshTcpPoolFailure } from "../ssh";
|
||||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
||||
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
||||
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
||||
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
import type { NodeObservabilityOptions } from "./entry";
|
||||
import { runNodeK3sArgs } from "./cleanup";
|
||||
import { nodeRuntimeExpected, nodeRuntimeLocalPostgresExpectedAbsent, parseNodeScopedDelegatedOptions } from "./plan";
|
||||
import { runTransWorkspaceStdinScript, runtimeSecretSpec } from "./public-exposure";
|
||||
import { joinUrlPath, nullableInteger } from "./render";
|
||||
import { compactPrometheusLines, compactRuntimeCommand, isWorkbenchBackendEventMetric, k8sServiceSelector, labelSelectorText, metricNameFromPrometheusLine, parseJsonRecordFromText, prometheusLineHasLabel, prometheusLineMatchesMetric, selectK8sPod } from "./runtime-common";
|
||||
import { compactCommandResultRedacted, record, shellQuote } from "./utils";
|
||||
import { readBootstrapAdminPasswordMaterial } from "./web-probe";
|
||||
import { webProbeCredential } from "./web-probe-observe";
|
||||
|
||||
export function nodeRuntimeControlPlanePlan(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
return {
|
||||
ok: true,
|
||||
command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
mode: "plan",
|
||||
mutation: false,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
expected: nodeRuntimeExpected(scoped.spec),
|
||||
checks: {
|
||||
nodeScopedTargetConfigured: true,
|
||||
externalPostgresDeclared: scoped.spec.externalPostgres !== undefined,
|
||||
secretValuesPrinted: false,
|
||||
runtimeNamespace: scoped.spec.runtimeNamespace,
|
||||
localPostgresExpectedAbsent: nodeRuntimeLocalPostgresExpectedAbsent(scoped.spec),
|
||||
publicExposureDeclared: scoped.spec.publicExposure !== null,
|
||||
},
|
||||
next: {
|
||||
infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
platformDbStatus: scoped.spec.externalPostgres === undefined
|
||||
? null
|
||||
: `bun scripts/cli.ts platform-db postgres status --config ${scoped.spec.externalPostgres.configRef}`,
|
||||
publicExposure: scoped.spec.publicExposure === null ? null : `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function runNodeObservability(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
if (options.action === "plan") return nodeObservabilityPlan(options);
|
||||
if (options.action === "apply") return nodeObservabilityApply(options);
|
||||
if (options.action === "status") return nodeObservabilityStatus(options);
|
||||
if (options.action === "performance-summary") return nodeObservabilityPerformanceSummary(options);
|
||||
return nodeObservabilityWorkbenchSummary(options);
|
||||
}
|
||||
|
||||
export function nodeObservabilityPlan(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
return {
|
||||
ok: configProblem === null,
|
||||
command: `hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
||||
mode: "node-observability-plan",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: nodeObservabilityExpected(options.spec),
|
||||
renderPlan: nodeObservabilityRenderPlan(options.spec.observability),
|
||||
degradedReason: configProblem,
|
||||
next: {
|
||||
dryRun: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
||||
apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityApply(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
const applyPlan = nodeObservabilityApplyPlan(options.spec.observability);
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: configProblem === null && applyPlan.supported === true,
|
||||
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
||||
mode: "node-observability-apply-dry-run",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: nodeObservabilityExpected(options.spec),
|
||||
applyPlan,
|
||||
degradedReason: configProblem ?? (applyPlan.supported === true ? undefined : "node-observability-apply-mode-unsupported"),
|
||||
};
|
||||
}
|
||||
if (configProblem !== null || applyPlan.supported !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
mode: "node-observability-apply",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
applyPlan,
|
||||
degradedReason: configProblem ?? "node-observability-apply-mode-unsupported",
|
||||
};
|
||||
}
|
||||
const status = nodeObservabilityStatus({ ...options, full: true });
|
||||
return {
|
||||
ok: status.ok === true,
|
||||
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
mode: "node-observability-apply",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
applyPlan,
|
||||
appliedResources: [],
|
||||
applyResult: {
|
||||
mode: applyPlan.mode,
|
||||
reason: applyPlan.reason,
|
||||
runtimeValidationOnly: true,
|
||||
},
|
||||
status: options.full ? status : summarizeNodeObservabilityStatus(status),
|
||||
degradedReason: status.ok === true ? undefined : "node-observability-status-not-ready",
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
const serviceStatus = nodeObservabilityServiceStatus(options);
|
||||
const publicRawMetrics = nodeObservabilityPublicRawMetricsProbe(options);
|
||||
const workbenchSummary = nodeObservabilityWorkbenchSummary(options, serviceStatus);
|
||||
const ok = configProblem === null && serviceStatus.ready === true && publicRawMetrics.ok === true && workbenchSummary.ok === true;
|
||||
const status = {
|
||||
ok,
|
||||
command: `hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
||||
mode: "node-observability-status",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: nodeObservabilityExpected(options.spec),
|
||||
service: serviceStatus,
|
||||
publicRawMetrics,
|
||||
workbenchSummary,
|
||||
degradedReason: configProblem
|
||||
?? (serviceStatus.ready === true
|
||||
? publicRawMetrics.ok === true
|
||||
? workbenchSummary.ok === true ? undefined : "node-observability-workbench-metrics-not-ready"
|
||||
: "node-observability-public-raw-metrics-boundary-failed"
|
||||
: "node-observability-service-not-ready"),
|
||||
next: {
|
||||
plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
return options.full ? status : summarizeNodeObservabilityStatus(status);
|
||||
}
|
||||
|
||||
export function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, existingServiceStatus?: Record<string, unknown>): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
const serviceStatus = existingServiceStatus ?? nodeObservabilityServiceStatus(options);
|
||||
const podName = typeof serviceStatus.podName === "string" && serviceStatus.podName.length > 0 ? serviceStatus.podName : null;
|
||||
const metricsProbe = podName === null ? null : nodeObservabilityMetricsProbe(options, podName);
|
||||
const metrics = metricsProbe?.summary ?? summarizeNodeObservabilityMetrics(options.spec.observability, "", null);
|
||||
const ok = configProblem === null && serviceStatus.ready === true && metricsProbe?.ok === true && metrics.ready === true;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
mode: "node-observability-workbench-summary",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
metricsEndpoint: nodeObservabilityEndpointSummary(options.spec.observability),
|
||||
service: {
|
||||
ready: serviceStatus.ready === true,
|
||||
namespace: serviceStatus.namespace,
|
||||
serviceName: serviceStatus.serviceName,
|
||||
podName,
|
||||
containerName: serviceStatus.containerName,
|
||||
},
|
||||
recordingRules: nodeObservabilityRecordingRuleSummaries(options.spec.observability),
|
||||
warningAlerts: nodeObservabilityWarningAlertSummaries(options.spec.observability),
|
||||
metrics,
|
||||
probe: metricsProbe === null ? null : {
|
||||
ok: metricsProbe.ok,
|
||||
httpStatus: metricsProbe.httpStatus,
|
||||
bodyBytes: metricsProbe.bodyBytes,
|
||||
result: compactRuntimeCommand(metricsProbe.result),
|
||||
error: metricsProbe.error,
|
||||
},
|
||||
degradedReason: configProblem
|
||||
?? (serviceStatus.ready === true
|
||||
? metricsProbe?.ok === true
|
||||
? metrics.ready === true ? undefined : "node-observability-required-series-missing"
|
||||
: "node-observability-metrics-endpoint-not-readable"
|
||||
: "node-observability-service-not-ready"),
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityPerformanceSummary(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const workbench = options.spec.observability.workbench;
|
||||
const summaryPath = workbench?.summaryPath ?? "/v1/web-performance/summary";
|
||||
const endpoint = joinUrlPath(options.spec.publicWebUrl, summaryPath);
|
||||
const secretSpec = runtimeSecretSpec({ node: options.node, lane: options.lane });
|
||||
const material = readBootstrapAdminPasswordMaterial(secretSpec);
|
||||
const credential = webProbeCredential(secretSpec, material);
|
||||
const command = `hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`;
|
||||
if (material.ok !== true || material.password === null) {
|
||||
return {
|
||||
ok: false,
|
||||
command,
|
||||
mode: "node-observability-performance-summary",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
target: {
|
||||
workspace: options.spec.workspace,
|
||||
webBaseUrl: options.spec.publicWebUrl,
|
||||
endpoint,
|
||||
},
|
||||
credential,
|
||||
degradedReason: material.error ?? "web-login-secret-unavailable",
|
||||
next: {
|
||||
secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name bootstrap-admin`,
|
||||
webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
const timeoutSeconds = Math.max(10, Math.min(options.timeoutSeconds, 55));
|
||||
const script = nodePerformanceSummaryRemoteScript(options.spec.publicWebUrl, summaryPath, secretSpec.bootstrapAdminUsername, material.password);
|
||||
const result = runTransWorkspaceStdinScript(options.node, options.spec.workspace, script, timeoutSeconds);
|
||||
const report = parseJsonRecordFromText(result.stdout);
|
||||
const performance = record(report.performance);
|
||||
const rows = nodePerformanceRows(performance);
|
||||
const analysis = nodePerformanceAnalysis(performance, rows);
|
||||
const ok = result.exitCode === 0 && report.ok === true && performance.ok !== false;
|
||||
return {
|
||||
ok,
|
||||
command,
|
||||
mode: "node-observability-performance-summary",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
target: {
|
||||
workspace: options.spec.workspace,
|
||||
webBaseUrl: options.spec.publicWebUrl,
|
||||
endpoint,
|
||||
summaryPath,
|
||||
lowSampleThreshold: workbench?.lowSampleThreshold ?? null,
|
||||
},
|
||||
credential,
|
||||
auth: record(report.auth),
|
||||
httpStatus: report.httpStatus ?? null,
|
||||
fetchAttempts: report.fetchAttempts ?? null,
|
||||
summary: performance,
|
||||
analysis,
|
||||
probe: compactCommandResultRedacted(result, [material.password]),
|
||||
degradedReason: ok
|
||||
? undefined
|
||||
: typeof report.error === "string" && report.error.length > 0
|
||||
? report.error
|
||||
: result.exitCode === 0 ? "web-performance-summary-not-ready" : "web-performance-summary-probe-failed",
|
||||
next: {
|
||||
webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
apiMetrics: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane} --full`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function nodePerformanceSummaryRemoteScript(baseUrl: string, summaryPath: string, username: string, password: string): string {
|
||||
const nodeScript = [
|
||||
"const baseUrl = process.env.HWLAB_WEB_BASE_URL;",
|
||||
"const summaryPath = process.env.HWLAB_PERFORMANCE_SUMMARY_PATH || '/v1/web-performance/summary';",
|
||||
"const username = process.env.HWLAB_WEB_USER;",
|
||||
"const password = process.env.HWLAB_WEB_PASS;",
|
||||
"function compactNumber(value) { const n = Number(value); return Number.isFinite(n) ? Math.round(n * 10) / 10 : null; }",
|
||||
"function compactSeconds(msValue, secondsValue) { const ms = Number(msValue); if (Number.isFinite(ms)) return Math.round((ms / 1000) * 10) / 10; return compactNumber(secondsValue); }",
|
||||
"function compactString(value) { return typeof value === 'string' && value.length > 0 ? value : null; }",
|
||||
"function compactRow(row) {",
|
||||
" if (!row || typeof row !== 'object' || Array.isArray(row)) return null;",
|
||||
" const route = compactString(row.route) || compactString(row.path) || compactString(row.name) || compactString(row.metric) || compactString(row.title);",
|
||||
" return {",
|
||||
" source: compactString(row.source) || compactString(row.kind) || compactString(row.category),",
|
||||
" metric: compactString(row.metric) || compactString(row.eventKind) || compactString(row.phase),",
|
||||
" route,",
|
||||
" problem: compactString(row.problem) || compactString(row.diagnosis) || compactString(row.reason),",
|
||||
" outcome: compactString(row.outcome) || compactString(row.result) || compactString(row.status),",
|
||||
" tone: compactString(row.tone) || compactString(row.severity),",
|
||||
" sampleState: compactString(row.sampleState) || compactString(row.sampleStatus),",
|
||||
" count: compactNumber(row.count ?? row.sampleCount ?? row.samples),",
|
||||
" durationUnit: 'seconds',",
|
||||
" p50Seconds: compactSeconds(row.p50Ms, row.p50Seconds ?? row.p50 ?? row.medianSeconds ?? row.median),",
|
||||
" p75Seconds: compactSeconds(row.p75Ms, row.p75Seconds ?? row.p75),",
|
||||
" p95Seconds: compactSeconds(row.p95Ms, row.p95Seconds ?? row.p95),",
|
||||
" maxSeconds: compactSeconds(row.maxMs, row.maxSeconds ?? row.max),",
|
||||
" };",
|
||||
"}",
|
||||
"function rowHasSignal(row) { return Boolean(row && (row.source || row.metric || row.route || row.problem || row.outcome)); }",
|
||||
"function compactRows(value, limit = 16) { return Array.isArray(value) ? value.map(compactRow).filter(rowHasSignal).slice(0, limit) : []; }",
|
||||
"function compactTopN(value) {",
|
||||
" if (!Array.isArray(value)) return [];",
|
||||
" return value.slice(0, 8).map((group) => {",
|
||||
" const rows = compactRows(group?.rows || group?.items || group?.data, 8);",
|
||||
" return { id: compactString(group?.id) || compactString(group?.key), title: compactString(group?.title) || compactString(group?.label), rows };",
|
||||
" }).filter((group) => group.rows.length > 0);",
|
||||
"}",
|
||||
"function compactCards(value) {",
|
||||
" if (!Array.isArray(value)) return [];",
|
||||
" return value.slice(0, 12).map((card) => ({ id: compactString(card?.id), title: compactString(card?.title), value: card?.value ?? null, tone: compactString(card?.tone) || compactString(card?.status), detail: compactString(card?.detail) || compactString(card?.hint) }));",
|
||||
"}",
|
||||
"function compactPerformance(body) {",
|
||||
" if (!body || typeof body !== 'object' || Array.isArray(body)) return { ok: false, degradedReason: 'non-json-summary' };",
|
||||
" const dashboard = body.dashboard && typeof body.dashboard === 'object' ? body.dashboard : {};",
|
||||
" const summary = body.summary && typeof body.summary === 'object' ? body.summary : {};",
|
||||
" const rows = compactRows(body.rows || body.tableRows || body.performanceRows, 24);",
|
||||
" const workbenchRows = compactRows(body.workbenchRows || body.workbench || body.workbenchTableRows, 12);",
|
||||
" const problems = compactRows(body.problems || body.problemRows || dashboard.problems, 24);",
|
||||
" const displayRows = rows.length > 0 ? rows.slice(0, 16) : problems.slice(0, 16);",
|
||||
" const displayProblems = rows.length > 0 ? [] : problems.slice(0, 16);",
|
||||
" const topN = compactTopN(dashboard.topN || body.topN);",
|
||||
" return {",
|
||||
" ok: true,",
|
||||
" schemaVersion: body.schemaVersion || dashboard.schemaVersion || null,",
|
||||
" source: body.source || dashboard.source || null,",
|
||||
" observedAt: body.observedAt || dashboard.observedAt || body.generatedAt || dashboard.generatedAt || null,",
|
||||
" window: body.window || body.sampleWindow || dashboard.window || null,",
|
||||
" freshness: body.freshness || dashboard.freshness || null,",
|
||||
" sampleCount: summary.sampleCount ?? body.sampleCount ?? dashboard.sampleCount ?? null,",
|
||||
" problemCount: summary.problemCount ?? body.problemCount ?? dashboard.problemCount ?? null,",
|
||||
" status: summary.status || body.status || dashboard.status || null,",
|
||||
" cards: compactCards(dashboard.cards || body.cards),",
|
||||
" rows: displayRows,",
|
||||
" workbenchRows,",
|
||||
" problems: displayProblems,",
|
||||
" topN,",
|
||||
" rowCount: rows.length,",
|
||||
" workbenchRowCount: workbenchRows.length,",
|
||||
" problemRowCount: problems.length,",
|
||||
" };",
|
||||
"}",
|
||||
"function splitSetCookie(raw) { return raw ? raw.split(/,(?=[^;,]+=)/g).map((item) => item.trim()).filter(Boolean) : []; }",
|
||||
"async function fetchWithTimeout(url, init, timeoutMs = 12000) {",
|
||||
" const controller = new AbortController();",
|
||||
" const timer = setTimeout(() => controller.abort(), timeoutMs);",
|
||||
" try {",
|
||||
" const response = await fetch(url, { ...init, signal: controller.signal });",
|
||||
" const text = await response.text();",
|
||||
" let body = null;",
|
||||
" try { body = JSON.parse(text); } catch {}",
|
||||
" return { response, text, body };",
|
||||
" } finally { clearTimeout(timer); }",
|
||||
"}",
|
||||
"function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }",
|
||||
"async function fetchWithRetry(url, init, attempts = 3) {",
|
||||
" let last = null;",
|
||||
" for (let attempt = 1; attempt <= attempts; attempt += 1) {",
|
||||
" try {",
|
||||
" const result = await fetchWithTimeout(url, init);",
|
||||
" result.attempt = attempt;",
|
||||
" last = result;",
|
||||
" if (![502, 503, 504].includes(result.response.status)) return result;",
|
||||
" } catch (error) {",
|
||||
" last = { attempt, error: error instanceof Error ? error.message : String(error), response: { status: 0, ok: false, headers: new Headers() }, body: null, text: '' };",
|
||||
" }",
|
||||
" if (attempt < attempts) await sleep(400 * attempt);",
|
||||
" }",
|
||||
" return last;",
|
||||
"}",
|
||||
"(async () => {",
|
||||
" try {",
|
||||
" const loginUrl = new URL('/auth/login', baseUrl).toString();",
|
||||
" const login = await fetchWithRetry(loginUrl, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ username, password }) });",
|
||||
" const setCookie = splitSetCookie(login.response.headers.get('set-cookie'));",
|
||||
" const cookie = setCookie.map((item) => item.split(';')[0].trim()).filter(Boolean).join('; ');",
|
||||
" const auth = { ok: login.response.ok && cookie.length > 0, httpStatus: login.response.status, cookiePresent: cookie.length > 0, attempts: login.attempt || 1 };",
|
||||
" if (!auth.ok) {",
|
||||
" console.log(JSON.stringify({ ok: false, auth, httpStatus: login.response.status, error: 'web-login-failed' }));",
|
||||
" process.exitCode = 1;",
|
||||
" return;",
|
||||
" }",
|
||||
" const summaryUrl = new URL(summaryPath, baseUrl).toString();",
|
||||
" const summary = await fetchWithRetry(summaryUrl, { method: 'GET', headers: { accept: 'application/json', cookie } });",
|
||||
" const performance = compactPerformance(summary.body);",
|
||||
" const ok = summary.response.ok && performance.ok === true;",
|
||||
" console.log(JSON.stringify({ ok, auth, httpStatus: summary.response.status, fetchAttempts: summary.attempt || 1, finalUrl: summaryUrl, performance, error: ok ? null : 'web-performance-summary-fetch-failed' }));",
|
||||
" process.exitCode = ok ? 0 : 1;",
|
||||
" } catch (error) {",
|
||||
" console.log(JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) }));",
|
||||
" process.exitCode = 1;",
|
||||
" }",
|
||||
"})();",
|
||||
].join("\n");
|
||||
return [
|
||||
"set -eu",
|
||||
`HWLAB_WEB_BASE_URL=${shellQuote(baseUrl)} HWLAB_PERFORMANCE_SUMMARY_PATH=${shellQuote(summaryPath)} HWLAB_WEB_USER=${shellQuote(username)} HWLAB_WEB_PASS=${shellQuote(password)} node <<'NODE'`,
|
||||
nodeScript,
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function nodePerformanceRows(performance: Record<string, unknown>): Record<string, unknown>[] {
|
||||
const rows = [
|
||||
...nodePerformanceArray(performance.problems),
|
||||
...nodePerformanceArray(performance.rows),
|
||||
...nodePerformanceArray(performance.workbenchRows),
|
||||
];
|
||||
const topN = Array.isArray(performance.topN) ? performance.topN.map(record) : [];
|
||||
for (const group of topN) {
|
||||
for (const row of nodePerformanceArray(group.rows)) rows.push(row);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
return rows.filter((row) => {
|
||||
const key = [
|
||||
nodePerformanceString(row.source),
|
||||
nodePerformanceString(row.metric),
|
||||
nodePerformanceString(row.route),
|
||||
nodePerformanceString(row.outcome),
|
||||
nodePerformanceString(row.problem),
|
||||
].join("|");
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export function nodePerformanceArray(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : [];
|
||||
}
|
||||
|
||||
export function nodePerformanceAnalysis(performance: Record<string, unknown>, rows: Record<string, unknown>[]): Record<string, unknown> {
|
||||
const actionableRows = rows.filter(nodePerformanceActionableProblem);
|
||||
const networkErrors = actionableRows.filter((row) => nodePerformanceString(row.outcome) === "network_error" || nodePerformanceString(row.problem) === "network_error" || nodePerformanceString(row.problem) === "network error");
|
||||
const blockedRows = actionableRows.filter((row) => nodePerformanceString(row.tone) === "blocked" || nodePerformanceString(row.status) === "blocked");
|
||||
const slowApiRows = rows.filter((row) => {
|
||||
const source = nodePerformanceString(row.source) ?? "";
|
||||
const metric = nodePerformanceString(row.metric) ?? "";
|
||||
const p95Seconds = nodePerformanceNumber(row.p95Seconds);
|
||||
return nodePerformanceActionableProblem(row)
|
||||
&& (source.includes("api") || metric.includes("api") || (nodePerformanceString(row.route) ?? "").startsWith("/v1") || (nodePerformanceString(row.route) ?? "").startsWith("/health"))
|
||||
&& (p95Seconds === null || p95Seconds >= 5);
|
||||
});
|
||||
const lcpRows = actionableRows.filter((row) => {
|
||||
const metric = `${nodePerformanceString(row.metric) ?? ""} ${nodePerformanceString(row.problem) ?? ""}`.toLowerCase();
|
||||
return metric.includes("lcp") || metric.includes("largest contentful") || metric.includes("最大内容");
|
||||
});
|
||||
const maxP95Seconds = Math.max(0, ...rows.map((row) => nodePerformanceNumber(row.p95Seconds) ?? 0));
|
||||
const headline = networkErrors.length > 0
|
||||
? "Public same-origin API has intermittent network_error samples; treat edge/proxy/API reachability as P0 before tuning page render."
|
||||
: lcpRows.length > 0
|
||||
? "Workbench page LCP is blocked, likely by first-load API waterfalls and late visible content."
|
||||
: slowApiRows.length > 0
|
||||
? "Same-origin API p95 is above the interactive budget; split edge wait, server work, and dependency latency."
|
||||
: "No blocking performance rows were returned by the current summary window.";
|
||||
const priorities = [
|
||||
networkErrors.length > 0
|
||||
? {
|
||||
priority: "P0",
|
||||
area: "public-edge-api-reachability",
|
||||
action: "Correlate network_error rows with Caddy/FRP/ingress, cloud-api pod restarts, response-header timeout, and browser Resource Timing failure stage.",
|
||||
evidence: nodePerformanceEvidence(networkErrors),
|
||||
}
|
||||
: null,
|
||||
slowApiRows.length > 0
|
||||
? {
|
||||
priority: "P1",
|
||||
area: "same-origin-api-latency",
|
||||
action: "Add server-side route histograms and dependency spans for slow /health, /v1/live-builds, /v1/workbench/sessions, trace events, hwpod specs, and hwpod-node-ops paths; then optimize the highest p95 route first.",
|
||||
evidence: nodePerformanceEvidence(slowApiRows),
|
||||
}
|
||||
: null,
|
||||
lcpRows.length > 0
|
||||
? {
|
||||
priority: "P1",
|
||||
area: "workbench-lcp",
|
||||
action: "Render a stable first screen before slow session/detail API calls complete, and split LCP by route plus blocking API waterfall.",
|
||||
evidence: nodePerformanceEvidence(lcpRows),
|
||||
}
|
||||
: null,
|
||||
blockedRows.length > 0
|
||||
? {
|
||||
priority: "P2",
|
||||
area: "alert-thresholds-and-sample-quality",
|
||||
action: "Keep low-sample rows visible but avoid closing incidents on count=1; require repeated windows or matching server metrics before labeling a regression fixed.",
|
||||
evidence: nodePerformanceEvidence(blockedRows),
|
||||
}
|
||||
: null,
|
||||
].filter(Boolean);
|
||||
return {
|
||||
headline,
|
||||
sampleCount: performance.sampleCount ?? null,
|
||||
problemCount: performance.problemCount ?? null,
|
||||
rowCount: rows.length,
|
||||
blockedRowCount: blockedRows.length,
|
||||
networkErrorRowCount: networkErrors.length,
|
||||
slowApiRowCount: slowApiRows.length,
|
||||
lcpRowCount: lcpRows.length,
|
||||
maxP95Seconds,
|
||||
priorities,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodePerformanceActionableProblem(row: Record<string, unknown>): boolean {
|
||||
const tone = nodePerformanceString(row.tone) ?? "";
|
||||
const status = nodePerformanceString(row.status) ?? "";
|
||||
if (tone === "blocked" || tone === "warn" || status === "blocked" || status === "warn") return true;
|
||||
const sampleState = nodePerformanceString(row.sampleState) ?? "";
|
||||
if (sampleState && sampleState !== "ok") return false;
|
||||
const problem = (nodePerformanceString(row.problem) ?? "").trim().toLowerCase();
|
||||
return Boolean(problem && problem !== "ok" && problem !== "low-sample" && problem !== "no-sample");
|
||||
}
|
||||
|
||||
export function nodePerformanceEvidence(rows: Record<string, unknown>[]): Record<string, unknown>[] {
|
||||
return rows.slice(0, 8).map((row) => ({
|
||||
source: nodePerformanceString(row.source),
|
||||
metric: nodePerformanceString(row.metric),
|
||||
route: nodePerformanceString(row.route),
|
||||
outcome: nodePerformanceString(row.outcome),
|
||||
problem: nodePerformanceString(row.problem),
|
||||
count: nodePerformanceNumber(row.count),
|
||||
durationUnit: nodePerformanceString(row.durationUnit) ?? "seconds",
|
||||
p50Seconds: nodePerformanceNumber(row.p50Seconds),
|
||||
p75Seconds: nodePerformanceNumber(row.p75Seconds),
|
||||
p95Seconds: nodePerformanceNumber(row.p95Seconds),
|
||||
}));
|
||||
}
|
||||
|
||||
export function nodePerformanceString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
export function nodePerformanceNumber(value: unknown): number | null {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
export function summarizeNodeObservabilityStatus(status: Record<string, unknown>): Record<string, unknown> {
|
||||
const service = record(status.service);
|
||||
const publicRawMetrics = record(status.publicRawMetrics);
|
||||
const workbenchSummary = record(status.workbenchSummary);
|
||||
const metrics = record(workbenchSummary.metrics);
|
||||
const recordingRules = Array.isArray(workbenchSummary.recordingRules) ? workbenchSummary.recordingRules : [];
|
||||
const warningAlerts = Array.isArray(workbenchSummary.warningAlerts) ? workbenchSummary.warningAlerts : [];
|
||||
return {
|
||||
ok: status.ok === true,
|
||||
command: status.command,
|
||||
mode: "node-observability-status-summary",
|
||||
mutation: false,
|
||||
node: status.node,
|
||||
lane: status.lane,
|
||||
degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
|
||||
service: {
|
||||
ready: service.ready === true,
|
||||
namespace: service.namespace ?? null,
|
||||
serviceName: service.serviceName ?? null,
|
||||
podName: service.podName ?? null,
|
||||
containerName: service.containerName ?? null,
|
||||
},
|
||||
publicRawMetrics: {
|
||||
ok: publicRawMetrics.ok === true,
|
||||
expected: publicRawMetrics.expected ?? null,
|
||||
httpStatus: publicRawMetrics.httpStatus ?? null,
|
||||
exposesPrometheus: publicRawMetrics.exposesPrometheus === true,
|
||||
},
|
||||
workbench: {
|
||||
ok: workbenchSummary.ok === true,
|
||||
httpStatus: metrics.httpStatus ?? null,
|
||||
bodyBytes: metrics.bodyBytes ?? null,
|
||||
recordingRuleCount: recordingRules.length,
|
||||
warningAlertCount: warningAlerts.length,
|
||||
seriesByPrefix: metrics.seriesByPrefix ?? {},
|
||||
missingSeries: metrics.missingSeries ?? [],
|
||||
topSlowDimensions: metrics.topSlowDimensions ?? [],
|
||||
deniedBackendEventLineCount: metrics.deniedBackendEventLineCount ?? null,
|
||||
maxDeniedBackendEventLines: metrics.maxDeniedBackendEventLines ?? null,
|
||||
},
|
||||
next: status.next,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
return {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
namespace: spec.runtimeNamespace,
|
||||
publicApiUrl: spec.publicApiUrl,
|
||||
observability: spec.observability,
|
||||
sourceOfTruth: hwlabRuntimeLaneConfigPath(),
|
||||
runtimeAsSourceOfTruth: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityRenderPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
||||
const endpoint = observability.metricsEndpoint;
|
||||
return {
|
||||
prometheusOperator: observability.prometheusOperator,
|
||||
metricsEndpoint: nodeObservabilityEndpointSummary(observability),
|
||||
workbench: observability.workbench ?? null,
|
||||
recordingRules: nodeObservabilityRecordingRuleSummaries(observability),
|
||||
warningAlerts: nodeObservabilityWarningAlertSummaries(observability),
|
||||
clusterResources: observability.prometheusOperator
|
||||
? {
|
||||
source: "HWLAB GitOps rendered manifests",
|
||||
serviceMonitor: "YAML-rendered only when declared by target lane",
|
||||
prometheusRule: "YAML-rendered only when declared by target lane",
|
||||
}
|
||||
: {
|
||||
source: "none",
|
||||
reason: endpoint?.scrapeMode === "pod-loopback" ? "target lane declares pod-loopback scrape mode" : "no YAML-declared scrape target",
|
||||
},
|
||||
publicRawMetrics: endpoint?.publicRawMetrics ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityApplyPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
||||
const endpoint = observability.metricsEndpoint;
|
||||
if (endpoint === undefined) {
|
||||
return {
|
||||
supported: false,
|
||||
mode: "unconfigured",
|
||||
reason: "metricsEndpoint is missing from YAML",
|
||||
};
|
||||
}
|
||||
if (!observability.prometheusOperator && endpoint.scrapeMode === "pod-loopback") {
|
||||
return {
|
||||
supported: true,
|
||||
mode: "validated-noop",
|
||||
reason: "target lane declares pod-loopback metrics collection and no Prometheus Operator resources",
|
||||
resources: [],
|
||||
validates: ["kubernetes-service", "selected-pod", "pod-loopback-metrics", "public-raw-metrics-boundary", "workbench-required-series"],
|
||||
};
|
||||
}
|
||||
if (observability.prometheusOperator) {
|
||||
return {
|
||||
supported: false,
|
||||
mode: "prometheus-operator",
|
||||
reason: "Prometheus Operator object rendering must be declared in the target lane GitOps YAML before this node-scoped apply can own it",
|
||||
};
|
||||
}
|
||||
return {
|
||||
supported: false,
|
||||
mode: endpoint.scrapeMode,
|
||||
reason: "unsupported metricsEndpoint.scrapeMode",
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityConfigProblem(observability: HwlabRuntimeObservabilitySpec): string | null {
|
||||
if (observability.metricsEndpoint === undefined) return "node-observability-metrics-endpoint-not-declared";
|
||||
if (observability.workbench === undefined) return "node-observability-workbench-not-declared";
|
||||
if (!observability.workbench.enabled) return "node-observability-workbench-disabled";
|
||||
return null;
|
||||
}
|
||||
|
||||
export function nodeObservabilityEndpointSummary(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> | null {
|
||||
const endpoint = observability.metricsEndpoint;
|
||||
if (endpoint === undefined) return null;
|
||||
return {
|
||||
serviceName: endpoint.serviceName,
|
||||
containerName: endpoint.containerName,
|
||||
port: endpoint.port,
|
||||
scheme: endpoint.scheme,
|
||||
path: endpoint.path,
|
||||
scrapeMode: endpoint.scrapeMode,
|
||||
publicRawMetrics: endpoint.publicRawMetrics,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityServiceStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const endpoint = options.spec.observability.metricsEndpoint;
|
||||
if (endpoint === undefined) {
|
||||
return {
|
||||
ready: false,
|
||||
namespace: options.spec.runtimeNamespace,
|
||||
serviceName: null,
|
||||
podName: null,
|
||||
containerName: null,
|
||||
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
||||
};
|
||||
}
|
||||
const namespace = runNodeK3sArgs(options.spec, ["kubectl", "get", "ns", options.spec.runtimeNamespace, "-o", "name"], 60);
|
||||
const namespaceExists = namespace.exitCode === 0;
|
||||
const service = namespaceExists
|
||||
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "svc", endpoint.serviceName, "-o", "json"], 60)
|
||||
: null;
|
||||
const serviceJson = service !== null && service.exitCode === 0 ? parseJsonRecordFromText(service.stdout) : {};
|
||||
const selector = k8sServiceSelector(serviceJson);
|
||||
const selectorText = labelSelectorText(selector);
|
||||
const pods = namespaceExists && service !== null && service.exitCode === 0 && selectorText.length > 0
|
||||
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "pods", "-l", selectorText, "-o", "json"], 60)
|
||||
: null;
|
||||
const pod = pods !== null && pods.exitCode === 0 ? selectK8sPod(parseJsonRecordFromText(pods.stdout), endpoint.containerName) : null;
|
||||
const ready = namespaceExists && service !== null && service.exitCode === 0 && pod?.ready === true;
|
||||
return {
|
||||
ready,
|
||||
namespace: options.spec.runtimeNamespace,
|
||||
namespaceExists,
|
||||
serviceName: endpoint.serviceName,
|
||||
serviceExists: service !== null && service.exitCode === 0,
|
||||
selector,
|
||||
selectorText: selectorText.length > 0 ? selectorText : null,
|
||||
podName: pod?.name ?? null,
|
||||
podPhase: pod?.phase ?? null,
|
||||
podReady: pod?.ready ?? false,
|
||||
containerName: endpoint.containerName,
|
||||
containerReady: pod?.containerReady ?? false,
|
||||
probes: {
|
||||
namespace: compactRuntimeCommand(namespace),
|
||||
service: service === null ? null : compactRuntimeCommand(service),
|
||||
pods: pods === null ? null : compactRuntimeCommand(pods),
|
||||
},
|
||||
degradedReason: ready ? undefined : namespaceExists ? "node-observability-pod-or-service-not-ready" : "node-observability-namespace-missing",
|
||||
};
|
||||
}
|
||||
|
||||
export interface NodeObservabilityMetricsProbeResult {
|
||||
readonly ok: boolean;
|
||||
readonly httpStatus: number | null;
|
||||
readonly bodyBytes: number;
|
||||
readonly summary: Record<string, unknown>;
|
||||
readonly result: CommandResult;
|
||||
readonly error: string | null;
|
||||
}
|
||||
|
||||
export function nodeObservabilityMetricsProbe(options: NodeObservabilityOptions, podName: string): NodeObservabilityMetricsProbeResult {
|
||||
const endpoint = options.spec.observability.metricsEndpoint;
|
||||
if (endpoint === undefined) throw new Error("metricsEndpoint is required before probing metrics");
|
||||
const workbench = options.spec.observability.workbench;
|
||||
const summaryConfig = Buffer.from(JSON.stringify({
|
||||
metricPrefixes: workbench?.metricPrefixes ?? [],
|
||||
requiredSeries: workbench?.requiredSeries ?? [],
|
||||
backendLabelDenylist: workbench?.backendLabelDenylist ?? [],
|
||||
maxDeniedBackendEventLines: workbench?.maxUnknownEventLines ?? 0,
|
||||
lowSampleThreshold: workbench?.lowSampleThreshold ?? 0,
|
||||
histogramMetrics: [
|
||||
{ id: "workbench_journey", metric: "hwlab_workbench_journey_duration_seconds", kind: "workbench_journey", dimensionLabels: ["namespace", "gitops_target", "journey", "route", "backend", "transport", "target_state", "cache", "source", "entry", "outcome"] },
|
||||
{ id: "workbench_event_phase", metric: "hwlab_workbench_event_phase_duration_seconds", kind: "workbench_event_phase", dimensionLabels: ["namespace", "gitops_target", "phase", "event_type", "backend", "transport", "outcome"] },
|
||||
{ id: "workbench_backend_event_visible", metric: "hwlab_workbench_backend_event_visible_latency_seconds", kind: "workbench_backend_event_visible", dimensionLabels: ["namespace", "gitops_target", "event_type", "backend", "transport", "outcome"] },
|
||||
{ id: "workbench_projection_lag_events", metric: "hwlab_workbench_projection_lag_events", kind: "workbench_projection_lag_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
||||
{ id: "workbench_projection_lag_seconds", metric: "hwlab_workbench_projection_lag_seconds", kind: "workbench_projection_lag_seconds", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
||||
{ id: "workbench_terminal_projection_delay", metric: "hwlab_workbench_terminal_projection_delay_seconds", kind: "workbench_terminal_projection_delay", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
||||
{ id: "workbench_turn_get", metric: "hwlab_workbench_turn_get_duration_seconds", kind: "workbench_turn_get", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "route", "status", "degraded_reason"] },
|
||||
{ id: "agentrun_result_duration", metric: "hwlab_agentrun_result_duration_seconds", kind: "agentrun_result_duration", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "event_count_bucket", "status"] },
|
||||
{ id: "agentrun_result_pages", metric: "hwlab_agentrun_result_pages_scanned", kind: "agentrun_result_pages", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
|
||||
{ id: "agentrun_result_events", metric: "hwlab_agentrun_result_events_scanned", kind: "agentrun_result_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
|
||||
{ id: "workbench_projector_batch", metric: "hwlab_workbench_projector_batch_duration_seconds", kind: "workbench_projector_batch", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "phase", "status"] },
|
||||
],
|
||||
}), "utf8").toString("base64");
|
||||
const source = [
|
||||
"const http = require('node:http');",
|
||||
"const port = Number(process.argv[1]);",
|
||||
"const path = process.argv[2];",
|
||||
"const config = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
|
||||
"function metricNameFromLine(line) { const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/.exec(line); return match ? match[1] : null; }",
|
||||
"function lineMatchesMetric(line, name) { if (!line.startsWith(name)) return false; const next = line.charAt(name.length); return next === '' || next === '{' || /\\s/.test(next); }",
|
||||
"function lineHasLabel(line, name, value) { return line.includes(`${name}=\\\"${String(value).replace(/\\\\/g, '\\\\\\\\').replace(/\\\"/g, '\\\\\\\"')}\\\"`); }",
|
||||
"function isBackendEventMetric(name) { return name.startsWith('hwlab_workbench_event_') || name.startsWith('hwlab_workbench_backend_event_'); }",
|
||||
"function compactLines(lines) { return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); }",
|
||||
"function parseLabels(raw) { const labels = {}; for (const part of String(raw || '').split(',')) { const index = part.indexOf('='); if (index <= 0) continue; const key = part.slice(0, index); let value = part.slice(index + 1); if (value.startsWith('\\\"') && value.endsWith('\\\"')) value = value.slice(1, -1); labels[key] = value.replace(/\\\\\\\"/g, '\\\"').replace(/\\\\\\\\/g, '\\\\'); } return labels; }",
|
||||
"function labelKey(metric, labels) { const copy = { ...labels }; delete copy.le; return `${metric}|${Object.keys(copy).sort().map((key) => `${key}=${copy[key]}`).join('|')}`; }",
|
||||
"function pickLabels(labels, names) { return Object.fromEntries(names.map((name) => [name, labels[name] || 'unknown'])); }",
|
||||
"function quantile(row, q) { const count = Number(row.count || 0); if (count <= 0) return 0; const rank = Math.max(1, Math.ceil(count * q)); const buckets = row.buckets.filter((item) => item.le !== '+Inf').sort((left, right) => Number(left.le) - Number(right.le)); for (const bucket of buckets) { if (bucket.value >= rank) return Number(bucket.le); } return buckets.length ? Number(buckets[buckets.length - 1].le) : 0; }",
|
||||
"function summarizeHistograms(sampleLines) {",
|
||||
" const metricConfig = new Map(config.histogramMetrics.map((item) => [item.metric, item]));",
|
||||
" const series = new Map();",
|
||||
" for (const line of sampleLines) {",
|
||||
" const match = /^([A-Za-z_:][A-Za-z0-9_:]*?)_(bucket|sum|count)\\{([^}]*)\\}\\s+([0-9.eE+-]+)/.exec(line);",
|
||||
" if (!match || !metricConfig.has(match[1])) continue;",
|
||||
" const metric = match[1];",
|
||||
" const suffix = match[2];",
|
||||
" const labels = parseLabels(match[3]);",
|
||||
" const value = Number(match[4]);",
|
||||
" if (!Number.isFinite(value)) continue;",
|
||||
" const key = labelKey(metric, labels);",
|
||||
" const row = series.get(key) || { metric, labels: { ...labels }, buckets: [], sum: 0, count: 0 };",
|
||||
" if (suffix === 'bucket') row.buckets.push({ le: labels.le || '+Inf', value });",
|
||||
" if (suffix === 'sum') row.sum = value;",
|
||||
" if (suffix === 'count') row.count = value;",
|
||||
" series.set(key, row);",
|
||||
" }",
|
||||
" const lowSampleThreshold = Number(config.lowSampleThreshold || 0);",
|
||||
" const groups = Object.fromEntries(config.histogramMetrics.map((item) => [item.id, []]));",
|
||||
" const rows = [];",
|
||||
" for (const row of series.values()) {",
|
||||
" const cfg = metricConfig.get(row.metric);",
|
||||
" const sampleCount = Number(row.count || 0);",
|
||||
" const output = { kind: cfg.kind, metric: row.metric, sampleCount, average: sampleCount > 0 ? Number((row.sum / sampleCount).toFixed(4)) : 0, p50: quantile(row, 0.5), p75: quantile(row, 0.75), p95: quantile(row, 0.95), lowSample: sampleCount > 0 && sampleCount < lowSampleThreshold, sampleState: sampleCount <= 0 ? 'empty' : sampleCount < lowSampleThreshold ? 'low-sample' : 'ok', dimensions: pickLabels(row.labels, cfg.dimensionLabels) };",
|
||||
" groups[cfg.id].push(output);",
|
||||
" rows.push(output);",
|
||||
" }",
|
||||
" for (const key of Object.keys(groups)) groups[key].sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
|
||||
" rows.sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
|
||||
" return { groups: Object.fromEntries(Object.entries(groups).map(([key, value]) => [key, value.slice(0, 24)])), topSlowDimensions: rows.slice(0, 12) };",
|
||||
"}",
|
||||
"function summarize(text, statusCode) {",
|
||||
" const lines = text.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean);",
|
||||
" const sampleLines = lines.filter((line) => !line.startsWith('#'));",
|
||||
" const seriesByPrefix = Object.fromEntries(config.metricPrefixes.map((prefix) => [prefix, sampleLines.filter((line) => (metricNameFromLine(line) || '').startsWith(prefix)).length]));",
|
||||
" const requiredSeries = config.requiredSeries.map((name) => { const matching = sampleLines.filter((line) => lineMatchesMetric(line, name)); return { name, present: matching.length > 0, sampleCount: matching.length }; });",
|
||||
" const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);",
|
||||
" const deniedBackendEventLines = sampleLines.filter((line) => { const name = metricNameFromLine(line) || ''; if (!isBackendEventMetric(name)) return false; return config.backendLabelDenylist.some((label) => lineHasLabel(line, 'backend', label)); });",
|
||||
" const histogramSummary = summarizeHistograms(sampleLines);",
|
||||
" const ready = statusCode >= 200 && statusCode < 300 && missingSeries.length === 0 && deniedBackendEventLines.length <= config.maxDeniedBackendEventLines;",
|
||||
" return {",
|
||||
" ready,",
|
||||
" httpStatus: statusCode,",
|
||||
" bodyBytes: Buffer.byteLength(text),",
|
||||
" lineCount: lines.length,",
|
||||
" sampleLineCount: sampleLines.length,",
|
||||
" metricPrefixes: config.metricPrefixes,",
|
||||
" seriesByPrefix,",
|
||||
" requiredSeries,",
|
||||
" missingSeries,",
|
||||
" backendLabelDenylist: config.backendLabelDenylist,",
|
||||
" lowSampleThreshold: config.lowSampleThreshold,",
|
||||
" workbenchHistograms: histogramSummary.groups,",
|
||||
" topSlowDimensions: histogramSummary.topSlowDimensions,",
|
||||
" deniedBackendEventLineCount: deniedBackendEventLines.length,",
|
||||
" maxDeniedBackendEventLines: config.maxDeniedBackendEventLines,",
|
||||
" deniedBackendEventLines: compactLines(deniedBackendEventLines),",
|
||||
" backendVisibleCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_backend_event_visible_latency_seconds_count'))),",
|
||||
" phaseCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_event_phase_duration_seconds_count'))),",
|
||||
" projectionLagCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projection_lag_events_count') || lineMatchesMetric(line, 'hwlab_workbench_projection_lag_seconds_count'))),",
|
||||
" turnGetCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_turn_get_duration_seconds_count'))),",
|
||||
" agentRunResultCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_agentrun_result_duration_seconds_count'))),",
|
||||
" projectorBatchCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_batch_duration_seconds_count'))),",
|
||||
" projectorLastSuccessLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_last_success_unixtime'))),",
|
||||
" degradedReason: ready ? undefined : missingSeries.length > 0 ? 'node-observability-required-series-missing' : 'node-observability-denied-backend-label-present',",
|
||||
" };",
|
||||
"}",
|
||||
"let settled = false;",
|
||||
"const finish = (payload, code = 0) => { if (settled) return; settled = true; console.log(JSON.stringify(payload)); process.exitCode = code; };",
|
||||
"const req = http.get({ host: '127.0.0.1', port, path, timeout: 8000 }, (res) => {",
|
||||
" let body = '';",
|
||||
" res.setEncoding('utf8');",
|
||||
" res.on('data', (chunk) => { body += chunk; if (body.length > 2000000) req.destroy(new Error('metrics body exceeded 2MB')); });",
|
||||
" res.on('end', () => finish({ ok: res.statusCode >= 200 && res.statusCode < 300, statusCode: res.statusCode, bodyBytes: Buffer.byteLength(body), summary: summarize(body, res.statusCode || 0) }));",
|
||||
"});",
|
||||
"req.on('timeout', () => req.destroy(new Error('timeout')));",
|
||||
"req.on('error', (error) => finish({ ok: false, statusCode: null, bodyBytes: 0, summary: null, error: String(error && error.message || error) }, 1));",
|
||||
].join("\n");
|
||||
const result = runNodeK3sArgs(options.spec, [
|
||||
"kubectl",
|
||||
"-n", options.spec.runtimeNamespace,
|
||||
"exec", podName,
|
||||
"-c", endpoint.containerName,
|
||||
"--",
|
||||
"node", "-e", source,
|
||||
String(endpoint.port),
|
||||
endpoint.path,
|
||||
summaryConfig,
|
||||
], options.timeoutSeconds);
|
||||
const payload = parseJsonRecordFromText(result.stdout);
|
||||
const httpStatus = typeof payload.statusCode === "number" ? payload.statusCode : null;
|
||||
const error = typeof payload.error === "string" ? payload.error : null;
|
||||
const summary = record(payload.summary);
|
||||
return {
|
||||
ok: result.exitCode === 0 && payload.ok === true && httpStatus !== null && httpStatus >= 200 && httpStatus < 300,
|
||||
httpStatus,
|
||||
bodyBytes: typeof payload.bodyBytes === "number" ? payload.bodyBytes : 0,
|
||||
summary,
|
||||
result,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeObservabilityPublicRawMetricsProbe(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const endpoint = options.spec.observability.metricsEndpoint;
|
||||
if (endpoint === undefined) {
|
||||
return {
|
||||
ok: false,
|
||||
expected: null,
|
||||
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
||||
};
|
||||
}
|
||||
const url = joinUrlPath(options.spec.publicApiUrl, endpoint.path);
|
||||
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "-", "-w", "\nUNIDESK_HTTP_STATUS:%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
|
||||
const parsed = splitCurlBodyStatus(result.stdout);
|
||||
const exposesPrometheus = looksLikePrometheusMetrics(parsed.body, options.spec.observability.workbench?.metricPrefixes ?? []);
|
||||
const denied = endpoint.publicRawMetrics === "denied" && (parsed.httpStatus === null || parsed.httpStatus >= 400 || !exposesPrometheus);
|
||||
return {
|
||||
ok: result.exitCode === 0 && denied,
|
||||
expected: endpoint.publicRawMetrics,
|
||||
url,
|
||||
httpStatus: parsed.httpStatus,
|
||||
exposesPrometheus,
|
||||
bodyBytes: Buffer.byteLength(parsed.body),
|
||||
result: compactRuntimeCommand(result),
|
||||
degradedReason: denied ? undefined : "node-observability-public-raw-metrics-exposed",
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeNodeObservabilityMetrics(observability: HwlabRuntimeObservabilitySpec, text: string, httpStatus: number | null): Record<string, unknown> {
|
||||
const workbench = observability.workbench;
|
||||
if (workbench === undefined) {
|
||||
return {
|
||||
ready: false,
|
||||
httpStatus,
|
||||
degradedReason: "node-observability-workbench-not-declared",
|
||||
};
|
||||
}
|
||||
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
const sampleLines = lines.filter((line) => !line.startsWith("#"));
|
||||
const seriesByPrefix = Object.fromEntries(workbench.metricPrefixes.map((prefix) => [
|
||||
prefix,
|
||||
sampleLines.filter((line) => (metricNameFromPrometheusLine(line) ?? "").startsWith(prefix)).length,
|
||||
]));
|
||||
const requiredSeries = workbench.requiredSeries.map((name) => {
|
||||
const matching = sampleLines.filter((line) => prometheusLineMatchesMetric(line, name));
|
||||
return {
|
||||
name,
|
||||
present: matching.length > 0,
|
||||
sampleCount: matching.length,
|
||||
};
|
||||
});
|
||||
const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);
|
||||
const deniedBackendEventLines = sampleLines.filter((line) => {
|
||||
const name = metricNameFromPrometheusLine(line) ?? "";
|
||||
if (!isWorkbenchBackendEventMetric(name)) return false;
|
||||
return workbench.backendLabelDenylist.some((label) => prometheusLineHasLabel(line, "backend", label));
|
||||
});
|
||||
const ready = httpStatus !== null
|
||||
&& httpStatus >= 200
|
||||
&& httpStatus < 300
|
||||
&& missingSeries.length === 0
|
||||
&& deniedBackendEventLines.length <= workbench.maxUnknownEventLines;
|
||||
return {
|
||||
ready,
|
||||
httpStatus,
|
||||
bodyBytes: Buffer.byteLength(text),
|
||||
lineCount: lines.length,
|
||||
sampleLineCount: sampleLines.length,
|
||||
metricPrefixes: workbench.metricPrefixes,
|
||||
seriesByPrefix,
|
||||
requiredSeries,
|
||||
missingSeries,
|
||||
backendLabelDenylist: workbench.backendLabelDenylist,
|
||||
deniedBackendEventLineCount: deniedBackendEventLines.length,
|
||||
maxDeniedBackendEventLines: workbench.maxUnknownEventLines,
|
||||
deniedBackendEventLines: compactPrometheusLines(deniedBackendEventLines),
|
||||
backendVisibleCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_backend_event_visible_latency_seconds_count"))),
|
||||
phaseCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_event_phase_duration_seconds_count"))),
|
||||
projectionLagCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_events_count") || prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_seconds_count"))),
|
||||
turnGetCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_turn_get_duration_seconds_count"))),
|
||||
agentRunResultCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_agentrun_result_duration_seconds_count"))),
|
||||
projectorBatchCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_batch_duration_seconds_count"))),
|
||||
projectorLastSuccessLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_last_success_unixtime"))),
|
||||
degradedReason: ready ? undefined : missingSeries.length > 0 ? "node-observability-required-series-missing" : "node-observability-denied-backend-label-present",
|
||||
};
|
||||
}
|
||||
|
||||
export function splitCurlBodyStatus(stdout: string): { body: string; httpStatus: number | null } {
|
||||
const marker = "\nUNIDESK_HTTP_STATUS:";
|
||||
const index = stdout.lastIndexOf(marker);
|
||||
if (index < 0) return { body: stdout, httpStatus: null };
|
||||
const rawStatus = stdout.slice(index + marker.length).trim();
|
||||
const status = nullableInteger(rawStatus);
|
||||
return {
|
||||
body: stdout.slice(0, index),
|
||||
httpStatus: status,
|
||||
};
|
||||
}
|
||||
|
||||
export function looksLikePrometheusMetrics(text: string, prefixes: readonly string[]): boolean {
|
||||
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
return lines.some((line) => {
|
||||
if (line.startsWith("# HELP ") || line.startsWith("# TYPE ")) return true;
|
||||
const name = metricNameFromPrometheusLine(line);
|
||||
return name !== null && (prefixes.length === 0 || prefixes.some((prefix) => name.startsWith(prefix)));
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user