Codex
1000 lines
57 KiB
TypeScript
1000 lines
57 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. observability module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:568-1599 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { NodeObservabilityOptions } from "./entry";
|
|
import { runNodeK3sArgs } from "./cleanup";
|
|
import { nodeRuntimeExpected, nodeRuntimeLocalPostgresExpectedAbsent, parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { runTransWorkspaceStdinScript, runtimeSecretSpec } from "./public-exposure";
|
|
import { joinUrlPath, nullableInteger } from "./render";
|
|
import { compactPrometheusLines, compactRuntimeCommand, isWorkbenchBackendEventMetric, k8sServiceSelector, labelSelectorText, metricNameFromPrometheusLine, parseJsonRecordFromText, prometheusLineHasLabel, prometheusLineMatchesMetric, selectK8sPod } from "./runtime-common";
|
|
import { compactCommandResultRedacted, record, shellQuote } from "./utils";
|
|
import { readBootstrapAdminPasswordMaterial } from "./web-probe";
|
|
import { webProbeCredential } from "./web-probe-observe";
|
|
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
|
|
|
|
export function nodeRuntimeControlPlanePlan(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(scoped.spec);
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
|
|
mode: "plan",
|
|
mutation: false,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
expected: nodeRuntimeExpected(scoped.spec),
|
|
checks: {
|
|
nodeScopedTargetConfigured: true,
|
|
externalPostgresDeclared: scoped.spec.externalPostgres !== undefined,
|
|
externalPostgresActive: activeExternalPostgres !== undefined,
|
|
secretValuesPrinted: false,
|
|
runtimeNamespace: scoped.spec.runtimeNamespace,
|
|
localPostgresExpectedAbsent: nodeRuntimeLocalPostgresExpectedAbsent(scoped.spec),
|
|
publicExposureDeclared: scoped.spec.publicExposure !== null,
|
|
},
|
|
next: {
|
|
infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
platformDbStatus: activeExternalPostgres === undefined
|
|
? null
|
|
: `bun scripts/cli.ts platform-db postgres status --config ${activeExternalPostgres.configRef}`,
|
|
publicExposure: scoped.spec.publicExposure === null ? null : `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runNodeObservability(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
if (options.action === "plan") return nodeObservabilityPlan(options);
|
|
if (options.action === "apply") return nodeObservabilityApply(options);
|
|
if (options.action === "status") return nodeObservabilityStatus(options);
|
|
if (options.action === "performance-summary") return nodeObservabilityPerformanceSummary(options);
|
|
return nodeObservabilityWorkbenchSummary(options);
|
|
}
|
|
|
|
export function nodeObservabilityPlan(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
return {
|
|
ok: configProblem === null,
|
|
command: `hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
|
mode: "node-observability-plan",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: nodeObservabilityExpected(options.spec),
|
|
renderPlan: nodeObservabilityRenderPlan(options.spec.observability),
|
|
degradedReason: configProblem,
|
|
next: {
|
|
dryRun: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
|
apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
|
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityApply(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
const applyPlan = nodeObservabilityApplyPlan(options.spec.observability);
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: configProblem === null && applyPlan.supported === true,
|
|
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
|
mode: "node-observability-apply-dry-run",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: nodeObservabilityExpected(options.spec),
|
|
applyPlan,
|
|
degradedReason: configProblem ?? (applyPlan.supported === true ? undefined : "node-observability-apply-mode-unsupported"),
|
|
};
|
|
}
|
|
if (configProblem !== null || applyPlan.supported !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
|
mode: "node-observability-apply",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
applyPlan,
|
|
degradedReason: configProblem ?? "node-observability-apply-mode-unsupported",
|
|
};
|
|
}
|
|
const status = nodeObservabilityStatus({ ...options, full: true });
|
|
return {
|
|
ok: status.ok === true,
|
|
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
|
mode: "node-observability-apply",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
applyPlan,
|
|
appliedResources: [],
|
|
applyResult: {
|
|
mode: applyPlan.mode,
|
|
reason: applyPlan.reason,
|
|
runtimeValidationOnly: true,
|
|
},
|
|
status: options.full ? status : summarizeNodeObservabilityStatus(status),
|
|
degradedReason: status.ok === true ? undefined : "node-observability-status-not-ready",
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
const serviceStatus = nodeObservabilityServiceStatus(options);
|
|
const publicRawMetrics = nodeObservabilityPublicRawMetricsProbe(options);
|
|
const workbenchSummary = nodeObservabilityWorkbenchSummary(options, serviceStatus);
|
|
const ok = configProblem === null && serviceStatus.ready === true && publicRawMetrics.ok === true && workbenchSummary.ok === true;
|
|
const status = {
|
|
ok,
|
|
command: `hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
|
mode: "node-observability-status",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: nodeObservabilityExpected(options.spec),
|
|
service: serviceStatus,
|
|
publicRawMetrics,
|
|
workbenchSummary,
|
|
degradedReason: configProblem
|
|
?? (serviceStatus.ready === true
|
|
? publicRawMetrics.ok === true
|
|
? workbenchSummary.ok === true ? undefined : "node-observability-workbench-metrics-not-ready"
|
|
: "node-observability-public-raw-metrics-boundary-failed"
|
|
: "node-observability-service-not-ready"),
|
|
next: {
|
|
plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
|
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
|
},
|
|
};
|
|
return options.full ? status : summarizeNodeObservabilityStatus(status);
|
|
}
|
|
|
|
export function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, existingServiceStatus?: Record<string, unknown>): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
const serviceStatus = existingServiceStatus ?? nodeObservabilityServiceStatus(options);
|
|
const podName = typeof serviceStatus.podName === "string" && serviceStatus.podName.length > 0 ? serviceStatus.podName : null;
|
|
const metricsProbe = podName === null ? null : nodeObservabilityMetricsProbe(options, podName);
|
|
const metrics = metricsProbe?.summary ?? summarizeNodeObservabilityMetrics(options.spec.observability, "", null);
|
|
const ok = configProblem === null && serviceStatus.ready === true && metricsProbe?.ok === true && metrics.ready === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
mode: "node-observability-workbench-summary",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
metricsEndpoint: nodeObservabilityEndpointSummary(options.spec.observability),
|
|
service: {
|
|
ready: serviceStatus.ready === true,
|
|
namespace: serviceStatus.namespace,
|
|
serviceName: serviceStatus.serviceName,
|
|
podName,
|
|
containerName: serviceStatus.containerName,
|
|
},
|
|
recordingRules: nodeObservabilityRecordingRuleSummaries(options.spec.observability),
|
|
warningAlerts: nodeObservabilityWarningAlertSummaries(options.spec.observability),
|
|
metrics,
|
|
probe: metricsProbe === null ? null : {
|
|
ok: metricsProbe.ok,
|
|
httpStatus: metricsProbe.httpStatus,
|
|
bodyBytes: metricsProbe.bodyBytes,
|
|
result: compactRuntimeCommand(metricsProbe.result),
|
|
error: metricsProbe.error,
|
|
},
|
|
degradedReason: configProblem
|
|
?? (serviceStatus.ready === true
|
|
? metricsProbe?.ok === true
|
|
? metrics.ready === true ? undefined : "node-observability-required-series-missing"
|
|
: "node-observability-metrics-endpoint-not-readable"
|
|
: "node-observability-service-not-ready"),
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityPerformanceSummary(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const workbench = options.spec.observability.workbench;
|
|
const summaryPath = workbench?.summaryPath ?? "/v1/web-performance/summary";
|
|
const endpoint = joinUrlPath(options.spec.publicWebUrl, summaryPath);
|
|
const secretSpec = runtimeSecretSpec({ node: options.node, lane: options.lane });
|
|
const material = readBootstrapAdminPasswordMaterial(secretSpec);
|
|
const credential = webProbeCredential(secretSpec, material);
|
|
const command = `hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`;
|
|
if (material.ok !== true || material.password === null) {
|
|
return {
|
|
ok: false,
|
|
command,
|
|
mode: "node-observability-performance-summary",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
target: {
|
|
workspace: options.spec.workspace,
|
|
webBaseUrl: options.spec.publicWebUrl,
|
|
endpoint,
|
|
},
|
|
credential,
|
|
degradedReason: material.error ?? "web-login-secret-unavailable",
|
|
next: {
|
|
secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name bootstrap-admin`,
|
|
webProbe: `bun scripts/cli.ts web-probe run --node ${options.node} --lane ${options.lane}`,
|
|
},
|
|
};
|
|
}
|
|
const timeoutSeconds = Math.max(10, Math.min(options.timeoutSeconds, 55));
|
|
const script = nodePerformanceSummaryRemoteScript(options.spec.publicWebUrl, summaryPath, material.username ?? secretSpec.bootstrapAdminUsername, material.password);
|
|
const result = runTransWorkspaceStdinScript(options.node, options.spec.workspace, script, timeoutSeconds);
|
|
const report = parseJsonRecordFromText(result.stdout);
|
|
const performance = record(report.performance);
|
|
const rows = nodePerformanceRows(performance);
|
|
const analysis = nodePerformanceAnalysis(performance, rows);
|
|
const ok = result.exitCode === 0 && report.ok === true && performance.ok !== false;
|
|
return {
|
|
ok,
|
|
command,
|
|
mode: "node-observability-performance-summary",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
target: {
|
|
workspace: options.spec.workspace,
|
|
webBaseUrl: options.spec.publicWebUrl,
|
|
endpoint,
|
|
summaryPath,
|
|
lowSampleThreshold: workbench?.lowSampleThreshold ?? null,
|
|
},
|
|
credential,
|
|
auth: record(report.auth),
|
|
httpStatus: report.httpStatus ?? null,
|
|
fetchAttempts: report.fetchAttempts ?? null,
|
|
summary: performance,
|
|
analysis,
|
|
probe: compactCommandResultRedacted(result, [material.password]),
|
|
degradedReason: ok
|
|
? undefined
|
|
: typeof report.error === "string" && report.error.length > 0
|
|
? report.error
|
|
: result.exitCode === 0 ? "web-performance-summary-not-ready" : "web-performance-summary-probe-failed",
|
|
next: {
|
|
webProbe: `bun scripts/cli.ts web-probe run --node ${options.node} --lane ${options.lane}`,
|
|
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
apiMetrics: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane} --full`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodePerformanceSummaryRemoteScript(baseUrl: string, summaryPath: string, username: string, password: string): string {
|
|
const nodeScript = [
|
|
"const baseUrl = process.env.HWLAB_WEB_BASE_URL;",
|
|
"const summaryPath = process.env.HWLAB_PERFORMANCE_SUMMARY_PATH || '/v1/web-performance/summary';",
|
|
"const username = process.env.HWLAB_WEB_USER;",
|
|
"const password = process.env.HWLAB_WEB_PASS;",
|
|
"function compactNumber(value) { const n = Number(value); return Number.isFinite(n) ? Math.round(n * 10) / 10 : null; }",
|
|
"function compactSeconds(msValue, secondsValue) { const ms = Number(msValue); if (Number.isFinite(ms)) return Math.round((ms / 1000) * 10) / 10; return compactNumber(secondsValue); }",
|
|
"function compactString(value) { return typeof value === 'string' && value.length > 0 ? value : null; }",
|
|
"function compactRow(row) {",
|
|
" if (!row || typeof row !== 'object' || Array.isArray(row)) return null;",
|
|
" const route = compactString(row.route) || compactString(row.path) || compactString(row.name) || compactString(row.metric) || compactString(row.title);",
|
|
" return {",
|
|
" source: compactString(row.source) || compactString(row.kind) || compactString(row.category),",
|
|
" metric: compactString(row.metric) || compactString(row.eventKind) || compactString(row.phase),",
|
|
" route,",
|
|
" problem: compactString(row.problem) || compactString(row.diagnosis) || compactString(row.reason),",
|
|
" outcome: compactString(row.outcome) || compactString(row.result) || compactString(row.status),",
|
|
" tone: compactString(row.tone) || compactString(row.severity),",
|
|
" sampleState: compactString(row.sampleState) || compactString(row.sampleStatus),",
|
|
" count: compactNumber(row.count ?? row.sampleCount ?? row.samples),",
|
|
" durationUnit: 'seconds',",
|
|
" p50Seconds: compactSeconds(row.p50Ms, row.p50Seconds ?? row.p50 ?? row.medianSeconds ?? row.median),",
|
|
" p75Seconds: compactSeconds(row.p75Ms, row.p75Seconds ?? row.p75),",
|
|
" p95Seconds: compactSeconds(row.p95Ms, row.p95Seconds ?? row.p95),",
|
|
" maxSeconds: compactSeconds(row.maxMs, row.maxSeconds ?? row.max),",
|
|
" };",
|
|
"}",
|
|
"function rowHasSignal(row) { return Boolean(row && (row.source || row.metric || row.route || row.problem || row.outcome)); }",
|
|
"function compactRows(value, limit = 16) { return Array.isArray(value) ? value.map(compactRow).filter(rowHasSignal).slice(0, limit) : []; }",
|
|
"function compactTopN(value) {",
|
|
" if (!Array.isArray(value)) return [];",
|
|
" return value.slice(0, 8).map((group) => {",
|
|
" const rows = compactRows(group?.rows || group?.items || group?.data, 8);",
|
|
" return { id: compactString(group?.id) || compactString(group?.key), title: compactString(group?.title) || compactString(group?.label), rows };",
|
|
" }).filter((group) => group.rows.length > 0);",
|
|
"}",
|
|
"function compactCards(value) {",
|
|
" if (!Array.isArray(value)) return [];",
|
|
" return value.slice(0, 12).map((card) => ({ id: compactString(card?.id), title: compactString(card?.title), value: card?.value ?? null, tone: compactString(card?.tone) || compactString(card?.status), detail: compactString(card?.detail) || compactString(card?.hint) }));",
|
|
"}",
|
|
"function compactPerformance(body) {",
|
|
" if (!body || typeof body !== 'object' || Array.isArray(body)) return { ok: false, degradedReason: 'non-json-summary' };",
|
|
" const dashboard = body.dashboard && typeof body.dashboard === 'object' ? body.dashboard : {};",
|
|
" const summary = body.summary && typeof body.summary === 'object' ? body.summary : {};",
|
|
" const rows = compactRows(body.rows || body.tableRows || body.performanceRows, 24);",
|
|
" const workbenchRows = compactRows(body.workbenchRows || body.workbench || body.workbenchTableRows, 12);",
|
|
" const problems = compactRows(body.problems || body.problemRows || dashboard.problems, 24);",
|
|
" const displayRows = rows.length > 0 ? rows.slice(0, 16) : problems.slice(0, 16);",
|
|
" const displayProblems = rows.length > 0 ? [] : problems.slice(0, 16);",
|
|
" const topN = compactTopN(dashboard.topN || body.topN);",
|
|
" return {",
|
|
" ok: true,",
|
|
" schemaVersion: body.schemaVersion || dashboard.schemaVersion || null,",
|
|
" source: body.source || dashboard.source || null,",
|
|
" observedAt: body.observedAt || dashboard.observedAt || body.generatedAt || dashboard.generatedAt || null,",
|
|
" window: body.window || body.sampleWindow || dashboard.window || null,",
|
|
" freshness: body.freshness || dashboard.freshness || null,",
|
|
" sampleCount: summary.sampleCount ?? body.sampleCount ?? dashboard.sampleCount ?? null,",
|
|
" problemCount: summary.problemCount ?? body.problemCount ?? dashboard.problemCount ?? null,",
|
|
" status: summary.status || body.status || dashboard.status || null,",
|
|
" cards: compactCards(dashboard.cards || body.cards),",
|
|
" rows: displayRows,",
|
|
" workbenchRows,",
|
|
" problems: displayProblems,",
|
|
" topN,",
|
|
" rowCount: rows.length,",
|
|
" workbenchRowCount: workbenchRows.length,",
|
|
" problemRowCount: problems.length,",
|
|
" };",
|
|
"}",
|
|
"function splitSetCookie(raw) { return raw ? raw.split(/,(?=[^;,]+=)/g).map((item) => item.trim()).filter(Boolean) : []; }",
|
|
"async function fetchWithTimeout(url, init, timeoutMs = 12000) {",
|
|
" const controller = new AbortController();",
|
|
" const timer = setTimeout(() => controller.abort(), timeoutMs);",
|
|
" try {",
|
|
" const response = await fetch(url, { ...init, signal: controller.signal });",
|
|
" const text = await response.text();",
|
|
" let body = null;",
|
|
" try { body = JSON.parse(text); } catch {}",
|
|
" return { response, text, body };",
|
|
" } finally { clearTimeout(timer); }",
|
|
"}",
|
|
"function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }",
|
|
"async function fetchWithRetry(url, init, attempts = 3) {",
|
|
" let last = null;",
|
|
" for (let attempt = 1; attempt <= attempts; attempt += 1) {",
|
|
" try {",
|
|
" const result = await fetchWithTimeout(url, init);",
|
|
" result.attempt = attempt;",
|
|
" last = result;",
|
|
" if (![502, 503, 504].includes(result.response.status)) return result;",
|
|
" } catch (error) {",
|
|
" last = { attempt, error: error instanceof Error ? error.message : String(error), response: { status: 0, ok: false, headers: new Headers() }, body: null, text: '' };",
|
|
" }",
|
|
" if (attempt < attempts) await sleep(400 * attempt);",
|
|
" }",
|
|
" return last;",
|
|
"}",
|
|
"(async () => {",
|
|
" try {",
|
|
" const loginUrl = new URL('/auth/login', baseUrl).toString();",
|
|
" const login = await fetchWithRetry(loginUrl, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ username, password }) });",
|
|
" const setCookie = splitSetCookie(login.response.headers.get('set-cookie'));",
|
|
" const cookie = setCookie.map((item) => item.split(';')[0].trim()).filter(Boolean).join('; ');",
|
|
" const auth = { ok: login.response.ok && cookie.length > 0, httpStatus: login.response.status, cookiePresent: cookie.length > 0, attempts: login.attempt || 1 };",
|
|
" if (!auth.ok) {",
|
|
" console.log(JSON.stringify({ ok: false, auth, httpStatus: login.response.status, error: 'web-login-failed' }));",
|
|
" process.exitCode = 1;",
|
|
" return;",
|
|
" }",
|
|
" const summaryUrl = new URL(summaryPath, baseUrl).toString();",
|
|
" const summary = await fetchWithRetry(summaryUrl, { method: 'GET', headers: { accept: 'application/json', cookie } });",
|
|
" const performance = compactPerformance(summary.body);",
|
|
" const ok = summary.response.ok && performance.ok === true;",
|
|
" console.log(JSON.stringify({ ok, auth, httpStatus: summary.response.status, fetchAttempts: summary.attempt || 1, finalUrl: summaryUrl, performance, error: ok ? null : 'web-performance-summary-fetch-failed' }));",
|
|
" process.exitCode = ok ? 0 : 1;",
|
|
" } catch (error) {",
|
|
" console.log(JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) }));",
|
|
" process.exitCode = 1;",
|
|
" }",
|
|
"})();",
|
|
].join("\n");
|
|
return [
|
|
"set -eu",
|
|
`HWLAB_WEB_BASE_URL=${shellQuote(baseUrl)} HWLAB_PERFORMANCE_SUMMARY_PATH=${shellQuote(summaryPath)} HWLAB_WEB_USER=${shellQuote(username)} HWLAB_WEB_PASS=${shellQuote(password)} node <<'NODE'`,
|
|
nodeScript,
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function nodePerformanceRows(performance: Record<string, unknown>): Record<string, unknown>[] {
|
|
const rows = [
|
|
...nodePerformanceArray(performance.problems),
|
|
...nodePerformanceArray(performance.rows),
|
|
...nodePerformanceArray(performance.workbenchRows),
|
|
];
|
|
const topN = Array.isArray(performance.topN) ? performance.topN.map(record) : [];
|
|
for (const group of topN) {
|
|
for (const row of nodePerformanceArray(group.rows)) rows.push(row);
|
|
}
|
|
const seen = new Set<string>();
|
|
return rows.filter((row) => {
|
|
const key = [
|
|
nodePerformanceString(row.source),
|
|
nodePerformanceString(row.metric),
|
|
nodePerformanceString(row.route),
|
|
nodePerformanceString(row.outcome),
|
|
nodePerformanceString(row.problem),
|
|
].join("|");
|
|
if (seen.has(key)) return false;
|
|
seen.add(key);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
export function nodePerformanceArray(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : [];
|
|
}
|
|
|
|
export function nodePerformanceAnalysis(performance: Record<string, unknown>, rows: Record<string, unknown>[]): Record<string, unknown> {
|
|
const actionableRows = rows.filter(nodePerformanceActionableProblem);
|
|
const networkErrors = actionableRows.filter((row) => nodePerformanceString(row.outcome) === "network_error" || nodePerformanceString(row.problem) === "network_error" || nodePerformanceString(row.problem) === "network error");
|
|
const blockedRows = actionableRows.filter((row) => nodePerformanceString(row.tone) === "blocked" || nodePerformanceString(row.status) === "blocked");
|
|
const slowApiRows = rows.filter((row) => {
|
|
const source = nodePerformanceString(row.source) ?? "";
|
|
const metric = nodePerformanceString(row.metric) ?? "";
|
|
const p95Seconds = nodePerformanceNumber(row.p95Seconds);
|
|
return nodePerformanceActionableProblem(row)
|
|
&& (source.includes("api") || metric.includes("api") || (nodePerformanceString(row.route) ?? "").startsWith("/v1") || (nodePerformanceString(row.route) ?? "").startsWith("/health"))
|
|
&& (p95Seconds === null || p95Seconds >= 5);
|
|
});
|
|
const lcpRows = actionableRows.filter((row) => {
|
|
const metric = `${nodePerformanceString(row.metric) ?? ""} ${nodePerformanceString(row.problem) ?? ""}`.toLowerCase();
|
|
return metric.includes("lcp") || metric.includes("largest contentful") || metric.includes("最大内容");
|
|
});
|
|
const maxP95Seconds = Math.max(0, ...rows.map((row) => nodePerformanceNumber(row.p95Seconds) ?? 0));
|
|
const headline = networkErrors.length > 0
|
|
? "Public same-origin API has intermittent network_error samples; treat edge/proxy/API reachability as P0 before tuning page render."
|
|
: lcpRows.length > 0
|
|
? "Workbench page LCP is blocked, likely by first-load API waterfalls and late visible content."
|
|
: slowApiRows.length > 0
|
|
? "Same-origin API p95 is above the interactive budget; split edge wait, server work, and dependency latency."
|
|
: "No blocking performance rows were returned by the current summary window.";
|
|
const priorities = [
|
|
networkErrors.length > 0
|
|
? {
|
|
priority: "P0",
|
|
area: "public-edge-api-reachability",
|
|
action: "Correlate network_error rows with Caddy/FRP/ingress, cloud-api pod restarts, response-header timeout, and browser Resource Timing failure stage.",
|
|
evidence: nodePerformanceEvidence(networkErrors),
|
|
}
|
|
: null,
|
|
slowApiRows.length > 0
|
|
? {
|
|
priority: "P1",
|
|
area: "same-origin-api-latency",
|
|
action: "Add server-side route histograms and dependency spans for slow /health, /v1/live-builds, /v1/workbench/sessions, trace events, hwpod specs, and hwpod-node-ops paths; then optimize the highest p95 route first.",
|
|
evidence: nodePerformanceEvidence(slowApiRows),
|
|
}
|
|
: null,
|
|
lcpRows.length > 0
|
|
? {
|
|
priority: "P1",
|
|
area: "workbench-lcp",
|
|
action: "Render a stable first screen before slow session/detail API calls complete, and split LCP by route plus blocking API waterfall.",
|
|
evidence: nodePerformanceEvidence(lcpRows),
|
|
}
|
|
: null,
|
|
blockedRows.length > 0
|
|
? {
|
|
priority: "P2",
|
|
area: "alert-thresholds-and-sample-quality",
|
|
action: "Keep low-sample rows visible but avoid closing incidents on count=1; require repeated windows or matching server metrics before labeling a regression fixed.",
|
|
evidence: nodePerformanceEvidence(blockedRows),
|
|
}
|
|
: null,
|
|
].filter(Boolean);
|
|
return {
|
|
headline,
|
|
sampleCount: performance.sampleCount ?? null,
|
|
problemCount: performance.problemCount ?? null,
|
|
rowCount: rows.length,
|
|
blockedRowCount: blockedRows.length,
|
|
networkErrorRowCount: networkErrors.length,
|
|
slowApiRowCount: slowApiRows.length,
|
|
lcpRowCount: lcpRows.length,
|
|
maxP95Seconds,
|
|
priorities,
|
|
};
|
|
}
|
|
|
|
export function nodePerformanceActionableProblem(row: Record<string, unknown>): boolean {
|
|
const tone = nodePerformanceString(row.tone) ?? "";
|
|
const status = nodePerformanceString(row.status) ?? "";
|
|
if (tone === "blocked" || tone === "warn" || status === "blocked" || status === "warn") return true;
|
|
const sampleState = nodePerformanceString(row.sampleState) ?? "";
|
|
if (sampleState && sampleState !== "ok") return false;
|
|
const problem = (nodePerformanceString(row.problem) ?? "").trim().toLowerCase();
|
|
return Boolean(problem && problem !== "ok" && problem !== "low-sample" && problem !== "no-sample");
|
|
}
|
|
|
|
export function nodePerformanceEvidence(rows: Record<string, unknown>[]): Record<string, unknown>[] {
|
|
return rows.slice(0, 8).map((row) => ({
|
|
source: nodePerformanceString(row.source),
|
|
metric: nodePerformanceString(row.metric),
|
|
route: nodePerformanceString(row.route),
|
|
outcome: nodePerformanceString(row.outcome),
|
|
problem: nodePerformanceString(row.problem),
|
|
count: nodePerformanceNumber(row.count),
|
|
durationUnit: nodePerformanceString(row.durationUnit) ?? "seconds",
|
|
p50Seconds: nodePerformanceNumber(row.p50Seconds),
|
|
p75Seconds: nodePerformanceNumber(row.p75Seconds),
|
|
p95Seconds: nodePerformanceNumber(row.p95Seconds),
|
|
}));
|
|
}
|
|
|
|
export function nodePerformanceString(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
export function nodePerformanceNumber(value: unknown): number | null {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
export function summarizeNodeObservabilityStatus(status: Record<string, unknown>): Record<string, unknown> {
|
|
const service = record(status.service);
|
|
const publicRawMetrics = record(status.publicRawMetrics);
|
|
const workbenchSummary = record(status.workbenchSummary);
|
|
const metrics = record(workbenchSummary.metrics);
|
|
const recordingRules = Array.isArray(workbenchSummary.recordingRules) ? workbenchSummary.recordingRules : [];
|
|
const warningAlerts = Array.isArray(workbenchSummary.warningAlerts) ? workbenchSummary.warningAlerts : [];
|
|
return {
|
|
ok: status.ok === true,
|
|
command: status.command,
|
|
mode: "node-observability-status-summary",
|
|
mutation: false,
|
|
node: status.node,
|
|
lane: status.lane,
|
|
degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
|
|
service: {
|
|
ready: service.ready === true,
|
|
namespace: service.namespace ?? null,
|
|
serviceName: service.serviceName ?? null,
|
|
podName: service.podName ?? null,
|
|
containerName: service.containerName ?? null,
|
|
},
|
|
publicRawMetrics: {
|
|
ok: publicRawMetrics.ok === true,
|
|
expected: publicRawMetrics.expected ?? null,
|
|
httpStatus: publicRawMetrics.httpStatus ?? null,
|
|
exposesPrometheus: publicRawMetrics.exposesPrometheus === true,
|
|
},
|
|
workbench: {
|
|
ok: workbenchSummary.ok === true,
|
|
httpStatus: metrics.httpStatus ?? null,
|
|
bodyBytes: metrics.bodyBytes ?? null,
|
|
recordingRuleCount: recordingRules.length,
|
|
warningAlertCount: warningAlerts.length,
|
|
seriesByPrefix: metrics.seriesByPrefix ?? {},
|
|
missingSeries: metrics.missingSeries ?? [],
|
|
topSlowDimensions: metrics.topSlowDimensions ?? [],
|
|
deniedBackendEventLineCount: metrics.deniedBackendEventLineCount ?? null,
|
|
maxDeniedBackendEventLines: metrics.maxDeniedBackendEventLines ?? null,
|
|
},
|
|
next: status.next,
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
return {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
namespace: spec.runtimeNamespace,
|
|
publicApiUrl: spec.publicApiUrl,
|
|
observability: spec.observability,
|
|
sourceOfTruth: hwlabRuntimeLaneConfigPath(),
|
|
runtimeAsSourceOfTruth: false,
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityRenderPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
|
const endpoint = observability.metricsEndpoint;
|
|
return {
|
|
prometheusOperator: observability.prometheusOperator,
|
|
metricsEndpoint: nodeObservabilityEndpointSummary(observability),
|
|
workbench: observability.workbench ?? null,
|
|
recordingRules: nodeObservabilityRecordingRuleSummaries(observability),
|
|
warningAlerts: nodeObservabilityWarningAlertSummaries(observability),
|
|
clusterResources: observability.prometheusOperator
|
|
? {
|
|
source: "HWLAB GitOps rendered manifests",
|
|
serviceMonitor: "YAML-rendered only when declared by target lane",
|
|
prometheusRule: "YAML-rendered only when declared by target lane",
|
|
}
|
|
: {
|
|
source: "none",
|
|
reason: endpoint?.scrapeMode === "pod-loopback" ? "target lane declares pod-loopback scrape mode" : "no YAML-declared scrape target",
|
|
},
|
|
publicRawMetrics: endpoint?.publicRawMetrics ?? null,
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityApplyPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
|
const endpoint = observability.metricsEndpoint;
|
|
if (endpoint === undefined) {
|
|
return {
|
|
supported: false,
|
|
mode: "unconfigured",
|
|
reason: "metricsEndpoint is missing from YAML",
|
|
};
|
|
}
|
|
if (!observability.prometheusOperator && endpoint.scrapeMode === "pod-loopback") {
|
|
return {
|
|
supported: true,
|
|
mode: "validated-noop",
|
|
reason: "target lane declares pod-loopback metrics collection and no Prometheus Operator resources",
|
|
resources: [],
|
|
validates: ["kubernetes-service", "selected-pod", "pod-loopback-metrics", "public-raw-metrics-boundary", "workbench-required-series"],
|
|
};
|
|
}
|
|
if (observability.prometheusOperator) {
|
|
return {
|
|
supported: false,
|
|
mode: "prometheus-operator",
|
|
reason: "Prometheus Operator object rendering must be declared in the target lane GitOps YAML before this node-scoped apply can own it",
|
|
};
|
|
}
|
|
return {
|
|
supported: false,
|
|
mode: endpoint.scrapeMode,
|
|
reason: "unsupported metricsEndpoint.scrapeMode",
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityConfigProblem(observability: HwlabRuntimeObservabilitySpec): string | null {
|
|
if (observability.metricsEndpoint === undefined) return "node-observability-metrics-endpoint-not-declared";
|
|
if (observability.workbench === undefined) return "node-observability-workbench-not-declared";
|
|
if (!observability.workbench.enabled) return "node-observability-workbench-disabled";
|
|
return null;
|
|
}
|
|
|
|
export function nodeObservabilityEndpointSummary(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> | null {
|
|
const endpoint = observability.metricsEndpoint;
|
|
if (endpoint === undefined) return null;
|
|
return {
|
|
serviceName: endpoint.serviceName,
|
|
containerName: endpoint.containerName,
|
|
port: endpoint.port,
|
|
scheme: endpoint.scheme,
|
|
path: endpoint.path,
|
|
scrapeMode: endpoint.scrapeMode,
|
|
publicRawMetrics: endpoint.publicRawMetrics,
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityServiceStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const endpoint = options.spec.observability.metricsEndpoint;
|
|
if (endpoint === undefined) {
|
|
return {
|
|
ready: false,
|
|
namespace: options.spec.runtimeNamespace,
|
|
serviceName: null,
|
|
podName: null,
|
|
containerName: null,
|
|
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
|
};
|
|
}
|
|
const namespace = runNodeK3sArgs(options.spec, ["kubectl", "get", "ns", options.spec.runtimeNamespace, "-o", "name"], 60);
|
|
const namespaceExists = namespace.exitCode === 0;
|
|
const service = namespaceExists
|
|
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "svc", endpoint.serviceName, "-o", "json"], 60)
|
|
: null;
|
|
const serviceJson = service !== null && service.exitCode === 0 ? parseJsonRecordFromText(service.stdout) : {};
|
|
const selector = k8sServiceSelector(serviceJson);
|
|
const selectorText = labelSelectorText(selector);
|
|
const pods = namespaceExists && service !== null && service.exitCode === 0 && selectorText.length > 0
|
|
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "pods", "-l", selectorText, "-o", "json"], 60)
|
|
: null;
|
|
const pod = pods !== null && pods.exitCode === 0 ? selectK8sPod(parseJsonRecordFromText(pods.stdout), endpoint.containerName) : null;
|
|
const ready = namespaceExists && service !== null && service.exitCode === 0 && pod?.ready === true;
|
|
return {
|
|
ready,
|
|
namespace: options.spec.runtimeNamespace,
|
|
namespaceExists,
|
|
serviceName: endpoint.serviceName,
|
|
serviceExists: service !== null && service.exitCode === 0,
|
|
selector,
|
|
selectorText: selectorText.length > 0 ? selectorText : null,
|
|
podName: pod?.name ?? null,
|
|
podPhase: pod?.phase ?? null,
|
|
podReady: pod?.ready ?? false,
|
|
containerName: endpoint.containerName,
|
|
containerReady: pod?.containerReady ?? false,
|
|
probes: {
|
|
namespace: compactRuntimeCommand(namespace),
|
|
service: service === null ? null : compactRuntimeCommand(service),
|
|
pods: pods === null ? null : compactRuntimeCommand(pods),
|
|
},
|
|
degradedReason: ready ? undefined : namespaceExists ? "node-observability-pod-or-service-not-ready" : "node-observability-namespace-missing",
|
|
};
|
|
}
|
|
|
|
export interface NodeObservabilityMetricsProbeResult {
|
|
readonly ok: boolean;
|
|
readonly httpStatus: number | null;
|
|
readonly bodyBytes: number;
|
|
readonly summary: Record<string, unknown>;
|
|
readonly result: CommandResult;
|
|
readonly error: string | null;
|
|
}
|
|
|
|
export function nodeObservabilityMetricsProbe(options: NodeObservabilityOptions, podName: string): NodeObservabilityMetricsProbeResult {
|
|
const endpoint = options.spec.observability.metricsEndpoint;
|
|
if (endpoint === undefined) throw new Error("metricsEndpoint is required before probing metrics");
|
|
const workbench = options.spec.observability.workbench;
|
|
const summaryConfig = Buffer.from(JSON.stringify({
|
|
metricPrefixes: workbench?.metricPrefixes ?? [],
|
|
requiredSeries: workbench?.requiredSeries ?? [],
|
|
backendLabelDenylist: workbench?.backendLabelDenylist ?? [],
|
|
maxDeniedBackendEventLines: workbench?.maxUnknownEventLines ?? 0,
|
|
lowSampleThreshold: workbench?.lowSampleThreshold ?? 0,
|
|
histogramMetrics: [
|
|
{ id: "workbench_journey", metric: "hwlab_workbench_journey_duration_seconds", kind: "workbench_journey", dimensionLabels: ["namespace", "gitops_target", "journey", "route", "backend", "transport", "target_state", "cache", "source", "entry", "outcome"] },
|
|
{ id: "workbench_event_phase", metric: "hwlab_workbench_event_phase_duration_seconds", kind: "workbench_event_phase", dimensionLabels: ["namespace", "gitops_target", "phase", "event_type", "backend", "transport", "outcome"] },
|
|
{ id: "workbench_backend_event_visible", metric: "hwlab_workbench_backend_event_visible_latency_seconds", kind: "workbench_backend_event_visible", dimensionLabels: ["namespace", "gitops_target", "event_type", "backend", "transport", "outcome"] },
|
|
{ id: "workbench_projection_lag_events", metric: "hwlab_workbench_projection_lag_events", kind: "workbench_projection_lag_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
|
{ id: "workbench_projection_lag_seconds", metric: "hwlab_workbench_projection_lag_seconds", kind: "workbench_projection_lag_seconds", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
|
{ id: "workbench_terminal_projection_delay", metric: "hwlab_workbench_terminal_projection_delay_seconds", kind: "workbench_terminal_projection_delay", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
|
{ id: "workbench_turn_get", metric: "hwlab_workbench_turn_get_duration_seconds", kind: "workbench_turn_get", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "route", "status", "degraded_reason"] },
|
|
{ id: "agentrun_result_duration", metric: "hwlab_agentrun_result_duration_seconds", kind: "agentrun_result_duration", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "event_count_bucket", "status"] },
|
|
{ id: "agentrun_result_pages", metric: "hwlab_agentrun_result_pages_scanned", kind: "agentrun_result_pages", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
|
|
{ id: "agentrun_result_events", metric: "hwlab_agentrun_result_events_scanned", kind: "agentrun_result_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
|
|
{ id: "workbench_projector_batch", metric: "hwlab_workbench_projector_batch_duration_seconds", kind: "workbench_projector_batch", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "phase", "status"] },
|
|
],
|
|
}), "utf8").toString("base64");
|
|
const source = [
|
|
"const http = require('node:http');",
|
|
"const port = Number(process.argv[1]);",
|
|
"const path = process.argv[2];",
|
|
"const config = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
|
|
"function metricNameFromLine(line) { const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/.exec(line); return match ? match[1] : null; }",
|
|
"function lineMatchesMetric(line, name) { if (!line.startsWith(name)) return false; const next = line.charAt(name.length); return next === '' || next === '{' || /\\s/.test(next); }",
|
|
"function lineHasLabel(line, name, value) { return line.includes(`${name}=\\\"${String(value).replace(/\\\\/g, '\\\\\\\\').replace(/\\\"/g, '\\\\\\\"')}\\\"`); }",
|
|
"function isBackendEventMetric(name) { return name.startsWith('hwlab_workbench_event_') || name.startsWith('hwlab_workbench_backend_event_'); }",
|
|
"function compactLines(lines) { return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); }",
|
|
"function parseLabels(raw) { const labels = {}; for (const part of String(raw || '').split(',')) { const index = part.indexOf('='); if (index <= 0) continue; const key = part.slice(0, index); let value = part.slice(index + 1); if (value.startsWith('\\\"') && value.endsWith('\\\"')) value = value.slice(1, -1); labels[key] = value.replace(/\\\\\\\"/g, '\\\"').replace(/\\\\\\\\/g, '\\\\'); } return labels; }",
|
|
"function labelKey(metric, labels) { const copy = { ...labels }; delete copy.le; return `${metric}|${Object.keys(copy).sort().map((key) => `${key}=${copy[key]}`).join('|')}`; }",
|
|
"function pickLabels(labels, names) { return Object.fromEntries(names.map((name) => [name, labels[name] || 'unknown'])); }",
|
|
"function quantile(row, q) { const count = Number(row.count || 0); if (count <= 0) return 0; const rank = Math.max(1, Math.ceil(count * q)); const buckets = row.buckets.filter((item) => item.le !== '+Inf').sort((left, right) => Number(left.le) - Number(right.le)); for (const bucket of buckets) { if (bucket.value >= rank) return Number(bucket.le); } return buckets.length ? Number(buckets[buckets.length - 1].le) : 0; }",
|
|
"function summarizeHistograms(sampleLines) {",
|
|
" const metricConfig = new Map(config.histogramMetrics.map((item) => [item.metric, item]));",
|
|
" const series = new Map();",
|
|
" for (const line of sampleLines) {",
|
|
" const match = /^([A-Za-z_:][A-Za-z0-9_:]*?)_(bucket|sum|count)\\{([^}]*)\\}\\s+([0-9.eE+-]+)/.exec(line);",
|
|
" if (!match || !metricConfig.has(match[1])) continue;",
|
|
" const metric = match[1];",
|
|
" const suffix = match[2];",
|
|
" const labels = parseLabels(match[3]);",
|
|
" const value = Number(match[4]);",
|
|
" if (!Number.isFinite(value)) continue;",
|
|
" const key = labelKey(metric, labels);",
|
|
" const row = series.get(key) || { metric, labels: { ...labels }, buckets: [], sum: 0, count: 0 };",
|
|
" if (suffix === 'bucket') row.buckets.push({ le: labels.le || '+Inf', value });",
|
|
" if (suffix === 'sum') row.sum = value;",
|
|
" if (suffix === 'count') row.count = value;",
|
|
" series.set(key, row);",
|
|
" }",
|
|
" const lowSampleThreshold = Number(config.lowSampleThreshold || 0);",
|
|
" const groups = Object.fromEntries(config.histogramMetrics.map((item) => [item.id, []]));",
|
|
" const rows = [];",
|
|
" for (const row of series.values()) {",
|
|
" const cfg = metricConfig.get(row.metric);",
|
|
" const sampleCount = Number(row.count || 0);",
|
|
" const output = { kind: cfg.kind, metric: row.metric, sampleCount, average: sampleCount > 0 ? Number((row.sum / sampleCount).toFixed(4)) : 0, p50: quantile(row, 0.5), p75: quantile(row, 0.75), p95: quantile(row, 0.95), lowSample: sampleCount > 0 && sampleCount < lowSampleThreshold, sampleState: sampleCount <= 0 ? 'empty' : sampleCount < lowSampleThreshold ? 'low-sample' : 'ok', dimensions: pickLabels(row.labels, cfg.dimensionLabels) };",
|
|
" groups[cfg.id].push(output);",
|
|
" rows.push(output);",
|
|
" }",
|
|
" for (const key of Object.keys(groups)) groups[key].sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
|
|
" rows.sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
|
|
" return { groups: Object.fromEntries(Object.entries(groups).map(([key, value]) => [key, value.slice(0, 24)])), topSlowDimensions: rows.slice(0, 12) };",
|
|
"}",
|
|
"function summarize(text, statusCode) {",
|
|
" const lines = text.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean);",
|
|
" const sampleLines = lines.filter((line) => !line.startsWith('#'));",
|
|
" const seriesByPrefix = Object.fromEntries(config.metricPrefixes.map((prefix) => [prefix, sampleLines.filter((line) => (metricNameFromLine(line) || '').startsWith(prefix)).length]));",
|
|
" const requiredSeries = config.requiredSeries.map((name) => { const matching = sampleLines.filter((line) => lineMatchesMetric(line, name)); return { name, present: matching.length > 0, sampleCount: matching.length }; });",
|
|
" const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);",
|
|
" const deniedBackendEventLines = sampleLines.filter((line) => { const name = metricNameFromLine(line) || ''; if (!isBackendEventMetric(name)) return false; return config.backendLabelDenylist.some((label) => lineHasLabel(line, 'backend', label)); });",
|
|
" const histogramSummary = summarizeHistograms(sampleLines);",
|
|
" const ready = statusCode >= 200 && statusCode < 300 && missingSeries.length === 0 && deniedBackendEventLines.length <= config.maxDeniedBackendEventLines;",
|
|
" return {",
|
|
" ready,",
|
|
" httpStatus: statusCode,",
|
|
" bodyBytes: Buffer.byteLength(text),",
|
|
" lineCount: lines.length,",
|
|
" sampleLineCount: sampleLines.length,",
|
|
" metricPrefixes: config.metricPrefixes,",
|
|
" seriesByPrefix,",
|
|
" requiredSeries,",
|
|
" missingSeries,",
|
|
" backendLabelDenylist: config.backendLabelDenylist,",
|
|
" lowSampleThreshold: config.lowSampleThreshold,",
|
|
" workbenchHistograms: histogramSummary.groups,",
|
|
" topSlowDimensions: histogramSummary.topSlowDimensions,",
|
|
" deniedBackendEventLineCount: deniedBackendEventLines.length,",
|
|
" maxDeniedBackendEventLines: config.maxDeniedBackendEventLines,",
|
|
" deniedBackendEventLines: compactLines(deniedBackendEventLines),",
|
|
" backendVisibleCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_backend_event_visible_latency_seconds_count'))),",
|
|
" phaseCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_event_phase_duration_seconds_count'))),",
|
|
" projectionLagCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projection_lag_events_count') || lineMatchesMetric(line, 'hwlab_workbench_projection_lag_seconds_count'))),",
|
|
" turnGetCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_turn_get_duration_seconds_count'))),",
|
|
" agentRunResultCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_agentrun_result_duration_seconds_count'))),",
|
|
" projectorBatchCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_batch_duration_seconds_count'))),",
|
|
" projectorLastSuccessLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_last_success_unixtime'))),",
|
|
" degradedReason: ready ? undefined : missingSeries.length > 0 ? 'node-observability-required-series-missing' : 'node-observability-denied-backend-label-present',",
|
|
" };",
|
|
"}",
|
|
"let settled = false;",
|
|
"const finish = (payload, code = 0) => { if (settled) return; settled = true; console.log(JSON.stringify(payload)); process.exitCode = code; };",
|
|
"const req = http.get({ host: '127.0.0.1', port, path, timeout: 8000 }, (res) => {",
|
|
" let body = '';",
|
|
" res.setEncoding('utf8');",
|
|
" res.on('data', (chunk) => { body += chunk; if (body.length > 2000000) req.destroy(new Error('metrics body exceeded 2MB')); });",
|
|
" res.on('end', () => finish({ ok: res.statusCode >= 200 && res.statusCode < 300, statusCode: res.statusCode, bodyBytes: Buffer.byteLength(body), summary: summarize(body, res.statusCode || 0) }));",
|
|
"});",
|
|
"req.on('timeout', () => req.destroy(new Error('timeout')));",
|
|
"req.on('error', (error) => finish({ ok: false, statusCode: null, bodyBytes: 0, summary: null, error: String(error && error.message || error) }, 1));",
|
|
].join("\n");
|
|
const result = runNodeK3sArgs(options.spec, [
|
|
"kubectl",
|
|
"-n", options.spec.runtimeNamespace,
|
|
"exec", podName,
|
|
"-c", endpoint.containerName,
|
|
"--",
|
|
"node", "-e", source,
|
|
String(endpoint.port),
|
|
endpoint.path,
|
|
summaryConfig,
|
|
], options.timeoutSeconds);
|
|
const payload = parseJsonRecordFromText(result.stdout);
|
|
const httpStatus = typeof payload.statusCode === "number" ? payload.statusCode : null;
|
|
const error = typeof payload.error === "string" ? payload.error : null;
|
|
const summary = record(payload.summary);
|
|
return {
|
|
ok: result.exitCode === 0 && payload.ok === true && httpStatus !== null && httpStatus >= 200 && httpStatus < 300,
|
|
httpStatus,
|
|
bodyBytes: typeof payload.bodyBytes === "number" ? payload.bodyBytes : 0,
|
|
summary,
|
|
result,
|
|
error,
|
|
};
|
|
}
|
|
|
|
export function nodeObservabilityPublicRawMetricsProbe(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const endpoint = options.spec.observability.metricsEndpoint;
|
|
if (endpoint === undefined) {
|
|
return {
|
|
ok: false,
|
|
expected: null,
|
|
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
|
};
|
|
}
|
|
const url = joinUrlPath(options.spec.publicApiUrl, endpoint.path);
|
|
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "-", "-w", "\nUNIDESK_HTTP_STATUS:%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
|
|
const parsed = splitCurlBodyStatus(result.stdout);
|
|
const exposesPrometheus = looksLikePrometheusMetrics(parsed.body, options.spec.observability.workbench?.metricPrefixes ?? []);
|
|
const denied = endpoint.publicRawMetrics === "denied" && (parsed.httpStatus === null || parsed.httpStatus >= 400 || !exposesPrometheus);
|
|
return {
|
|
ok: result.exitCode === 0 && denied,
|
|
expected: endpoint.publicRawMetrics,
|
|
url,
|
|
httpStatus: parsed.httpStatus,
|
|
exposesPrometheus,
|
|
bodyBytes: Buffer.byteLength(parsed.body),
|
|
result: compactRuntimeCommand(result),
|
|
degradedReason: denied ? undefined : "node-observability-public-raw-metrics-exposed",
|
|
};
|
|
}
|
|
|
|
export function summarizeNodeObservabilityMetrics(observability: HwlabRuntimeObservabilitySpec, text: string, httpStatus: number | null): Record<string, unknown> {
|
|
const workbench = observability.workbench;
|
|
if (workbench === undefined) {
|
|
return {
|
|
ready: false,
|
|
httpStatus,
|
|
degradedReason: "node-observability-workbench-not-declared",
|
|
};
|
|
}
|
|
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
const sampleLines = lines.filter((line) => !line.startsWith("#"));
|
|
const seriesByPrefix = Object.fromEntries(workbench.metricPrefixes.map((prefix) => [
|
|
prefix,
|
|
sampleLines.filter((line) => (metricNameFromPrometheusLine(line) ?? "").startsWith(prefix)).length,
|
|
]));
|
|
const requiredSeries = workbench.requiredSeries.map((name) => {
|
|
const matching = sampleLines.filter((line) => prometheusLineMatchesMetric(line, name));
|
|
return {
|
|
name,
|
|
present: matching.length > 0,
|
|
sampleCount: matching.length,
|
|
};
|
|
});
|
|
const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);
|
|
const deniedBackendEventLines = sampleLines.filter((line) => {
|
|
const name = metricNameFromPrometheusLine(line) ?? "";
|
|
if (!isWorkbenchBackendEventMetric(name)) return false;
|
|
return workbench.backendLabelDenylist.some((label) => prometheusLineHasLabel(line, "backend", label));
|
|
});
|
|
const ready = httpStatus !== null
|
|
&& httpStatus >= 200
|
|
&& httpStatus < 300
|
|
&& missingSeries.length === 0
|
|
&& deniedBackendEventLines.length <= workbench.maxUnknownEventLines;
|
|
return {
|
|
ready,
|
|
httpStatus,
|
|
bodyBytes: Buffer.byteLength(text),
|
|
lineCount: lines.length,
|
|
sampleLineCount: sampleLines.length,
|
|
metricPrefixes: workbench.metricPrefixes,
|
|
seriesByPrefix,
|
|
requiredSeries,
|
|
missingSeries,
|
|
backendLabelDenylist: workbench.backendLabelDenylist,
|
|
deniedBackendEventLineCount: deniedBackendEventLines.length,
|
|
maxDeniedBackendEventLines: workbench.maxUnknownEventLines,
|
|
deniedBackendEventLines: compactPrometheusLines(deniedBackendEventLines),
|
|
backendVisibleCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_backend_event_visible_latency_seconds_count"))),
|
|
phaseCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_event_phase_duration_seconds_count"))),
|
|
projectionLagCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_events_count") || prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_seconds_count"))),
|
|
turnGetCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_turn_get_duration_seconds_count"))),
|
|
agentRunResultCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_agentrun_result_duration_seconds_count"))),
|
|
projectorBatchCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_batch_duration_seconds_count"))),
|
|
projectorLastSuccessLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_last_success_unixtime"))),
|
|
degradedReason: ready ? undefined : missingSeries.length > 0 ? "node-observability-required-series-missing" : "node-observability-denied-backend-label-present",
|
|
};
|
|
}
|
|
|
|
export function splitCurlBodyStatus(stdout: string): { body: string; httpStatus: number | null } {
|
|
const marker = "\nUNIDESK_HTTP_STATUS:";
|
|
const index = stdout.lastIndexOf(marker);
|
|
if (index < 0) return { body: stdout, httpStatus: null };
|
|
const rawStatus = stdout.slice(index + marker.length).trim();
|
|
const status = nullableInteger(rawStatus);
|
|
return {
|
|
body: stdout.slice(0, index),
|
|
httpStatus: status,
|
|
};
|
|
}
|
|
|
|
export function looksLikePrometheusMetrics(text: string, prefixes: readonly string[]): boolean {
|
|
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
return lines.some((line) => {
|
|
if (line.startsWith("# HELP ") || line.startsWith("# TYPE ")) return true;
|
|
const name = metricNameFromPrometheusLine(line);
|
|
return name !== null && (prefixes.length === 0 || prefixes.some((prefix) => name.startsWith(prefix)));
|
|
});
|
|
}
|