diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 328304f5..4d7853b6 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -1,3 +1,4 @@ +# SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. version: 1 kind: HwlabNodeLaneConfig metadata: @@ -157,6 +158,28 @@ lanes: XDG_CONFIG_HOME: /tekton/home/.config observability: prometheusOperator: false + metricsEndpoint: + serviceName: hwlab-cloud-api + containerName: hwlab-cloud-api + port: 6667 + scheme: http + path: /v1/web-performance/metrics + scrapeMode: pod-loopback + publicRawMetrics: denied + workbench: + enabled: true + summaryPath: /v1/web-performance/summary + metricPrefixes: + - hwlab_workbench_ + - hwlab_webui_ + - hwlab_http_ + requiredSeries: + - hwlab_workbench_journey_total + - hwlab_workbench_event_phase_duration_seconds_count + - hwlab_workbench_backend_event_visible_latency_seconds_count + backendLabelDenylist: + - unknown + maxUnknownEventLines: 0 runtimeImageRewrites: - source: fatedier/frpc:v0.68.1 target: 127.0.0.1:5000/hwlab/frpc:v0.68.1 diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 5b8ccbb7..e5067df2 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -1,3 +1,5 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: YAML source-of-truth parsing for HWLAB node/lane Workbench observability. import { readFileSync } from "node:fs"; import { rootPath } from "./config"; @@ -83,6 +85,27 @@ export interface HwlabRuntimeBuildkitSpec { export interface HwlabRuntimeObservabilitySpec { readonly prometheusOperator: boolean; + readonly metricsEndpoint?: HwlabRuntimeObservabilityMetricsEndpointSpec; + readonly workbench?: HwlabRuntimeObservabilityWorkbenchSpec; +} + +export interface HwlabRuntimeObservabilityMetricsEndpointSpec { + readonly serviceName: string; + readonly containerName: string; + readonly port: number; + readonly scheme: "http"; + readonly path: string; + readonly scrapeMode: "pod-loopback"; + readonly publicRawMetrics: "denied"; +} + +export interface HwlabRuntimeObservabilityWorkbenchSpec { + readonly enabled: boolean; + readonly summaryPath: string; + readonly metricPrefixes: readonly string[]; + readonly requiredSeries: readonly string[]; + readonly backendLabelDenylist: readonly string[]; + readonly maxUnknownEventLines: number; } export interface HwlabRuntimeImageRewriteSpec { @@ -266,6 +289,12 @@ function booleanField(obj: Record, key: string, path: string): return value; } +function nonNegativeIntegerField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`); + return value; +} + function sortedRecordEntries(value: unknown, path: string): Array<[string, Record]> { return Object.entries(asRecord(value, path)).map(([key, item]) => [key, asRecord(item, `${path}.${key}`)]); } @@ -557,6 +586,41 @@ function observabilityConfig(value: unknown, path: string): HwlabRuntimeObservab const raw = asRecord(value, path); return { prometheusOperator: booleanField(raw, "prometheusOperator", path), + metricsEndpoint: observabilityMetricsEndpointConfig(raw.metricsEndpoint, `${path}.metricsEndpoint`), + workbench: observabilityWorkbenchConfig(raw.workbench, `${path}.workbench`), + }; +} + +function observabilityMetricsEndpointConfig(value: unknown, path: string): HwlabRuntimeObservabilityMetricsEndpointSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + const scheme = stringField(raw, "scheme", path); + if (scheme !== "http") throw new Error(`${path}.scheme must be http`); + const scrapeMode = stringField(raw, "scrapeMode", path); + if (scrapeMode !== "pod-loopback") throw new Error(`${path}.scrapeMode must be pod-loopback`); + const publicRawMetrics = stringField(raw, "publicRawMetrics", path); + if (publicRawMetrics !== "denied") throw new Error(`${path}.publicRawMetrics must be denied`); + return { + serviceName: stringField(raw, "serviceName", path), + containerName: stringField(raw, "containerName", path), + port: numberField(raw, "port", path), + scheme, + path: stringField(raw, "path", path), + scrapeMode, + publicRawMetrics, + }; +} + +function observabilityWorkbenchConfig(value: unknown, path: string): HwlabRuntimeObservabilityWorkbenchSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + return { + enabled: booleanField(raw, "enabled", path), + summaryPath: stringField(raw, "summaryPath", path), + metricPrefixes: stringArrayField(raw, "metricPrefixes", path), + requiredSeries: stringArrayField(raw, "requiredSeries", path), + backendLabelDenylist: stringArrayField(raw, "backendLabelDenylist", path), + maxUnknownEventLines: nonNegativeIntegerField(raw, "maxUnknownEventLines", path), }; } diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 131b3452..092bb413 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -1,3 +1,5 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -7,7 +9,7 @@ import { startJob } from "./jobs"; import { classifySshTcpPoolFailure } from "./ssh"; import { runHwlabG14Command } from "./hwlab-g14"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; -import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes"; +import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes"; type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; @@ -51,6 +53,19 @@ interface NodeWebProbeScriptOptions { type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions; +type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary"; + +interface NodeObservabilityOptions { + action: NodeObservabilityAction; + node: string; + lane: HwlabRuntimeLane; + confirm: boolean; + dryRun: boolean; + full: boolean; + timeoutSeconds: number; + spec: HwlabRuntimeLaneSpec; +} + interface NodeRuntimeRenderResult { readonly result: CommandResult; readonly renderDir: string; @@ -191,12 +206,16 @@ export async function runHwlabNodeCommand(_config: Config, args: string[]): Prom if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeWebProbeHelp(); return runNodeWebProbe(parseNodeWebProbeOptions(args.slice(1))); } + if (domain === "observability") { + if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeObservabilityHelp(); + return runNodeObservability(parseNodeObservabilityOptions(args.slice(1))); + } if (args.includes("--help") || args.includes("-h")) return hwlabNodeHelp(); if (domain === "control-plane" || domain === "git-mirror") { return runNodeDelegatedDomain(_config, domain, args.slice(1)); } if (domain !== "secret") { - return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret, hwlab nodes test-accounts, hwlab nodes web-probe" }; + return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes observability, hwlab nodes secret, hwlab nodes test-accounts, hwlab nodes web-probe" }; } const options = parseSecretOptions(args.slice(1)); return runNodeSecret(options); @@ -247,6 +266,10 @@ export function hwlabNodeHelp(): Record { "bun scripts/cli.ts hwlab nodes test-accounts status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes test-accounts sync --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000", + "bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03", ], }; } @@ -268,6 +291,58 @@ export function hwlabNodeWebProbeHelp(): Record { }; } +export function hwlabNodeObservabilityHelp(): Record { + return { + ok: true, + command: "hwlab nodes observability", + description: "YAML-first node/lane observability control for HWLAB Workbench metrics. Runtime is queried only for presence, boundary, and summary evidence.", + configPath: hwlabRuntimeLaneConfigPath(), + examples: [ + "bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03", + ], + actions: { + plan: "Render the YAML-declared collection mode, scrape target, boundary, and required Workbench metric series.", + apply: "Apply the YAML-declared observability control plane. D601 pod-loopback mode validates the boundary and metrics without creating ad-hoc cluster objects.", + status: "Check Kubernetes service/pod presence, public raw metrics denial, and Workbench metric readiness.", + "workbench-summary": "Read the YAML-declared metrics endpoint via pod loopback and summarize required Workbench series.", + }, + }; +} + +function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions { + const [actionRaw] = args; + if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) { + throw new Error("observability usage: observability ACTION --node NODE --lane vNN [--dry-run|--confirm]"); + } + if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary") { + throw new Error(`observability action must be plan, apply, status, or workbench-summary; got ${actionRaw}`); + } + assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--full", "--raw"])); + const node = requiredOption(args, "--node"); + assertNodeId(node); + const laneRaw = requiredOption(args, "--lane"); + if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`); + const confirm = args.includes("--confirm"); + const dryRun = args.includes("--dry-run"); + if (confirm && dryRun) throw new Error("observability accepts only one of --confirm or --dry-run"); + if (actionRaw === "apply" && !confirm && !dryRun) throw new Error("observability apply requires --dry-run or --confirm"); + if (actionRaw !== "apply" && (confirm || dryRun)) throw new Error(`observability ${actionRaw} is read-only and does not accept --confirm or --dry-run`); + return { + action: actionRaw, + node, + lane: laneRaw, + confirm, + dryRun, + full: args.includes("--full") || args.includes("--raw"), + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), + spec: hwlabRuntimeLaneSpecForNode(laneRaw, node), + }; +} + async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise> { const scoped = parseNodeScopedDelegatedOptions(domain, args); const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane); @@ -487,6 +562,593 @@ function nodeRuntimeControlPlanePlan(scoped: ReturnType { + if (options.action === "plan") return nodeObservabilityPlan(options); + if (options.action === "apply") return nodeObservabilityApply(options); + if (options.action === "status") return nodeObservabilityStatus(options); + return nodeObservabilityWorkbenchSummary(options); +} + +function nodeObservabilityPlan(options: NodeObservabilityOptions): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + return { + ok: configProblem === null, + command: `hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`, + mode: "node-observability-plan", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + expected: nodeObservabilityExpected(options.spec), + renderPlan: nodeObservabilityRenderPlan(options.spec.observability), + degradedReason: configProblem, + next: { + dryRun: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`, + apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`, + status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`, + workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`, + }, + }; +} + +function nodeObservabilityApply(options: NodeObservabilityOptions): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + const applyPlan = nodeObservabilityApplyPlan(options.spec.observability); + if (options.dryRun) { + return { + ok: configProblem === null && applyPlan.supported === true, + command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`, + mode: "node-observability-apply-dry-run", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + expected: nodeObservabilityExpected(options.spec), + applyPlan, + degradedReason: configProblem ?? (applyPlan.supported === true ? undefined : "node-observability-apply-mode-unsupported"), + }; + } + if (configProblem !== null || applyPlan.supported !== true) { + return { + ok: false, + command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`, + mode: "node-observability-apply", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + applyPlan, + degradedReason: configProblem ?? "node-observability-apply-mode-unsupported", + }; + } + const status = nodeObservabilityStatus({ ...options, full: true }); + return { + ok: status.ok === true, + command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`, + mode: "node-observability-apply", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + applyPlan, + appliedResources: [], + applyResult: { + mode: applyPlan.mode, + reason: applyPlan.reason, + runtimeValidationOnly: true, + }, + status: options.full ? status : summarizeNodeObservabilityStatus(status), + degradedReason: status.ok === true ? undefined : "node-observability-status-not-ready", + }; +} + +function nodeObservabilityStatus(options: NodeObservabilityOptions): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + const serviceStatus = nodeObservabilityServiceStatus(options); + const publicRawMetrics = nodeObservabilityPublicRawMetricsProbe(options); + const workbenchSummary = nodeObservabilityWorkbenchSummary(options, serviceStatus); + const ok = configProblem === null && serviceStatus.ready === true && publicRawMetrics.ok === true && workbenchSummary.ok === true; + const status = { + ok, + command: `hwlab nodes observability status --node ${options.node} --lane ${options.lane}`, + mode: "node-observability-status", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + expected: nodeObservabilityExpected(options.spec), + service: serviceStatus, + publicRawMetrics, + workbenchSummary, + degradedReason: configProblem + ?? (serviceStatus.ready === true + ? publicRawMetrics.ok === true + ? workbenchSummary.ok === true ? undefined : "node-observability-workbench-metrics-not-ready" + : "node-observability-public-raw-metrics-boundary-failed" + : "node-observability-service-not-ready"), + next: { + plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`, + workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`, + }, + }; + return options.full ? status : summarizeNodeObservabilityStatus(status); +} + +function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, existingServiceStatus?: Record): Record { + const configProblem = nodeObservabilityConfigProblem(options.spec.observability); + const serviceStatus = existingServiceStatus ?? nodeObservabilityServiceStatus(options); + const podName = typeof serviceStatus.podName === "string" && serviceStatus.podName.length > 0 ? serviceStatus.podName : null; + const metricsProbe = podName === null ? null : nodeObservabilityMetricsProbe(options, podName); + const metrics = metricsProbe?.summary ?? summarizeNodeObservabilityMetrics(options.spec.observability, "", null); + const ok = configProblem === null && serviceStatus.ready === true && metricsProbe?.ok === true && metrics.ready === true; + return { + ok, + command: `hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`, + mode: "node-observability-workbench-summary", + mutation: false, + node: options.node, + lane: options.lane, + configPath: hwlabRuntimeLaneConfigPath(), + metricsEndpoint: nodeObservabilityEndpointSummary(options.spec.observability), + service: { + ready: serviceStatus.ready === true, + namespace: serviceStatus.namespace, + serviceName: serviceStatus.serviceName, + podName, + containerName: serviceStatus.containerName, + }, + metrics, + probe: metricsProbe === null ? null : { + ok: metricsProbe.ok, + httpStatus: metricsProbe.httpStatus, + bodyBytes: metricsProbe.bodyBytes, + result: compactRuntimeCommand(metricsProbe.result), + error: metricsProbe.error, + }, + degradedReason: configProblem + ?? (serviceStatus.ready === true + ? metricsProbe?.ok === true + ? metrics.ready === true ? undefined : "node-observability-required-series-missing" + : "node-observability-metrics-endpoint-not-readable" + : "node-observability-service-not-ready"), + }; +} + +function summarizeNodeObservabilityStatus(status: Record): Record { + const service = record(status.service); + const publicRawMetrics = record(status.publicRawMetrics); + const workbenchSummary = record(status.workbenchSummary); + const metrics = record(workbenchSummary.metrics); + return { + ok: status.ok === true, + command: status.command, + mode: "node-observability-status-summary", + mutation: false, + node: status.node, + lane: status.lane, + degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null, + service: { + ready: service.ready === true, + namespace: service.namespace ?? null, + serviceName: service.serviceName ?? null, + podName: service.podName ?? null, + containerName: service.containerName ?? null, + }, + publicRawMetrics: { + ok: publicRawMetrics.ok === true, + expected: publicRawMetrics.expected ?? null, + httpStatus: publicRawMetrics.httpStatus ?? null, + exposesPrometheus: publicRawMetrics.exposesPrometheus === true, + }, + workbench: { + ok: workbenchSummary.ok === true, + httpStatus: metrics.httpStatus ?? null, + bodyBytes: metrics.bodyBytes ?? null, + seriesByPrefix: metrics.seriesByPrefix ?? {}, + missingSeries: metrics.missingSeries ?? [], + deniedBackendEventLineCount: metrics.deniedBackendEventLineCount ?? null, + maxDeniedBackendEventLines: metrics.maxDeniedBackendEventLines ?? null, + }, + next: status.next, + }; +} + +function nodeObservabilityExpected(spec: HwlabRuntimeLaneSpec): Record { + return { + node: spec.nodeId, + lane: spec.lane, + namespace: spec.runtimeNamespace, + publicApiUrl: spec.publicApiUrl, + observability: spec.observability, + sourceOfTruth: hwlabRuntimeLaneConfigPath(), + runtimeAsSourceOfTruth: false, + }; +} + +function nodeObservabilityRenderPlan(observability: HwlabRuntimeObservabilitySpec): Record { + const endpoint = observability.metricsEndpoint; + return { + prometheusOperator: observability.prometheusOperator, + metricsEndpoint: nodeObservabilityEndpointSummary(observability), + workbench: observability.workbench ?? null, + clusterResources: observability.prometheusOperator + ? { + source: "HWLAB GitOps rendered manifests", + serviceMonitor: "YAML-rendered only when declared by target lane", + prometheusRule: "YAML-rendered only when declared by target lane", + } + : { + source: "none", + reason: endpoint?.scrapeMode === "pod-loopback" ? "target lane declares pod-loopback scrape mode" : "no YAML-declared scrape target", + }, + publicRawMetrics: endpoint?.publicRawMetrics ?? null, + }; +} + +function nodeObservabilityApplyPlan(observability: HwlabRuntimeObservabilitySpec): Record { + const endpoint = observability.metricsEndpoint; + if (endpoint === undefined) { + return { + supported: false, + mode: "unconfigured", + reason: "metricsEndpoint is missing from YAML", + }; + } + if (!observability.prometheusOperator && endpoint.scrapeMode === "pod-loopback") { + return { + supported: true, + mode: "validated-noop", + reason: "target lane declares pod-loopback metrics collection and no Prometheus Operator resources", + resources: [], + validates: ["kubernetes-service", "selected-pod", "pod-loopback-metrics", "public-raw-metrics-boundary", "workbench-required-series"], + }; + } + if (observability.prometheusOperator) { + return { + supported: false, + mode: "prometheus-operator", + reason: "Prometheus Operator object rendering must be declared in the target lane GitOps YAML before this node-scoped apply can own it", + }; + } + return { + supported: false, + mode: endpoint.scrapeMode, + reason: "unsupported metricsEndpoint.scrapeMode", + }; +} + +function nodeObservabilityConfigProblem(observability: HwlabRuntimeObservabilitySpec): string | null { + if (observability.metricsEndpoint === undefined) return "node-observability-metrics-endpoint-not-declared"; + if (observability.workbench === undefined) return "node-observability-workbench-not-declared"; + if (!observability.workbench.enabled) return "node-observability-workbench-disabled"; + return null; +} + +function nodeObservabilityEndpointSummary(observability: HwlabRuntimeObservabilitySpec): Record | null { + const endpoint = observability.metricsEndpoint; + if (endpoint === undefined) return null; + return { + serviceName: endpoint.serviceName, + containerName: endpoint.containerName, + port: endpoint.port, + scheme: endpoint.scheme, + path: endpoint.path, + scrapeMode: endpoint.scrapeMode, + publicRawMetrics: endpoint.publicRawMetrics, + }; +} + +function nodeObservabilityServiceStatus(options: NodeObservabilityOptions): Record { + const endpoint = options.spec.observability.metricsEndpoint; + if (endpoint === undefined) { + return { + ready: false, + namespace: options.spec.runtimeNamespace, + serviceName: null, + podName: null, + containerName: null, + degradedReason: "node-observability-metrics-endpoint-not-declared", + }; + } + const namespace = runNodeK3sArgs(options.spec, ["kubectl", "get", "ns", options.spec.runtimeNamespace, "-o", "name"], 60); + const namespaceExists = namespace.exitCode === 0; + const service = namespaceExists + ? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "svc", endpoint.serviceName, "-o", "json"], 60) + : null; + const serviceJson = service !== null && service.exitCode === 0 ? parseJsonRecordFromText(service.stdout) : {}; + const selector = k8sServiceSelector(serviceJson); + const selectorText = labelSelectorText(selector); + const pods = namespaceExists && service !== null && service.exitCode === 0 && selectorText.length > 0 + ? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "pods", "-l", selectorText, "-o", "json"], 60) + : null; + const pod = pods !== null && pods.exitCode === 0 ? selectK8sPod(parseJsonRecordFromText(pods.stdout), endpoint.containerName) : null; + const ready = namespaceExists && service !== null && service.exitCode === 0 && pod?.ready === true; + return { + ready, + namespace: options.spec.runtimeNamespace, + namespaceExists, + serviceName: endpoint.serviceName, + serviceExists: service !== null && service.exitCode === 0, + selector, + selectorText: selectorText.length > 0 ? selectorText : null, + podName: pod?.name ?? null, + podPhase: pod?.phase ?? null, + podReady: pod?.ready ?? false, + containerName: endpoint.containerName, + containerReady: pod?.containerReady ?? false, + probes: { + namespace: compactRuntimeCommand(namespace), + service: service === null ? null : compactRuntimeCommand(service), + pods: pods === null ? null : compactRuntimeCommand(pods), + }, + degradedReason: ready ? undefined : namespaceExists ? "node-observability-pod-or-service-not-ready" : "node-observability-namespace-missing", + }; +} + +interface NodeObservabilityMetricsProbeResult { + readonly ok: boolean; + readonly httpStatus: number | null; + readonly bodyBytes: number; + readonly summary: Record; + readonly result: CommandResult; + readonly error: string | null; +} + +function nodeObservabilityMetricsProbe(options: NodeObservabilityOptions, podName: string): NodeObservabilityMetricsProbeResult { + const endpoint = options.spec.observability.metricsEndpoint; + if (endpoint === undefined) throw new Error("metricsEndpoint is required before probing metrics"); + const workbench = options.spec.observability.workbench; + const summaryConfig = Buffer.from(JSON.stringify({ + metricPrefixes: workbench?.metricPrefixes ?? [], + requiredSeries: workbench?.requiredSeries ?? [], + backendLabelDenylist: workbench?.backendLabelDenylist ?? [], + maxDeniedBackendEventLines: workbench?.maxUnknownEventLines ?? 0, + }), "utf8").toString("base64"); + const source = [ + "const http = require('node:http');", + "const port = Number(process.argv[1]);", + "const path = process.argv[2];", + "const config = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));", + "function metricNameFromLine(line) { const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/.exec(line); return match ? match[1] : null; }", + "function lineMatchesMetric(line, name) { if (!line.startsWith(name)) return false; const next = line.charAt(name.length); return next === '' || next === '{' || /\\s/.test(next); }", + "function lineHasLabel(line, name, value) { return line.includes(`${name}=\\\"${String(value).replace(/\\\\/g, '\\\\\\\\').replace(/\\\"/g, '\\\\\\\"')}\\\"`); }", + "function isBackendEventMetric(name) { return name.startsWith('hwlab_workbench_event_') || name.startsWith('hwlab_workbench_backend_event_'); }", + "function compactLines(lines) { return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); }", + "function summarize(text, statusCode) {", + " const lines = text.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean);", + " const sampleLines = lines.filter((line) => !line.startsWith('#'));", + " const seriesByPrefix = Object.fromEntries(config.metricPrefixes.map((prefix) => [prefix, sampleLines.filter((line) => (metricNameFromLine(line) || '').startsWith(prefix)).length]));", + " const requiredSeries = config.requiredSeries.map((name) => { const matching = sampleLines.filter((line) => lineMatchesMetric(line, name)); return { name, present: matching.length > 0, sampleCount: matching.length }; });", + " const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);", + " const deniedBackendEventLines = sampleLines.filter((line) => { const name = metricNameFromLine(line) || ''; if (!isBackendEventMetric(name)) return false; return config.backendLabelDenylist.some((label) => lineHasLabel(line, 'backend', label)); });", + " const ready = statusCode >= 200 && statusCode < 300 && missingSeries.length === 0 && deniedBackendEventLines.length <= config.maxDeniedBackendEventLines;", + " return {", + " ready,", + " httpStatus: statusCode,", + " bodyBytes: Buffer.byteLength(text),", + " lineCount: lines.length,", + " sampleLineCount: sampleLines.length,", + " metricPrefixes: config.metricPrefixes,", + " seriesByPrefix,", + " requiredSeries,", + " missingSeries,", + " backendLabelDenylist: config.backendLabelDenylist,", + " deniedBackendEventLineCount: deniedBackendEventLines.length,", + " maxDeniedBackendEventLines: config.maxDeniedBackendEventLines,", + " deniedBackendEventLines: compactLines(deniedBackendEventLines),", + " backendVisibleCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_backend_event_visible_latency_seconds_count'))),", + " phaseCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_event_phase_duration_seconds_count'))),", + " degradedReason: ready ? undefined : missingSeries.length > 0 ? 'node-observability-required-series-missing' : 'node-observability-denied-backend-label-present',", + " };", + "}", + "let settled = false;", + "const finish = (payload, code = 0) => { if (settled) return; settled = true; console.log(JSON.stringify(payload)); process.exitCode = code; };", + "const req = http.get({ host: '127.0.0.1', port, path, timeout: 8000 }, (res) => {", + " let body = '';", + " res.setEncoding('utf8');", + " res.on('data', (chunk) => { body += chunk; if (body.length > 2000000) req.destroy(new Error('metrics body exceeded 2MB')); });", + " res.on('end', () => finish({ ok: res.statusCode >= 200 && res.statusCode < 300, statusCode: res.statusCode, bodyBytes: Buffer.byteLength(body), summary: summarize(body, res.statusCode || 0) }));", + "});", + "req.on('timeout', () => req.destroy(new Error('timeout')));", + "req.on('error', (error) => finish({ ok: false, statusCode: null, bodyBytes: 0, summary: null, error: String(error && error.message || error) }, 1));", + ].join("\n"); + const result = runNodeK3sArgs(options.spec, [ + "kubectl", + "-n", options.spec.runtimeNamespace, + "exec", podName, + "-c", endpoint.containerName, + "--", + "node", "-e", source, + String(endpoint.port), + endpoint.path, + summaryConfig, + ], options.timeoutSeconds); + const payload = parseJsonRecordFromText(result.stdout); + const httpStatus = typeof payload.statusCode === "number" ? payload.statusCode : null; + const error = typeof payload.error === "string" ? payload.error : null; + const summary = record(payload.summary); + return { + ok: result.exitCode === 0 && payload.ok === true && httpStatus !== null && httpStatus >= 200 && httpStatus < 300, + httpStatus, + bodyBytes: typeof payload.bodyBytes === "number" ? payload.bodyBytes : 0, + summary, + result, + error, + }; +} + +function nodeObservabilityPublicRawMetricsProbe(options: NodeObservabilityOptions): Record { + const endpoint = options.spec.observability.metricsEndpoint; + if (endpoint === undefined) { + return { + ok: false, + expected: null, + degradedReason: "node-observability-metrics-endpoint-not-declared", + }; + } + const url = joinUrlPath(options.spec.publicApiUrl, endpoint.path); + const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "-", "-w", "\nUNIDESK_HTTP_STATUS:%{http_code}", url], repoRoot, { timeoutMs: 15_000 }); + const parsed = splitCurlBodyStatus(result.stdout); + const exposesPrometheus = looksLikePrometheusMetrics(parsed.body, options.spec.observability.workbench?.metricPrefixes ?? []); + const denied = endpoint.publicRawMetrics === "denied" && (parsed.httpStatus === null || parsed.httpStatus >= 400 || !exposesPrometheus); + return { + ok: result.exitCode === 0 && denied, + expected: endpoint.publicRawMetrics, + url, + httpStatus: parsed.httpStatus, + exposesPrometheus, + bodyBytes: Buffer.byteLength(parsed.body), + result: compactRuntimeCommand(result), + degradedReason: denied ? undefined : "node-observability-public-raw-metrics-exposed", + }; +} + +function summarizeNodeObservabilityMetrics(observability: HwlabRuntimeObservabilitySpec, text: string, httpStatus: number | null): Record { + const workbench = observability.workbench; + if (workbench === undefined) { + return { + ready: false, + httpStatus, + degradedReason: "node-observability-workbench-not-declared", + }; + } + const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + const sampleLines = lines.filter((line) => !line.startsWith("#")); + const seriesByPrefix = Object.fromEntries(workbench.metricPrefixes.map((prefix) => [ + prefix, + sampleLines.filter((line) => (metricNameFromPrometheusLine(line) ?? "").startsWith(prefix)).length, + ])); + const requiredSeries = workbench.requiredSeries.map((name) => { + const matching = sampleLines.filter((line) => prometheusLineMatchesMetric(line, name)); + return { + name, + present: matching.length > 0, + sampleCount: matching.length, + }; + }); + const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name); + const deniedBackendEventLines = sampleLines.filter((line) => { + const name = metricNameFromPrometheusLine(line) ?? ""; + if (!isWorkbenchBackendEventMetric(name)) return false; + return workbench.backendLabelDenylist.some((label) => prometheusLineHasLabel(line, "backend", label)); + }); + const ready = httpStatus !== null + && httpStatus >= 200 + && httpStatus < 300 + && missingSeries.length === 0 + && deniedBackendEventLines.length <= workbench.maxUnknownEventLines; + return { + ready, + httpStatus, + bodyBytes: Buffer.byteLength(text), + lineCount: lines.length, + sampleLineCount: sampleLines.length, + metricPrefixes: workbench.metricPrefixes, + seriesByPrefix, + requiredSeries, + missingSeries, + backendLabelDenylist: workbench.backendLabelDenylist, + deniedBackendEventLineCount: deniedBackendEventLines.length, + maxDeniedBackendEventLines: workbench.maxUnknownEventLines, + deniedBackendEventLines: compactPrometheusLines(deniedBackendEventLines), + backendVisibleCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_backend_event_visible_latency_seconds_count"))), + phaseCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_event_phase_duration_seconds_count"))), + degradedReason: ready ? undefined : missingSeries.length > 0 ? "node-observability-required-series-missing" : "node-observability-denied-backend-label-present", + }; +} + +function splitCurlBodyStatus(stdout: string): { body: string; httpStatus: number | null } { + const marker = "\nUNIDESK_HTTP_STATUS:"; + const index = stdout.lastIndexOf(marker); + if (index < 0) return { body: stdout, httpStatus: null }; + const rawStatus = stdout.slice(index + marker.length).trim(); + const status = nullableInteger(rawStatus); + return { + body: stdout.slice(0, index), + httpStatus: status, + }; +} + +function looksLikePrometheusMetrics(text: string, prefixes: readonly string[]): boolean { + const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + return lines.some((line) => { + if (line.startsWith("# HELP ") || line.startsWith("# TYPE ")) return true; + const name = metricNameFromPrometheusLine(line); + return name !== null && (prefixes.length === 0 || prefixes.some((prefix) => name.startsWith(prefix))); + }); +} + +function metricNameFromPrometheusLine(line: string): string | null { + const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line); + return match?.[1] ?? null; +} + +function prometheusLineMatchesMetric(line: string, name: string): boolean { + if (!line.startsWith(name)) return false; + const next = line.charAt(name.length); + return next === "" || next === "{" || /\s/u.test(next); +} + +function prometheusLineHasLabel(line: string, name: string, value: string): boolean { + return line.includes(`${name}="${value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"')}"`); +} + +function isWorkbenchBackendEventMetric(name: string): boolean { + return name.startsWith("hwlab_workbench_event_") || name.startsWith("hwlab_workbench_backend_event_"); +} + +function compactPrometheusLines(lines: readonly string[]): string[] { + return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); +} + +function parseJsonRecordFromText(text: string): Record { + const trimmed = text.trim(); + if (trimmed.length === 0) return {}; + try { + const parsed = JSON.parse(trimmed) as unknown; + return record(parsed); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { + return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown); + } catch {} + } + } + return {}; +} + +function k8sServiceSelector(serviceJson: Record): Record { + const spec = record(serviceJson.spec); + const selector = record(spec.selector); + return Object.fromEntries(Object.entries(selector).filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].length > 0)); +} + +function labelSelectorText(selector: Record): string { + return Object.entries(selector).map(([key, value]) => `${key}=${value}`).join(","); +} + +function selectK8sPod(podsJson: Record, containerName: string): { name: string; phase: string | null; ready: boolean; containerReady: boolean } | null { + const items = Array.isArray(podsJson.items) ? podsJson.items.map(record) : []; + const pods = items.map((item) => { + const metadata = record(item.metadata); + const status = record(item.status); + const name = typeof metadata.name === "string" ? metadata.name : ""; + const phase = typeof status.phase === "string" ? status.phase : null; + const containerStatuses = Array.isArray(status.containerStatuses) ? status.containerStatuses.map(record) : []; + const targetContainer = containerStatuses.find((container) => container.name === containerName); + const containerReady = targetContainer?.ready === true; + return { + name, + phase, + ready: name.length > 0 && phase === "Running" && containerReady, + containerReady, + }; + }).filter((pod) => pod.name.length > 0); + return pods.find((pod) => pod.ready) ?? pods.find((pod) => pod.phase === "Running") ?? pods[0] ?? null; +} + function compactRuntimeCommand(result: CommandResult): Record { return { exitCode: result.exitCode,