feat: add node workbench observability ops
This commit is contained in:
Codex
@@ -1,3 +1,5 @@
|
||||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: YAML source-of-truth parsing for HWLAB node/lane Workbench observability.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { rootPath } from "./config";
|
||||
|
||||
@@ -83,6 +85,27 @@ export interface HwlabRuntimeBuildkitSpec {
|
||||
|
||||
export interface HwlabRuntimeObservabilitySpec {
|
||||
readonly prometheusOperator: boolean;
|
||||
readonly metricsEndpoint?: HwlabRuntimeObservabilityMetricsEndpointSpec;
|
||||
readonly workbench?: HwlabRuntimeObservabilityWorkbenchSpec;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeObservabilityMetricsEndpointSpec {
|
||||
readonly serviceName: string;
|
||||
readonly containerName: string;
|
||||
readonly port: number;
|
||||
readonly scheme: "http";
|
||||
readonly path: string;
|
||||
readonly scrapeMode: "pod-loopback";
|
||||
readonly publicRawMetrics: "denied";
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeObservabilityWorkbenchSpec {
|
||||
readonly enabled: boolean;
|
||||
readonly summaryPath: string;
|
||||
readonly metricPrefixes: readonly string[];
|
||||
readonly requiredSeries: readonly string[];
|
||||
readonly backendLabelDenylist: readonly string[];
|
||||
readonly maxUnknownEventLines: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeImageRewriteSpec {
|
||||
@@ -266,6 +289,12 @@ function booleanField(obj: Record<string, unknown>, key: string, path: string):
|
||||
return value;
|
||||
}
|
||||
|
||||
function nonNegativeIntegerField(obj: Record<string, unknown>, key: string, path: string): number {
|
||||
const value = obj[key];
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function sortedRecordEntries(value: unknown, path: string): Array<[string, Record<string, unknown>]> {
|
||||
return Object.entries(asRecord(value, path)).map(([key, item]) => [key, asRecord(item, `${path}.${key}`)]);
|
||||
}
|
||||
@@ -557,6 +586,41 @@ function observabilityConfig(value: unknown, path: string): HwlabRuntimeObservab
|
||||
const raw = asRecord(value, path);
|
||||
return {
|
||||
prometheusOperator: booleanField(raw, "prometheusOperator", path),
|
||||
metricsEndpoint: observabilityMetricsEndpointConfig(raw.metricsEndpoint, `${path}.metricsEndpoint`),
|
||||
workbench: observabilityWorkbenchConfig(raw.workbench, `${path}.workbench`),
|
||||
};
|
||||
}
|
||||
|
||||
function observabilityMetricsEndpointConfig(value: unknown, path: string): HwlabRuntimeObservabilityMetricsEndpointSpec | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
const raw = asRecord(value, path);
|
||||
const scheme = stringField(raw, "scheme", path);
|
||||
if (scheme !== "http") throw new Error(`${path}.scheme must be http`);
|
||||
const scrapeMode = stringField(raw, "scrapeMode", path);
|
||||
if (scrapeMode !== "pod-loopback") throw new Error(`${path}.scrapeMode must be pod-loopback`);
|
||||
const publicRawMetrics = stringField(raw, "publicRawMetrics", path);
|
||||
if (publicRawMetrics !== "denied") throw new Error(`${path}.publicRawMetrics must be denied`);
|
||||
return {
|
||||
serviceName: stringField(raw, "serviceName", path),
|
||||
containerName: stringField(raw, "containerName", path),
|
||||
port: numberField(raw, "port", path),
|
||||
scheme,
|
||||
path: stringField(raw, "path", path),
|
||||
scrapeMode,
|
||||
publicRawMetrics,
|
||||
};
|
||||
}
|
||||
|
||||
function observabilityWorkbenchConfig(value: unknown, path: string): HwlabRuntimeObservabilityWorkbenchSpec | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
const raw = asRecord(value, path);
|
||||
return {
|
||||
enabled: booleanField(raw, "enabled", path),
|
||||
summaryPath: stringField(raw, "summaryPath", path),
|
||||
metricPrefixes: stringArrayField(raw, "metricPrefixes", path),
|
||||
requiredSeries: stringArrayField(raw, "requiredSeries", path),
|
||||
backendLabelDenylist: stringArrayField(raw, "backendLabelDenylist", path),
|
||||
maxUnknownEventLines: nonNegativeIntegerField(raw, "maxUnknownEventLines", path),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+664
-2
@@ -1,3 +1,5 @@
|
||||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
@@ -7,7 +9,7 @@ import { startJob } from "./jobs";
|
||||
import { classifySshTcpPoolFailure } from "./ssh";
|
||||
import { runHwlabG14Command } from "./hwlab-g14";
|
||||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
|
||||
|
||||
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete";
|
||||
type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
|
||||
@@ -51,6 +53,19 @@ interface NodeWebProbeScriptOptions {
|
||||
|
||||
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions;
|
||||
|
||||
type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary";
|
||||
|
||||
interface NodeObservabilityOptions {
|
||||
action: NodeObservabilityAction;
|
||||
node: string;
|
||||
lane: HwlabRuntimeLane;
|
||||
confirm: boolean;
|
||||
dryRun: boolean;
|
||||
full: boolean;
|
||||
timeoutSeconds: number;
|
||||
spec: HwlabRuntimeLaneSpec;
|
||||
}
|
||||
|
||||
interface NodeRuntimeRenderResult {
|
||||
readonly result: CommandResult;
|
||||
readonly renderDir: string;
|
||||
@@ -191,12 +206,16 @@ export async function runHwlabNodeCommand(_config: Config, args: string[]): Prom
|
||||
if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeWebProbeHelp();
|
||||
return runNodeWebProbe(parseNodeWebProbeOptions(args.slice(1)));
|
||||
}
|
||||
if (domain === "observability") {
|
||||
if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeObservabilityHelp();
|
||||
return runNodeObservability(parseNodeObservabilityOptions(args.slice(1)));
|
||||
}
|
||||
if (args.includes("--help") || args.includes("-h")) return hwlabNodeHelp();
|
||||
if (domain === "control-plane" || domain === "git-mirror") {
|
||||
return runNodeDelegatedDomain(_config, domain, args.slice(1));
|
||||
}
|
||||
if (domain !== "secret") {
|
||||
return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret, hwlab nodes test-accounts, hwlab nodes web-probe" };
|
||||
return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes observability, hwlab nodes secret, hwlab nodes test-accounts, hwlab nodes web-probe" };
|
||||
}
|
||||
const options = parseSecretOptions(args.slice(1));
|
||||
return runNodeSecret(options);
|
||||
@@ -247,6 +266,10 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab nodes test-accounts status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes test-accounts sync --node D601 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000",
|
||||
"bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03",
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -268,6 +291,58 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
export function hwlabNodeObservabilityHelp(): Record<string, unknown> {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab nodes observability",
|
||||
description: "YAML-first node/lane observability control for HWLAB Workbench metrics. Runtime is queried only for presence, boundary, and summary evidence.",
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
examples: [
|
||||
"bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --confirm",
|
||||
"bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03",
|
||||
],
|
||||
actions: {
|
||||
plan: "Render the YAML-declared collection mode, scrape target, boundary, and required Workbench metric series.",
|
||||
apply: "Apply the YAML-declared observability control plane. D601 pod-loopback mode validates the boundary and metrics without creating ad-hoc cluster objects.",
|
||||
status: "Check Kubernetes service/pod presence, public raw metrics denial, and Workbench metric readiness.",
|
||||
"workbench-summary": "Read the YAML-declared metrics endpoint via pod loopback and summarize required Workbench series.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions {
|
||||
const [actionRaw] = args;
|
||||
if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) {
|
||||
throw new Error("observability usage: observability ACTION --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
}
|
||||
if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary") {
|
||||
throw new Error(`observability action must be plan, apply, status, or workbench-summary; got ${actionRaw}`);
|
||||
}
|
||||
assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--full", "--raw"]));
|
||||
const node = requiredOption(args, "--node");
|
||||
assertNodeId(node);
|
||||
const laneRaw = requiredOption(args, "--lane");
|
||||
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`);
|
||||
const confirm = args.includes("--confirm");
|
||||
const dryRun = args.includes("--dry-run");
|
||||
if (confirm && dryRun) throw new Error("observability accepts only one of --confirm or --dry-run");
|
||||
if (actionRaw === "apply" && !confirm && !dryRun) throw new Error("observability apply requires --dry-run or --confirm");
|
||||
if (actionRaw !== "apply" && (confirm || dryRun)) throw new Error(`observability ${actionRaw} is read-only and does not accept --confirm or --dry-run`);
|
||||
return {
|
||||
action: actionRaw,
|
||||
node,
|
||||
lane: laneRaw,
|
||||
confirm,
|
||||
dryRun,
|
||||
full: args.includes("--full") || args.includes("--raw"),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
spec: hwlabRuntimeLaneSpecForNode(laneRaw, node),
|
||||
};
|
||||
}
|
||||
|
||||
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown>> {
|
||||
const scoped = parseNodeScopedDelegatedOptions(domain, args);
|
||||
const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane);
|
||||
@@ -487,6 +562,593 @@ function nodeRuntimeControlPlanePlan(scoped: ReturnType<typeof parseNodeScopedDe
|
||||
};
|
||||
}
|
||||
|
||||
function runNodeObservability(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
if (options.action === "plan") return nodeObservabilityPlan(options);
|
||||
if (options.action === "apply") return nodeObservabilityApply(options);
|
||||
if (options.action === "status") return nodeObservabilityStatus(options);
|
||||
return nodeObservabilityWorkbenchSummary(options);
|
||||
}
|
||||
|
||||
function nodeObservabilityPlan(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
return {
|
||||
ok: configProblem === null,
|
||||
command: `hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
||||
mode: "node-observability-plan",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: nodeObservabilityExpected(options.spec),
|
||||
renderPlan: nodeObservabilityRenderPlan(options.spec.observability),
|
||||
degradedReason: configProblem,
|
||||
next: {
|
||||
dryRun: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
||||
apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityApply(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
const applyPlan = nodeObservabilityApplyPlan(options.spec.observability);
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: configProblem === null && applyPlan.supported === true,
|
||||
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
||||
mode: "node-observability-apply-dry-run",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: nodeObservabilityExpected(options.spec),
|
||||
applyPlan,
|
||||
degradedReason: configProblem ?? (applyPlan.supported === true ? undefined : "node-observability-apply-mode-unsupported"),
|
||||
};
|
||||
}
|
||||
if (configProblem !== null || applyPlan.supported !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
mode: "node-observability-apply",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
applyPlan,
|
||||
degradedReason: configProblem ?? "node-observability-apply-mode-unsupported",
|
||||
};
|
||||
}
|
||||
const status = nodeObservabilityStatus({ ...options, full: true });
|
||||
return {
|
||||
ok: status.ok === true,
|
||||
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
mode: "node-observability-apply",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
applyPlan,
|
||||
appliedResources: [],
|
||||
applyResult: {
|
||||
mode: applyPlan.mode,
|
||||
reason: applyPlan.reason,
|
||||
runtimeValidationOnly: true,
|
||||
},
|
||||
status: options.full ? status : summarizeNodeObservabilityStatus(status),
|
||||
degradedReason: status.ok === true ? undefined : "node-observability-status-not-ready",
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
const serviceStatus = nodeObservabilityServiceStatus(options);
|
||||
const publicRawMetrics = nodeObservabilityPublicRawMetricsProbe(options);
|
||||
const workbenchSummary = nodeObservabilityWorkbenchSummary(options, serviceStatus);
|
||||
const ok = configProblem === null && serviceStatus.ready === true && publicRawMetrics.ok === true && workbenchSummary.ok === true;
|
||||
const status = {
|
||||
ok,
|
||||
command: `hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
||||
mode: "node-observability-status",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: nodeObservabilityExpected(options.spec),
|
||||
service: serviceStatus,
|
||||
publicRawMetrics,
|
||||
workbenchSummary,
|
||||
degradedReason: configProblem
|
||||
?? (serviceStatus.ready === true
|
||||
? publicRawMetrics.ok === true
|
||||
? workbenchSummary.ok === true ? undefined : "node-observability-workbench-metrics-not-ready"
|
||||
: "node-observability-public-raw-metrics-boundary-failed"
|
||||
: "node-observability-service-not-ready"),
|
||||
next: {
|
||||
plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
return options.full ? status : summarizeNodeObservabilityStatus(status);
|
||||
}
|
||||
|
||||
function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, existingServiceStatus?: Record<string, unknown>): Record<string, unknown> {
|
||||
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
||||
const serviceStatus = existingServiceStatus ?? nodeObservabilityServiceStatus(options);
|
||||
const podName = typeof serviceStatus.podName === "string" && serviceStatus.podName.length > 0 ? serviceStatus.podName : null;
|
||||
const metricsProbe = podName === null ? null : nodeObservabilityMetricsProbe(options, podName);
|
||||
const metrics = metricsProbe?.summary ?? summarizeNodeObservabilityMetrics(options.spec.observability, "", null);
|
||||
const ok = configProblem === null && serviceStatus.ready === true && metricsProbe?.ok === true && metrics.ready === true;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
mode: "node-observability-workbench-summary",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
metricsEndpoint: nodeObservabilityEndpointSummary(options.spec.observability),
|
||||
service: {
|
||||
ready: serviceStatus.ready === true,
|
||||
namespace: serviceStatus.namespace,
|
||||
serviceName: serviceStatus.serviceName,
|
||||
podName,
|
||||
containerName: serviceStatus.containerName,
|
||||
},
|
||||
metrics,
|
||||
probe: metricsProbe === null ? null : {
|
||||
ok: metricsProbe.ok,
|
||||
httpStatus: metricsProbe.httpStatus,
|
||||
bodyBytes: metricsProbe.bodyBytes,
|
||||
result: compactRuntimeCommand(metricsProbe.result),
|
||||
error: metricsProbe.error,
|
||||
},
|
||||
degradedReason: configProblem
|
||||
?? (serviceStatus.ready === true
|
||||
? metricsProbe?.ok === true
|
||||
? metrics.ready === true ? undefined : "node-observability-required-series-missing"
|
||||
: "node-observability-metrics-endpoint-not-readable"
|
||||
: "node-observability-service-not-ready"),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeNodeObservabilityStatus(status: Record<string, unknown>): Record<string, unknown> {
|
||||
const service = record(status.service);
|
||||
const publicRawMetrics = record(status.publicRawMetrics);
|
||||
const workbenchSummary = record(status.workbenchSummary);
|
||||
const metrics = record(workbenchSummary.metrics);
|
||||
return {
|
||||
ok: status.ok === true,
|
||||
command: status.command,
|
||||
mode: "node-observability-status-summary",
|
||||
mutation: false,
|
||||
node: status.node,
|
||||
lane: status.lane,
|
||||
degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
|
||||
service: {
|
||||
ready: service.ready === true,
|
||||
namespace: service.namespace ?? null,
|
||||
serviceName: service.serviceName ?? null,
|
||||
podName: service.podName ?? null,
|
||||
containerName: service.containerName ?? null,
|
||||
},
|
||||
publicRawMetrics: {
|
||||
ok: publicRawMetrics.ok === true,
|
||||
expected: publicRawMetrics.expected ?? null,
|
||||
httpStatus: publicRawMetrics.httpStatus ?? null,
|
||||
exposesPrometheus: publicRawMetrics.exposesPrometheus === true,
|
||||
},
|
||||
workbench: {
|
||||
ok: workbenchSummary.ok === true,
|
||||
httpStatus: metrics.httpStatus ?? null,
|
||||
bodyBytes: metrics.bodyBytes ?? null,
|
||||
seriesByPrefix: metrics.seriesByPrefix ?? {},
|
||||
missingSeries: metrics.missingSeries ?? [],
|
||||
deniedBackendEventLineCount: metrics.deniedBackendEventLineCount ?? null,
|
||||
maxDeniedBackendEventLines: metrics.maxDeniedBackendEventLines ?? null,
|
||||
},
|
||||
next: status.next,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
return {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
namespace: spec.runtimeNamespace,
|
||||
publicApiUrl: spec.publicApiUrl,
|
||||
observability: spec.observability,
|
||||
sourceOfTruth: hwlabRuntimeLaneConfigPath(),
|
||||
runtimeAsSourceOfTruth: false,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityRenderPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
||||
const endpoint = observability.metricsEndpoint;
|
||||
return {
|
||||
prometheusOperator: observability.prometheusOperator,
|
||||
metricsEndpoint: nodeObservabilityEndpointSummary(observability),
|
||||
workbench: observability.workbench ?? null,
|
||||
clusterResources: observability.prometheusOperator
|
||||
? {
|
||||
source: "HWLAB GitOps rendered manifests",
|
||||
serviceMonitor: "YAML-rendered only when declared by target lane",
|
||||
prometheusRule: "YAML-rendered only when declared by target lane",
|
||||
}
|
||||
: {
|
||||
source: "none",
|
||||
reason: endpoint?.scrapeMode === "pod-loopback" ? "target lane declares pod-loopback scrape mode" : "no YAML-declared scrape target",
|
||||
},
|
||||
publicRawMetrics: endpoint?.publicRawMetrics ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityApplyPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
||||
const endpoint = observability.metricsEndpoint;
|
||||
if (endpoint === undefined) {
|
||||
return {
|
||||
supported: false,
|
||||
mode: "unconfigured",
|
||||
reason: "metricsEndpoint is missing from YAML",
|
||||
};
|
||||
}
|
||||
if (!observability.prometheusOperator && endpoint.scrapeMode === "pod-loopback") {
|
||||
return {
|
||||
supported: true,
|
||||
mode: "validated-noop",
|
||||
reason: "target lane declares pod-loopback metrics collection and no Prometheus Operator resources",
|
||||
resources: [],
|
||||
validates: ["kubernetes-service", "selected-pod", "pod-loopback-metrics", "public-raw-metrics-boundary", "workbench-required-series"],
|
||||
};
|
||||
}
|
||||
if (observability.prometheusOperator) {
|
||||
return {
|
||||
supported: false,
|
||||
mode: "prometheus-operator",
|
||||
reason: "Prometheus Operator object rendering must be declared in the target lane GitOps YAML before this node-scoped apply can own it",
|
||||
};
|
||||
}
|
||||
return {
|
||||
supported: false,
|
||||
mode: endpoint.scrapeMode,
|
||||
reason: "unsupported metricsEndpoint.scrapeMode",
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityConfigProblem(observability: HwlabRuntimeObservabilitySpec): string | null {
|
||||
if (observability.metricsEndpoint === undefined) return "node-observability-metrics-endpoint-not-declared";
|
||||
if (observability.workbench === undefined) return "node-observability-workbench-not-declared";
|
||||
if (!observability.workbench.enabled) return "node-observability-workbench-disabled";
|
||||
return null;
|
||||
}
|
||||
|
||||
function nodeObservabilityEndpointSummary(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> | null {
|
||||
const endpoint = observability.metricsEndpoint;
|
||||
if (endpoint === undefined) return null;
|
||||
return {
|
||||
serviceName: endpoint.serviceName,
|
||||
containerName: endpoint.containerName,
|
||||
port: endpoint.port,
|
||||
scheme: endpoint.scheme,
|
||||
path: endpoint.path,
|
||||
scrapeMode: endpoint.scrapeMode,
|
||||
publicRawMetrics: endpoint.publicRawMetrics,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityServiceStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const endpoint = options.spec.observability.metricsEndpoint;
|
||||
if (endpoint === undefined) {
|
||||
return {
|
||||
ready: false,
|
||||
namespace: options.spec.runtimeNamespace,
|
||||
serviceName: null,
|
||||
podName: null,
|
||||
containerName: null,
|
||||
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
||||
};
|
||||
}
|
||||
const namespace = runNodeK3sArgs(options.spec, ["kubectl", "get", "ns", options.spec.runtimeNamespace, "-o", "name"], 60);
|
||||
const namespaceExists = namespace.exitCode === 0;
|
||||
const service = namespaceExists
|
||||
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "svc", endpoint.serviceName, "-o", "json"], 60)
|
||||
: null;
|
||||
const serviceJson = service !== null && service.exitCode === 0 ? parseJsonRecordFromText(service.stdout) : {};
|
||||
const selector = k8sServiceSelector(serviceJson);
|
||||
const selectorText = labelSelectorText(selector);
|
||||
const pods = namespaceExists && service !== null && service.exitCode === 0 && selectorText.length > 0
|
||||
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "pods", "-l", selectorText, "-o", "json"], 60)
|
||||
: null;
|
||||
const pod = pods !== null && pods.exitCode === 0 ? selectK8sPod(parseJsonRecordFromText(pods.stdout), endpoint.containerName) : null;
|
||||
const ready = namespaceExists && service !== null && service.exitCode === 0 && pod?.ready === true;
|
||||
return {
|
||||
ready,
|
||||
namespace: options.spec.runtimeNamespace,
|
||||
namespaceExists,
|
||||
serviceName: endpoint.serviceName,
|
||||
serviceExists: service !== null && service.exitCode === 0,
|
||||
selector,
|
||||
selectorText: selectorText.length > 0 ? selectorText : null,
|
||||
podName: pod?.name ?? null,
|
||||
podPhase: pod?.phase ?? null,
|
||||
podReady: pod?.ready ?? false,
|
||||
containerName: endpoint.containerName,
|
||||
containerReady: pod?.containerReady ?? false,
|
||||
probes: {
|
||||
namespace: compactRuntimeCommand(namespace),
|
||||
service: service === null ? null : compactRuntimeCommand(service),
|
||||
pods: pods === null ? null : compactRuntimeCommand(pods),
|
||||
},
|
||||
degradedReason: ready ? undefined : namespaceExists ? "node-observability-pod-or-service-not-ready" : "node-observability-namespace-missing",
|
||||
};
|
||||
}
|
||||
|
||||
interface NodeObservabilityMetricsProbeResult {
|
||||
readonly ok: boolean;
|
||||
readonly httpStatus: number | null;
|
||||
readonly bodyBytes: number;
|
||||
readonly summary: Record<string, unknown>;
|
||||
readonly result: CommandResult;
|
||||
readonly error: string | null;
|
||||
}
|
||||
|
||||
function nodeObservabilityMetricsProbe(options: NodeObservabilityOptions, podName: string): NodeObservabilityMetricsProbeResult {
|
||||
const endpoint = options.spec.observability.metricsEndpoint;
|
||||
if (endpoint === undefined) throw new Error("metricsEndpoint is required before probing metrics");
|
||||
const workbench = options.spec.observability.workbench;
|
||||
const summaryConfig = Buffer.from(JSON.stringify({
|
||||
metricPrefixes: workbench?.metricPrefixes ?? [],
|
||||
requiredSeries: workbench?.requiredSeries ?? [],
|
||||
backendLabelDenylist: workbench?.backendLabelDenylist ?? [],
|
||||
maxDeniedBackendEventLines: workbench?.maxUnknownEventLines ?? 0,
|
||||
}), "utf8").toString("base64");
|
||||
const source = [
|
||||
"const http = require('node:http');",
|
||||
"const port = Number(process.argv[1]);",
|
||||
"const path = process.argv[2];",
|
||||
"const config = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
|
||||
"function metricNameFromLine(line) { const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/.exec(line); return match ? match[1] : null; }",
|
||||
"function lineMatchesMetric(line, name) { if (!line.startsWith(name)) return false; const next = line.charAt(name.length); return next === '' || next === '{' || /\\s/.test(next); }",
|
||||
"function lineHasLabel(line, name, value) { return line.includes(`${name}=\\\"${String(value).replace(/\\\\/g, '\\\\\\\\').replace(/\\\"/g, '\\\\\\\"')}\\\"`); }",
|
||||
"function isBackendEventMetric(name) { return name.startsWith('hwlab_workbench_event_') || name.startsWith('hwlab_workbench_backend_event_'); }",
|
||||
"function compactLines(lines) { return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); }",
|
||||
"function summarize(text, statusCode) {",
|
||||
" const lines = text.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean);",
|
||||
" const sampleLines = lines.filter((line) => !line.startsWith('#'));",
|
||||
" const seriesByPrefix = Object.fromEntries(config.metricPrefixes.map((prefix) => [prefix, sampleLines.filter((line) => (metricNameFromLine(line) || '').startsWith(prefix)).length]));",
|
||||
" const requiredSeries = config.requiredSeries.map((name) => { const matching = sampleLines.filter((line) => lineMatchesMetric(line, name)); return { name, present: matching.length > 0, sampleCount: matching.length }; });",
|
||||
" const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);",
|
||||
" const deniedBackendEventLines = sampleLines.filter((line) => { const name = metricNameFromLine(line) || ''; if (!isBackendEventMetric(name)) return false; return config.backendLabelDenylist.some((label) => lineHasLabel(line, 'backend', label)); });",
|
||||
" const ready = statusCode >= 200 && statusCode < 300 && missingSeries.length === 0 && deniedBackendEventLines.length <= config.maxDeniedBackendEventLines;",
|
||||
" return {",
|
||||
" ready,",
|
||||
" httpStatus: statusCode,",
|
||||
" bodyBytes: Buffer.byteLength(text),",
|
||||
" lineCount: lines.length,",
|
||||
" sampleLineCount: sampleLines.length,",
|
||||
" metricPrefixes: config.metricPrefixes,",
|
||||
" seriesByPrefix,",
|
||||
" requiredSeries,",
|
||||
" missingSeries,",
|
||||
" backendLabelDenylist: config.backendLabelDenylist,",
|
||||
" deniedBackendEventLineCount: deniedBackendEventLines.length,",
|
||||
" maxDeniedBackendEventLines: config.maxDeniedBackendEventLines,",
|
||||
" deniedBackendEventLines: compactLines(deniedBackendEventLines),",
|
||||
" backendVisibleCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_backend_event_visible_latency_seconds_count'))),",
|
||||
" phaseCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_event_phase_duration_seconds_count'))),",
|
||||
" degradedReason: ready ? undefined : missingSeries.length > 0 ? 'node-observability-required-series-missing' : 'node-observability-denied-backend-label-present',",
|
||||
" };",
|
||||
"}",
|
||||
"let settled = false;",
|
||||
"const finish = (payload, code = 0) => { if (settled) return; settled = true; console.log(JSON.stringify(payload)); process.exitCode = code; };",
|
||||
"const req = http.get({ host: '127.0.0.1', port, path, timeout: 8000 }, (res) => {",
|
||||
" let body = '';",
|
||||
" res.setEncoding('utf8');",
|
||||
" res.on('data', (chunk) => { body += chunk; if (body.length > 2000000) req.destroy(new Error('metrics body exceeded 2MB')); });",
|
||||
" res.on('end', () => finish({ ok: res.statusCode >= 200 && res.statusCode < 300, statusCode: res.statusCode, bodyBytes: Buffer.byteLength(body), summary: summarize(body, res.statusCode || 0) }));",
|
||||
"});",
|
||||
"req.on('timeout', () => req.destroy(new Error('timeout')));",
|
||||
"req.on('error', (error) => finish({ ok: false, statusCode: null, bodyBytes: 0, summary: null, error: String(error && error.message || error) }, 1));",
|
||||
].join("\n");
|
||||
const result = runNodeK3sArgs(options.spec, [
|
||||
"kubectl",
|
||||
"-n", options.spec.runtimeNamespace,
|
||||
"exec", podName,
|
||||
"-c", endpoint.containerName,
|
||||
"--",
|
||||
"node", "-e", source,
|
||||
String(endpoint.port),
|
||||
endpoint.path,
|
||||
summaryConfig,
|
||||
], options.timeoutSeconds);
|
||||
const payload = parseJsonRecordFromText(result.stdout);
|
||||
const httpStatus = typeof payload.statusCode === "number" ? payload.statusCode : null;
|
||||
const error = typeof payload.error === "string" ? payload.error : null;
|
||||
const summary = record(payload.summary);
|
||||
return {
|
||||
ok: result.exitCode === 0 && payload.ok === true && httpStatus !== null && httpStatus >= 200 && httpStatus < 300,
|
||||
httpStatus,
|
||||
bodyBytes: typeof payload.bodyBytes === "number" ? payload.bodyBytes : 0,
|
||||
summary,
|
||||
result,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityPublicRawMetricsProbe(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const endpoint = options.spec.observability.metricsEndpoint;
|
||||
if (endpoint === undefined) {
|
||||
return {
|
||||
ok: false,
|
||||
expected: null,
|
||||
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
||||
};
|
||||
}
|
||||
const url = joinUrlPath(options.spec.publicApiUrl, endpoint.path);
|
||||
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "-", "-w", "\nUNIDESK_HTTP_STATUS:%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
|
||||
const parsed = splitCurlBodyStatus(result.stdout);
|
||||
const exposesPrometheus = looksLikePrometheusMetrics(parsed.body, options.spec.observability.workbench?.metricPrefixes ?? []);
|
||||
const denied = endpoint.publicRawMetrics === "denied" && (parsed.httpStatus === null || parsed.httpStatus >= 400 || !exposesPrometheus);
|
||||
return {
|
||||
ok: result.exitCode === 0 && denied,
|
||||
expected: endpoint.publicRawMetrics,
|
||||
url,
|
||||
httpStatus: parsed.httpStatus,
|
||||
exposesPrometheus,
|
||||
bodyBytes: Buffer.byteLength(parsed.body),
|
||||
result: compactRuntimeCommand(result),
|
||||
degradedReason: denied ? undefined : "node-observability-public-raw-metrics-exposed",
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeNodeObservabilityMetrics(observability: HwlabRuntimeObservabilitySpec, text: string, httpStatus: number | null): Record<string, unknown> {
|
||||
const workbench = observability.workbench;
|
||||
if (workbench === undefined) {
|
||||
return {
|
||||
ready: false,
|
||||
httpStatus,
|
||||
degradedReason: "node-observability-workbench-not-declared",
|
||||
};
|
||||
}
|
||||
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
const sampleLines = lines.filter((line) => !line.startsWith("#"));
|
||||
const seriesByPrefix = Object.fromEntries(workbench.metricPrefixes.map((prefix) => [
|
||||
prefix,
|
||||
sampleLines.filter((line) => (metricNameFromPrometheusLine(line) ?? "").startsWith(prefix)).length,
|
||||
]));
|
||||
const requiredSeries = workbench.requiredSeries.map((name) => {
|
||||
const matching = sampleLines.filter((line) => prometheusLineMatchesMetric(line, name));
|
||||
return {
|
||||
name,
|
||||
present: matching.length > 0,
|
||||
sampleCount: matching.length,
|
||||
};
|
||||
});
|
||||
const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);
|
||||
const deniedBackendEventLines = sampleLines.filter((line) => {
|
||||
const name = metricNameFromPrometheusLine(line) ?? "";
|
||||
if (!isWorkbenchBackendEventMetric(name)) return false;
|
||||
return workbench.backendLabelDenylist.some((label) => prometheusLineHasLabel(line, "backend", label));
|
||||
});
|
||||
const ready = httpStatus !== null
|
||||
&& httpStatus >= 200
|
||||
&& httpStatus < 300
|
||||
&& missingSeries.length === 0
|
||||
&& deniedBackendEventLines.length <= workbench.maxUnknownEventLines;
|
||||
return {
|
||||
ready,
|
||||
httpStatus,
|
||||
bodyBytes: Buffer.byteLength(text),
|
||||
lineCount: lines.length,
|
||||
sampleLineCount: sampleLines.length,
|
||||
metricPrefixes: workbench.metricPrefixes,
|
||||
seriesByPrefix,
|
||||
requiredSeries,
|
||||
missingSeries,
|
||||
backendLabelDenylist: workbench.backendLabelDenylist,
|
||||
deniedBackendEventLineCount: deniedBackendEventLines.length,
|
||||
maxDeniedBackendEventLines: workbench.maxUnknownEventLines,
|
||||
deniedBackendEventLines: compactPrometheusLines(deniedBackendEventLines),
|
||||
backendVisibleCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_backend_event_visible_latency_seconds_count"))),
|
||||
phaseCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_event_phase_duration_seconds_count"))),
|
||||
degradedReason: ready ? undefined : missingSeries.length > 0 ? "node-observability-required-series-missing" : "node-observability-denied-backend-label-present",
|
||||
};
|
||||
}
|
||||
|
||||
function splitCurlBodyStatus(stdout: string): { body: string; httpStatus: number | null } {
|
||||
const marker = "\nUNIDESK_HTTP_STATUS:";
|
||||
const index = stdout.lastIndexOf(marker);
|
||||
if (index < 0) return { body: stdout, httpStatus: null };
|
||||
const rawStatus = stdout.slice(index + marker.length).trim();
|
||||
const status = nullableInteger(rawStatus);
|
||||
return {
|
||||
body: stdout.slice(0, index),
|
||||
httpStatus: status,
|
||||
};
|
||||
}
|
||||
|
||||
function looksLikePrometheusMetrics(text: string, prefixes: readonly string[]): boolean {
|
||||
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
return lines.some((line) => {
|
||||
if (line.startsWith("# HELP ") || line.startsWith("# TYPE ")) return true;
|
||||
const name = metricNameFromPrometheusLine(line);
|
||||
return name !== null && (prefixes.length === 0 || prefixes.some((prefix) => name.startsWith(prefix)));
|
||||
});
|
||||
}
|
||||
|
||||
function metricNameFromPrometheusLine(line: string): string | null {
|
||||
const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function prometheusLineMatchesMetric(line: string, name: string): boolean {
|
||||
if (!line.startsWith(name)) return false;
|
||||
const next = line.charAt(name.length);
|
||||
return next === "" || next === "{" || /\s/u.test(next);
|
||||
}
|
||||
|
||||
function prometheusLineHasLabel(line: string, name: string, value: string): boolean {
|
||||
return line.includes(`${name}="${value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"')}"`);
|
||||
}
|
||||
|
||||
function isWorkbenchBackendEventMetric(name: string): boolean {
|
||||
return name.startsWith("hwlab_workbench_event_") || name.startsWith("hwlab_workbench_backend_event_");
|
||||
}
|
||||
|
||||
function compactPrometheusLines(lines: readonly string[]): string[] {
|
||||
return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line);
|
||||
}
|
||||
|
||||
function parseJsonRecordFromText(text: string): Record<string, unknown> {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length === 0) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
return record(parsed);
|
||||
} catch {
|
||||
const start = trimmed.indexOf("{");
|
||||
const end = trimmed.lastIndexOf("}");
|
||||
if (start >= 0 && end > start) {
|
||||
try {
|
||||
return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function k8sServiceSelector(serviceJson: Record<string, unknown>): Record<string, string> {
|
||||
const spec = record(serviceJson.spec);
|
||||
const selector = record(spec.selector);
|
||||
return Object.fromEntries(Object.entries(selector).filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].length > 0));
|
||||
}
|
||||
|
||||
function labelSelectorText(selector: Record<string, string>): string {
|
||||
return Object.entries(selector).map(([key, value]) => `${key}=${value}`).join(",");
|
||||
}
|
||||
|
||||
function selectK8sPod(podsJson: Record<string, unknown>, containerName: string): { name: string; phase: string | null; ready: boolean; containerReady: boolean } | null {
|
||||
const items = Array.isArray(podsJson.items) ? podsJson.items.map(record) : [];
|
||||
const pods = items.map((item) => {
|
||||
const metadata = record(item.metadata);
|
||||
const status = record(item.status);
|
||||
const name = typeof metadata.name === "string" ? metadata.name : "";
|
||||
const phase = typeof status.phase === "string" ? status.phase : null;
|
||||
const containerStatuses = Array.isArray(status.containerStatuses) ? status.containerStatuses.map(record) : [];
|
||||
const targetContainer = containerStatuses.find((container) => container.name === containerName);
|
||||
const containerReady = targetContainer?.ready === true;
|
||||
return {
|
||||
name,
|
||||
phase,
|
||||
ready: name.length > 0 && phase === "Running" && containerReady,
|
||||
containerReady,
|
||||
};
|
||||
}).filter((pod) => pod.name.length > 0);
|
||||
return pods.find((pod) => pod.ready) ?? pods.find((pod) => pod.phase === "Running") ?? pods[0] ?? null;
|
||||
}
|
||||
|
||||
function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
|
||||
return {
|
||||
exitCode: result.exitCode,
|
||||
|
||||
Reference in New Issue
Block a user