Codex
1316 lines
63 KiB
TypeScript
1316 lines
63 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. observability module for scripts/src/hwlab-g14.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-g14.ts:6778-8074 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult, G14ObservabilityOptions, ShellSection } from "./types";
|
|
import { commandErrorSummary, tailText } from "./cleanup";
|
|
import { statusText } from "./pr-monitor";
|
|
import { commandJson, compactCommandMetadata, compactCommandResult, g14K3s, isCommandSuccess, nested, record, shellQuote, stringOrNull } from "./remote";
|
|
import { G14_OBSERVABILITY_FIELD_MANAGER, G14_OBSERVABILITY_NAMESPACE, G14_PROMETHEUS_NAME, G14_PROMETHEUS_OPERATOR_RELEASE_ASSET, G14_PROMETHEUS_OPERATOR_VERSION, G14_PROMETHEUS_SERVICE, G14_PROMETHEUS_SERVICE_ACCOUNT, G14_PROMETHEUS_VERSION, V02_CLOUD_API_URL, V02_CLOUD_WEB_URL, V02_OBSERVABILITY_BOOLEAN_QUERY_NAMES, V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES, V02_OBSERVABILITY_EXPECTED_TARGET_COUNT, V02_OBSERVABILITY_QUERIES, V02_OBSERVABILITY_SERVICE_IDS, V02_RUNTIME_NAMESPACE } from "./types";
|
|
import { numericValue, parseShellSections, shellSectionOk } from "./v02-status";
|
|
|
|
export function observabilityLabels(component: string): Record<string, string> {
|
|
return {
|
|
"app.kubernetes.io/part-of": "devops-infra",
|
|
"app.kubernetes.io/component": "observability",
|
|
"g14.pikastech.local/observability-component": component,
|
|
};
|
|
}
|
|
|
|
export function observabilityNamespaceLabel(): Record<string, string> {
|
|
return {
|
|
"g14.pikastech.local/observability-discovery": "enabled",
|
|
};
|
|
}
|
|
|
|
export function g14PrometheusManifest(): Record<string, unknown> {
|
|
const namespaceSelector = { matchLabels: observabilityNamespaceLabel() };
|
|
const monitorSelector = { matchLabels: { "hwlab.pikastech.local/monitoring": "enabled" } };
|
|
return {
|
|
apiVersion: "v1",
|
|
kind: "List",
|
|
items: [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: G14_OBSERVABILITY_NAMESPACE,
|
|
labels: {
|
|
...observabilityNamespaceLabel(),
|
|
...observabilityLabels("namespace"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: V02_RUNTIME_NAMESPACE,
|
|
labels: observabilityNamespaceLabel(),
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ServiceAccount",
|
|
metadata: {
|
|
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
labels: observabilityLabels("prometheus"),
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "ClusterRole",
|
|
metadata: {
|
|
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
|
|
labels: observabilityLabels("prometheus"),
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes", "nodes/metrics", "services", "endpoints", "pods"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["discovery.k8s.io"],
|
|
resources: ["endpointslices"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["networking.k8s.io"],
|
|
resources: ["ingresses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
nonResourceURLs: ["/metrics"],
|
|
verbs: ["get"],
|
|
},
|
|
],
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "ClusterRoleBinding",
|
|
metadata: {
|
|
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
|
|
labels: observabilityLabels("prometheus"),
|
|
},
|
|
roleRef: {
|
|
apiGroup: "rbac.authorization.k8s.io",
|
|
kind: "ClusterRole",
|
|
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
|
|
},
|
|
subjects: [{
|
|
kind: "ServiceAccount",
|
|
name: G14_PROMETHEUS_SERVICE_ACCOUNT,
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
}],
|
|
},
|
|
{
|
|
apiVersion: "monitoring.coreos.com/v1",
|
|
kind: "Prometheus",
|
|
metadata: {
|
|
name: G14_PROMETHEUS_NAME,
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
labels: observabilityLabels("prometheus"),
|
|
},
|
|
spec: {
|
|
replicas: 1,
|
|
version: G14_PROMETHEUS_VERSION,
|
|
serviceAccountName: G14_PROMETHEUS_SERVICE_ACCOUNT,
|
|
scrapeInterval: "30s",
|
|
evaluationInterval: "30s",
|
|
retention: "7d",
|
|
resources: {
|
|
requests: { cpu: "100m", memory: "256Mi" },
|
|
limits: { cpu: "500m", memory: "1Gi" },
|
|
},
|
|
storage: {
|
|
volumeClaimTemplate: {
|
|
spec: {
|
|
storageClassName: "local-path",
|
|
accessModes: ["ReadWriteOnce"],
|
|
resources: { requests: { storage: "10Gi" } },
|
|
},
|
|
},
|
|
},
|
|
serviceMonitorSelector: monitorSelector,
|
|
serviceMonitorNamespaceSelector: namespaceSelector,
|
|
podMonitorSelector: monitorSelector,
|
|
podMonitorNamespaceSelector: namespaceSelector,
|
|
ruleSelector: monitorSelector,
|
|
ruleNamespaceSelector: namespaceSelector,
|
|
probeSelector: monitorSelector,
|
|
probeNamespaceSelector: namespaceSelector,
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
metadata: {
|
|
name: G14_PROMETHEUS_SERVICE,
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
labels: observabilityLabels("query"),
|
|
},
|
|
spec: {
|
|
type: "ClusterIP",
|
|
selector: {
|
|
prometheus: G14_PROMETHEUS_NAME,
|
|
},
|
|
ports: [{
|
|
name: "web",
|
|
port: 9090,
|
|
targetPort: "web",
|
|
}],
|
|
},
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
export function parseSectionJson(section: ShellSection | undefined): Record<string, unknown> {
|
|
const text = String(section?.stdout ?? "").trim();
|
|
if (text.length === 0) return {};
|
|
try {
|
|
return record(JSON.parse(text) as unknown);
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
export function parseSectionJsonArray(section: ShellSection | undefined): Record<string, unknown>[] {
|
|
const parsed = parseSectionJson(section);
|
|
const items = parsed.kind === "List" && Array.isArray(parsed.items)
|
|
? parsed.items
|
|
: Array.isArray(parsed.items)
|
|
? parsed.items
|
|
: [];
|
|
return items.map((item) => record(item));
|
|
}
|
|
|
|
export function sectionLines(section: ShellSection | undefined): string[] {
|
|
return String(section?.stdout ?? "")
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
export function podRows(section: ShellSection | undefined): Record<string, unknown>[] {
|
|
return sectionLines(section).map((line) => {
|
|
const [name = "", phase = "", readyRaw = ""] = line.split("\t");
|
|
const readyValues = readyRaw.split(",").map((item) => item.trim()).filter(Boolean);
|
|
return {
|
|
name,
|
|
phase: phase || null,
|
|
ready: readyValues.length > 0 ? readyValues.every((item) => item === "true") : null,
|
|
};
|
|
}).filter((item) => String(item.name).length > 0);
|
|
}
|
|
|
|
export function conditionStatus(items: Record<string, unknown>[], type: string): string | null {
|
|
for (const item of items) {
|
|
if (item.type === type) return typeof item.status === "string" ? item.status : null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function deploymentReady(deployment: Record<string, unknown>): boolean {
|
|
const spec = record(deployment.spec);
|
|
const status = record(deployment.status);
|
|
const desired = numericValue(spec.replicas) ?? 1;
|
|
const ready = numericValue(status.readyReplicas) ?? 0;
|
|
const available = numericValue(status.availableReplicas) ?? 0;
|
|
return ready >= desired && available >= desired;
|
|
}
|
|
|
|
export function prometheusReady(prometheus: Record<string, unknown>): boolean {
|
|
const conditions = Array.isArray(record(prometheus.status).conditions)
|
|
? record(prometheus.status).conditions.map((item) => record(item))
|
|
: [];
|
|
const available = conditionStatus(conditions, "Available");
|
|
const reconciled = conditionStatus(conditions, "Reconciled");
|
|
return available === "True" || reconciled === "True";
|
|
}
|
|
|
|
export function observabilityQueryPath(promql: string): string {
|
|
return `/api/v1/namespaces/${G14_OBSERVABILITY_NAMESPACE}/services/http:${G14_PROMETHEUS_SERVICE}:9090/proxy/api/v1/query?query=${encodeURIComponent(promql)}`;
|
|
}
|
|
|
|
export function arrayRecords(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.map((item) => record(item)) : [];
|
|
}
|
|
|
|
export function prometheusResultItems(parsed: unknown): Record<string, unknown>[] {
|
|
return arrayRecords(nested(parsed, ["data", "result"]));
|
|
}
|
|
|
|
export function prometheusSeriesValue(series: Record<string, unknown>): string | null {
|
|
const value = series.value;
|
|
if (Array.isArray(value) && value.length >= 2) return String(value[1]);
|
|
if (typeof value === "string" || typeof value === "number") return String(value);
|
|
return null;
|
|
}
|
|
|
|
export function prometheusSeriesTimestamp(series: Record<string, unknown>): number | null {
|
|
const value = series.value;
|
|
if (!Array.isArray(value) || value.length === 0) return null;
|
|
return numericValue(value[0]);
|
|
}
|
|
|
|
export function parseK8sCpuMillicores(value: unknown): number | null {
|
|
const raw = stringOrNull(value);
|
|
if (raw === null) return null;
|
|
const text = raw.trim();
|
|
if (text.length === 0) return null;
|
|
const match = text.match(/^([0-9]+(?:\.[0-9]+)?)(n|u|m)?$/u);
|
|
if (match === null) return null;
|
|
const amount = Number(match[1]);
|
|
if (!Number.isFinite(amount)) return null;
|
|
const unit = match[2] ?? "";
|
|
if (unit === "n") return amount / 1_000_000;
|
|
if (unit === "u") return amount / 1_000;
|
|
if (unit === "m") return amount;
|
|
return amount * 1000;
|
|
}
|
|
|
|
export function parseK8sMemoryMiB(value: unknown): number | null {
|
|
const raw = stringOrNull(value);
|
|
if (raw === null) return null;
|
|
const text = raw.trim();
|
|
if (text.length === 0) return null;
|
|
const match = text.match(/^([0-9]+(?:\.[0-9]+)?)(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)?$/u);
|
|
if (match === null) return null;
|
|
const amount = Number(match[1]);
|
|
if (!Number.isFinite(amount)) return null;
|
|
const unit = match[2] ?? "";
|
|
const binaryFactors: Record<string, number> = {
|
|
Ki: 1 / 1024,
|
|
Mi: 1,
|
|
Gi: 1024,
|
|
Ti: 1024 * 1024,
|
|
Pi: 1024 * 1024 * 1024,
|
|
Ei: 1024 * 1024 * 1024 * 1024,
|
|
};
|
|
const decimalByteFactors: Record<string, number> = {
|
|
"": 1,
|
|
K: 1000,
|
|
M: 1000 * 1000,
|
|
G: 1000 * 1000 * 1000,
|
|
T: 1000 * 1000 * 1000 * 1000,
|
|
P: 1000 * 1000 * 1000 * 1000 * 1000,
|
|
E: 1000 * 1000 * 1000 * 1000 * 1000 * 1000,
|
|
};
|
|
if (binaryFactors[unit] !== undefined) return amount * binaryFactors[unit];
|
|
const byteFactor = decimalByteFactors[unit];
|
|
return byteFactor === undefined ? null : (amount * byteFactor) / (1024 * 1024);
|
|
}
|
|
|
|
export function prometheusSeriesSummary(series: Record<string, unknown>): Record<string, unknown> {
|
|
const metric = record(series.metric);
|
|
return {
|
|
metricName: stringOrNull(metric.__name__),
|
|
serviceId: stringOrNull(metric.hwlab_pikastech_local_service_id),
|
|
service: stringOrNull(metric.service),
|
|
job: stringOrNull(metric.job),
|
|
pod: stringOrNull(metric.pod),
|
|
namespace: stringOrNull(metric.namespace),
|
|
container: stringOrNull(metric.container),
|
|
instance: stringOrNull(metric.instance),
|
|
value: prometheusSeriesValue(series),
|
|
timestamp: prometheusSeriesTimestamp(series),
|
|
};
|
|
}
|
|
|
|
export function prometheusSeriesSummaries(parsed: unknown, limit = 30): Record<string, unknown>[] {
|
|
return prometheusResultItems(parsed).slice(0, limit).map(prometheusSeriesSummary);
|
|
}
|
|
|
|
export function metricValueMatches(actual: string | null, expected: string): boolean {
|
|
if (actual === expected) return true;
|
|
const actualNumber = actual === null ? Number.NaN : Number(actual);
|
|
const expectedNumber = Number(expected);
|
|
return Number.isFinite(actualNumber) && Number.isFinite(expectedNumber) && actualNumber === expectedNumber;
|
|
}
|
|
|
|
export function g14ObservabilityQueryAssertion(parsed: unknown, expectedCount?: number, expectedValue?: string): Record<string, unknown> | null {
|
|
if (expectedCount === undefined && expectedValue === undefined) return null;
|
|
const series = prometheusResultItems(parsed);
|
|
const actualCount = series.length;
|
|
const countOk = expectedCount === undefined || actualCount === expectedCount;
|
|
const badValues = expectedValue === undefined
|
|
? []
|
|
: series
|
|
.filter((item) => !metricValueMatches(prometheusSeriesValue(item), expectedValue))
|
|
.map(prometheusSeriesSummary);
|
|
const missingCount = expectedCount === undefined ? 0 : Math.max(0, expectedCount - actualCount);
|
|
const extraCount = expectedCount === undefined ? 0 : Math.max(0, actualCount - expectedCount);
|
|
const queryStatus = stringOrNull(record(parsed).status);
|
|
return {
|
|
ok: queryStatus === "success" && countOk && badValues.length === 0,
|
|
queryStatus,
|
|
expectedCount: expectedCount ?? null,
|
|
actualCount,
|
|
expectedValue: expectedValue ?? null,
|
|
countOk,
|
|
valueOk: expectedValue === undefined ? null : badValues.length === 0,
|
|
badValueCount: badValues.length,
|
|
badValues: badValues.slice(0, 20),
|
|
missingSeries: missingCount > 0 ? [{ expectedCount, actualCount, missingCount }] : [],
|
|
extraSeries: extraCount > 0 ? prometheusSeriesSummaries(parsed, Math.min(extraCount, 20)) : [],
|
|
};
|
|
}
|
|
|
|
export function observabilitySemanticName(promql: string): string | null {
|
|
if (promql === V02_OBSERVABILITY_QUERIES.scrapeReachable) return "scrapeReachable";
|
|
if (promql === V02_OBSERVABILITY_QUERIES.sidecarServing) return "sidecarServing";
|
|
if (promql === V02_OBSERVABILITY_QUERIES.businessHealthProbe) return "businessHealthProbe";
|
|
return null;
|
|
}
|
|
|
|
export function k8sItems(parsed: unknown): Record<string, unknown>[] {
|
|
const root = record(parsed);
|
|
return Array.isArray(root.items) ? root.items.map((item) => record(item)) : [];
|
|
}
|
|
|
|
export function k8sMetadataName(item: Record<string, unknown>): string | null {
|
|
return stringOrNull(record(item.metadata).name);
|
|
}
|
|
|
|
export function k8sLabels(item: Record<string, unknown>): Record<string, unknown> {
|
|
return record(record(item.metadata).labels);
|
|
}
|
|
|
|
export function podReadyCondition(pod: Record<string, unknown>): boolean | null {
|
|
const conditions = arrayRecords(record(pod.status).conditions);
|
|
const ready = conditions.find((item) => item.type === "Ready");
|
|
return ready === undefined ? null : ready.status === "True";
|
|
}
|
|
|
|
export function podMetricsSidecarSummary(pod: Record<string, unknown>): Record<string, unknown> {
|
|
const statuses = arrayRecords(record(pod.status).containerStatuses);
|
|
const metrics = statuses.find((item) => item.name === "hwlab-metrics")
|
|
?? statuses.find((item) => String(item.name ?? "").includes("metrics"));
|
|
return {
|
|
pod: k8sMetadataName(pod),
|
|
namespace: stringOrNull(record(pod.metadata).namespace),
|
|
serviceId: stringOrNull(k8sLabels(pod)["hwlab.pikastech.local/service-id"])
|
|
?? stringOrNull(k8sLabels(pod)["app.kubernetes.io/name"]),
|
|
phase: stringOrNull(record(pod.status).phase),
|
|
podReady: podReadyCondition(pod),
|
|
sidecarFound: metrics !== undefined,
|
|
sidecarName: metrics === undefined ? null : stringOrNull(metrics.name),
|
|
sidecarReady: metrics === undefined ? null : metrics.ready === true,
|
|
restartCount: metrics === undefined ? null : numericValue(metrics.restartCount),
|
|
};
|
|
}
|
|
|
|
export function resourceUsageSummary(containers: Record<string, unknown>[]): Record<string, unknown> {
|
|
const rows = containers.map((container) => {
|
|
const usage = record(container.usage);
|
|
return {
|
|
name: stringOrNull(container.name),
|
|
cpuMillicores: parseK8sCpuMillicores(usage.cpu),
|
|
memoryMiB: parseK8sMemoryMiB(usage.memory),
|
|
};
|
|
});
|
|
const sum = (key: "cpuMillicores" | "memoryMiB"): number | null => {
|
|
const values = rows.map((item) => item[key]).filter((item): item is number => item !== null);
|
|
return values.length === 0 ? null : values.reduce((acc, item) => acc + item, 0);
|
|
};
|
|
return {
|
|
cpuMillicores: sum("cpuMillicores"),
|
|
memoryMiB: sum("memoryMiB"),
|
|
containers: rows,
|
|
};
|
|
}
|
|
|
|
export function podResourceMetricSummary(item: Record<string, unknown>): Record<string, unknown> {
|
|
const containers = arrayRecords(item.containers);
|
|
const metricsSidecars = containers.filter((container) => stringOrNull(container.name) === "hwlab-metrics");
|
|
const businessContainers = containers.filter((container) => stringOrNull(container.name) !== "hwlab-metrics");
|
|
return {
|
|
pod: k8sMetadataName(item),
|
|
namespace: stringOrNull(record(item.metadata).namespace),
|
|
serviceId: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/service-id"])
|
|
?? stringOrNull(k8sLabels(item)["app.kubernetes.io/name"]),
|
|
timestamp: stringOrNull(item.timestamp),
|
|
window: stringOrNull(item.window),
|
|
total: resourceUsageSummary(containers),
|
|
business: resourceUsageSummary(businessContainers),
|
|
metricsSidecar: resourceUsageSummary(metricsSidecars),
|
|
};
|
|
}
|
|
|
|
export function monitorSummaries(items: Record<string, unknown>[]): Record<string, unknown>[] {
|
|
return items.map((item) => ({
|
|
kind: stringOrNull(item.kind),
|
|
name: k8sMetadataName(item),
|
|
labels: {
|
|
monitoring: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/monitoring"]),
|
|
serviceId: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/service-id"]),
|
|
gitopsTarget: stringOrNull(k8sLabels(item)["hwlab.pikastech.local/gitops-target"]),
|
|
},
|
|
}));
|
|
}
|
|
|
|
export function monitoredServiceIds(monitors: Record<string, unknown>[], metricSeries: Record<string, Record<string, unknown>[]>): string[] {
|
|
const ids = new Set<string>();
|
|
for (const item of monitorSummaries(monitors)) {
|
|
const kind = stringOrNull(item.kind);
|
|
const serviceId = stringOrNull(record(item.labels).serviceId);
|
|
if ((kind === "ServiceMonitor" || kind === "PodMonitor") && serviceId !== null) ids.add(serviceId);
|
|
}
|
|
for (const seriesItems of Object.values(metricSeries)) {
|
|
for (const item of seriesItems.map(prometheusSeriesSummary)) {
|
|
const serviceId = stringOrNull(item.serviceId) ?? stringOrNull(item.service) ?? stringOrNull(item.job);
|
|
if (serviceId !== null) ids.add(serviceId);
|
|
}
|
|
}
|
|
return [...ids].sort();
|
|
}
|
|
|
|
export function attachResourceMetricsToTargets(targets: Record<string, unknown>[], resources: Record<string, unknown>[]): Record<string, unknown>[] {
|
|
const byPod = new Map(resources.map((item) => [String(item.pod ?? ""), item]));
|
|
const byServiceId = new Map(resources.map((item) => [String(item.serviceId ?? ""), item]));
|
|
return targets.map((target) => {
|
|
const resource = byPod.get(String(target.pod ?? "")) ?? byServiceId.get(String(target.serviceId ?? ""));
|
|
return resource === undefined ? target : { ...target, resource };
|
|
});
|
|
}
|
|
|
|
export function targetMetricValue(target: Record<string, unknown>, metricName: string): number | null {
|
|
const metric = record(record(target.metrics)[metricName]);
|
|
return numericValue(metric.value);
|
|
}
|
|
|
|
export function targetMetricString(target: Record<string, unknown>, metricName: string): string | null {
|
|
return stringOrNull(record(record(target.metrics)[metricName]).value);
|
|
}
|
|
|
|
export function targetMetricTimestamp(target: Record<string, unknown>, metricName: string): number | null {
|
|
return numericValue(record(record(target.metrics)[metricName]).timestamp);
|
|
}
|
|
|
|
export function sortedNumeric(values: number[]): number[] {
|
|
return [...values].filter((item) => Number.isFinite(item)).sort((a, b) => a - b);
|
|
}
|
|
|
|
export function percentile(values: number[], p: number): number | null {
|
|
const sorted = sortedNumeric(values);
|
|
if (sorted.length === 0) return null;
|
|
if (sorted.length === 1) return sorted[0];
|
|
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
|
return sorted[index];
|
|
}
|
|
|
|
export function secondsStats(values: number[]): Record<string, unknown> {
|
|
const sorted = sortedNumeric(values);
|
|
if (sorted.length === 0) return { count: 0, minSeconds: null, maxSeconds: null, avgSeconds: null, p50Seconds: null, p95Seconds: null, p99Seconds: null };
|
|
const sum = sorted.reduce((acc, item) => acc + item, 0);
|
|
return {
|
|
count: sorted.length,
|
|
minSeconds: sorted[0],
|
|
maxSeconds: sorted[sorted.length - 1],
|
|
avgSeconds: sum / sorted.length,
|
|
p50Seconds: percentile(sorted, 50),
|
|
p95Seconds: percentile(sorted, 95),
|
|
p99Seconds: percentile(sorted, 99),
|
|
};
|
|
}
|
|
|
|
export function numberStats(values: number[]): Record<string, unknown> {
|
|
const sorted = sortedNumeric(values);
|
|
if (sorted.length === 0) return { count: 0, min: null, max: null, avg: null, p50: null, p95: null, p99: null };
|
|
const sum = sorted.reduce((acc, item) => acc + item, 0);
|
|
return {
|
|
count: sorted.length,
|
|
min: sorted[0],
|
|
max: sorted[sorted.length - 1],
|
|
avg: sum / sorted.length,
|
|
p50: percentile(sorted, 50),
|
|
p95: percentile(sorted, 95),
|
|
p99: percentile(sorted, 99),
|
|
};
|
|
}
|
|
|
|
export function buildObservabilityLevelSummary(targets: Record<string, unknown>[]): Record<string, unknown> {
|
|
const observedTargets = targets.filter((target) => Object.keys(record(target.metrics)).length > 0);
|
|
const healthDurations = observedTargets.map((target) => targetMetricValue(target, "healthProbeDurationSeconds")).filter((item): item is number => item !== null);
|
|
const scrapeDurations = observedTargets.map((target) => targetMetricValue(target, "scrapeDurationSeconds")).filter((item): item is number => item !== null);
|
|
const samplesScraped = observedTargets.map((target) => targetMetricValue(target, "scrapeSamplesScraped")).filter((item): item is number => item !== null);
|
|
const resourceSnapshots = observedTargets.map((target) => record(target.resource)).filter((item) => Object.keys(item).length > 0);
|
|
const resourceVisible = resourceSnapshots.length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT;
|
|
const resourceValue = (snapshot: Record<string, unknown>, path: string[]): number | null => numericValue(nested(snapshot, path));
|
|
const resourceTotals = {
|
|
totalCpuMillicores: resourceSnapshots.map((item) => resourceValue(item, ["total", "cpuMillicores"])).filter((item): item is number => item !== null),
|
|
totalMemoryMiB: resourceSnapshots.map((item) => resourceValue(item, ["total", "memoryMiB"])).filter((item): item is number => item !== null),
|
|
businessCpuMillicores: resourceSnapshots.map((item) => resourceValue(item, ["business", "cpuMillicores"])).filter((item): item is number => item !== null),
|
|
businessMemoryMiB: resourceSnapshots.map((item) => resourceValue(item, ["business", "memoryMiB"])).filter((item): item is number => item !== null),
|
|
metricsSidecarCpuMillicores: resourceSnapshots.map((item) => resourceValue(item, ["metricsSidecar", "cpuMillicores"])).filter((item): item is number => item !== null),
|
|
metricsSidecarMemoryMiB: resourceSnapshots.map((item) => resourceValue(item, ["metricsSidecar", "memoryMiB"])).filter((item): item is number => item !== null),
|
|
};
|
|
const healthRows = observedTargets.map((target) => ({
|
|
serviceId: target.serviceId ?? target.service ?? target.job ?? target.key,
|
|
scrapeReachable: targetMetricString(target, "scrapeReachable"),
|
|
sidecarServing: targetMetricString(target, "sidecarServing"),
|
|
businessHealthProbe: targetMetricString(target, "businessHealthProbe"),
|
|
statusCode: targetMetricValue(target, "healthProbeStatusCode"),
|
|
healthProbeDurationSeconds: targetMetricValue(target, "healthProbeDurationSeconds"),
|
|
scrapeDurationSeconds: targetMetricValue(target, "scrapeDurationSeconds"),
|
|
scrapeSamplesScraped: targetMetricValue(target, "scrapeSamplesScraped"),
|
|
processUptimeSeconds: targetMetricValue(target, "processUptimeSeconds"),
|
|
totalCpuMillicores: resourceValue(record(target.resource), ["total", "cpuMillicores"]),
|
|
totalMemoryMiB: resourceValue(record(target.resource), ["total", "memoryMiB"]),
|
|
businessCpuMillicores: resourceValue(record(target.resource), ["business", "cpuMillicores"]),
|
|
businessMemoryMiB: resourceValue(record(target.resource), ["business", "memoryMiB"]),
|
|
metricsSidecarCpuMillicores: resourceValue(record(target.resource), ["metricsSidecar", "cpuMillicores"]),
|
|
metricsSidecarMemoryMiB: resourceValue(record(target.resource), ["metricsSidecar", "memoryMiB"]),
|
|
lastSampleTimestamp: targetMetricTimestamp(target, "scrapeReachable") ?? targetMetricTimestamp(target, "businessHealthProbe"),
|
|
})).sort((a, b) => String(a.serviceId).localeCompare(String(b.serviceId)));
|
|
const unsupportedSignals = [
|
|
"request_rate",
|
|
"error_rate",
|
|
"route_latency_p95",
|
|
"route_latency_p99",
|
|
"business_operation_latency",
|
|
...(resourceVisible ? [] : ["cpu_memory_node_pod_usage"]),
|
|
];
|
|
return {
|
|
observedTargetCount: observedTargets.length,
|
|
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
|
|
healthProbeDuration: secondsStats(healthDurations),
|
|
scrapeDuration: secondsStats(scrapeDurations),
|
|
scrapeSamplesScraped: {
|
|
count: samplesScraped.length,
|
|
min: sortedNumeric(samplesScraped)[0] ?? null,
|
|
max: sortedNumeric(samplesScraped)[sortedNumeric(samplesScraped).length - 1] ?? null,
|
|
values: samplesScraped,
|
|
},
|
|
resourceUsage: {
|
|
source: "metrics.k8s.io/v1beta1",
|
|
observedTargetCount: resourceSnapshots.length,
|
|
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
|
|
totalCpuMillicores: numberStats(resourceTotals.totalCpuMillicores),
|
|
totalMemoryMiB: numberStats(resourceTotals.totalMemoryMiB),
|
|
businessCpuMillicores: numberStats(resourceTotals.businessCpuMillicores),
|
|
businessMemoryMiB: numberStats(resourceTotals.businessMemoryMiB),
|
|
metricsSidecarCpuMillicores: numberStats(resourceTotals.metricsSidecarCpuMillicores),
|
|
metricsSidecarMemoryMiB: numberStats(resourceTotals.metricsSidecarMemoryMiB),
|
|
},
|
|
services: healthRows,
|
|
interpretation: {
|
|
visible: [
|
|
"scrape availability",
|
|
"metrics sidecar availability",
|
|
"business health endpoint success",
|
|
"business health endpoint status code",
|
|
"business health probe duration",
|
|
"Prometheus scrape duration",
|
|
"metrics sidecar uptime",
|
|
...(resourceVisible ? ["current pod/container CPU and memory snapshot from metrics.k8s.io"] : []),
|
|
],
|
|
notYetVisible: unsupportedSignals,
|
|
conclusion: unsupportedSignals.length > 0
|
|
? "current metrics are sufficient for availability, synthetic health latency, and current resource snapshot when metrics.k8s.io is available, but not yet sufficient for real request throughput, route latency, or error rate analysis"
|
|
: "current metrics cover the requested performance surface",
|
|
},
|
|
};
|
|
}
|
|
|
|
export function compactObservabilityTarget(target: Record<string, unknown>): Record<string, unknown> {
|
|
const sidecar = record(target.sidecar);
|
|
return {
|
|
serviceId: target.serviceId ?? target.service ?? target.job ?? target.key,
|
|
pod: target.pod ?? sidecar.pod ?? null,
|
|
serviceMonitors: target.serviceMonitors ?? [],
|
|
sidecar: {
|
|
found: sidecar.sidecarFound ?? null,
|
|
ready: sidecar.sidecarReady ?? null,
|
|
podReady: sidecar.podReady ?? null,
|
|
restartCount: sidecar.restartCount ?? null,
|
|
},
|
|
metrics: {
|
|
scrapeReachable: targetMetricString(target, "scrapeReachable"),
|
|
sidecarServing: targetMetricString(target, "sidecarServing"),
|
|
businessHealthProbe: targetMetricString(target, "businessHealthProbe"),
|
|
healthProbeConfigured: targetMetricString(target, "healthProbeConfigured"),
|
|
healthProbeStatusCode: targetMetricValue(target, "healthProbeStatusCode"),
|
|
healthProbeDurationSeconds: targetMetricValue(target, "healthProbeDurationSeconds"),
|
|
scrapeDurationSeconds: targetMetricValue(target, "scrapeDurationSeconds"),
|
|
scrapeSamplesScraped: targetMetricValue(target, "scrapeSamplesScraped"),
|
|
processUptimeSeconds: targetMetricValue(target, "processUptimeSeconds"),
|
|
lastSampleTimestamp: targetMetricTimestamp(target, "scrapeReachable") ?? targetMetricTimestamp(target, "businessHealthProbe"),
|
|
},
|
|
resource: target.resource ?? null,
|
|
};
|
|
}
|
|
|
|
export function mergeObservabilityTargets(
|
|
pods: Record<string, unknown>[],
|
|
monitors: Record<string, unknown>[],
|
|
metricSeries: Record<string, Record<string, unknown>[]>,
|
|
): Record<string, unknown>[] {
|
|
const byKey = new Map<string, Record<string, unknown>>();
|
|
const ensure = (key: string): Record<string, unknown> => {
|
|
const existing = byKey.get(key);
|
|
if (existing !== undefined) return existing;
|
|
const next = { key, serviceId: null, service: null, job: null, pod: null, sidecar: null, metrics: {} as Record<string, unknown> };
|
|
byKey.set(key, next);
|
|
return next;
|
|
};
|
|
for (const pod of pods.map(podMetricsSidecarSummary)) {
|
|
const key = String(pod.serviceId ?? pod.pod ?? "unknown");
|
|
const target = ensure(key);
|
|
target.serviceId = target.serviceId ?? pod.serviceId;
|
|
target.pod = target.pod ?? pod.pod;
|
|
target.sidecar = pod;
|
|
}
|
|
for (const [metricName, seriesItems] of Object.entries(metricSeries)) {
|
|
for (const item of seriesItems.map(prometheusSeriesSummary)) {
|
|
const key = String(item.serviceId ?? item.service ?? item.job ?? item.pod ?? "unknown");
|
|
const target = ensure(key);
|
|
target.serviceId = target.serviceId ?? item.serviceId;
|
|
target.service = target.service ?? item.service;
|
|
target.job = target.job ?? item.job;
|
|
target.pod = target.pod ?? item.pod;
|
|
record(target.metrics)[metricName] = {
|
|
value: item.value,
|
|
timestamp: item.timestamp,
|
|
instance: item.instance,
|
|
container: item.container,
|
|
};
|
|
}
|
|
}
|
|
const serviceIds = new Set(monitoredServiceIds(monitors, metricSeries));
|
|
const monitorNames = monitorSummaries(monitors);
|
|
return [...byKey.values()]
|
|
.filter((target) => {
|
|
const key = String(target.serviceId ?? target.service ?? target.job ?? target.key ?? "");
|
|
return serviceIds.has(key);
|
|
})
|
|
.map((target) => ({
|
|
...target,
|
|
serviceMonitors: monitorNames
|
|
.filter((item) => stringOrNull(record(item.labels).serviceId) === stringOrNull(target.serviceId))
|
|
.map((item) => stringOrNull(item.name))
|
|
.filter((name): name is string => name !== null)
|
|
.slice(0, 5),
|
|
}))
|
|
.sort((a, b) => String(a.serviceId ?? a.service ?? a.job ?? a.pod).localeCompare(String(b.serviceId ?? b.service ?? b.job ?? b.pod)));
|
|
}
|
|
|
|
export function publicMetricsProbe(name: string, url: string): Record<string, unknown> {
|
|
const result = commandJson([
|
|
"curl",
|
|
"-sS",
|
|
"-L",
|
|
"--max-time",
|
|
"20",
|
|
"--range",
|
|
"0-4095",
|
|
"-w",
|
|
"\n__UNIDESK_HTTP_CODE__:%{http_code}\n__UNIDESK_CONTENT_TYPE__:%{content_type}\n",
|
|
url,
|
|
], 30_000);
|
|
const text = result.stdout;
|
|
const codeMatch = text.match(/\n__UNIDESK_HTTP_CODE__:(\d{3})\n/u);
|
|
const contentTypeMatch = text.match(/\n__UNIDESK_CONTENT_TYPE__:(.*)\n?$/u);
|
|
const body = codeMatch === null ? text : text.slice(0, codeMatch.index).trim();
|
|
const httpCode = codeMatch === null ? null : Number(codeMatch[1]);
|
|
const contentType = contentTypeMatch === null ? "" : contentTypeMatch[1].trim();
|
|
const prometheusText = looksLikePrometheusText(body, contentType);
|
|
const exposed = httpCode !== null && httpCode >= 200 && httpCode < 300 && prometheusText;
|
|
const denied = httpCode !== null && !exposed && (httpCode >= 400 || !prometheusText);
|
|
return {
|
|
ok: result.ok && denied,
|
|
name,
|
|
url,
|
|
httpCode,
|
|
contentType,
|
|
exposed,
|
|
denied,
|
|
reason: result.ok
|
|
? exposed
|
|
? "prometheus-text-exposed"
|
|
: httpCode !== null && httpCode >= 400
|
|
? "http-error-denied"
|
|
: "non-prometheus-body"
|
|
: "curl-failed",
|
|
bodyPreview: body.slice(0, 240),
|
|
commandResult: compactCommandMetadata(result, !result.ok || exposed),
|
|
};
|
|
}
|
|
|
|
export function looksLikePrometheusText(body: string, contentType: string): boolean {
|
|
const type = contentType.toLowerCase();
|
|
if (type.includes("openmetrics")) return true;
|
|
const hasMetricLine = /(?:^|\n)(?:#\s+(?:HELP|TYPE)\s+[A-Za-z_:][A-Za-z0-9_:]*|[A-Za-z_:][A-Za-z0-9_:]*(?:\{[^}\n]*\})?\s+[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?)\b/u.test(body);
|
|
if (!hasMetricLine) return false;
|
|
return type.includes("text/plain") || type.length === 0;
|
|
}
|
|
|
|
export function g14ObservabilityStatus(): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const queryPath = `/api/v1/namespaces/${G14_OBSERVABILITY_NAMESPACE}/services/http:${G14_PROMETHEUS_SERVICE}:9090/proxy/api/v1/query?query=${encodeURIComponent("up")}`;
|
|
const crds = [
|
|
"servicemonitors.monitoring.coreos.com",
|
|
"podmonitors.monitoring.coreos.com",
|
|
"prometheusrules.monitoring.coreos.com",
|
|
"prometheuses.monitoring.coreos.com",
|
|
"alertmanagers.monitoring.coreos.com",
|
|
];
|
|
const script = [
|
|
"set +e",
|
|
"section() {",
|
|
" name=\"$1\"",
|
|
" shift",
|
|
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
|
" \"$@\"",
|
|
" code=$?",
|
|
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
|
"}",
|
|
`section namespace kubectl get namespace ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -o json`,
|
|
`section discoveryNamespace kubectl get namespace ${shellQuote(V02_RUNTIME_NAMESPACE)} -o json`,
|
|
`section crds kubectl get crd ${crds.map(shellQuote).join(" ")} -o 'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}'`,
|
|
`section operator kubectl get deploy -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} prometheus-operator -o json`,
|
|
`section operatorPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l app.kubernetes.io/name=prometheus-operator -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .status.containerStatuses[*]}{.ready}{","}{end}{"\\n"}{end}'`,
|
|
`section prometheus kubectl get prometheus -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} ${shellQuote(G14_PROMETHEUS_NAME)} -o json`,
|
|
`section prometheusPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l prometheus=${shellQuote(G14_PROMETHEUS_NAME)} -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .status.containerStatuses[*]}{.ready}{","}{end}{"\\n"}{end}'`,
|
|
`section prometheusService kubectl get service -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} ${shellQuote(G14_PROMETHEUS_SERVICE)} -o json`,
|
|
`section workloadMonitors kubectl get servicemonitor,prometheusrule -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -l hwlab.pikastech.local/monitoring=enabled -o json`,
|
|
`section query kubectl get --raw ${shellQuote(queryPath)}`,
|
|
].join("\n");
|
|
const bundle = g14K3s(["sh", "--", script], 120_000);
|
|
const sections = parseShellSections(statusText(bundle));
|
|
const namespace = parseSectionJson(sections.namespace);
|
|
const discoveryNamespace = parseSectionJson(sections.discoveryNamespace);
|
|
const crdNames = sectionLines(sections.crds);
|
|
const operator = parseSectionJson(sections.operator);
|
|
const operatorPods = podRows(sections.operatorPods);
|
|
const prometheus = parseSectionJson(sections.prometheus);
|
|
const prometheusPods = podRows(sections.prometheusPods);
|
|
const prometheusService = parseSectionJson(sections.prometheusService);
|
|
const workloadMonitorItems = parseSectionJsonArray(sections.workloadMonitors);
|
|
const query = parseSectionJson(sections.query);
|
|
const requiredCrdsPresent = crds.every((name) => crdNames.includes(name));
|
|
const namespaceLabel = stringOrNull(record(record(namespace.metadata).labels)["g14.pikastech.local/observability-discovery"]);
|
|
const workloadNamespaceLabel = stringOrNull(record(record(discoveryNamespace.metadata).labels)["g14.pikastech.local/observability-discovery"]);
|
|
const operatorIsReady = Object.keys(operator).length > 0 && deploymentReady(operator);
|
|
const prometheusExists = Object.keys(prometheus).length > 0;
|
|
const prometheusIsReady = prometheusExists && (
|
|
prometheusReady(prometheus)
|
|
|| prometheusPods.some((pod) => pod.ready === true)
|
|
);
|
|
const queryOk = sections.query?.exitCode === 0 && query.status === "success";
|
|
const ok = isCommandSuccess(bundle) && requiredCrdsPresent && operatorIsReady && prometheusExists && prometheusIsReady && queryOk;
|
|
return {
|
|
ok,
|
|
command: "hwlab g14 observability status",
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
mode: "status",
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
versions: {
|
|
prometheusOperator: G14_PROMETHEUS_OPERATOR_VERSION,
|
|
prometheus: G14_PROMETHEUS_VERSION,
|
|
operatorBundle: G14_PROMETHEUS_OPERATOR_RELEASE_ASSET,
|
|
},
|
|
discovery: {
|
|
namespaceLabel,
|
|
workloadNamespace: V02_RUNTIME_NAMESPACE,
|
|
workloadNamespaceLabel,
|
|
selectorLabel: "hwlab.pikastech.local/monitoring=enabled",
|
|
},
|
|
crds: {
|
|
ok: requiredCrdsPresent,
|
|
required: crds,
|
|
present: crdNames,
|
|
missing: crds.filter((name) => !crdNames.includes(name)),
|
|
sectionOk: shellSectionOk(sections.crds),
|
|
},
|
|
operator: {
|
|
ok: operatorIsReady,
|
|
deployment: stringOrNull(record(operator.metadata).name),
|
|
desiredReplicas: numericValue(record(operator.spec).replicas) ?? 1,
|
|
readyReplicas: numericValue(record(operator.status).readyReplicas) ?? 0,
|
|
availableReplicas: numericValue(record(operator.status).availableReplicas) ?? 0,
|
|
pods: operatorPods.map((pod) => ({
|
|
name: stringOrNull(pod.name),
|
|
phase: stringOrNull(pod.phase),
|
|
ready: pod.ready ?? null,
|
|
})),
|
|
sectionOk: shellSectionOk(sections.operator),
|
|
},
|
|
prometheus: {
|
|
ok: prometheusExists && prometheusIsReady,
|
|
name: G14_PROMETHEUS_NAME,
|
|
service: G14_PROMETHEUS_SERVICE,
|
|
serviceExists: Object.keys(prometheusService).length > 0,
|
|
ready: prometheusIsReady,
|
|
conditions: Array.isArray(record(prometheus.status).conditions) ? record(prometheus.status).conditions : [],
|
|
pods: prometheusPods.map((pod) => ({
|
|
name: stringOrNull(pod.name),
|
|
phase: stringOrNull(pod.phase),
|
|
ready: pod.ready ?? null,
|
|
})),
|
|
sectionOk: shellSectionOk(sections.prometheus),
|
|
},
|
|
workloadMonitors: {
|
|
ok: shellSectionOk(sections.workloadMonitors),
|
|
namespace: V02_RUNTIME_NAMESPACE,
|
|
count: workloadMonitorItems.length,
|
|
items: workloadMonitorItems.map((item) => ({
|
|
kind: item.kind ?? null,
|
|
name: stringOrNull(record(item.metadata).name),
|
|
})),
|
|
stderr: shellSectionOk(sections.workloadMonitors) ? "" : commandErrorSummary(bundle),
|
|
},
|
|
query: {
|
|
ok: queryOk,
|
|
promql: "up",
|
|
serviceProxyPath: queryPath,
|
|
resultType: nested(query, ["data", "resultType"]) ?? null,
|
|
resultCount: Array.isArray(nested(query, ["data", "result"])) ? (nested(query, ["data", "result"]) as unknown[]).length : null,
|
|
status: query.status ?? null,
|
|
sectionOk: shellSectionOk(sections.query),
|
|
stderr: shellSectionOk(sections.query) ? "" : commandErrorSummary(bundle),
|
|
},
|
|
result: compactCommandMetadata(bundle, !ok),
|
|
next: requiredCrdsPresent && operatorIsReady && prometheusExists
|
|
? { query: 'bun scripts/cli.ts hwlab g14 observability query --promql \'up{namespace="hwlab-v02"}\'' }
|
|
: { apply: "bun scripts/cli.ts hwlab g14 observability apply --confirm" },
|
|
};
|
|
}
|
|
|
|
export function g14ObservabilityApplyScript(options: G14ObservabilityOptions, manifestB64: string): string {
|
|
const dryRunArg = options.dryRun ? "--dry-run=server" : "";
|
|
const stackDryRunCommand = options.dryRun
|
|
? [
|
|
"core_stack_path=\"$tmpdir/g14-prometheus-core-stack.json\"",
|
|
"node - \"$stack_path\" \"$core_stack_path\" <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const input = process.argv[2];",
|
|
"const output = process.argv[3];",
|
|
"const stack = JSON.parse(fs.readFileSync(input, 'utf8'));",
|
|
"stack.items = (stack.items || []).filter((item) => item.kind !== 'Prometheus');",
|
|
"fs.writeFileSync(output, JSON.stringify(stack));",
|
|
"NODE",
|
|
"kubectl apply --dry-run=client --validate=false -f \"$core_stack_path\"",
|
|
"echo prometheus_cr_dry_run=skipped_until_monitoring_crds_are_installed",
|
|
].join("\n")
|
|
: `kubectl apply --server-side --force-conflicts --field-manager=${shellQuote(G14_OBSERVABILITY_FIELD_MANAGER)} -f "$stack_path"`;
|
|
const preStackWaitCommands = options.dryRun
|
|
? "echo observability_wait=skipped_dry_run"
|
|
: [
|
|
"kubectl wait --for=condition=Established --timeout=45s crd/servicemonitors.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com crd/prometheuses.monitoring.coreos.com",
|
|
`kubectl -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} get deploy/prometheus-operator -o name`,
|
|
].join("\n");
|
|
const postStackWaitCommands = options.dryRun
|
|
? "echo prometheus_wait=skipped_dry_run"
|
|
: "echo prometheus_wait=deferred_to_status_command";
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shellQuote(G14_OBSERVABILITY_NAMESPACE)}`,
|
|
`bundle_url=${shellQuote(G14_PROMETHEUS_OPERATOR_RELEASE_ASSET)}`,
|
|
`operator_version=${shellQuote(G14_PROMETHEUS_OPERATOR_VERSION)}`,
|
|
`prometheus_version=${shellQuote(G14_PROMETHEUS_VERSION)}`,
|
|
`stack_b64=${shellQuote(manifestB64)}`,
|
|
"tmpdir=$(mktemp -d /tmp/g14-observability-XXXXXX)",
|
|
"cleanup() { rm -rf \"$tmpdir\"; }",
|
|
"trap cleanup EXIT",
|
|
"bundle_path=\"$tmpdir/operator-bundle.yaml\"",
|
|
"operator_path=\"$tmpdir/operator-rendered.yaml\"",
|
|
"stack_path=\"$tmpdir/g14-prometheus-stack.json\"",
|
|
"printf '%s' \"$stack_b64\" | base64 -d > \"$stack_path\"",
|
|
"export HTTP_PROXY=${HTTP_PROXY:-http://127.0.0.1:10808}",
|
|
"export HTTPS_PROXY=${HTTPS_PROXY:-http://127.0.0.1:10808}",
|
|
"export http_proxy=$HTTP_PROXY",
|
|
"export https_proxy=$HTTPS_PROXY",
|
|
"export NO_PROXY=${NO_PROXY:-localhost,127.0.0.1,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc}",
|
|
"export no_proxy=$NO_PROXY",
|
|
"curl -fsSL --connect-timeout 20 --retry 3 --retry-delay 2 -o \"$bundle_path\" \"$bundle_url\"",
|
|
"cat > \"$tmpdir/kustomization.yaml\" <<'YAML'",
|
|
"apiVersion: kustomize.config.k8s.io/v1beta1",
|
|
"kind: Kustomization",
|
|
`namespace: ${G14_OBSERVABILITY_NAMESPACE}`,
|
|
"resources:",
|
|
"- operator-bundle.yaml",
|
|
"YAML",
|
|
"kubectl kustomize \"$tmpdir\" > \"$operator_path\"",
|
|
"grep -q 'namespace: devops-infra' \"$operator_path\"",
|
|
`kubectl create namespace ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${shellQuote(G14_OBSERVABILITY_FIELD_MANAGER)} ${dryRunArg} -f -`,
|
|
`kubectl apply --server-side --force-conflicts --field-manager=${shellQuote(G14_OBSERVABILITY_FIELD_MANAGER)} ${dryRunArg} -f "$operator_path"`,
|
|
preStackWaitCommands,
|
|
stackDryRunCommand,
|
|
postStackWaitCommands,
|
|
`printf 'observability_apply=ok namespace=%s operator=%s prometheus=%s dryRun=%s\\n' "$namespace" "$operator_version" "$prometheus_version" ${shellQuote(String(options.dryRun))}`,
|
|
].join("\n");
|
|
}
|
|
|
|
export function runG14ObservabilityApply(options: G14ObservabilityOptions): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const manifest = g14PrometheusManifest();
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
const script = g14ObservabilityApplyScript(options, manifestB64);
|
|
const result = g14K3s(["sh", "--", script], options.timeoutSeconds * 1000 + 90_000);
|
|
const ok = isCommandSuccess(result);
|
|
return {
|
|
ok,
|
|
command: "hwlab g14 observability apply",
|
|
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
versions: {
|
|
prometheusOperator: G14_PROMETHEUS_OPERATOR_VERSION,
|
|
prometheus: G14_PROMETHEUS_VERSION,
|
|
operatorBundle: G14_PROMETHEUS_OPERATOR_RELEASE_ASSET,
|
|
},
|
|
manifest: options.dryRun ? manifest : undefined,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
result: compactCommandResult(result),
|
|
status: ok && !options.dryRun ? g14ObservabilityStatus() : undefined,
|
|
next: options.dryRun
|
|
? { apply: "bun scripts/cli.ts hwlab g14 observability apply --confirm" }
|
|
: { status: "bun scripts/cli.ts hwlab g14 observability status", query: 'bun scripts/cli.ts hwlab g14 observability query --promql \'up{namespace="hwlab-v02"}\'' },
|
|
};
|
|
}
|
|
|
|
export function g14PrometheusQuery(promql: string, timeoutMs: number): { serviceProxyPath: string; result: CommandJsonResult; parsed: Record<string, unknown>; ok: boolean } {
|
|
const serviceProxyPath = observabilityQueryPath(promql);
|
|
const result = g14K3s(["kubectl", "get", "--raw", serviceProxyPath], timeoutMs);
|
|
const parsed = (() => {
|
|
try {
|
|
return record(JSON.parse(statusText(result)) as unknown);
|
|
} catch {
|
|
return {};
|
|
}
|
|
})();
|
|
return { serviceProxyPath, result, parsed, ok: isCommandSuccess(result) && parsed.status === "success" };
|
|
}
|
|
|
|
export function runG14ObservabilityQuery(options: G14ObservabilityOptions): Record<string, unknown> {
|
|
const query = g14PrometheusQuery(options.query, options.timeoutSeconds * 1000);
|
|
const assertion = g14ObservabilityQueryAssertion(query.parsed, options.expectCount, options.expectValue);
|
|
const semantic = observabilitySemanticName(options.query);
|
|
const resultCount = prometheusResultItems(query.parsed).length;
|
|
const hasAssertion = options.expectCount !== undefined || options.expectValue !== undefined;
|
|
const includeData = !hasAssertion && resultCount <= 20;
|
|
const ok = query.ok && (assertion === null || assertion.ok === true);
|
|
return {
|
|
ok,
|
|
command: "hwlab g14 observability query",
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
service: G14_PROMETHEUS_SERVICE,
|
|
promql: options.query,
|
|
semantic,
|
|
serviceProxyPath: query.serviceProxyPath,
|
|
status: query.parsed.status ?? null,
|
|
resultType: nested(query.parsed, ["data", "resultType"]) ?? null,
|
|
resultCount,
|
|
series: prometheusSeriesSummaries(query.parsed, 50),
|
|
assertion,
|
|
data: includeData ? query.parsed.data ?? null : undefined,
|
|
dataOmitted: includeData ? undefined : {
|
|
reason: hasAssertion ? "assertion-mode-uses-series-summary" : "result-count-exceeds-default-inline-limit",
|
|
resultCount,
|
|
inlineLimit: 20,
|
|
summaryField: "series",
|
|
queryCommand: `bun scripts/cli.ts hwlab g14 observability query --promql ${shellQuote(options.query)}`,
|
|
},
|
|
raw: Object.keys(query.parsed).length === 0 ? tailText(statusText(query.result), 4000) : undefined,
|
|
commandResult: compactCommandMetadata(query.result, !query.ok, !query.ok),
|
|
degradedReason: ok ? undefined : assertion !== null && assertion.ok !== true ? "assertion-failed" : "query-failed",
|
|
next: assertion !== null && assertion.ok !== true
|
|
? { targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02", closeout: "bun scripts/cli.ts hwlab g14 observability closeout --lane v02" }
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
export function runG14ObservabilityTargets(options: G14ObservabilityOptions): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const queryCommands = Object.entries(V02_OBSERVABILITY_QUERIES).map(([name, promql]) => `section query_${name} kubectl get --raw ${shellQuote(observabilityQueryPath(promql))}`);
|
|
const script = [
|
|
"set +e",
|
|
"section() {",
|
|
" name=\"$1\"",
|
|
" shift",
|
|
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
|
" \"$@\"",
|
|
" code=$?",
|
|
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
|
"}",
|
|
`section pods kubectl get pods -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -o json`,
|
|
`section monitors kubectl get servicemonitor,podmonitor,prometheusrule -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -l hwlab.pikastech.local/monitoring=enabled -o json`,
|
|
`section podResourceMetrics kubectl get --raw ${shellQuote(`/apis/metrics.k8s.io/v1beta1/namespaces/${V02_RUNTIME_NAMESPACE}/pods`)}`,
|
|
...queryCommands,
|
|
].join("\n");
|
|
const bundle = g14K3s(["sh", "--", script], options.timeoutSeconds * 1000);
|
|
const sections = parseShellSections(statusText(bundle));
|
|
const pods = k8sItems(parseSectionJson(sections.pods));
|
|
const monitors = k8sItems(parseSectionJson(sections.monitors));
|
|
const podResourceMetrics = k8sItems(parseSectionJson(sections.podResourceMetrics)).map(podResourceMetricSummary);
|
|
const parsedQueries = Object.fromEntries(Object.entries(V02_OBSERVABILITY_QUERIES).map(([name]) => [name, parseSectionJson(sections[`query_${name}`])]));
|
|
const metricSeries = Object.fromEntries(Object.entries(parsedQueries).map(([name, parsed]) => [name, prometheusResultItems(parsed)]));
|
|
const booleanQueryNames = new Set<string>(V02_OBSERVABILITY_BOOLEAN_QUERY_NAMES);
|
|
const closeoutQueryNames = new Set<string>(V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES);
|
|
const assertions = Object.fromEntries(Object.entries(parsedQueries).map(([name, parsed]) => [
|
|
name,
|
|
g14ObservabilityQueryAssertion(parsed, V02_OBSERVABILITY_EXPECTED_TARGET_COUNT, booleanQueryNames.has(name) ? "1" : undefined),
|
|
]));
|
|
const targets = attachResourceMetricsToTargets(mergeObservabilityTargets(pods, monitors, metricSeries), podResourceMetrics);
|
|
const compactTargets = targets.map(compactObservabilityTarget);
|
|
const levelSummary = buildObservabilityLevelSummary(targets);
|
|
const sidecars = targets
|
|
.map((target) => record(target.sidecar))
|
|
.filter((sidecar) => Object.keys(sidecar).length > 0);
|
|
const notReadySidecars = sidecars.filter((sidecar) => sidecar.sidecarReady !== true || sidecar.podReady !== true);
|
|
const restartProblems = sidecars.filter((sidecar) => (numericValue(sidecar.restartCount) ?? 0) > 0);
|
|
const queryOk = V02_OBSERVABILITY_CLOSEOUT_QUERY_NAMES.every((name) => record(assertions[name]).ok === true);
|
|
const sidecarOk = targets.length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT
|
|
&& sidecars.length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT
|
|
&& notReadySidecars.length === 0;
|
|
const monitorsOk = shellSectionOk(sections.monitors) && monitors.length > 0;
|
|
const resourceOk = shellSectionOk(sections.podResourceMetrics)
|
|
&& podResourceMetrics.filter((item) => {
|
|
const serviceId = stringOrNull(item.serviceId);
|
|
return serviceId !== null && V02_OBSERVABILITY_SERVICE_IDS.includes(serviceId);
|
|
}).length === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT;
|
|
const ok = isCommandSuccess(bundle) && shellSectionOk(sections.pods) && monitorsOk && queryOk && sidecarOk && resourceOk;
|
|
return {
|
|
ok,
|
|
command: "hwlab g14 observability targets",
|
|
lane: options.lane,
|
|
namespace: V02_RUNTIME_NAMESPACE,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
|
|
targetCount: targets.length,
|
|
levelSummary,
|
|
targets: compactTargets,
|
|
sidecars: {
|
|
ok: sidecarOk,
|
|
observedCount: sidecars.length,
|
|
readyCount: sidecars.filter((sidecar) => sidecar.sidecarReady === true && sidecar.podReady === true).length,
|
|
notReady: notReadySidecars,
|
|
restartProblems,
|
|
},
|
|
monitors: {
|
|
ok: monitorsOk,
|
|
count: monitors.length,
|
|
items: monitorSummaries(monitors),
|
|
sectionOk: shellSectionOk(sections.monitors),
|
|
},
|
|
resourceSnapshot: {
|
|
ok: resourceOk,
|
|
source: "metrics.k8s.io/v1beta1",
|
|
observedTargetCount: podResourceMetrics.filter((item) => {
|
|
const serviceId = stringOrNull(item.serviceId);
|
|
return serviceId !== null && V02_OBSERVABILITY_SERVICE_IDS.includes(serviceId);
|
|
}).length,
|
|
expectedTargetCount: V02_OBSERVABILITY_EXPECTED_TARGET_COUNT,
|
|
sectionOk: shellSectionOk(sections.podResourceMetrics),
|
|
stderr: shellSectionOk(sections.podResourceMetrics) ? "" : commandErrorSummary(bundle),
|
|
},
|
|
queries: Object.fromEntries(Object.entries(parsedQueries).map(([name, parsed]) => [name, {
|
|
promql: record(V02_OBSERVABILITY_QUERIES)[name],
|
|
resultType: nested(parsed, ["data", "resultType"]) ?? null,
|
|
resultCount: prometheusResultItems(parsed).length,
|
|
assertion: assertions[name],
|
|
seriesOmitted: {
|
|
reason: closeoutQueryNames.has(name) ? "target-summary-uses-assertion-and-target-rows" : "auxiliary-metric-summarized-in-levelSummary",
|
|
summaryField: closeoutQueryNames.has(name) ? "targets" : "levelSummary",
|
|
drillDownCommand: closeoutQueryNames.has(name)
|
|
? `bun scripts/cli.ts hwlab g14 observability query --promql ${shellQuote(String(record(V02_OBSERVABILITY_QUERIES)[name]))} --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`
|
|
: undefined,
|
|
},
|
|
sectionOk: shellSectionOk(sections[`query_${name}`]),
|
|
}])),
|
|
result: compactCommandMetadata(bundle, !ok),
|
|
degradedReason: ok ? undefined : "target-health-failed",
|
|
next: ok ? { closeout: "bun scripts/cli.ts hwlab g14 observability closeout --lane v02" } : { boundary: "bun scripts/cli.ts hwlab g14 observability boundary --lane v02" },
|
|
};
|
|
}
|
|
|
|
export function runG14ObservabilityBoundary(options: G14ObservabilityOptions): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const script = [
|
|
"set +e",
|
|
"section() {",
|
|
" name=\"$1\"",
|
|
" shift",
|
|
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
|
" \"$@\"",
|
|
" code=$?",
|
|
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
|
"}",
|
|
`section workloadMonitoring kubectl get servicemonitor,podmonitor,prometheusrule,prometheus,alertmanager -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -o json`,
|
|
`section infraControlPlane kubectl get prometheus,alertmanager -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -o json`,
|
|
].join("\n");
|
|
const bundle = g14K3s(["sh", "--", script], options.timeoutSeconds * 1000);
|
|
const sections = parseShellSections(statusText(bundle));
|
|
const workloadItems = k8sItems(parseSectionJson(sections.workloadMonitoring));
|
|
const infraItems = k8sItems(parseSectionJson(sections.infraControlPlane));
|
|
const forbiddenWorkloadControlPlane = workloadItems.filter((item) => item.kind === "Prometheus" || item.kind === "Alertmanager");
|
|
const allowedWorkloadDeclarations = workloadItems.filter((item) => item.kind === "ServiceMonitor" || item.kind === "PodMonitor" || item.kind === "PrometheusRule");
|
|
const namespaceBoundaryOk = shellSectionOk(sections.workloadMonitoring) && forbiddenWorkloadControlPlane.length === 0;
|
|
const infraControlPlaneOk = shellSectionOk(sections.infraControlPlane) && infraItems.some((item) => item.kind === "Prometheus" && k8sMetadataName(item) === G14_PROMETHEUS_NAME);
|
|
const publicProbes = [
|
|
publicMetricsProbe("hwlab-v02-cloud-web", `${V02_CLOUD_WEB_URL}/metrics`),
|
|
publicMetricsProbe("hwlab-v02-cloud-api", `${V02_CLOUD_API_URL}/metrics`),
|
|
];
|
|
const publicMetricsExposureOk = publicProbes.every((probe) => probe.ok === true && probe.exposed !== true);
|
|
const ok = isCommandSuccess(bundle) && namespaceBoundaryOk && infraControlPlaneOk && publicMetricsExposureOk;
|
|
return {
|
|
ok,
|
|
command: "hwlab g14 observability boundary",
|
|
lane: options.lane,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
namespaceBoundary: {
|
|
ok: namespaceBoundaryOk,
|
|
workloadNamespace: V02_RUNTIME_NAMESPACE,
|
|
allowedKinds: ["ServiceMonitor", "PodMonitor", "PrometheusRule"],
|
|
allowedDeclarationCount: allowedWorkloadDeclarations.length,
|
|
allowedDeclarations: monitorSummaries(allowedWorkloadDeclarations),
|
|
forbiddenKinds: ["Prometheus", "Alertmanager"],
|
|
forbiddenControlPlaneCount: forbiddenWorkloadControlPlane.length,
|
|
forbiddenControlPlane: forbiddenWorkloadControlPlane.map((item) => ({ kind: item.kind, name: k8sMetadataName(item) })),
|
|
sectionOk: shellSectionOk(sections.workloadMonitoring),
|
|
},
|
|
infraControlPlane: {
|
|
ok: infraControlPlaneOk,
|
|
namespace: G14_OBSERVABILITY_NAMESPACE,
|
|
items: infraItems.map((item) => ({ kind: item.kind, name: k8sMetadataName(item) })),
|
|
sectionOk: shellSectionOk(sections.infraControlPlane),
|
|
},
|
|
publicMetricsExposure: {
|
|
ok: publicMetricsExposureOk,
|
|
expected: "denied",
|
|
probes: publicProbes,
|
|
},
|
|
result: compactCommandMetadata(bundle, !ok),
|
|
degradedReason: ok ? undefined : "boundary-failed",
|
|
next: ok ? { closeout: "bun scripts/cli.ts hwlab g14 observability closeout --lane v02" } : { targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02" },
|
|
};
|
|
}
|
|
|
|
export function passFail(value: boolean): "pass" | "fail" {
|
|
return value ? "pass" : "fail";
|
|
}
|
|
|
|
export function closeoutAdvice(summary: Record<string, unknown>): string[] {
|
|
const advice: string[] = [];
|
|
if (summary.platformReady !== "pass") advice.push("platformReady failed -> run observability status/apply and inspect Prometheus Operator plus Prometheus readiness in devops-infra");
|
|
if (summary.scrapeReachable !== "pass") advice.push("scrapeReachable failed -> check ServiceMonitor labels, metrics sidecar port name, and Prometheus target discovery");
|
|
if (summary.sidecarServing !== "pass") advice.push("sidecarServing failed -> check hwlab-metrics sidecar readiness, restartCount, and metrics script/container logs");
|
|
if (summary.businessHealthProbe !== "pass") advice.push("businessHealthProbe failed -> up=1 but health_probe=0 usually means sidecar can be scraped but cannot reach the business health endpoint");
|
|
if (summary.resourceSnapshot !== "pass") advice.push("resourceSnapshot failed -> check metrics.k8s.io APIService and metrics-server availability on G14 k3s");
|
|
if (summary.namespaceControlPlaneBoundary !== "pass") advice.push("namespaceControlPlaneBoundary failed -> remove Prometheus/Alertmanager from workload namespace; shared control plane belongs in devops-infra");
|
|
if (summary.publicMetricsExposure !== "pass") advice.push("publicMetricsExposure failed -> public /metrics returned Prometheus text; remove FRP/edge exposure or add an authenticated internal-only route");
|
|
return advice;
|
|
}
|
|
|
|
export function runG14ObservabilityCloseout(options: G14ObservabilityOptions): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const status = g14ObservabilityStatus();
|
|
const targets = runG14ObservabilityTargets(options);
|
|
const boundary = runG14ObservabilityBoundary(options);
|
|
const targetQueries = record(targets.queries);
|
|
const queryOk = (name: string): boolean => record(record(targetQueries[name]).assertion).ok === true;
|
|
const sidecarsOk = record(record(targets.sidecars)).ok === true;
|
|
const platformReady = record(status.crds).ok === true && record(status.operator).ok === true && record(status.prometheus).ok === true && record(status.query).ok === true;
|
|
const namespaceBoundaryOk = record(record(boundary.namespaceBoundary)).ok === true && record(record(boundary.infraControlPlane)).ok === true;
|
|
const publicDenied = record(record(boundary.publicMetricsExposure)).ok === true;
|
|
const resourceOk = record(record(targets.resourceSnapshot)).ok === true;
|
|
const summary = {
|
|
platformReady: passFail(platformReady),
|
|
workloadMonitorCount: numericValue(record(status.workloadMonitors).count) ?? numericValue(record(targets.monitors).count) ?? 0,
|
|
observedTargetCount: numericValue(targets.targetCount) ?? 0,
|
|
scrapeReachable: passFail(queryOk("scrapeReachable")),
|
|
sidecarServing: passFail(queryOk("sidecarServing") && sidecarsOk),
|
|
businessHealthProbe: passFail(queryOk("businessHealthProbe")),
|
|
sidecarReady: passFail(sidecarsOk),
|
|
resourceSnapshot: passFail(resourceOk),
|
|
namespaceControlPlaneBoundary: passFail(namespaceBoundaryOk),
|
|
publicMetricsExposure: passFail(publicDenied),
|
|
publicMetricsExposureState: publicDenied ? "denied" : "exposed-or-unknown",
|
|
};
|
|
const ok = Object.entries(summary).every(([key, value]) => key === "workloadMonitorCount"
|
|
? numericValue(value) !== null && Number(value) > 0
|
|
: key === "observedTargetCount"
|
|
? numericValue(value) === V02_OBSERVABILITY_EXPECTED_TARGET_COUNT
|
|
: key === "publicMetricsExposureState"
|
|
? value === "denied"
|
|
: value === "pass");
|
|
return {
|
|
ok,
|
|
command: "hwlab g14 observability closeout",
|
|
lane: options.lane,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
summary,
|
|
advice: closeoutAdvice(summary),
|
|
evidence: {
|
|
platform: {
|
|
ok: status.ok,
|
|
crdsOk: record(status.crds).ok === true,
|
|
operatorReady: record(status.operator).ok === true,
|
|
prometheusReady: record(status.prometheus).ok === true,
|
|
workloadMonitorCount: summary.workloadMonitorCount,
|
|
upResultCount: record(status.query).resultCount ?? null,
|
|
},
|
|
targets: {
|
|
ok: targets.ok,
|
|
expectedTargetCount: targets.expectedTargetCount,
|
|
observedTargetCount: targets.targetCount,
|
|
readySidecarCount: record(targets.sidecars).readyCount ?? null,
|
|
healthProbeDuration: record(record(targets.levelSummary).healthProbeDuration),
|
|
scrapeDuration: record(record(targets.levelSummary).scrapeDuration),
|
|
resourceSnapshot: record(targets.resourceSnapshot),
|
|
resourceUsage: record(record(targets.levelSummary).resourceUsage),
|
|
services: arrayRecords(record(targets.levelSummary).services).map((service) => ({
|
|
serviceId: service.serviceId ?? null,
|
|
scrapeReachable: service.scrapeReachable ?? null,
|
|
sidecarServing: service.sidecarServing ?? null,
|
|
businessHealthProbe: service.businessHealthProbe ?? null,
|
|
statusCode: service.statusCode ?? null,
|
|
totalCpuMillicores: service.totalCpuMillicores ?? null,
|
|
totalMemoryMiB: service.totalMemoryMiB ?? null,
|
|
businessCpuMillicores: service.businessCpuMillicores ?? null,
|
|
businessMemoryMiB: service.businessMemoryMiB ?? null,
|
|
})),
|
|
},
|
|
boundary: {
|
|
ok: boundary.ok,
|
|
forbiddenControlPlaneCount: numericValue(record(boundary.namespaceBoundary).forbiddenControlPlaneCount) ?? 0,
|
|
publicMetricsExposure: record(boundary.publicMetricsExposure),
|
|
},
|
|
drillDown: {
|
|
status: "bun scripts/cli.ts hwlab g14 observability status",
|
|
targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02",
|
|
boundary: "bun scripts/cli.ts hwlab g14 observability boundary --lane v02",
|
|
scrapeReachable: `bun scripts/cli.ts hwlab g14 observability query --promql 'up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
|
sidecarServing: `bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_up{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
|
businessHealthProbe: `bun scripts/cli.ts hwlab g14 observability query --promql 'hwlab_service_health_probe_success{namespace=\"hwlab-v02\"}' --expect-count ${V02_OBSERVABILITY_EXPECTED_TARGET_COUNT} --expect-value 1`,
|
|
},
|
|
},
|
|
degradedReason: ok ? undefined : "observability-closeout-failed",
|
|
next: {
|
|
targets: "bun scripts/cli.ts hwlab g14 observability targets --lane v02",
|
|
boundary: "bun scripts/cli.ts hwlab g14 observability boundary --lane v02",
|
|
controlPlaneCloseout: "bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit <full-sha>",
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runG14Observability(options: G14ObservabilityOptions): Record<string, unknown> {
|
|
if (options.action === "status") return g14ObservabilityStatus();
|
|
if (options.action === "query") return runG14ObservabilityQuery(options);
|
|
if (options.action === "targets") return runG14ObservabilityTargets(options);
|
|
if (options.action === "boundary") return runG14ObservabilityBoundary(options);
|
|
if (options.action === "closeout") return runG14ObservabilityCloseout(options);
|
|
return runG14ObservabilityApply(options);
|
|
}
|