feat: 扩展共享 Prometheus 可观测性
This commit is contained in:
Codex
@@ -21,6 +21,10 @@ images:
|
||||
repository: docker.m.daocloud.io/grafana/tempo
|
||||
tag: 2.8.1
|
||||
pullPolicy: IfNotPresent
|
||||
prometheus:
|
||||
repository: docker.m.daocloud.io/prom/prometheus
|
||||
tag: v3.5.0
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
targets:
|
||||
- id: NC01
|
||||
@@ -66,6 +70,38 @@ traceBackend:
|
||||
mode: emptyDir
|
||||
retention: 24h
|
||||
|
||||
metricsBackend:
|
||||
type: prometheus
|
||||
enabled: true
|
||||
deploymentName: prometheus
|
||||
serviceName: prometheus
|
||||
configMapName: prometheus-config
|
||||
serviceAccountName: prometheus
|
||||
clusterRoleName: platform-infra-prometheus
|
||||
clusterRoleBindingName: platform-infra-prometheus
|
||||
persistentVolumeClaimName: prometheus-data
|
||||
replicas: 1
|
||||
httpPort: 9090
|
||||
storage:
|
||||
mode: persistentVolumeClaim
|
||||
size: 20Gi
|
||||
retention: 15d
|
||||
discovery:
|
||||
namespaces: []
|
||||
labelSelector: unidesk.ai/metrics=true
|
||||
annotationKeys:
|
||||
scrape: prometheus.io/scrape
|
||||
path: prometheus.io/path
|
||||
port: prometheus.io/port
|
||||
defaultPath: /metrics
|
||||
scrapeInterval: 30s
|
||||
scrapeTimeout: 10s
|
||||
query:
|
||||
path: /api/v1/query
|
||||
timeoutSeconds: 15
|
||||
maxSeries: 200
|
||||
maxResponseBytes: 262144
|
||||
|
||||
sampling:
|
||||
mode: parentbased_traceidratio
|
||||
ratio: 1
|
||||
@@ -185,3 +221,7 @@ probes:
|
||||
service: otel-collector
|
||||
portName: health
|
||||
path: /
|
||||
- name: prometheus-ready
|
||||
service: prometheus
|
||||
portName: http
|
||||
path: /-/ready
|
||||
|
||||
@@ -2,6 +2,15 @@
|
||||
|
||||
UniDesk 的可观测性优先级高于静默成功。CLI、服务日志、Docker 日志和数据库状态都必须能通过短命令查询。
|
||||
|
||||
## 共享 Prometheus 指标面
|
||||
|
||||
- `config/platform-infra/observability.yaml` 同时拥有 Collector、Tempo 和 Prometheus 的 image、retention、storage、port、服务发现与查询预算；TypeScript 只校验和渲染。
|
||||
- Prometheus 使用 Kubernetes 原生 pod 服务发现。服务通过稳定 label `unidesk.ai/metrics=true` 与 `prometheus.io/scrape=true`、`prometheus.io/path`、`prometheus.io/port` annotation 声明 `/metrics` 抓取，不允许业务专属分支。
|
||||
- `bun scripts/cli.ts platform-infra observability plan --target <NODE>` 展示 Prometheus enabled/disabled、Service、发现选择器和 manifest 摘要。
|
||||
- `bun scripts/cli.ts platform-infra observability status --target <NODE>` 与 `validate` 将 Prometheus readiness 单列为 warning；不得改变业务 readiness 或交付成功结论。
|
||||
- `bun scripts/cli.ts platform-infra observability metrics-query --target <NODE> --query <promql>` 只执行 YAML 预算约束的瞬时 PromQL 查询；默认有界，`--full` 展开结构，`--raw` 仅用于解析诊断。
|
||||
- Collector + Tempo 继续是 trace authority；Prometheus 只承担 metrics，不成为第二业务状态库或交付门禁。
|
||||
|
||||
## Capability Boundary
|
||||
|
||||
版本、commit 和微服务契约漂移只属于可观测性信号。OTel、日志、diagnose 和 status 必须把它们投影为非阻塞 warning，并继续以真实业务结果为权威；不得把漂移告警升级为业务失败、admission blocker、readiness failure 或 CI/CD gate。完整边界见 [DevOps Hygiene](devops-hygiene.md#prohibited-deployment-truth)。
|
||||
|
||||
@@ -13,6 +13,8 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
expect(Buffer.byteLength(compact.renderedText, "utf8")).toBeLessThan(10_240);
|
||||
expect(compact.renderedText).toContain("serviceConnections=8");
|
||||
expect(compact.renderedText).toContain("configRefs=6/6 missing=0");
|
||||
expect(compact.renderedText).toContain("prometheus=true service=prometheus");
|
||||
expect(compact.renderedText).toContain("scrapeSelector=unidesk.ai/metrics=true");
|
||||
expect(compact.renderedText).toContain("--full");
|
||||
expect(compact.renderedText).toContain("--raw");
|
||||
|
||||
@@ -25,6 +27,7 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
renderPlan: { target: { id: "NC01", route: "NC01:k3s", namespace: "platform-infra" } },
|
||||
});
|
||||
expect((expanded as Record<string, any>).renderPlan.instrumentation).toHaveLength(8);
|
||||
expect((expanded as Record<string, any>).config.metricsBackend).toMatchObject({ type: "prometheus", enabled: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -38,8 +41,8 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
expect(result.stdout).not.toContain("/tmp/unidesk-cli-output");
|
||||
});
|
||||
|
||||
test("search, trace and diagnose-code-agent expose only scoped help", () => {
|
||||
for (const operation of ["search", "trace", "diagnose-code-agent"]) {
|
||||
test("query commands expose only scoped help", () => {
|
||||
for (const operation of ["search", "trace", "diagnose-code-agent", "metrics-query"]) {
|
||||
const result = cli(["platform-infra", "observability", operation, "--help"]);
|
||||
expect(result.status).toBe(0);
|
||||
const payload = JSON.parse(result.stdout) as Record<string, any>;
|
||||
@@ -57,7 +60,7 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
expect(result.status).toBe(0);
|
||||
const payload = JSON.parse(result.stdout) as Record<string, any>;
|
||||
expect(payload.data.operations.map((item: Record<string, unknown>) => item.name)).toEqual([
|
||||
"plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent",
|
||||
"plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent", "metrics-query",
|
||||
]);
|
||||
expect(payload.data.scopedHelp).toContain("<operation> --help");
|
||||
expect(payload.data.usage).toBeUndefined();
|
||||
|
||||
@@ -69,6 +69,8 @@ export function plan(options: CommonOptions): Record<string, unknown> {
|
||||
collectorGrpcEndpoint: `${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.grpcPort}`,
|
||||
collectorHttpEndpoint: `http://${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.httpPort}`,
|
||||
backendGrpcEndpoint: `${observability.traceBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.traceBackend.otlp.grpcPort}`,
|
||||
prometheusHttpEndpoint: observability.metricsBackend.enabled ? `http://${observability.metricsBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.metricsBackend.httpPort}` : null,
|
||||
scrapeDiscovery: observability.metricsBackend.discovery,
|
||||
},
|
||||
instrumentation: observability.instrumentation.serviceConnections,
|
||||
resourceAttributes: observability.resourceAttributes,
|
||||
@@ -80,6 +82,7 @@ export function plan(options: CommonOptions): Record<string, unknown> {
|
||||
status: `bun scripts/cli.ts platform-infra observability status --target ${target.id}`,
|
||||
validate: `bun scripts/cli.ts platform-infra observability validate --target ${target.id}`,
|
||||
trace: `bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id <traceId>`,
|
||||
metricsQuery: `bun scripts/cli.ts platform-infra observability metrics-query --target ${target.id} --query <promql>`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -39,6 +39,11 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
const traceBackend = objectField(root, "traceBackend", "");
|
||||
const traceBackendOtlp = objectField(traceBackend, "otlp", "traceBackend");
|
||||
const traceBackendStorage = objectField(traceBackend, "storage", "traceBackend");
|
||||
const metricsBackend = objectField(root, "metricsBackend", "");
|
||||
const metricsStorage = objectField(metricsBackend, "storage", "metricsBackend");
|
||||
const metricsDiscovery = objectField(metricsBackend, "discovery", "metricsBackend");
|
||||
const metricsAnnotations = objectField(metricsDiscovery, "annotationKeys", "metricsBackend.discovery");
|
||||
const metricsQuery = objectField(metricsBackend, "query", "metricsBackend");
|
||||
const sampling = objectField(root, "sampling", "");
|
||||
const instrumentation = objectField(root, "instrumentation", "");
|
||||
const resourceAttributes = objectField(root, "resourceAttributes", "");
|
||||
@@ -56,6 +61,7 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
images: {
|
||||
collector: imageSpec(objectField(images, "collector", "images"), "images.collector"),
|
||||
tempo: imageSpec(objectField(images, "tempo", "images"), "images.tempo"),
|
||||
prometheus: imageSpec(objectField(images, "prometheus", "images"), "images.prometheus"),
|
||||
},
|
||||
targets: arrayOfRecords(root.targets, "targets").map(parseTarget),
|
||||
collector: {
|
||||
@@ -79,6 +85,42 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
retention: stringField(traceBackendStorage, "retention", "traceBackend.storage"),
|
||||
},
|
||||
},
|
||||
metricsBackend: {
|
||||
type: enumField(metricsBackend, "type", "metricsBackend", ["prometheus"] as const),
|
||||
enabled: booleanField(metricsBackend, "enabled", "metricsBackend"),
|
||||
deploymentName: kubernetesNameField(metricsBackend, "deploymentName", "metricsBackend"),
|
||||
serviceName: kubernetesNameField(metricsBackend, "serviceName", "metricsBackend"),
|
||||
configMapName: kubernetesNameField(metricsBackend, "configMapName", "metricsBackend"),
|
||||
serviceAccountName: kubernetesNameField(metricsBackend, "serviceAccountName", "metricsBackend"),
|
||||
clusterRoleName: kubernetesNameField(metricsBackend, "clusterRoleName", "metricsBackend"),
|
||||
clusterRoleBindingName: kubernetesNameField(metricsBackend, "clusterRoleBindingName", "metricsBackend"),
|
||||
persistentVolumeClaimName: kubernetesNameField(metricsBackend, "persistentVolumeClaimName", "metricsBackend"),
|
||||
replicas: integerField(metricsBackend, "replicas", "metricsBackend"),
|
||||
httpPort: portField(metricsBackend, "httpPort", "metricsBackend"),
|
||||
storage: {
|
||||
mode: enumField(metricsStorage, "mode", "metricsBackend.storage", ["persistentVolumeClaim"] as const),
|
||||
size: stringField(metricsStorage, "size", "metricsBackend.storage"),
|
||||
retention: stringField(metricsStorage, "retention", "metricsBackend.storage"),
|
||||
},
|
||||
discovery: {
|
||||
namespaces: stringArrayField(metricsDiscovery, "namespaces", "metricsBackend.discovery"),
|
||||
labelSelector: stringField(metricsDiscovery, "labelSelector", "metricsBackend.discovery"),
|
||||
annotationKeys: {
|
||||
scrape: stringField(metricsAnnotations, "scrape", "metricsBackend.discovery.annotationKeys"),
|
||||
path: stringField(metricsAnnotations, "path", "metricsBackend.discovery.annotationKeys"),
|
||||
port: stringField(metricsAnnotations, "port", "metricsBackend.discovery.annotationKeys"),
|
||||
},
|
||||
defaultPath: apiPathField(metricsDiscovery, "defaultPath", "metricsBackend.discovery"),
|
||||
scrapeInterval: stringField(metricsDiscovery, "scrapeInterval", "metricsBackend.discovery"),
|
||||
scrapeTimeout: stringField(metricsDiscovery, "scrapeTimeout", "metricsBackend.discovery"),
|
||||
},
|
||||
query: {
|
||||
path: apiPathField(metricsQuery, "path", "metricsBackend.query"),
|
||||
timeoutSeconds: integerField(metricsQuery, "timeoutSeconds", "metricsBackend.query"),
|
||||
maxSeries: integerField(metricsQuery, "maxSeries", "metricsBackend.query"),
|
||||
maxResponseBytes: integerField(metricsQuery, "maxResponseBytes", "metricsBackend.query"),
|
||||
},
|
||||
},
|
||||
sampling: {
|
||||
mode: enumField(sampling, "mode", "sampling", ["parentbased_traceidratio"] as const),
|
||||
ratio: numberField(sampling, "ratio", "sampling"),
|
||||
@@ -99,7 +141,8 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
};
|
||||
if (config.targets.length === 0) throw new Error(`${configLabel}.targets must not be empty`);
|
||||
assertKnownEnabledTarget(config.targets, config.defaults.targetId, "defaults.targetId");
|
||||
if (config.collector.replicas < 0 || config.traceBackend.replicas < 0) throw new Error(`${configLabel} replicas must be >= 0`);
|
||||
if (config.collector.replicas < 0 || config.traceBackend.replicas < 0 || config.metricsBackend.replicas < 0) throw new Error(`${configLabel} replicas must be >= 0`);
|
||||
if (config.metricsBackend.query.timeoutSeconds < 1 || config.metricsBackend.query.maxSeries < 1 || config.metricsBackend.query.maxResponseBytes < 1024) throw new Error(`${configLabel}.metricsBackend.query budgets must be positive and maxResponseBytes must be >= 1024`);
|
||||
if (config.sampling.ratio < 0 || config.sampling.ratio > 1) throw new Error(`${configLabel}.sampling.ratio must be between 0 and 1`);
|
||||
if (!config.probes.traceQueryPathTemplate.includes("{{traceId}}")) throw new Error(`${configLabel}.probes.traceQueryPathTemplate must include {{traceId}}`);
|
||||
return config;
|
||||
|
||||
@@ -19,18 +19,19 @@ import {
|
||||
capture,
|
||||
} from "../platform-infra-ops-library";
|
||||
|
||||
import type { ApplyOptions, CommonOptions, DiagnoseCodeAgentOptions, SearchOptions, TraceOptions } from "./types";
|
||||
import type { ApplyOptions, CommonOptions, DiagnoseCodeAgentOptions, MetricsQueryOptions, SearchOptions, TraceOptions } from "./types";
|
||||
import { apply, plan, status, validate } from "./actions";
|
||||
import { diagnoseCodeAgent, search, trace } from "./render";
|
||||
import { renderObservabilityPlan, renderObservabilityPlanFailure } from "./plan-render";
|
||||
import { metricsQuery } from "./prometheus";
|
||||
|
||||
const observabilityOperations = ["plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent"] as const;
|
||||
const observabilityOperations = ["plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent", "metrics-query"] as const;
|
||||
|
||||
export function observabilityHelp(scope: string | null = null): Record<string, unknown> {
|
||||
const scoped = scope === null ? null : observabilityOperationHelp(scope);
|
||||
if (scoped !== null) return scoped;
|
||||
return {
|
||||
command: "platform-infra observability plan|apply|status|validate|trace|search|diagnose-code-agent",
|
||||
command: "platform-infra observability plan|apply|status|validate|trace|search|diagnose-code-agent|metrics-query",
|
||||
output: "默认输出为有界摘要；--full/--raw 显式披露完整结构",
|
||||
configTruth: "config/platform-infra/observability.yaml",
|
||||
spec: "PJ2026-01060501 OTel追踪 draft-2026-06-19-p0",
|
||||
@@ -39,7 +40,7 @@ export function observabilityHelp(scope: string | null = null): Record<string, u
|
||||
help: `bun scripts/cli.ts platform-infra observability ${name} --help`,
|
||||
})),
|
||||
scopedHelp: "bun scripts/cli.ts platform-infra observability <operation> --help",
|
||||
boundary: "Prometheus 仍是指标来源；本命令只负责 platform-infra OTel Collector、Tempo readiness 与 trace 查询。",
|
||||
boundary: "Collector + Tempo 是 trace authority；Prometheus 仅承担 metrics，故障和漂移保持非阻塞 warning。",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -62,6 +63,7 @@ export async function runPlatformObservabilityCommand(config: UniDeskConfig, arg
|
||||
if (action === "trace") return await trace(config, parseTraceOptions(args.slice(1)));
|
||||
if (action === "search") return await search(config, parseSearchOptions(args.slice(1)));
|
||||
if (action === "diagnose-code-agent") return await diagnoseCodeAgent(config, parseDiagnoseCodeAgentOptions(args.slice(1)));
|
||||
if (action === "metrics-query") return await metricsQuery(config, parseMetricsQueryOptions(args.slice(1)));
|
||||
return { ok: false, error: "unsupported-platform-infra-observability-command", args, help: observabilityHelp() };
|
||||
}
|
||||
|
||||
@@ -105,6 +107,12 @@ function observabilityOperationHelp(scope: string): Record<string, unknown> | nu
|
||||
],
|
||||
options: ["--trace-id <32-hex>", "--target <NODE>", "--grep <text>", "--limit <1..500>", "--full", "--raw"],
|
||||
};
|
||||
if (scope === "metrics-query") return {
|
||||
...common,
|
||||
command: "platform-infra observability metrics-query",
|
||||
usage: ["bun scripts/cli.ts platform-infra observability metrics-query --target <NODE> --query <promql> [--full|--raw]"],
|
||||
options: ["--target <NODE>", "--query <promql>", "--full", "--raw"],
|
||||
};
|
||||
if (scope === "search") return {
|
||||
...common,
|
||||
command: "platform-infra observability search",
|
||||
@@ -126,6 +134,27 @@ function observabilityOperationHelp(scope: string): Record<string, unknown> | nu
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseMetricsQueryOptions(args: string[]): MetricsQueryOptions {
|
||||
const commonArgs: string[] = [];
|
||||
let query: string | null = null;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--query") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error("--query requires a value");
|
||||
query = value;
|
||||
index += 1;
|
||||
} else {
|
||||
commonArgs.push(arg);
|
||||
if (arg === "--target") {
|
||||
commonArgs.push(args[index + 1] ?? "");
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ...parseCommonOptions(commonArgs), query };
|
||||
}
|
||||
|
||||
export function parseCommonOptions(args: string[]): CommonOptions {
|
||||
let targetId: string | null = null;
|
||||
let full = false;
|
||||
|
||||
@@ -8,6 +8,7 @@ export function renderObservabilityPlan(result: Record<string, unknown>): Render
|
||||
const images = record(config.images);
|
||||
const collector = record(config.collector);
|
||||
const traceBackend = record(config.traceBackend);
|
||||
const metricsBackend = record(config.metricsBackend);
|
||||
const renderPlan = record(result.renderPlan);
|
||||
const otlp = record(renderPlan.otlp);
|
||||
const connections = records(config.serviceConnections);
|
||||
@@ -39,6 +40,8 @@ export function renderObservabilityPlan(result: Record<string, unknown>): Render
|
||||
`target=${textValue(target.id)} route=${textValue(target.route)} namespace=${textValue(target.namespace)} role=${textValue(target.role)}`,
|
||||
`collector=${textValue(collector.serviceName)} replicas=${textValue(collector.replicas)} image=${textValue(images.collector)}`,
|
||||
`tempo=${textValue(traceBackend.serviceName)} replicas=${textValue(traceBackend.replicas)} retention=${textValue(record(traceBackend.storage).retention)} image=${textValue(images.traceBackend)}`,
|
||||
`prometheus=${textValue(metricsBackend.enabled)} service=${textValue(metricsBackend.serviceName)} replicas=${textValue(metricsBackend.replicas)} retention=${textValue(record(metricsBackend.storage).retention)} image=${textValue(images.metricsBackend)}`,
|
||||
`scrapeSelector=${textValue(record(metricsBackend.discovery).labelSelector)} namespaces=${values(record(metricsBackend.discovery).namespaces).length === 0 ? "all" : values(record(metricsBackend.discovery).namespaces).join(",")} queryBudget=${textValue(record(metricsBackend.query).timeoutSeconds)}s/${textValue(record(metricsBackend.query).maxSeries)}series/${textValue(record(metricsBackend.query).maxResponseBytes)}bytes`,
|
||||
`serviceConnections=${connections.length} services=${serviceNames.size} nodes=${nodes.size} configRefs=${presentConfigRefs.length}/${configRefs.length} missing=${missingConfigRefs.length} objects=${objects.length}`,
|
||||
"",
|
||||
"Service connections:",
|
||||
@@ -77,6 +80,7 @@ export function renderObservabilityPlan(result: Record<string, unknown>): Render
|
||||
` collector-grpc: ${textValue(otlp.collectorGrpcEndpoint)}`,
|
||||
` collector-http: ${textValue(otlp.collectorHttpEndpoint)}`,
|
||||
` tempo-grpc: ${textValue(otlp.backendGrpcEndpoint)}`,
|
||||
` prometheus-http: ${textValue(otlp.prometheusHttpEndpoint)}`,
|
||||
"",
|
||||
"Next:",
|
||||
` bun scripts/cli.ts platform-infra observability plan --target ${textValue(target.id)} --full`,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
import { capture, compactCapture, parseJsonOutput, redactSensitiveUnknown, shQuote } from "../platform-infra-ops-library";
|
||||
import { resolveTarget } from "./actions";
|
||||
import { readObservabilityConfig } from "./config";
|
||||
import { formatTable, shortenEnd, textValue } from "./manifest";
|
||||
import { imageReference, indent, targetSummary } from "./summary";
|
||||
import type { MetricsQueryOptions, ObservabilityConfig, ObservabilityTarget } from "./types";
|
||||
|
||||
export function prometheusManifests(observability: ObservabilityConfig, target: ObservabilityTarget): string[] {
|
||||
const prometheus = observability.metricsBackend;
|
||||
if (!prometheus.enabled) return [];
|
||||
const namespaceNames = prometheus.discovery.namespaces.length > 0
|
||||
? `\n namespaces:\n names:\n${prometheus.discovery.namespaces.map((name) => ` - ${name}`).join("\n")}`
|
||||
: "";
|
||||
const config = `global:\n scrape_interval: ${prometheus.discovery.scrapeInterval}\n scrape_timeout: ${prometheus.discovery.scrapeTimeout}\nscrape_configs:\n - job_name: kubernetes-pods\n kubernetes_sd_configs:\n - role: pod${namespaceNames}\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.scrape)}]\n action: keep\n regex: \"true\"\n - source_labels: [__meta_kubernetes_pod_label_${labelMeta(prometheus.discovery.labelSelector.split("=")[0] ?? "")} ]\n action: keep\n regex: \"${prometheus.discovery.labelSelector.split("=")[1] ?? ".+"}\"\n - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: (.+)\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.port)}]\n action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: \$1:\$2\n target_label: __address__\n - target_label: __metrics_path__\n replacement: ${prometheus.discovery.defaultPath}\n`;
|
||||
const renderedConfig = config.replace(
|
||||
" - target_label: __metrics_path__\n replacement:",
|
||||
` - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: ^$\n replacement:`,
|
||||
);
|
||||
const labels = ` app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n app.kubernetes.io/part-of: platform-infra\n app.kubernetes.io/managed-by: unidesk`;
|
||||
return [
|
||||
`apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: ${prometheus.serviceAccountName}\n namespace: ${target.namespace}\n labels:\n${labels}\n`,
|
||||
`apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: ${prometheus.clusterRoleName}\nrules:\n - apiGroups: [\"\"]\n resources: [\"nodes\", \"nodes/proxy\", \"services\", \"endpoints\", \"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n`,
|
||||
`apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: ${prometheus.clusterRoleBindingName}\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: ${prometheus.clusterRoleName}\nsubjects:\n - kind: ServiceAccount\n name: ${prometheus.serviceAccountName}\n namespace: ${target.namespace}\n`,
|
||||
`apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: ${prometheus.configMapName}\n namespace: ${target.namespace}\n labels:\n${labels}\ndata:\n prometheus.yml: |\n${indent(renderedConfig, 4)}\n`,
|
||||
`apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: ${prometheus.persistentVolumeClaimName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n accessModes: [ReadWriteOnce]\n resources:\n requests:\n storage: ${prometheus.storage.size}\n`,
|
||||
`apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: ${prometheus.deploymentName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n replicas: ${prometheus.replicas}\n selector:\n matchLabels:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n template:\n metadata:\n labels:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n app.kubernetes.io/part-of: platform-infra\n spec:\n serviceAccountName: ${prometheus.serviceAccountName}\n containers:\n - name: prometheus\n image: ${imageReference(observability.images.prometheus)}\n imagePullPolicy: ${observability.images.prometheus.pullPolicy}\n args:\n - --config.file=/etc/prometheus/prometheus.yml\n - --storage.tsdb.path=/prometheus\n - --storage.tsdb.retention.time=${prometheus.storage.retention}\n ports:\n - name: http\n containerPort: ${prometheus.httpPort}\n readinessProbe:\n httpGet:\n path: /-/ready\n port: http\n volumeMounts:\n - name: config\n mountPath: /etc/prometheus/prometheus.yml\n subPath: prometheus.yml\n readOnly: true\n - name: data\n mountPath: /prometheus\n volumes:\n - name: config\n configMap:\n name: ${prometheus.configMapName}\n - name: data\n persistentVolumeClaim:\n claimName: ${prometheus.persistentVolumeClaimName}\n`,
|
||||
`apiVersion: v1\nkind: Service\nmetadata:\n name: ${prometheus.serviceName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n type: ClusterIP\n selector:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n ports:\n - name: http\n port: ${prometheus.httpPort}\n targetPort: http\n`,
|
||||
];
|
||||
}
|
||||
|
||||
export async function metricsQuery(config: UniDeskConfig, options: MetricsQueryOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (options.query === null || options.query.trim() === "") throw new Error("observability metrics-query requires --query <promql>");
|
||||
const observability = readObservabilityConfig();
|
||||
const target = resolveTarget(observability, options.targetId);
|
||||
const prometheus = observability.metricsBackend;
|
||||
if (!prometheus.enabled) return { ok: false, action: "platform-infra-observability-metrics-query", mutation: false, target: targetSummary(target), metrics: { enabled: false }, warnings: ["Prometheus is disabled in config/platform-infra/observability.yaml"] };
|
||||
const path = `${prometheus.query.path}?query=${encodeURIComponent(options.query)}`;
|
||||
const script = `set -u\npython3 - <<'PY'\nimport json, subprocess\npath = ${JSON.stringify(`/api/v1/namespaces/${target.namespace}/services/http:${prometheus.serviceName}:http/proxy${path}`)}\nproc = subprocess.run([\"kubectl\", \"get\", \"--raw\", path], text=True, capture_output=True, timeout=${prometheus.query.timeoutSeconds})\nbody = proc.stdout[:${prometheus.query.maxResponseBytes}]\ntry:\n parsed = json.loads(body) if body else None\nexcept Exception:\n parsed = None\nresult = parsed.get(\"data\", {}).get(\"result\", []) if isinstance(parsed, dict) else []\nprint(json.dumps({\"ok\": proc.returncode == 0 and isinstance(parsed, dict) and parsed.get(\"status\") == \"success\", \"exitCode\": proc.returncode, \"truncated\": len(proc.stdout) > ${prometheus.query.maxResponseBytes} or len(result) > ${prometheus.query.maxSeries}, \"resultType\": parsed.get(\"data\", {}).get(\"resultType\") if isinstance(parsed, dict) else None, \"result\": result[:${prometheus.query.maxSeries}], \"stderrTail\": proc.stderr[-2000:]}, ensure_ascii=False))\nPY`;
|
||||
const result = await capture(config, target.route, ["sh"], script);
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
const ok = result.exitCode === 0 && parsed?.ok === true;
|
||||
if (options.full || options.raw) return { ok, action: "platform-infra-observability-metrics-query", mutation: false, target: targetSummary(target), query: options.query, budgets: prometheus.query, result: parsed === null ? compactCapture(result, { full: true }) : redactSensitiveUnknown(parsed) };
|
||||
const rows = Array.isArray(parsed?.result) ? parsed.result.slice(0, 20).map((item: unknown) => [shortenEnd(JSON.stringify(item), 160)]) : [];
|
||||
return { ok, command: "platform-infra observability metrics-query", contentType: "text/plain", renderedText: [`platform-infra observability metrics-query (${ok ? "ok" : "not-ok"})`, "", `target=${target.id} prometheus=${prometheus.serviceName}.${target.namespace}.svc.cluster.local:${prometheus.httpPort}`, `query=${options.query}`, `budget=timeout:${prometheus.query.timeoutSeconds}s series:${prometheus.query.maxSeries} bytes:${prometheus.query.maxResponseBytes} truncated=${textValue(parsed?.truncated)}`, "", formatTable(["RESULT"], rows.length > 0 ? rows : [["-"]]), "", `Next: bun scripts/cli.ts platform-infra observability metrics-query --target ${target.id} --query ${shQuote(options.query)} --full`].join("\n") };
|
||||
}
|
||||
|
||||
function annotationMeta(value: string): string { return value.replaceAll(".", "_").replaceAll("/", "_"); }
|
||||
function labelMeta(value: string): string { return value.replaceAll(".", "_").replaceAll("/", "_").replaceAll("-", "_"); }
|
||||
@@ -102,7 +102,13 @@ PY
|
||||
}
|
||||
|
||||
export function statusScript(observability: ObservabilityConfig, target: ObservabilityTarget, full: boolean, generateValidationTrace = false): string {
|
||||
const endpointsJson = JSON.stringify(observability.probes.statusEndpoints);
|
||||
const enabledEndpoints = observability.metricsBackend.enabled
|
||||
? observability.probes.statusEndpoints
|
||||
: observability.probes.statusEndpoints.filter((endpoint) => endpoint.service !== observability.metricsBackend.serviceName);
|
||||
const endpointsJson = JSON.stringify(enabledEndpoints);
|
||||
const metricsCaptures = observability.metricsBackend.enabled
|
||||
? `capture_json metrics_deployments kubectl -n ${shQuote(target.namespace)} get deployment ${shQuote(observability.metricsBackend.deploymentName)} -o json\ncapture_json metrics_services kubectl -n ${shQuote(target.namespace)} get service ${shQuote(observability.metricsBackend.serviceName)} -o json\ncapture_json metrics_pods kubectl -n ${shQuote(target.namespace)} get pods -l ${shQuote(`app.kubernetes.io/name=${observability.metricsBackend.deploymentName}`)} -o json`
|
||||
: "";
|
||||
return `
|
||||
set -u
|
||||
tmp="$(mktemp -d)"
|
||||
@@ -123,6 +129,7 @@ capture_json namespace kubectl get namespace ${shQuote(target.namespace)} -o jso
|
||||
capture_json deployments kubectl -n ${shQuote(target.namespace)} get deployment ${shQuote(observability.collector.deploymentName)} ${shQuote(observability.traceBackend.deploymentName)} -o json
|
||||
capture_json services kubectl -n ${shQuote(target.namespace)} get service ${shQuote(observability.collector.serviceName)} ${shQuote(observability.traceBackend.serviceName)} -o json
|
||||
capture_json pods kubectl -n ${shQuote(target.namespace)} get pods -l ${shQuote(`app.kubernetes.io/name in (${observability.collector.deploymentName},${observability.traceBackend.deploymentName})`)} -o json
|
||||
${metricsCaptures}
|
||||
capture_json events kubectl -n ${shQuote(target.namespace)} get events -o json
|
||||
python3 - "$tmp" '${endpointsJson.replaceAll("'", "'\"'\"'")}' <<'PY'
|
||||
import json, random, socket, subprocess, sys, time, urllib.error, urllib.request
|
||||
@@ -304,7 +311,9 @@ def generate_test_trace():
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=3)
|
||||
runtime_ready = rc("namespace") == 0 and rc("deployments") == 0 and rc("services") == 0 and all(item["ok"] for item in probe_results)
|
||||
blocking_probes = [item for item in probe_results if item.get("service") != "${observability.metricsBackend.serviceName}"]
|
||||
metrics_probes = [item for item in probe_results if item.get("service") == "${observability.metricsBackend.serviceName}"]
|
||||
runtime_ready = rc("namespace") == 0 and rc("deployments") == 0 and rc("services") == 0 and all(item["ok"] for item in blocking_probes)
|
||||
validation_trace = generate_test_trace() if (${generateValidationTrace ? "True" : "False"} and runtime_ready) else None
|
||||
payload = {
|
||||
"ok": runtime_ready and (validation_trace is None or validation_trace.get("ok") is True),
|
||||
@@ -316,8 +325,12 @@ payload = {
|
||||
"services": compact_section("services"),
|
||||
"pods": compact_section("pods"),
|
||||
"events": compact_section("events"),
|
||||
"metricsDeployments": compact_section("metrics_deployments") if ${observability.metricsBackend.enabled ? "True" : "False"} else None,
|
||||
"metricsServices": compact_section("metrics_services") if ${observability.metricsBackend.enabled ? "True" : "False"} else None,
|
||||
"metricsPods": compact_section("metrics_pods") if ${observability.metricsBackend.enabled ? "True" : "False"} else None,
|
||||
},
|
||||
"probes": probe_results,
|
||||
"warnings": ([{"code": "prometheus-not-ready", "detail": item} for item in metrics_probes if item.get("ok") is not True] if ${observability.metricsBackend.enabled ? "True" : "False"} else [{"code": "prometheus-disabled"}]),
|
||||
"validationTrace": validation_trace,
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2 if ${full ? "True" : "False"} else None))
|
||||
|
||||
@@ -32,9 +32,11 @@ export function configSummary(observability: ObservabilityConfig, target: Observ
|
||||
images: {
|
||||
collector: imageReference(observability.images.collector),
|
||||
traceBackend: imageReference(observability.images.tempo),
|
||||
metricsBackend: imageReference(observability.images.prometheus),
|
||||
},
|
||||
collector: observability.collector,
|
||||
traceBackend: observability.traceBackend,
|
||||
metricsBackend: observability.metricsBackend,
|
||||
sampling: observability.sampling,
|
||||
serviceConnections: observability.instrumentation.serviceConnections.map((item) => ({
|
||||
serviceName: item.serviceName,
|
||||
@@ -62,8 +64,8 @@ export function targetSummary(target: ObservabilityTarget): Record<string, unkno
|
||||
|
||||
export function policyChecks(yaml: string, target: ObservabilityTarget): Array<Record<string, unknown>> {
|
||||
return [
|
||||
{ name: "yaml-source-of-truth", ok: true, detail: "All concrete images, routes, namespace, ports, retention and sampling values are read from config/platform-infra/observability.yaml." },
|
||||
{ name: "clusterip-only", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(yaml), detail: "Collector and trace backend stay ClusterIP-only." },
|
||||
{ name: "yaml-source-of-truth", ok: true, detail: "All concrete images, routes, namespace, ports, retention, storage, scrape discovery and query budgets are read from config/platform-infra/observability.yaml." },
|
||||
{ name: "clusterip-only", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(yaml), detail: "Collector, Tempo and Prometheus stay ClusterIP-only." },
|
||||
{ name: "no-ingress", ok: !/^\s*kind:\s*Ingress\s*$/mu.test(yaml), detail: "No public ingress is rendered for the first tracing backend." },
|
||||
{ name: "no-host-network", ok: !/^\s*hostNetwork:\s*true\s*$/mu.test(yaml), detail: "Pods must not use host network." },
|
||||
{ name: "allow-all-network-policy", ok: yaml.includes("kind: NetworkPolicy") && yaml.includes("name: allow-all") && yaml.includes(`namespace: ${target.namespace}`), detail: `NetworkPolicy/allow-all is rendered in ${target.namespace}.` },
|
||||
@@ -76,6 +78,9 @@ export function statusSummary(payload: Record<string, unknown>): Record<string,
|
||||
const services = objectList(sectionJson(sections, "services"));
|
||||
const pods = objectList(sectionJson(sections, "pods"));
|
||||
const events = objectList(sectionJson(sections, "events"));
|
||||
const metricsDeployments = objectList(optionalSectionJson(sections, "metricsDeployments"));
|
||||
const metricsServices = objectList(optionalSectionJson(sections, "metricsServices"));
|
||||
const metricsPods = objectList(optionalSectionJson(sections, "metricsPods"));
|
||||
const podEvents = latestPodEvents(events);
|
||||
const probes = Array.isArray(payload.probes) ? payload.probes as Array<Record<string, unknown>> : [];
|
||||
const readyDeployments = deployments.map((item) => deploymentSummary(item));
|
||||
@@ -92,6 +97,12 @@ export function statusSummary(payload: Record<string, unknown>): Record<string,
|
||||
path: item.path,
|
||||
stderrTail: item.ok === true ? "" : item.stderrTail,
|
||||
})),
|
||||
warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
|
||||
metrics: {
|
||||
deployments: metricsDeployments.map((item) => deploymentSummary(item)),
|
||||
services: metricsServices.map((item) => metadataName(item)),
|
||||
pods: metricsPods.map((item) => podSummary(item, podEvents)),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -106,6 +117,12 @@ export function sectionJson(sections: Record<string, unknown>, name: string): un
|
||||
return section.json;
|
||||
}
|
||||
|
||||
function optionalSectionJson(sections: Record<string, unknown>, name: string): unknown {
|
||||
const value = sections[name];
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
||||
return (value as Record<string, unknown>).json;
|
||||
}
|
||||
|
||||
export function objectList(value: unknown): Record<string, unknown>[] {
|
||||
if (typeof value !== "object" || value === null) return [];
|
||||
const items = (value as Record<string, unknown>).items;
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
import type { ObservabilityConfig, ObservabilityTarget } from "./types";
|
||||
import { tempoService } from "./search-script";
|
||||
import { imageReference, indent } from "./summary";
|
||||
import { prometheusManifests } from "./prometheus";
|
||||
|
||||
export function asArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
@@ -43,6 +44,7 @@ export function renderManifest(observability: ObservabilityConfig, target: Obser
|
||||
tempoConfigMap(observability, target),
|
||||
tempoDeployment(observability, target, tempoImage),
|
||||
tempoService(observability, target),
|
||||
...prometheusManifests(observability, target),
|
||||
].filter((item) => item.trim().length > 0).join("\n---\n");
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ export interface ObservabilityConfig {
|
||||
images: {
|
||||
collector: ImageSpec;
|
||||
tempo: ImageSpec;
|
||||
prometheus: ImageSpec;
|
||||
};
|
||||
targets: ObservabilityTarget[];
|
||||
collector: {
|
||||
@@ -71,6 +72,7 @@ export interface ObservabilityConfig {
|
||||
otlp: OtlpPorts;
|
||||
storage: { mode: "emptyDir"; retention: string };
|
||||
};
|
||||
metricsBackend: PrometheusConfig;
|
||||
sampling: { mode: "parentbased_traceidratio"; ratio: number };
|
||||
instrumentation: {
|
||||
contextPropagation: string[];
|
||||
@@ -87,6 +89,30 @@ export interface ObservabilityConfig {
|
||||
};
|
||||
}
|
||||
|
||||
export interface PrometheusConfig {
|
||||
type: "prometheus";
|
||||
enabled: boolean;
|
||||
deploymentName: string;
|
||||
serviceName: string;
|
||||
configMapName: string;
|
||||
serviceAccountName: string;
|
||||
clusterRoleName: string;
|
||||
clusterRoleBindingName: string;
|
||||
persistentVolumeClaimName: string;
|
||||
replicas: number;
|
||||
httpPort: number;
|
||||
storage: { mode: "persistentVolumeClaim"; size: string; retention: string };
|
||||
discovery: {
|
||||
namespaces: string[];
|
||||
labelSelector: string;
|
||||
annotationKeys: { scrape: string; path: string; port: string };
|
||||
defaultPath: string;
|
||||
scrapeInterval: string;
|
||||
scrapeTimeout: string;
|
||||
};
|
||||
query: { path: string; timeoutSeconds: number; maxSeries: number; maxResponseBytes: number };
|
||||
}
|
||||
|
||||
export interface ImageSpec {
|
||||
repository: string;
|
||||
tag: string;
|
||||
@@ -165,3 +191,7 @@ export interface DiagnoseCodeAgentOptions extends CommonOptions {
|
||||
candidateLimit: number;
|
||||
lookbackMinutes: number;
|
||||
}
|
||||
|
||||
export interface MetricsQueryOptions extends CommonOptions {
|
||||
query: string | null;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user