diff --git a/config/platform-infra/observability.yaml b/config/platform-infra/observability.yaml index 045b2760..1c4e3dd8 100644 --- a/config/platform-infra/observability.yaml +++ b/config/platform-infra/observability.yaml @@ -73,6 +73,8 @@ traceBackend: metricsBackend: type: prometheus enabled: true + targetIds: + - NC01 deploymentName: prometheus serviceName: prometheus configMapName: prometheus-config @@ -88,7 +90,9 @@ metricsBackend: retention: 15d discovery: namespaces: [] - labelSelector: unidesk.ai/metrics=true + labelSelector: + key: unidesk.ai/metrics + value: "true" annotationKeys: scrape: prometheus.io/scrape path: prometheus.io/path diff --git a/docs/reference/observability.md b/docs/reference/observability.md index 4ae7d50c..760b58cc 100644 --- a/docs/reference/observability.md +++ b/docs/reference/observability.md @@ -4,8 +4,9 @@ UniDesk 的可观测性优先级高于静默成功。CLI、服务日志、Docker ## 共享 Prometheus 指标面 -- `config/platform-infra/observability.yaml` 同时拥有 Collector、Tempo 和 Prometheus 的 image、retention、storage、port、服务发现与查询预算;TypeScript 只校验和渲染。 -- Prometheus 使用 Kubernetes 原生 pod 服务发现。服务通过稳定 label `unidesk.ai/metrics=true` 与 `prometheus.io/scrape=true`、`prometheus.io/path`、`prometheus.io/port` annotation 声明 `/metrics` 抓取,不允许业务专属分支。 +- `config/platform-infra/observability.yaml` 同时拥有 Collector、Tempo 和 Prometheus 的 image、retention、storage、port、目标 `targetIds`、服务发现与查询预算;TypeScript 只校验和渲染。 +- Prometheus 仅作用于 `metricsBackend.targetIds` 选中的 observability target;其他 target 的 plan 显示 `disabled-for-target`,manifest、status probe/capture 和 `metrics-query` 均不访问 Prometheus。 +- Prometheus 使用 Kubernetes 原生 pod 服务发现。YAML 通过结构化 `labelSelector.key/value` 声明稳定 label,并用 `prometheus.io/scrape=true`、`prometheus.io/path`、`prometheus.io/port` annotation 声明抓取;显式 path 优先,仅在 annotation 缺失时使用 `defaultPath`,不允许业务专属分支。 - `bun scripts/cli.ts platform-infra observability plan --target ` 展示 Prometheus enabled/disabled、Service、发现选择器和 manifest 摘要。 - `bun scripts/cli.ts platform-infra observability status --target ` 与 `validate` 将 Prometheus readiness 单列为 warning;不得改变业务 readiness 或交付成功结论。 - `bun scripts/cli.ts platform-infra observability metrics-query --target --query ` 只执行 YAML 预算约束的瞬时 PromQL 查询;默认有界,`--full` 展开结构,`--raw` 仅用于解析诊断。 diff --git a/scripts/src/platform-infra-observability.test.ts b/scripts/src/platform-infra-observability.test.ts index e86c0421..6022844d 100644 --- a/scripts/src/platform-infra-observability.test.ts +++ b/scripts/src/platform-infra-observability.test.ts @@ -3,7 +3,11 @@ import { spawnSync } from "node:child_process"; import { rootPath } from "./config"; import { isRenderedCliResult } from "./output"; import { runPlatformObservabilityCommand } from "./platform-infra-observability"; +import { resolveTarget } from "./platform-infra-observability/actions"; +import { readObservabilityConfig } from "./platform-infra-observability/config"; import { renderObservabilityPlanFailure } from "./platform-infra-observability/plan-render"; +import { statusScript } from "./platform-infra-observability/search-script"; +import { renderManifest } from "./platform-infra-observability/trace-script"; describe("platform-infra observability progressive disclosure", () => { test("default plan is a bounded table while full and raw preserve the complete plan", async () => { @@ -13,7 +17,7 @@ describe("platform-infra observability progressive disclosure", () => { expect(Buffer.byteLength(compact.renderedText, "utf8")).toBeLessThan(10_240); expect(compact.renderedText).toContain("serviceConnections=8"); expect(compact.renderedText).toContain("configRefs=6/6 missing=0"); - expect(compact.renderedText).toContain("prometheus=true service=prometheus"); + expect(compact.renderedText).toContain("prometheus=enabled service=prometheus targets=NC01"); expect(compact.renderedText).toContain("scrapeSelector=unidesk.ai/metrics=true"); expect(compact.renderedText).toContain("--full"); expect(compact.renderedText).toContain("--raw"); @@ -41,6 +45,44 @@ describe("platform-infra observability progressive disclosure", () => { expect(result.stdout).not.toContain("/tmp/unidesk-cli-output"); }); + test("Prometheus manifests and runtime access only apply to YAML-selected targets", async () => { + const observability = readObservabilityConfig(); + const nc01 = resolveTarget(observability, "NC01"); + const d518 = resolveTarget(observability, "D518"); + const jd01 = resolveTarget(observability, "JD01"); + const nc01Manifest = renderManifest(observability, nc01); + for (const target of [d518, jd01]) { + const manifest = renderManifest(observability, target); + expect(manifest).not.toContain("kind: ClusterRole\nmetadata:\n name: platform-infra-prometheus"); + expect(manifest).not.toContain("kind: PersistentVolumeClaim\nmetadata:\n name: prometheus-data"); + expect(manifest).not.toContain("app.kubernetes.io/component: metrics-backend"); + const plan = await runPlatformObservabilityCommand({} as never, ["plan", "--target", target.id]); + expect(isRenderedCliResult(plan)).toBe(true); + if (!isRenderedCliResult(plan)) throw new Error("expected rendered plan"); + expect(plan.renderedText).toContain("prometheus=disabled-for-target"); + expect(plan.renderedText).toContain("objects=8"); + const script = statusScript(observability, target, false); + expect(script).not.toContain("capture_json metrics_deployments"); + expect(script).not.toContain("prometheus-ready"); + const query = await runPlatformObservabilityCommand({} as never, ["metrics-query", "--target", target.id, "--query", "up", "--full"]); + expect(query).toMatchObject({ metrics: { enabled: false, disposition: "disabled-for-target", targetIds: ["NC01"] } }); + } + expect(nc01Manifest).toContain("kind: ClusterRole\nmetadata:\n name: platform-infra-prometheus"); + expect(nc01Manifest).toContain("kind: PersistentVolumeClaim\nmetadata:\n name: prometheus-data"); + }); + + test("Prometheus relabel rules prefer an explicit path and only default when absent", () => { + const observability = readObservabilityConfig(); + const manifest = renderManifest(observability, resolveTarget(observability, "NC01")); + const explicitPath = "source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: (/.+)"; + const defaultPath = "source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: ^$\n replacement: /metrics"; + expect(manifest).toContain("source_labels: [__meta_kubernetes_pod_label_unidesk_ai_metrics]"); + expect(manifest).toContain('regex: "true"'); + expect(manifest).toContain(explicitPath); + expect(manifest).toContain(defaultPath); + expect(manifest.indexOf(explicitPath)).toBeLessThan(manifest.indexOf(defaultPath)); + }); + test("query commands expose only scoped help", () => { for (const operation of ["search", "trace", "diagnose-code-agent", "metrics-query"]) { const result = cli(["platform-infra", "observability", operation, "--help"]); diff --git a/scripts/src/platform-infra-observability/actions.ts b/scripts/src/platform-infra-observability/actions.ts index 24369aa0..ba8cbd14 100644 --- a/scripts/src/platform-infra-observability/actions.ts +++ b/scripts/src/platform-infra-observability/actions.ts @@ -27,6 +27,7 @@ import { compactStatus, configSummary, manifestObjectSummary, policyChecks, stat import { renderManifest } from "./trace-script"; import { formatTable, shortenEnd, textValue } from "./manifest"; import { apiPathField, configLabel, kubernetesNameField, stringField } from "./types"; +import { metricsEnabledForTarget, metricsTargetDisposition } from "./metrics-target"; export function parseStatusEndpoint(record: Record, index: number): StatusEndpoint { const path = `probes.statusEndpoints[${index}]`; @@ -57,6 +58,7 @@ export function plan(options: CommonOptions): Record { const target = resolveTarget(observability, options.targetId); const yaml = renderManifest(observability, target); const policy = policyChecks(yaml, target); + const metricsDisposition = metricsTargetDisposition(observability, target); return { ok: policy.every((check) => check.ok), action: "platform-infra-observability-plan", @@ -65,11 +67,15 @@ export function plan(options: CommonOptions): Record { renderPlan: { target: targetSummary(target), objects: manifestObjectSummary(yaml), + metrics: { + disposition: metricsDisposition, + targetIds: observability.metricsBackend.targetIds, + }, otlp: { collectorGrpcEndpoint: `${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.grpcPort}`, collectorHttpEndpoint: `http://${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.httpPort}`, backendGrpcEndpoint: `${observability.traceBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.traceBackend.otlp.grpcPort}`, - prometheusHttpEndpoint: observability.metricsBackend.enabled ? `http://${observability.metricsBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.metricsBackend.httpPort}` : null, + prometheusHttpEndpoint: metricsEnabledForTarget(observability, target) ? `http://${observability.metricsBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.metricsBackend.httpPort}` : null, scrapeDiscovery: observability.metricsBackend.discovery, }, instrumentation: observability.instrumentation.serviceConnections, diff --git a/scripts/src/platform-infra-observability/config.ts b/scripts/src/platform-infra-observability/config.ts index a941c7be..33b1fb10 100644 --- a/scripts/src/platform-infra-observability/config.ts +++ b/scripts/src/platform-infra-observability/config.ts @@ -42,6 +42,7 @@ export function readObservabilityConfig(): ObservabilityConfig { const metricsBackend = objectField(root, "metricsBackend", ""); const metricsStorage = objectField(metricsBackend, "storage", "metricsBackend"); const metricsDiscovery = objectField(metricsBackend, "discovery", "metricsBackend"); + const metricsLabelSelector = objectField(metricsDiscovery, "labelSelector", "metricsBackend.discovery"); const metricsAnnotations = objectField(metricsDiscovery, "annotationKeys", "metricsBackend.discovery"); const metricsQuery = objectField(metricsBackend, "query", "metricsBackend"); const sampling = objectField(root, "sampling", ""); @@ -88,6 +89,7 @@ export function readObservabilityConfig(): ObservabilityConfig { metricsBackend: { type: enumField(metricsBackend, "type", "metricsBackend", ["prometheus"] as const), enabled: booleanField(metricsBackend, "enabled", "metricsBackend"), + targetIds: stringArrayField(metricsBackend, "targetIds", "metricsBackend"), deploymentName: kubernetesNameField(metricsBackend, "deploymentName", "metricsBackend"), serviceName: kubernetesNameField(metricsBackend, "serviceName", "metricsBackend"), configMapName: kubernetesNameField(metricsBackend, "configMapName", "metricsBackend"), @@ -104,7 +106,10 @@ export function readObservabilityConfig(): ObservabilityConfig { }, discovery: { namespaces: stringArrayField(metricsDiscovery, "namespaces", "metricsBackend.discovery"), - labelSelector: stringField(metricsDiscovery, "labelSelector", "metricsBackend.discovery"), + labelSelector: { + key: stringField(metricsLabelSelector, "key", "metricsBackend.discovery.labelSelector"), + value: stringField(metricsLabelSelector, "value", "metricsBackend.discovery.labelSelector"), + }, annotationKeys: { scrape: stringField(metricsAnnotations, "scrape", "metricsBackend.discovery.annotationKeys"), path: stringField(metricsAnnotations, "path", "metricsBackend.discovery.annotationKeys"), @@ -141,6 +146,11 @@ export function readObservabilityConfig(): ObservabilityConfig { }; if (config.targets.length === 0) throw new Error(`${configLabel}.targets must not be empty`); assertKnownEnabledTarget(config.targets, config.defaults.targetId, "defaults.targetId"); + for (const targetId of config.metricsBackend.targetIds) assertKnownEnabledTarget(config.targets, targetId, "metricsBackend.targetIds"); + if (config.metricsBackend.enabled && config.metricsBackend.targetIds.length === 0) throw new Error(`${configLabel}.metricsBackend.targetIds must not be empty when metricsBackend.enabled is true`); + if (new Set(config.metricsBackend.targetIds.map((targetId) => targetId.toLowerCase())).size !== config.metricsBackend.targetIds.length) throw new Error(`${configLabel}.metricsBackend.targetIds must not contain duplicates`); + if (!/^[A-Za-z0-9]([A-Za-z0-9._/-]*[A-Za-z0-9])?$/u.test(config.metricsBackend.discovery.labelSelector.key)) throw new Error(`${configLabel}.metricsBackend.discovery.labelSelector.key must be a Kubernetes label key`); + if (!/^([A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?)?$/u.test(config.metricsBackend.discovery.labelSelector.value) || config.metricsBackend.discovery.labelSelector.value.length > 63) throw new Error(`${configLabel}.metricsBackend.discovery.labelSelector.value must be a Kubernetes label value`); if (config.collector.replicas < 0 || config.traceBackend.replicas < 0 || config.metricsBackend.replicas < 0) throw new Error(`${configLabel} replicas must be >= 0`); if (config.metricsBackend.query.timeoutSeconds < 1 || config.metricsBackend.query.maxSeries < 1 || config.metricsBackend.query.maxResponseBytes < 1024) throw new Error(`${configLabel}.metricsBackend.query budgets must be positive and maxResponseBytes must be >= 1024`); if (config.sampling.ratio < 0 || config.sampling.ratio > 1) throw new Error(`${configLabel}.sampling.ratio must be between 0 and 1`); diff --git a/scripts/src/platform-infra-observability/metrics-target.ts b/scripts/src/platform-infra-observability/metrics-target.ts new file mode 100644 index 00000000..1fa27a3b --- /dev/null +++ b/scripts/src/platform-infra-observability/metrics-target.ts @@ -0,0 +1,12 @@ +import type { ObservabilityConfig, ObservabilityTarget } from "./types"; + +export type MetricsTargetDisposition = "enabled" | "disabled" | "disabled-for-target"; + +export function metricsTargetDisposition(observability: ObservabilityConfig, target: ObservabilityTarget): MetricsTargetDisposition { + if (!observability.metricsBackend.enabled) return "disabled"; + return observability.metricsBackend.targetIds.some((targetId) => targetId.toLowerCase() === target.id.toLowerCase()) ? "enabled" : "disabled-for-target"; +} + +export function metricsEnabledForTarget(observability: ObservabilityConfig, target: ObservabilityTarget): boolean { + return metricsTargetDisposition(observability, target) === "enabled"; +} diff --git a/scripts/src/platform-infra-observability/plan-render.ts b/scripts/src/platform-infra-observability/plan-render.ts index aa02c981..c41f365e 100644 --- a/scripts/src/platform-infra-observability/plan-render.ts +++ b/scripts/src/platform-infra-observability/plan-render.ts @@ -10,6 +10,7 @@ export function renderObservabilityPlan(result: Record): Render const traceBackend = record(config.traceBackend); const metricsBackend = record(config.metricsBackend); const renderPlan = record(result.renderPlan); + const metricsPlan = record(renderPlan.metrics); const otlp = record(renderPlan.otlp); const connections = records(config.serviceConnections); const objects = records(renderPlan.objects); @@ -40,8 +41,8 @@ export function renderObservabilityPlan(result: Record): Render `target=${textValue(target.id)} route=${textValue(target.route)} namespace=${textValue(target.namespace)} role=${textValue(target.role)}`, `collector=${textValue(collector.serviceName)} replicas=${textValue(collector.replicas)} image=${textValue(images.collector)}`, `tempo=${textValue(traceBackend.serviceName)} replicas=${textValue(traceBackend.replicas)} retention=${textValue(record(traceBackend.storage).retention)} image=${textValue(images.traceBackend)}`, - `prometheus=${textValue(metricsBackend.enabled)} service=${textValue(metricsBackend.serviceName)} replicas=${textValue(metricsBackend.replicas)} retention=${textValue(record(metricsBackend.storage).retention)} image=${textValue(images.metricsBackend)}`, - `scrapeSelector=${textValue(record(metricsBackend.discovery).labelSelector)} namespaces=${values(record(metricsBackend.discovery).namespaces).length === 0 ? "all" : values(record(metricsBackend.discovery).namespaces).join(",")} queryBudget=${textValue(record(metricsBackend.query).timeoutSeconds)}s/${textValue(record(metricsBackend.query).maxSeries)}series/${textValue(record(metricsBackend.query).maxResponseBytes)}bytes`, + `prometheus=${textValue(metricsPlan.disposition)} service=${textValue(metricsBackend.serviceName)} targets=${values(metricsPlan.targetIds).join(",")} replicas=${textValue(metricsBackend.replicas)} retention=${textValue(record(metricsBackend.storage).retention)} image=${textValue(images.metricsBackend)}`, + `scrapeSelector=${textValue(record(record(metricsBackend.discovery).labelSelector).key)}=${textValue(record(record(metricsBackend.discovery).labelSelector).value)} namespaces=${values(record(metricsBackend.discovery).namespaces).length === 0 ? "all" : values(record(metricsBackend.discovery).namespaces).join(",")} queryBudget=${textValue(record(metricsBackend.query).timeoutSeconds)}s/${textValue(record(metricsBackend.query).maxSeries)}series/${textValue(record(metricsBackend.query).maxResponseBytes)}bytes`, `serviceConnections=${connections.length} services=${serviceNames.size} nodes=${nodes.size} configRefs=${presentConfigRefs.length}/${configRefs.length} missing=${missingConfigRefs.length} objects=${objects.length}`, "", "Service connections:", diff --git a/scripts/src/platform-infra-observability/prometheus.ts b/scripts/src/platform-infra-observability/prometheus.ts index dfd3536b..bb28d59c 100644 --- a/scripts/src/platform-infra-observability/prometheus.ts +++ b/scripts/src/platform-infra-observability/prometheus.ts @@ -6,24 +6,21 @@ import { readObservabilityConfig } from "./config"; import { formatTable, shortenEnd, textValue } from "./manifest"; import { imageReference, indent, targetSummary } from "./summary"; import type { MetricsQueryOptions, ObservabilityConfig, ObservabilityTarget } from "./types"; +import { metricsEnabledForTarget, metricsTargetDisposition } from "./metrics-target"; export function prometheusManifests(observability: ObservabilityConfig, target: ObservabilityTarget): string[] { const prometheus = observability.metricsBackend; - if (!prometheus.enabled) return []; + if (!metricsEnabledForTarget(observability, target)) return []; const namespaceNames = prometheus.discovery.namespaces.length > 0 ? `\n namespaces:\n names:\n${prometheus.discovery.namespaces.map((name) => ` - ${name}`).join("\n")}` : ""; - const config = `global:\n scrape_interval: ${prometheus.discovery.scrapeInterval}\n scrape_timeout: ${prometheus.discovery.scrapeTimeout}\nscrape_configs:\n - job_name: kubernetes-pods\n kubernetes_sd_configs:\n - role: pod${namespaceNames}\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.scrape)}]\n action: keep\n regex: \"true\"\n - source_labels: [__meta_kubernetes_pod_label_${labelMeta(prometheus.discovery.labelSelector.split("=")[0] ?? "")} ]\n action: keep\n regex: \"${prometheus.discovery.labelSelector.split("=")[1] ?? ".+"}\"\n - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: (.+)\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.port)}]\n action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: \$1:\$2\n target_label: __address__\n - target_label: __metrics_path__\n replacement: ${prometheus.discovery.defaultPath}\n`; - const renderedConfig = config.replace( - " - target_label: __metrics_path__\n replacement:", - ` - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: ^$\n replacement:`, - ); + const config = `global:\n scrape_interval: ${prometheus.discovery.scrapeInterval}\n scrape_timeout: ${prometheus.discovery.scrapeTimeout}\nscrape_configs:\n - job_name: kubernetes-pods\n kubernetes_sd_configs:\n - role: pod${namespaceNames}\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.scrape)}]\n action: keep\n regex: \"true\"\n - source_labels: [__meta_kubernetes_pod_label_${labelMeta(prometheus.discovery.labelSelector.key)}]\n action: keep\n regex: \"${prometheus.discovery.labelSelector.value}\"\n - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: (/.+)\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.port)}]\n action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: \$1:\$2\n target_label: __address__\n - source_labels: [__meta_kubernetes_pod_annotation_${annotationMeta(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: ^$\n replacement: ${prometheus.discovery.defaultPath}\n`; const labels = ` app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n app.kubernetes.io/part-of: platform-infra\n app.kubernetes.io/managed-by: unidesk`; return [ `apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: ${prometheus.serviceAccountName}\n namespace: ${target.namespace}\n labels:\n${labels}\n`, `apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: ${prometheus.clusterRoleName}\nrules:\n - apiGroups: [\"\"]\n resources: [\"nodes\", \"nodes/proxy\", \"services\", \"endpoints\", \"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n`, `apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: ${prometheus.clusterRoleBindingName}\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: ${prometheus.clusterRoleName}\nsubjects:\n - kind: ServiceAccount\n name: ${prometheus.serviceAccountName}\n namespace: ${target.namespace}\n`, - `apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: ${prometheus.configMapName}\n namespace: ${target.namespace}\n labels:\n${labels}\ndata:\n prometheus.yml: |\n${indent(renderedConfig, 4)}\n`, + `apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: ${prometheus.configMapName}\n namespace: ${target.namespace}\n labels:\n${labels}\ndata:\n prometheus.yml: |\n${indent(config, 4)}\n`, `apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: ${prometheus.persistentVolumeClaimName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n accessModes: [ReadWriteOnce]\n resources:\n requests:\n storage: ${prometheus.storage.size}\n`, `apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: ${prometheus.deploymentName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n replicas: ${prometheus.replicas}\n selector:\n matchLabels:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n template:\n metadata:\n labels:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n app.kubernetes.io/part-of: platform-infra\n spec:\n serviceAccountName: ${prometheus.serviceAccountName}\n containers:\n - name: prometheus\n image: ${imageReference(observability.images.prometheus)}\n imagePullPolicy: ${observability.images.prometheus.pullPolicy}\n args:\n - --config.file=/etc/prometheus/prometheus.yml\n - --storage.tsdb.path=/prometheus\n - --storage.tsdb.retention.time=${prometheus.storage.retention}\n ports:\n - name: http\n containerPort: ${prometheus.httpPort}\n readinessProbe:\n httpGet:\n path: /-/ready\n port: http\n volumeMounts:\n - name: config\n mountPath: /etc/prometheus/prometheus.yml\n subPath: prometheus.yml\n readOnly: true\n - name: data\n mountPath: /prometheus\n volumes:\n - name: config\n configMap:\n name: ${prometheus.configMapName}\n - name: data\n persistentVolumeClaim:\n claimName: ${prometheus.persistentVolumeClaimName}\n`, `apiVersion: v1\nkind: Service\nmetadata:\n name: ${prometheus.serviceName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n type: ClusterIP\n selector:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n ports:\n - name: http\n port: ${prometheus.httpPort}\n targetPort: http\n`, @@ -35,7 +32,8 @@ export async function metricsQuery(config: UniDeskConfig, options: MetricsQueryO const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const prometheus = observability.metricsBackend; - if (!prometheus.enabled) return { ok: false, action: "platform-infra-observability-metrics-query", mutation: false, target: targetSummary(target), metrics: { enabled: false }, warnings: ["Prometheus is disabled in config/platform-infra/observability.yaml"] }; + const disposition = metricsTargetDisposition(observability, target); + if (disposition !== "enabled") return { ok: false, action: "platform-infra-observability-metrics-query", mutation: false, target: targetSummary(target), metrics: { enabled: false, disposition, targetIds: prometheus.targetIds }, warnings: [`Prometheus is ${disposition} in config/platform-infra/observability.yaml`] }; const path = `${prometheus.query.path}?query=${encodeURIComponent(options.query)}`; const script = `set -u\npython3 - <<'PY'\nimport json, subprocess\npath = ${JSON.stringify(`/api/v1/namespaces/${target.namespace}/services/http:${prometheus.serviceName}:http/proxy${path}`)}\nproc = subprocess.run([\"kubectl\", \"get\", \"--raw\", path], text=True, capture_output=True, timeout=${prometheus.query.timeoutSeconds})\nbody = proc.stdout[:${prometheus.query.maxResponseBytes}]\ntry:\n parsed = json.loads(body) if body else None\nexcept Exception:\n parsed = None\nresult = parsed.get(\"data\", {}).get(\"result\", []) if isinstance(parsed, dict) else []\nprint(json.dumps({\"ok\": proc.returncode == 0 and isinstance(parsed, dict) and parsed.get(\"status\") == \"success\", \"exitCode\": proc.returncode, \"truncated\": len(proc.stdout) > ${prometheus.query.maxResponseBytes} or len(result) > ${prometheus.query.maxSeries}, \"resultType\": parsed.get(\"data\", {}).get(\"resultType\") if isinstance(parsed, dict) else None, \"result\": result[:${prometheus.query.maxSeries}], \"stderrTail\": proc.stderr[-2000:]}, ensure_ascii=False))\nPY`; const result = await capture(config, target.route, ["sh"], script); diff --git a/scripts/src/platform-infra-observability/search-script.ts b/scripts/src/platform-infra-observability/search-script.ts index 1bd8df23..83295560 100644 --- a/scripts/src/platform-infra-observability/search-script.ts +++ b/scripts/src/platform-infra-observability/search-script.ts @@ -21,6 +21,7 @@ import { import type { ObservabilityConfig, ObservabilityTarget } from "./types"; import { fieldManager } from "./types"; +import { metricsEnabledForTarget, metricsTargetDisposition } from "./metrics-target"; export function tempoService(observability: ObservabilityConfig, target: ObservabilityTarget): string { return `apiVersion: v1 @@ -102,11 +103,13 @@ PY } export function statusScript(observability: ObservabilityConfig, target: ObservabilityTarget, full: boolean, generateValidationTrace = false): string { - const enabledEndpoints = observability.metricsBackend.enabled + const metricsEnabled = metricsEnabledForTarget(observability, target); + const metricsDisposition = metricsTargetDisposition(observability, target); + const enabledEndpoints = metricsEnabled ? observability.probes.statusEndpoints : observability.probes.statusEndpoints.filter((endpoint) => endpoint.service !== observability.metricsBackend.serviceName); const endpointsJson = JSON.stringify(enabledEndpoints); - const metricsCaptures = observability.metricsBackend.enabled + const metricsCaptures = metricsEnabled ? `capture_json metrics_deployments kubectl -n ${shQuote(target.namespace)} get deployment ${shQuote(observability.metricsBackend.deploymentName)} -o json\ncapture_json metrics_services kubectl -n ${shQuote(target.namespace)} get service ${shQuote(observability.metricsBackend.serviceName)} -o json\ncapture_json metrics_pods kubectl -n ${shQuote(target.namespace)} get pods -l ${shQuote(`app.kubernetes.io/name=${observability.metricsBackend.deploymentName}`)} -o json` : ""; return ` @@ -325,12 +328,12 @@ payload = { "services": compact_section("services"), "pods": compact_section("pods"), "events": compact_section("events"), - "metricsDeployments": compact_section("metrics_deployments") if ${observability.metricsBackend.enabled ? "True" : "False"} else None, - "metricsServices": compact_section("metrics_services") if ${observability.metricsBackend.enabled ? "True" : "False"} else None, - "metricsPods": compact_section("metrics_pods") if ${observability.metricsBackend.enabled ? "True" : "False"} else None, + "metricsDeployments": compact_section("metrics_deployments") if ${metricsEnabled ? "True" : "False"} else None, + "metricsServices": compact_section("metrics_services") if ${metricsEnabled ? "True" : "False"} else None, + "metricsPods": compact_section("metrics_pods") if ${metricsEnabled ? "True" : "False"} else None, }, "probes": probe_results, - "warnings": ([{"code": "prometheus-not-ready", "detail": item} for item in metrics_probes if item.get("ok") is not True] if ${observability.metricsBackend.enabled ? "True" : "False"} else [{"code": "prometheus-disabled"}]), + "warnings": ([{"code": "prometheus-not-ready", "detail": item} for item in metrics_probes if item.get("ok") is not True] if ${metricsEnabled ? "True" : "False"} else [{"code": "prometheus-${metricsDisposition}"}]), "validationTrace": validation_trace, } print(json.dumps(payload, ensure_ascii=False, indent=2 if ${full ? "True" : "False"} else None)) diff --git a/scripts/src/platform-infra-observability/types.ts b/scripts/src/platform-infra-observability/types.ts index 2ce889df..629ff0d7 100644 --- a/scripts/src/platform-infra-observability/types.ts +++ b/scripts/src/platform-infra-observability/types.ts @@ -92,6 +92,7 @@ export interface ObservabilityConfig { export interface PrometheusConfig { type: "prometheus"; enabled: boolean; + targetIds: string[]; deploymentName: string; serviceName: string; configMapName: string; @@ -104,7 +105,7 @@ export interface PrometheusConfig { storage: { mode: "persistentVolumeClaim"; size: string; retention: string }; discovery: { namespaces: string[]; - labelSelector: string; + labelSelector: { key: string; value: string }; annotationKeys: { scrape: string; path: string; port: string }; defaultPath: string; scrapeInterval: string;