From 3875a8f08e20be5d5b617f08e2cc26232da9a8ee Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 5 Jun 2026 00:23:22 +0000 Subject: [PATCH] fix: keep G14 observability apply bounded --- scripts/src/hwlab-g14.ts | 60 +++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index f8474311..8581661f 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -4337,6 +4337,25 @@ function parseSectionJsonArray(section: ShellSection | undefined): Record record(item)); } +function sectionLines(section: ShellSection | undefined): string[] { + return String(section?.stdout ?? "") + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); +} + +function podRows(section: ShellSection | undefined): Record[] { + return sectionLines(section).map((line) => { + const [name = "", phase = "", readyRaw = ""] = line.split("\t"); + const readyValues = readyRaw.split(",").map((item) => item.trim()).filter(Boolean); + return { + name, + phase: phase || null, + ready: readyValues.length > 0 ? readyValues.every((item) => item === "true") : null, + }; + }).filter((item) => String(item.name).length > 0); +} + function conditionStatus(items: Record[], type: string): string | null { for (const item of items) { if (item.type === type) return typeof item.status === "string" ? item.status : null; @@ -4384,11 +4403,11 @@ function g14ObservabilityStatus(): Record { "}", `section namespace kubectl get namespace ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -o json`, `section discoveryNamespace kubectl get namespace ${shellQuote(V02_RUNTIME_NAMESPACE)} -o json`, - `section crds kubectl get crd ${crds.map(shellQuote).join(" ")} -o json`, + `section crds kubectl get crd ${crds.map(shellQuote).join(" ")} -o 'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}'`, `section operator kubectl get deploy -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} prometheus-operator -o json`, - `section operatorPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l app.kubernetes.io/name=prometheus-operator -o json`, + `section operatorPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l app.kubernetes.io/name=prometheus-operator -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .status.containerStatuses[*]}{.ready}{","}{end}{"\\n"}{end}'`, `section prometheus kubectl get prometheus -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} ${shellQuote(G14_PROMETHEUS_NAME)} -o json`, - `section prometheusPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l prometheus=${shellQuote(G14_PROMETHEUS_NAME)} -o json`, + `section prometheusPods kubectl get pods -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} -l prometheus=${shellQuote(G14_PROMETHEUS_NAME)} -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{range .status.containerStatuses[*]}{.ready}{","}{end}{"\\n"}{end}'`, `section prometheusService kubectl get service -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} ${shellQuote(G14_PROMETHEUS_SERVICE)} -o json`, `section workloadMonitors kubectl get servicemonitor,prometheusrule -n ${shellQuote(V02_RUNTIME_NAMESPACE)} -l hwlab.pikastech.local/monitoring=enabled -o json`, `section query kubectl get --raw ${shellQuote(queryPath)}`, @@ -4397,12 +4416,11 @@ function g14ObservabilityStatus(): Record { const sections = parseShellSections(statusText(bundle)); const namespace = parseSectionJson(sections.namespace); const discoveryNamespace = parseSectionJson(sections.discoveryNamespace); - const crdItems = parseSectionJsonArray(sections.crds); - const crdNames = crdItems.map((item) => String(record(item.metadata).name ?? "")).filter(Boolean); + const crdNames = sectionLines(sections.crds); const operator = parseSectionJson(sections.operator); - const operatorPods = parseSectionJsonArray(sections.operatorPods); + const operatorPods = podRows(sections.operatorPods); const prometheus = parseSectionJson(sections.prometheus); - const prometheusPods = parseSectionJsonArray(sections.prometheusPods); + const prometheusPods = podRows(sections.prometheusPods); const prometheusService = parseSectionJson(sections.prometheusService); const workloadMonitorItems = parseSectionJsonArray(sections.workloadMonitors); const query = parseSectionJson(sections.query); @@ -4413,10 +4431,7 @@ function g14ObservabilityStatus(): Record { const prometheusExists = Object.keys(prometheus).length > 0; const prometheusIsReady = prometheusExists && ( prometheusReady(prometheus) - || prometheusPods.some((pod) => { - const statuses = Array.isArray(record(pod.status).containerStatuses) ? record(pod.status).containerStatuses.map((item) => record(item)) : []; - return statuses.length > 0 && statuses.every((status) => status.ready === true); - }) + || prometheusPods.some((pod) => pod.ready === true) ); const queryOk = sections.query?.exitCode === 0 && query.status === "success"; return { @@ -4450,8 +4465,9 @@ function g14ObservabilityStatus(): Record { readyReplicas: numericValue(record(operator.status).readyReplicas) ?? 0, availableReplicas: numericValue(record(operator.status).availableReplicas) ?? 0, pods: operatorPods.map((pod) => ({ - name: stringOrNull(record(pod.metadata).name), - phase: stringOrNull(record(pod.status).phase), + name: stringOrNull(pod.name), + phase: stringOrNull(pod.phase), + ready: pod.ready ?? null, })), sectionOk: shellSectionOk(sections.operator), }, @@ -4463,11 +4479,9 @@ function g14ObservabilityStatus(): Record { ready: prometheusIsReady, conditions: Array.isArray(record(prometheus.status).conditions) ? record(prometheus.status).conditions : [], pods: prometheusPods.map((pod) => ({ - name: stringOrNull(record(pod.metadata).name), - phase: stringOrNull(record(pod.status).phase), - ready: Array.isArray(record(pod.status).containerStatuses) - ? record(pod.status).containerStatuses.map((item) => record(item)).every((status) => status.ready === true) - : null, + name: stringOrNull(pod.name), + phase: stringOrNull(pod.phase), + ready: pod.ready ?? null, })), sectionOk: shellSectionOk(sections.prometheus), }, @@ -4518,16 +4532,12 @@ function g14ObservabilityApplyScript(options: G14ObservabilityOptions, manifestB const preStackWaitCommands = options.dryRun ? "echo observability_wait=skipped_dry_run" : [ - "kubectl wait --for=condition=Established --timeout=120s crd/servicemonitors.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com crd/prometheuses.monitoring.coreos.com", - `kubectl -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} rollout status deploy/prometheus-operator --timeout=${options.timeoutSeconds}s`, + "kubectl wait --for=condition=Established --timeout=45s crd/servicemonitors.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com crd/prometheuses.monitoring.coreos.com", + `kubectl -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} get deploy/prometheus-operator -o name`, ].join("\n"); const postStackWaitCommands = options.dryRun ? "echo prometheus_wait=skipped_dry_run" - : [ - `kubectl -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} wait --for=condition=Available --timeout=${options.timeoutSeconds}s prometheus/${G14_PROMETHEUS_NAME} || true`, - `kubectl -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} get deploy,pod,svc,prometheus -l app.kubernetes.io/component=observability -o wide || true`, - `kubectl -n ${shellQuote(G14_OBSERVABILITY_NAMESPACE)} get pod -l prometheus=${shellQuote(G14_PROMETHEUS_NAME)} -o wide || true`, - ].join("\n"); + : "echo prometheus_wait=deferred_to_status_command"; return [ "set -eu", `namespace=${shellQuote(G14_OBSERVABILITY_NAMESPACE)}`,