diff --git a/config/platform-infra/observability.yaml b/config/platform-infra/observability.yaml index 3319fef1..e03b8fed 100644 --- a/config/platform-infra/observability.yaml +++ b/config/platform-infra/observability.yaml @@ -13,11 +13,11 @@ defaults: images: collector: - repository: otel/opentelemetry-collector-contrib + repository: docker.m.daocloud.io/otel/opentelemetry-collector-contrib tag: 0.130.1 pullPolicy: IfNotPresent tempo: - repository: grafana/tempo + repository: docker.m.daocloud.io/grafana/tempo tag: 2.8.1 pullPolicy: IfNotPresent diff --git a/scripts/src/ci.ts b/scripts/src/ci.ts index ecbbce31..6b5b8784 100644 --- a/scripts/src/ci.ts +++ b/scripts/src/ci.ts @@ -155,6 +155,13 @@ interface CiLogsOptions { capture: "dispatch-task" | "ssh-stream"; } +interface CiCleanupRunsOptions { + target: CiTarget; + minAgeMinutes: number; + limit: number; + confirm: boolean; +} + type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required" | "registry-not-installed" | "registry-unhealthy" | "remote-command-timeout" | "ssh-helper-command-shape-incompatible" | "ci-runner-not-ready"; type PublishPreflightControlChannel = "backend-core" | "database" | "provider" | "registry"; type PublishPreflightDetailedChannel = "backend-core-api" | "provider-dispatch" | "provider-host-ssh" | "database" | "artifact-registry"; @@ -275,6 +282,19 @@ function ciLogsOptions(args: string[]): CiLogsOptions { return { tailLines, capture: args.includes("--dispatch-task") ? "dispatch-task" : "ssh-stream" }; } +function ciCleanupRunsOptions(args: string[]): CiCleanupRunsOptions { + const limit = numberOption(args, "--limit", 50); + if (limit <= 0 || limit > 500) throw new Error("ci cleanup-runs --limit must be an integer between 1 and 500"); + const confirm = boolFlag(args, "--confirm"); + if (confirm && boolFlag(args, "--dry-run")) throw new Error("ci cleanup-runs accepts only one of --confirm or --dry-run"); + return { + target: ciTarget(providerIdOption(args)), + minAgeMinutes: numberOption(args, "--min-age-minutes", 60), + limit, + confirm, + }; +} + function countTextLines(value: string): number { if (value.length === 0) return 0; const breaks = value.match(/\r\n|\r|\n/gu)?.length ?? 0; @@ -956,7 +976,7 @@ async function runRemoteKubectl(script: string, waitMs = 60_000, remoteTimeoutMs async function runRemoteKubectlRaw(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise { const command = [ - "set -euo pipefail", + "set -eu", ...ciTargetGuardShellLines(target, { passOutput: "stderr" }), script, ].join("\n"); @@ -1177,6 +1197,118 @@ async function status(target = ciTarget(null)): Promise> }; } +interface CiCleanupPodCandidate { + name: string; + phase: "Succeeded" | "Failed"; + createdAt: string; + ageMinutes: number; + selected: boolean; + selectedReason: string; +} + +function cleanupRunsQueryScript(): string { + const jsonPath = "{range .items[*]}{.metadata.name}{\"\\t\"}{.status.phase}{\"\\t\"}{.metadata.creationTimestamp}{\"\\n\"}{end}"; + return [ + "set -eu", + `kubectl get pods -n unidesk-ci -o jsonpath=${shellQuote(jsonPath)} || true`, + ].join("\n"); +} + +function parseCleanupPodCandidates(stdout: string, generatedAtMs: number, minAgeMinutes: number, limit: number): CiCleanupPodCandidate[] { + const byName = new Map>(); + for (const line of stdout.split(/\r?\n/u)) { + const [name, phaseRaw, createdAt] = line.trim().split("\t"); + if (!name || !createdAt) continue; + if (phaseRaw !== "Succeeded" && phaseRaw !== "Failed") continue; + if (!/^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$/u.test(name)) continue; + const createdAtMs = Date.parse(createdAt); + if (!Number.isFinite(createdAtMs)) continue; + const ageMinutes = Math.max(0, Math.floor((generatedAtMs - createdAtMs) / 60_000)); + if (ageMinutes < minAgeMinutes) continue; + byName.set(name, { name, phase: phaseRaw, createdAt, ageMinutes }); + } + const sorted = Array.from(byName.values()).sort((left, right) => { + const byAge = right.ageMinutes - left.ageMinutes; + return byAge !== 0 ? byAge : left.name.localeCompare(right.name); + }); + const selectedNames = new Set(sorted.slice(0, limit).map((item) => item.name)); + return sorted.map((item) => ({ + ...item, + selected: selectedNames.has(item.name), + selectedReason: selectedNames.has(item.name) ? "terminal-pod-age-within-limit" : "over-limit", + })); +} + +function cleanupRunsDeleteScript(names: string[]): string { + const lines = ["set -eu"]; + for (let index = 0; index < names.length; index += 50) { + const chunk = names.slice(index, index + 50).map(shellQuote).join(" "); + lines.push(`kubectl -n unidesk-ci delete pod ${chunk} --ignore-not-found=true --wait=false`); + } + return lines.join("\n"); +} + +async function runCleanupCapture(config: UniDeskConfig, target: CiTarget, script: string) { + const command = [ + "set -eu", + ...ciTargetGuardShellLines(target, { passOutput: "stderr" }), + script, + ].join("\n"); + return runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], command); +} + +async function cleanupRuns(config: UniDeskConfig, options: CiCleanupRunsOptions): Promise> { + const generatedAt = new Date(); + const query = await runCleanupCapture(config, options.target, cleanupRunsQueryScript()); + const queryOk = "ok" in query ? query.ok : query.exitCode === 0; + if (!queryOk) { + return { + ok: false, + command: "ci cleanup-runs", + providerId: options.target.providerId, + namespace: "unidesk-ci", + mutation: false, + failureKind: "ci-cleanup-query-failed", + query: { + exitCode: query.exitCode, + stdoutTail: tailTextLines(query.stdout, 80), + stderrTail: tailTextLines(query.stderr, 80), + }, + }; + } + const candidates = parseCleanupPodCandidates(query.stdout, generatedAt.getTime(), options.minAgeMinutes, options.limit); + const selected = candidates.filter((item) => item.selected); + const deletion = options.confirm && selected.length > 0 + ? await runCleanupCapture(config, options.target, cleanupRunsDeleteScript(selected.map((item) => item.name))) + : null; + const deletionOk = deletion === null ? true : ("ok" in deletion ? deletion.ok : deletion.exitCode === 0); + const ok = deletionOk; + return { + ok, + command: "ci cleanup-runs", + providerId: options.target.providerId, + namespace: "unidesk-ci", + generatedAt: generatedAt.toISOString(), + mode: options.confirm ? "confirmed-cleanup" : "dry-run", + minAgeMinutes: options.minAgeMinutes, + limit: options.limit, + mutation: options.confirm, + candidateCount: candidates.length, + selectedPodCount: selected.length, + candidates: candidates.slice(0, Math.min(candidates.length, 120)), + truncated: candidates.length > 120, + deletedPodCount: deletionOk && deletion !== null ? selected.length : 0, + deletion: deletion === null ? null : { + exitCode: deletion.exitCode, + stdoutTail: tailTextLines(deletion.stdout, 120), + stderrTail: tailTextLines(deletion.stderr, 80), + }, + next: options.confirm + ? { status: `bun scripts/cli.ts ci status --provider-id ${options.target.providerId}` } + : { confirm: `bun scripts/cli.ts ci cleanup-runs --provider-id ${options.target.providerId} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` }, + }; +} + async function install(config: UniDeskConfig, options: CiInstallOptions): Promise> { if (!existsSync(rootPath(options.target.pipelineManifest))) { throw new Error("CI manifests are missing"); @@ -3048,7 +3180,7 @@ function catalogArtifactDescriptor(artifact: CiCatalogArtifact): Record { const catalog = loadCiCatalog(); return { - command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs", + command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs|cleanup-runs", description: "Manage native k3s Tekton CI on D601 or G14. CI may publish commit-pinned image artifacts, but it intentionally does not deploy CD.", examples: [ "bun scripts/cli.ts ci install", @@ -3068,6 +3200,8 @@ export function ciHelp(): Record { "bun scripts/cli.ts ci publish-user-service --service frontend --commit ", "bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000", "bun scripts/cli.ts ci logs [--provider-id G14] [--tail-lines 80]", + "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --dry-run", + "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --confirm", ], tekton: { pipelineVersion: tektonPipelineVersion, @@ -3131,6 +3265,12 @@ export function ciHelp(): Record { tailLines: "1..2000", reason: "stream capture avoids backend-core task-result compactJson truncating CI log text before the CLI can extract failure hints", }, + cleanupRuns: { + defaultMode: "dry-run", + namespace: "unidesk-ci", + selector: "status.phase in Succeeded,Failed", + guard: "--confirm required for deletion; Running/Pending pods are never selected", + }, }; } @@ -3238,7 +3378,8 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi }); } if (action === "logs") return logs(config, nameArg ?? "", ciTarget(providerIdOption(args)), ciLogsOptions(args)); - throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs"); + if (action === "cleanup-runs") return cleanupRuns(config, ciCleanupRunsOptions(args)); + throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs, cleanup-runs"); } export function startCiInstallJob(providerId = d601ProviderId): Record { diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index e95e235e..7d4d14a1 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -4,7 +4,6 @@ import { Buffer } from "node:buffer"; import { readFileSync } from "node:fs"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; -import { startJob } from "./jobs"; import { compactCapture, compactUnknown, @@ -212,7 +211,7 @@ function parseTraceOptions(args: string[]): TraceOptions { if (arg === "--trace-id") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--trace-id requires a value"); - if (!/^[A-Za-z0-9._:-]+$/u.test(value)) throw new Error("--trace-id has an unsupported format"); + if (!/^[0-9a-fA-F]{32}$/u.test(value)) throw new Error("--trace-id must be a 32-character OpenTelemetry trace id encoded as hex"); traceId = value; index += 1; } else { @@ -409,27 +408,6 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise check.ok)) return { ok: false, action: "platform-infra-observability-apply", mode: "policy-blocked", policy }; - if (options.confirm && !options.wait) { - const job = startJob( - `platform_infra_observability_apply_${target.id.toLowerCase()}`, - ["bun", "scripts/cli.ts", "platform-infra", "observability", "apply", "--target", target.id, "--confirm", "--wait"], - `Apply ${target.id} platform-infra OTel Collector and trace backend through the controlled UniDesk CLI`, - ); - return { - ok: true, - action: "platform-infra-observability-apply", - mode: "async-job", - mutation: true, - target: targetSummary(target), - job, - statusCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, - next: { - status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, - rollout: `bun scripts/cli.ts platform-infra observability status --target ${target.id}`, - validate: `bun scripts/cli.ts platform-infra observability validate --target ${target.id}`, - }, - }; - } const result = await capture(config, target.route, ["sh"], applyScript({ yaml, target, @@ -474,20 +452,27 @@ async function status(config: UniDeskConfig, options: CommonOptions): Promise> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); - const result = await capture(config, target.route, ["sh"], statusScript(observability, target, options.full)); + const result = await capture(config, target.route, ["sh"], statusScript(observability, target, options.full, true)); const parsed = parseJsonOutput(result.stdout); const summary = parsed === null ? null : statusSummary(parsed); const ready = summary?.ready === true; + const validationTrace = parsed !== null && typeof parsed.validationTrace === "object" && parsed.validationTrace !== null && !Array.isArray(parsed.validationTrace) + ? parsed.validationTrace as Record + : null; + const validationTraceId = typeof validationTrace?.traceId === "string" ? validationTrace.traceId : null; + const validationTraceOk = validationTrace?.ok === true; return { - ok: result.exitCode === 0 && ready, + ok: result.exitCode === 0 && ready && validationTraceOk, action: "platform-infra-observability-validate", mutation: false, target: targetSummary(target), summary, validation: { readiness: ready ? "passed" : "failed", - testTrace: "not-generated-by-this-stage", - traceQuery: ready ? `bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id ` : "blocked-until-runtime-ready", + testTrace: validationTrace ?? "not-generated-runtime-not-ready", + traceQuery: validationTraceId !== null + ? `bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id ${validationTraceId}` + : "blocked-until-validation-trace-generated", metricsBoundary: "Prometheus/RUM remains outside this trace readiness check.", }, remote: options.raw ? parsed : compactStatus(parsed, options.full) ?? compactCapture(result, { full: true }), @@ -621,6 +606,11 @@ metadata: app.kubernetes.io/managed-by: unidesk spec: replicas: ${observability.collector.replicas} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: ${observability.collector.deploymentName} @@ -748,6 +738,11 @@ metadata: app.kubernetes.io/managed-by: unidesk spec: replicas: ${observability.traceBackend.replicas} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: ${observability.traceBackend.deploymentName} @@ -833,15 +828,7 @@ function applyScript(params: { }): string { const encoded = Buffer.from(params.yaml, "utf8").toString("base64"); const dryRunArg = params.dryRun ? "--dry-run=server" : ""; - const wait = params.dryRun || !params.wait - ? "wait_disposition=skipped" - : [ - `kubectl -n ${shQuote(params.target.namespace)} rollout status deployment/${shQuote(params.collectorDeploymentName)} --timeout=180s >"$tmp/collector-rollout.out" 2>"$tmp/collector-rollout.err"`, - "collector_rollout_rc=$?", - `kubectl -n ${shQuote(params.target.namespace)} rollout status deployment/${shQuote(params.backendDeploymentName)} --timeout=180s >"$tmp/backend-rollout.out" 2>"$tmp/backend-rollout.err"`, - "backend_rollout_rc=$?", - "wait_disposition=executed", - ].join("\n"); + const waitDisposition = params.wait ? "skipped-short-connection-use-status" : "skipped"; return ` set -u tmp="$(mktemp -d)" @@ -852,7 +839,7 @@ kubectl apply --server-side --field-manager=${fieldManager} ${dryRunArg} -f "$ma apply_rc=$? collector_rollout_rc=0 backend_rollout_rc=0 -${wait} +wait_disposition=${waitDisposition} python3 - "$apply_rc" "$collector_rollout_rc" "$backend_rollout_rc" "$wait_disposition" "$tmp/apply.out" "$tmp/apply.err" "$tmp/collector-rollout.out" "$tmp/collector-rollout.err" "$tmp/backend-rollout.out" "$tmp/backend-rollout.err" <<'PY' import json, os, sys def text(path, limit=6000): @@ -881,7 +868,7 @@ PY `; } -function statusScript(observability: ObservabilityConfig, target: ObservabilityTarget, full: boolean): string { +function statusScript(observability: ObservabilityConfig, target: ObservabilityTarget, full: boolean, generateValidationTrace = false): string { const endpointsJson = JSON.stringify(observability.probes.statusEndpoints); return ` set -u @@ -903,8 +890,9 @@ capture_json namespace kubectl get namespace ${shQuote(target.namespace)} -o jso capture_json deployments kubectl -n ${shQuote(target.namespace)} get deployment ${shQuote(observability.collector.deploymentName)} ${shQuote(observability.traceBackend.deploymentName)} -o json capture_json services kubectl -n ${shQuote(target.namespace)} get service ${shQuote(observability.collector.serviceName)} ${shQuote(observability.traceBackend.serviceName)} -o json capture_json pods kubectl -n ${shQuote(target.namespace)} get pods -l ${shQuote(`app.kubernetes.io/name in (${observability.collector.deploymentName},${observability.traceBackend.deploymentName})`)} -o json +capture_json events kubectl -n ${shQuote(target.namespace)} get events -o json python3 - "$tmp" '${endpointsJson.replaceAll("'", "'\"'\"'")}' <<'PY' -import json, subprocess, sys +import json, random, socket, subprocess, sys, time, urllib.error, urllib.request tmp = sys.argv[1] endpoints = json.loads(sys.argv[2]) namespace = "${target.namespace}" @@ -952,8 +940,141 @@ for ep in endpoints: "stderrTail": proc.stderr[-1000:], "ok": proc.returncode == 0, }) +def free_port(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return port +def parse_body(raw): + try: + return json.loads(raw) if raw else None + except Exception: + return raw[-2000:] +def http_request(method, url, body=None, timeout=10): + data = None + headers = {} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read(200000).decode("utf-8", errors="replace") + status = int(getattr(resp, "status", 0) or 0) + return {"ok": 200 <= status < 300, "status": status, "body": parse_body(raw)} + except urllib.error.HTTPError as exc: + raw = exc.read(200000).decode("utf-8", errors="replace") + return {"ok": False, "status": exc.code, "body": parse_body(raw)} + except Exception as exc: + return {"ok": False, "error": type(exc).__name__ + ": " + str(exc)} +def wait_proxy(base_url): + last = None + for _ in range(30): + last = http_request("GET", base_url + "/version", timeout=1) + if last.get("ok") is True: + return last + time.sleep(0.2) + return last or {"ok": False, "error": "kubectl proxy did not answer"} +def compact_http(result): + if not isinstance(result, dict): + return {"ok": False, "error": "missing-result"} + compact = {"ok": result.get("ok") is True} + if "status" in result: + compact["status"] = result.get("status") + if "error" in result: + compact["error"] = result.get("error") + body = result.get("body") + if isinstance(body, dict): + compact["bodyKeys"] = sorted(list(body.keys()))[:12] + elif isinstance(body, str) and body: + compact["bodyTail"] = body[-1000:] + return compact +def validation_trace_payload(trace_id, span_id): + now = time.time_ns() + return { + "resourceSpans": [{ + "resource": {"attributes": [ + {"key": "service.name", "value": {"stringValue": "unidesk-observability-validate"}}, + {"key": "deployment.environment", "value": {"stringValue": "${target.id}"}}, + {"key": "unidesk.node", "value": {"stringValue": "${target.id}"}}, + {"key": "k8s.namespace.name", "value": {"stringValue": namespace}}, + ]}, + "scopeSpans": [{ + "scope": {"name": "unidesk-cli-platform-infra-observability"}, + "spans": [{ + "traceId": trace_id, + "spanId": span_id, + "name": "unidesk.observability.validate", + "kind": 1, + "startTimeUnixNano": str(now), + "endTimeUnixNano": str(now + 1000000), + "attributes": [ + {"key": "unidesk.issue", "value": {"stringValue": "489"}}, + {"key": "unidesk.spec", "value": {"stringValue": "${observability.metadata.spec}"}}, + {"key": "unidesk.validation", "value": {"stringValue": "platform-infra-observability"}}, + ], + "status": {"code": 1}, + }], + }], + }], + } +def generate_test_trace(): + trace_id = ("%032x" % random.getrandbits(128)) + span_id = ("%016x" % random.getrandbits(64)) + port = free_port() + base_url = "http://127.0.0.1:%s" % port + stdout_path = f"{tmp}/kubectl-proxy.out" + stderr_path = f"{tmp}/kubectl-proxy.err" + with open(stdout_path, "w", encoding="utf-8") as stdout, open(stderr_path, "w", encoding="utf-8") as stderr: + proc = subprocess.Popen( + ["kubectl", "proxy", "--address=127.0.0.1", "--port", str(port), "--accept-hosts=^127\\\\.0\\\\.0\\\\.1$"], + stdout=stdout, + stderr=stderr, + text=True, + ) + try: + proxy_ready = wait_proxy(base_url) + if proxy_ready.get("ok") is not True: + return { + "ok": False, + "stage": "proxy-start", + "traceId": trace_id, + "proxy": compact_http(proxy_ready), + "proxyStderrTail": read(stderr_path, limit=2000), + } + collector_url = base_url + "/api/v1/namespaces/%s/services/http:${observability.collector.serviceName}:otlp-http/proxy/v1/traces" % namespace + tempo_url = base_url + "/api/v1/namespaces/%s/services/http:${observability.traceBackend.serviceName}:http/proxy/api/traces/%s" % (namespace, trace_id) + ingest = http_request("POST", collector_url, validation_trace_payload(trace_id, span_id), timeout=15) + query = None + if ingest.get("ok") is True: + for _ in range(20): + query = http_request("GET", tempo_url, timeout=10) + if query.get("ok") is True: + break + time.sleep(1) + return { + "ok": ingest.get("ok") is True and isinstance(query, dict) and query.get("ok") is True, + "stage": "query" if ingest.get("ok") is True else "ingest", + "traceId": trace_id, + "spanId": span_id, + "collectorEndpoint": "${observability.collector.serviceName}:otlp-http", + "backendEndpoint": "${observability.traceBackend.serviceName}:http", + "ingest": compact_http(ingest), + "query": compact_http(query), + "valuesPrinted": False, + } + finally: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=3) +runtime_ready = rc("namespace") == 0 and rc("deployments") == 0 and rc("services") == 0 and all(item["ok"] for item in probe_results) +validation_trace = generate_test_trace() if (${generateValidationTrace ? "True" : "False"} and runtime_ready) else None payload = { - "ok": rc("namespace") == 0 and rc("deployments") == 0 and rc("services") == 0 and all(item["ok"] for item in probe_results), + "ok": runtime_ready and (validation_trace is None or validation_trace.get("ok") is True), "target": "${target.id}", "namespace": namespace, "sections": { @@ -961,8 +1082,10 @@ payload = { "deployments": compact_section("deployments"), "services": compact_section("services"), "pods": compact_section("pods"), + "events": compact_section("events"), }, "probes": probe_results, + "validationTrace": validation_trace, } print(json.dumps(payload, ensure_ascii=False, indent=2 if ${full ? "True" : "False"} else None)) sys.exit(0 if payload["ok"] else 1) @@ -1043,6 +1166,8 @@ function statusSummary(payload: Record): Record> : []; const readyDeployments = deployments.map((item) => deploymentSummary(item)); return { @@ -1050,7 +1175,7 @@ function statusSummary(payload: Record): Record metadataName(item)), - pods: pods.map((item) => podSummary(item)), + pods: pods.map((item) => podSummary(item, podEvents)), probes: probes.map((item) => ({ name: item.name, ok: item.ok === true, @@ -1092,14 +1217,88 @@ function deploymentSummary(item: Record): Record): Record { +function podSummary(item: Record, podEvents: Map>): Record { const status = item.status as Record | undefined; + const waiting = firstWaitingContainer(status); + const scheduled = conditionSummary(status, "PodScheduled"); + const ready = conditionSummary(status, "Ready"); + const name = metadataName(item); + const showEvent = status?.phase !== "Running" || waiting !== null || scheduled !== null || ready !== null; + const latestEvent = showEvent && name !== null ? podEvents.get(name) ?? null : null; return { - name: metadataName(item), + name, phase: status?.phase ?? null, + reason: waiting?.reason ?? scheduled?.reason ?? ready?.reason ?? null, + message: waiting?.message ?? scheduled?.message ?? ready?.message ?? null, + latestEvent: latestEvent === null ? null : { + reason: stringValue(latestEvent.reason), + message: stringValue(latestEvent.message), + type: stringValue(latestEvent.type), + count: typeof latestEvent.count === "number" ? latestEvent.count : null, + timestamp: eventTimestamp(latestEvent), + }, }; } +function latestPodEvents(events: Record[]): Map> { + const byPod = new Map>(); + for (const item of events) { + const involved = item.involvedObject; + if (typeof involved !== "object" || involved === null || Array.isArray(involved)) continue; + const podName = (involved as Record).name; + const kind = (involved as Record).kind; + if (kind !== "Pod" || typeof podName !== "string") continue; + const current = byPod.get(podName); + if (current === undefined || eventTimestamp(item) > eventTimestamp(current)) byPod.set(podName, item); + } + return byPod; +} + +function eventTimestamp(item: Record): string | null { + const candidate = stringValue(item.eventTime) + ?? stringValue(item.lastTimestamp) + ?? stringValue(item.deprecatedLastTimestamp) + ?? (typeof item.metadata === "object" && item.metadata !== null && !Array.isArray(item.metadata) + ? stringValue((item.metadata as Record).creationTimestamp) + : null); + return candidate; +} + +function stringValue(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function firstWaitingContainer(status: Record | undefined): { reason: string | null; message: string | null } | null { + const statuses = Array.isArray(status?.containerStatuses) ? status.containerStatuses : []; + for (const item of statuses) { + if (typeof item !== "object" || item === null || Array.isArray(item)) continue; + const state = (item as Record).state; + if (typeof state !== "object" || state === null || Array.isArray(state)) continue; + const waiting = (state as Record).waiting; + if (typeof waiting !== "object" || waiting === null || Array.isArray(waiting)) continue; + const record = waiting as Record; + return { + reason: typeof record.reason === "string" ? record.reason : null, + message: typeof record.message === "string" ? record.message : null, + }; + } + return null; +} + +function conditionSummary(status: Record | undefined, type: string): { reason: string | null; message: string | null } | null { + const conditions = Array.isArray(status?.conditions) ? status.conditions : []; + for (const item of conditions) { + if (typeof item !== "object" || item === null || Array.isArray(item)) continue; + const record = item as Record; + if (record.type !== type || record.status === "True") continue; + return { + reason: typeof record.reason === "string" ? record.reason : null, + message: typeof record.message === "string" ? record.message : null, + }; + } + return null; +} + function metadataName(item: Record): string | null { const metadata = item.metadata as Record | undefined; return typeof metadata?.name === "string" ? metadata.name : null;