// SPEC: PJ2026-01060501 OTel追踪 draft-2026-06-19-p0.
// Responsibility: YAML-first platform-infra OpenTelemetry tracing control commands.
import { Buffer } from "node:buffer";
import { readFileSync } from "node:fs";
import type { UniDeskConfig } from "./config";
import { rootPath } from "./config";
import type { RenderedCliResult } from "./output";
import {
  compactCapture,
  createYamlFieldReader,
  numberField,
  parseJsonOutput,
  redactSensitiveUnknown,
  shQuote,
  capture,
} from "./platform-infra-ops-library";

const configFile = rootPath("config", "platform-infra", "observability.yaml");
const configLabel = "config/platform-infra/observability.yaml";
const fieldManager = "unidesk-platform-observability";
const {
  asRecord,
  objectField,
  arrayOfRecords,
  stringField,
  integerField,
  booleanField,
  stringArrayField,
  numberArrayField,
  enumField,
  kubernetesNameField,
  portField,
  apiPathField,
} = createYamlFieldReader(configLabel);

interface ObservabilityConfig {
  version: number;
  kind: "platform-infra-observability";
  metadata: { id: string; owner: string; spec: string; relatedIssues: number[] };
  defaults: { targetId: string };
  images: {
    collector: ImageSpec;
    tempo: ImageSpec;
  };
  targets: ObservabilityTarget[];
  collector: {
    deploymentName: string;
    serviceName: string;
    configMapName: string;
    replicas: number;
    healthPort: number;
    otlp: OtlpPorts;
  };
  traceBackend: {
    type: "tempo";
    deploymentName: string;
    serviceName: string;
    configMapName: string;
    replicas: number;
    httpPort: number;
    otlp: OtlpPorts;
    storage: { mode: "emptyDir"; retention: string };
  };
  sampling: { mode: "parentbased_traceidratio"; ratio: number };
  instrumentation: {
    contextPropagation: string[];
    serviceConnections: ServiceConnection[];
  };
  resourceAttributes: {
    required: string[];
    businessCorrelationAttributes: string[];
  };
  probes: {
    readinessPath: string;
    traceQueryPathTemplate: string;
    statusEndpoints: StatusEndpoint[];
  };
}

interface ImageSpec {
  repository: string;
  tag: string;
  pullPolicy: "Always" | "IfNotPresent" | "Never";
}

interface ObservabilityTarget {
  id: string;
  route: string;
  namespace: string;
  role: "active" | "standby";
  enabled: boolean;
  createNamespace: boolean;
}

interface OtlpPorts {
  grpcPort: number;
  httpPort: number;
}

interface ServiceConnection {
  serviceName: string;
  owningRepo: string;
  targetNode: string;
  lane: string;
  namespace: string;
  requiredSpans: string[];
}

interface StatusEndpoint {
  name: string;
  service: string;
  portName: string;
  path: string;
}

interface CommonOptions {
  targetId: string | null;
  full: boolean;
  raw: boolean;
}

interface ApplyOptions extends CommonOptions {
  confirm: boolean;
  dryRun: boolean;
  wait: boolean;
}

interface TraceOptions extends CommonOptions {
  traceId: string | null;
  grep: string | null;
  limit: number;
}

interface SearchOptions extends CommonOptions {
  grep: string | null;
  query: string | null;
  path: string | null;
  status: number | null;
  limit: number;
  candidateLimit: number;
  lookbackMinutes: number;
  endAt: string | null;
}

interface DiagnoseCodeAgentOptions extends CommonOptions {
  businessTraceId: string | null;
  traceId: string | null;
  limit: number;
  candidateLimit: number;
  lookbackMinutes: number;
}

export function observabilityHelp(): Record<string, unknown> {
  return {
    command: "platform-infra observability plan|apply|status|validate|trace|search|diagnose-code-agent",
    output: "text/table by default for search and diagnose-code-agent; use --full or --raw for structured JSON",
    configTruth: "config/platform-infra/observability.yaml",
    spec: "PJ2026-01060501 OTel追踪 draft-2026-06-19-p0",
    usage: [
      "bun scripts/cli.ts platform-infra observability plan --target D601",
      "bun scripts/cli.ts platform-infra observability apply --target D601 --dry-run",
      "bun scripts/cli.ts platform-infra observability apply --target D601 --confirm",
      "bun scripts/cli.ts platform-infra observability status --target D601 [--full|--raw]",
      "bun scripts/cli.ts platform-infra observability validate --target D601 [--full|--raw]",
      "bun scripts/cli.ts platform-infra observability trace --target D601 --trace-id <traceId> [--grep provider-stream-disconnected] [--limit 40] [--full|--raw]",
      "bun scripts/cli.ts platform-infra observability search --target D601 --grep 'no rollout found' [--lookback-minutes 360] [--candidate-limit 80] [--limit 20] [--full|--raw]",
      "bun scripts/cli.ts platform-infra observability search --target D601 --path /v1/workbench/sessions --status 502 [--lookback-minutes 120] [--full|--raw]",
      "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --business-trace-id <trc_...> [--full|--raw]",
    ],
    boundary: "Prometheus remains the metrics source; this command owns only platform-infra OTel Collector, trace backend readiness, and trace lookup.",
  };
}

export async function runPlatformObservabilityCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
  const [action = "plan"] = args;
  if (action === "plan") return plan(parseCommonOptions(args.slice(1)));
  if (action === "apply") return await apply(config, parseApplyOptions(args.slice(1)));
  if (action === "status") return await status(config, parseCommonOptions(args.slice(1)));
  if (action === "validate") return await validate(config, parseCommonOptions(args.slice(1)));
  if (action === "trace") return await trace(config, parseTraceOptions(args.slice(1)));
  if (action === "search") return await search(config, parseSearchOptions(args.slice(1)));
  if (action === "diagnose-code-agent") return await diagnoseCodeAgent(config, parseDiagnoseCodeAgentOptions(args.slice(1)));
  return { ok: false, error: "unsupported-platform-infra-observability-command", args, help: observabilityHelp() };
}

function parseCommonOptions(args: string[]): CommonOptions {
  let targetId: string | null = null;
  let full = false;
  let raw = false;
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (arg === "--target") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
      if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error("--target must be a simple target id");
      targetId = value;
      index += 1;
    } else if (arg === "--full") {
      full = true;
    } else if (arg === "--raw") {
      raw = true;
      full = true;
    } else {
      throw new Error(`unsupported observability option: ${arg}`);
    }
  }
  return { targetId, full, raw };
}

function parseApplyOptions(args: string[]): ApplyOptions {
  const commonArgs: string[] = [];
  let confirm = false;
  let dryRun = false;
  let wait = false;
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (arg === "--confirm") confirm = true;
    else if (arg === "--dry-run") dryRun = true;
    else if (arg === "--wait") wait = true;
    else {
      commonArgs.push(arg);
      if (arg === "--target") {
        commonArgs.push(args[index + 1] ?? "");
        index += 1;
      }
    }
  }
  if (confirm && dryRun) throw new Error("observability apply accepts only one of --confirm or --dry-run");
  return { ...parseCommonOptions(commonArgs), confirm, dryRun: dryRun || !confirm, wait };
}

function parseTraceOptions(args: string[]): TraceOptions {
  const commonArgs: string[] = [];
  let traceId: string | null = null;
  let grep: string | null = null;
  let limit = 40;
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (arg === "--trace-id") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--trace-id requires a value");
      if (!/^[0-9a-fA-F]{32}$/u.test(value)) throw new Error("--trace-id must be a 32-character OpenTelemetry trace id encoded as hex");
      traceId = value;
      index += 1;
    } else if (arg === "--grep") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--grep requires a value");
      grep = value;
      index += 1;
    } else if (arg === "--limit") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--limit requires a value");
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 500) throw new Error("--limit must be an integer from 1 to 500");
      limit = parsed;
      index += 1;
    } else {
      commonArgs.push(arg);
      if (arg === "--target") {
        commonArgs.push(args[index + 1] ?? "");
        index += 1;
      }
    }
  }
  return { ...parseCommonOptions(commonArgs), traceId, grep, limit };
}

function parseSearchOptions(args: string[]): SearchOptions {
  const commonArgs: string[] = [];
  let grep: string | null = null;
  let query: string | null = null;
  let path: string | null = null;
  let status: number | null = null;
  let limit = 20;
  let candidateLimit = 80;
  let lookbackMinutes = 360;
  let endAt: string | null = null;
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (arg === "--grep") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--grep requires a value");
      grep = value;
      index += 1;
    } else if (arg === "--query" || arg === "--q") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
      query = value;
      index += 1;
    } else if (arg === "--path") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--path requires a value");
      if (!value.startsWith("/") || value.includes("\n") || value.length > 300) throw new Error("--path must be an absolute HTTP path up to 300 characters");
      path = value;
      index += 1;
    } else if (arg === "--status") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--status requires a value");
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 100 || parsed > 599) throw new Error("--status must be an integer HTTP status from 100 to 599");
      status = parsed;
      index += 1;
    } else if (arg === "--limit") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--limit requires a value");
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 100) throw new Error("--limit must be an integer from 1 to 100");
      limit = parsed;
      index += 1;
    } else if (arg === "--candidate-limit") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--candidate-limit requires a value");
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 500) throw new Error("--candidate-limit must be an integer from 1 to 500");
      candidateLimit = parsed;
      index += 1;
    } else if (arg === "--lookback-minutes" || arg === "--since-minutes") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 10080) throw new Error(`${arg} must be an integer from 1 to 10080`);
      lookbackMinutes = parsed;
      index += 1;
    } else if (arg === "--end-at") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--end-at requires an ISO timestamp value");
      const parsed = Date.parse(value);
      if (!Number.isFinite(parsed)) throw new Error("--end-at must be an ISO timestamp parseable by Date.parse");
      endAt = new Date(parsed).toISOString();
      index += 1;
    } else {
      commonArgs.push(arg);
      if (arg === "--target") {
        commonArgs.push(args[index + 1] ?? "");
        index += 1;
      }
    }
  }
  if (grep === null && query === null && path === null && status === null) throw new Error("observability search requires --grep <text>, --query <tempo-query>, --path <route>, or --status <code>");
  return { ...parseCommonOptions(commonArgs), grep, query, path, status, limit, candidateLimit, lookbackMinutes, endAt };
}

function parseDiagnoseCodeAgentOptions(args: string[]): DiagnoseCodeAgentOptions {
  const commonArgs: string[] = [];
  let businessTraceId: string | null = null;
  let traceId: string | null = null;
  let limit = 40;
  let candidateLimit = 300;
  let lookbackMinutes = 10080;
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (arg === "--business-trace-id" || arg === "--trace") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
      if (!/^trc_[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${arg} must look like trc_<id>`);
      businessTraceId = value;
      index += 1;
    } else if (arg === "--trace-id" || arg === "--otel-trace-id") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
      if (!/^[0-9a-fA-F]{32}$/u.test(value)) throw new Error(`${arg} must be a 32-character OpenTelemetry trace id encoded as hex`);
      traceId = value.toLowerCase();
      index += 1;
    } else if (arg === "--limit") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--limit requires a value");
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 200) throw new Error("--limit must be an integer from 1 to 200");
      limit = parsed;
      index += 1;
    } else if (arg === "--candidate-limit") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--candidate-limit requires a value");
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 500) throw new Error("--candidate-limit must be an integer from 1 to 500");
      candidateLimit = parsed;
      index += 1;
    } else if (arg === "--lookback-minutes" || arg === "--since-minutes") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
      const parsed = Number(value);
      if (!Number.isInteger(parsed) || parsed < 1 || parsed > 10080) throw new Error(`${arg} must be an integer from 1 to 10080`);
      lookbackMinutes = parsed;
      index += 1;
    } else {
      commonArgs.push(arg);
      if (arg === "--target") {
        commonArgs.push(args[index + 1] ?? "");
        index += 1;
      }
    }
  }
  if (businessTraceId === null && traceId === null) throw new Error("observability diagnose-code-agent requires --business-trace-id <trc_...> or --trace-id <otelTraceId>");
  return { ...parseCommonOptions(commonArgs), businessTraceId, traceId, limit, candidateLimit, lookbackMinutes };
}

function readObservabilityConfig(): ObservabilityConfig {
  const parsed = Bun.YAML.parse(readFileSync(configFile, "utf8")) as unknown;
  const root = asRecord(parsed, configLabel);
  const version = integerField(root, "version", "");
  const kind = stringField(root, "kind", "");
  if (kind !== "platform-infra-observability") throw new Error(`${configLabel}.kind must be platform-infra-observability`);
  const metadata = objectField(root, "metadata", "");
  const defaults = objectField(root, "defaults", "");
  const images = objectField(root, "images", "");
  const collector = objectField(root, "collector", "");
  const collectorOtlp = objectField(collector, "otlp", "collector");
  const traceBackend = objectField(root, "traceBackend", "");
  const traceBackendOtlp = objectField(traceBackend, "otlp", "traceBackend");
  const traceBackendStorage = objectField(traceBackend, "storage", "traceBackend");
  const sampling = objectField(root, "sampling", "");
  const instrumentation = objectField(root, "instrumentation", "");
  const resourceAttributes = objectField(root, "resourceAttributes", "");
  const probes = objectField(root, "probes", "");
  const config: ObservabilityConfig = {
    version,
    kind,
    metadata: {
      id: stringField(metadata, "id", "metadata"),
      owner: stringField(metadata, "owner", "metadata"),
      spec: stringField(metadata, "spec", "metadata"),
      relatedIssues: numberArrayField(metadata, "relatedIssues", "metadata"),
    },
    defaults: { targetId: stringField(defaults, "targetId", "defaults") },
    images: {
      collector: imageSpec(objectField(images, "collector", "images"), "images.collector"),
      tempo: imageSpec(objectField(images, "tempo", "images"), "images.tempo"),
    },
    targets: arrayOfRecords(root.targets, "targets").map(parseTarget),
    collector: {
      deploymentName: kubernetesNameField(collector, "deploymentName", "collector"),
      serviceName: kubernetesNameField(collector, "serviceName", "collector"),
      configMapName: kubernetesNameField(collector, "configMapName", "collector"),
      replicas: integerField(collector, "replicas", "collector"),
      healthPort: portField(collector, "healthPort", "collector"),
      otlp: parseOtlpPorts(collectorOtlp, "collector.otlp"),
    },
    traceBackend: {
      type: enumField(traceBackend, "type", "traceBackend", ["tempo"] as const),
      deploymentName: kubernetesNameField(traceBackend, "deploymentName", "traceBackend"),
      serviceName: kubernetesNameField(traceBackend, "serviceName", "traceBackend"),
      configMapName: kubernetesNameField(traceBackend, "configMapName", "traceBackend"),
      replicas: integerField(traceBackend, "replicas", "traceBackend"),
      httpPort: portField(traceBackend, "httpPort", "traceBackend"),
      otlp: parseOtlpPorts(traceBackendOtlp, "traceBackend.otlp"),
      storage: {
        mode: enumField(traceBackendStorage, "mode", "traceBackend.storage", ["emptyDir"] as const),
        retention: stringField(traceBackendStorage, "retention", "traceBackend.storage"),
      },
    },
    sampling: {
      mode: enumField(sampling, "mode", "sampling", ["parentbased_traceidratio"] as const),
      ratio: numberField(sampling, "ratio", "sampling"),
    },
    instrumentation: {
      contextPropagation: stringArrayField(instrumentation, "contextPropagation", "instrumentation"),
      serviceConnections: arrayOfRecords(instrumentation.serviceConnections, "instrumentation.serviceConnections").map(parseServiceConnection),
    },
    resourceAttributes: {
      required: stringArrayField(resourceAttributes, "required", "resourceAttributes"),
      businessCorrelationAttributes: stringArrayField(resourceAttributes, "businessCorrelationAttributes", "resourceAttributes"),
    },
    probes: {
      readinessPath: apiPathField(probes, "readinessPath", "probes"),
      traceQueryPathTemplate: stringField(probes, "traceQueryPathTemplate", "probes"),
      statusEndpoints: arrayOfRecords(probes.statusEndpoints, "probes.statusEndpoints").map(parseStatusEndpoint),
    },
  };
  if (config.targets.length === 0) throw new Error(`${configLabel}.targets must not be empty`);
  assertKnownEnabledTarget(config.targets, config.defaults.targetId, "defaults.targetId");
  if (config.collector.replicas < 0 || config.traceBackend.replicas < 0) throw new Error(`${configLabel} replicas must be >= 0`);
  if (config.sampling.ratio < 0 || config.sampling.ratio > 1) throw new Error(`${configLabel}.sampling.ratio must be between 0 and 1`);
  if (!config.probes.traceQueryPathTemplate.includes("{{traceId}}")) throw new Error(`${configLabel}.probes.traceQueryPathTemplate must include {{traceId}}`);
  return config;
}

function imageSpec(record: Record<string, unknown>, path: string): ImageSpec {
  const image = {
    repository: stringField(record, "repository", path),
    tag: stringField(record, "tag", path),
    pullPolicy: enumField(record, "pullPolicy", path, ["Always", "IfNotPresent", "Never"] as const),
  };
  if (!/^[A-Za-z0-9._/:@-]+$/u.test(`${image.repository}:${image.tag}`)) throw new Error(`${configLabel}.${path} must render a valid image reference`);
  return image;
}

function parseTarget(record: Record<string, unknown>, index: number): ObservabilityTarget {
  const path = `targets[${index}]`;
  return {
    id: stringField(record, "id", path),
    route: stringField(record, "route", path),
    namespace: kubernetesNameField(record, "namespace", path),
    role: enumField(record, "role", path, ["active", "standby"] as const),
    enabled: booleanField(record, "enabled", path),
    createNamespace: booleanField(record, "createNamespace", path),
  };
}

function parseOtlpPorts(record: Record<string, unknown>, path: string): OtlpPorts {
  return {
    grpcPort: portField(record, "grpcPort", path),
    httpPort: portField(record, "httpPort", path),
  };
}

function parseServiceConnection(record: Record<string, unknown>, index: number): ServiceConnection {
  const path = `instrumentation.serviceConnections[${index}]`;
  return {
    serviceName: stringField(record, "serviceName", path),
    owningRepo: stringField(record, "owningRepo", path),
    targetNode: stringField(record, "targetNode", path),
    lane: stringField(record, "lane", path),
    namespace: kubernetesNameField(record, "namespace", path),
    requiredSpans: stringArrayField(record, "requiredSpans", path),
  };
}

function parseStatusEndpoint(record: Record<string, unknown>, index: number): StatusEndpoint {
  const path = `probes.statusEndpoints[${index}]`;
  return {
    name: stringField(record, "name", path),
    service: kubernetesNameField(record, "service", path),
    portName: stringField(record, "portName", path),
    path: apiPathField(record, "path", path),
  };
}

function assertKnownEnabledTarget(targets: ObservabilityTarget[], targetId: string, path: string): void {
  const target = targets.find((item) => item.id.toLowerCase() === targetId.toLowerCase());
  if (target === undefined) throw new Error(`${configLabel}.${path} references unknown target ${targetId}; known targets: ${targets.map((item) => item.id).join(", ")}`);
  if (!target.enabled) throw new Error(`${configLabel}.${path} references disabled target ${target.id}`);
}

function resolveTarget(observability: ObservabilityConfig, targetId: string | null): ObservabilityTarget {
  const resolved = targetId ?? observability.defaults.targetId;
  const target = observability.targets.find((item) => item.id.toLowerCase() === resolved.toLowerCase());
  if (target === undefined) throw new Error(`unknown observability target ${resolved}; known targets: ${observability.targets.map((item) => item.id).join(", ")}`);
  if (!target.enabled) throw new Error(`observability target ${target.id} is disabled in ${configLabel}`);
  return target;
}

function plan(options: CommonOptions): Record<string, unknown> {
  const observability = readObservabilityConfig();
  const target = resolveTarget(observability, options.targetId);
  const yaml = renderManifest(observability, target);
  const policy = policyChecks(yaml, target);
  return {
    ok: policy.every((check) => check.ok),
    action: "platform-infra-observability-plan",
    mutation: false,
    config: configSummary(observability, target),
    renderPlan: {
      target: targetSummary(target),
      objects: manifestObjectSummary(yaml),
      otlp: {
        collectorGrpcEndpoint: `${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.grpcPort}`,
        collectorHttpEndpoint: `http://${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.httpPort}`,
        backendGrpcEndpoint: `${observability.traceBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.traceBackend.otlp.grpcPort}`,
      },
      instrumentation: observability.instrumentation.serviceConnections,
      resourceAttributes: observability.resourceAttributes,
    },
    policy,
    next: {
      dryRun: `bun scripts/cli.ts platform-infra observability apply --target ${target.id} --dry-run`,
      apply: `bun scripts/cli.ts platform-infra observability apply --target ${target.id} --confirm`,
      status: `bun scripts/cli.ts platform-infra observability status --target ${target.id}`,
      validate: `bun scripts/cli.ts platform-infra observability validate --target ${target.id}`,
      trace: `bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id <traceId>`,
    },
  };
}

async function apply(config: UniDeskConfig, options: ApplyOptions): Promise<Record<string, unknown>> {
  const observability = readObservabilityConfig();
  const target = resolveTarget(observability, options.targetId);
  const yaml = renderManifest(observability, target);
  const policy = policyChecks(yaml, target);
  if (!policy.every((check) => check.ok)) return { ok: false, action: "platform-infra-observability-apply", mode: "policy-blocked", policy };
  const result = await capture(config, target.route, ["sh"], applyScript({
    yaml,
    target,
    dryRun: options.dryRun,
    wait: options.wait,
    collectorDeploymentName: observability.collector.deploymentName,
    backendDeploymentName: observability.traceBackend.deploymentName,
  }));
  const parsed = parseJsonOutput(result.stdout);
  return {
    ok: result.exitCode === 0 && parsed?.ok === true,
    action: "platform-infra-observability-apply",
    mode: options.dryRun ? "dry-run" : "confirmed",
    mutation: !options.dryRun,
    target: targetSummary(target),
    policy,
    remote: parsed ?? compactCapture(result, { full: true }),
  };
}

async function status(config: UniDeskConfig, options: CommonOptions): Promise<Record<string, unknown>> {
  const observability = readObservabilityConfig();
  const target = resolveTarget(observability, options.targetId);
  const result = await capture(config, target.route, ["sh"], statusScript(observability, target, options.full));
  const parsed = parseJsonOutput(result.stdout);
  const summary = parsed === null ? null : statusSummary(parsed);
  return {
    ok: result.exitCode === 0 && summary?.ready === true,
    action: "platform-infra-observability-status",
    mutation: false,
    target: targetSummary(target),
    summary,
    remote: options.raw ? parsed : compactStatus(parsed, options.full) ?? compactCapture(result, { full: true }),
    next: {
      plan: `bun scripts/cli.ts platform-infra observability plan --target ${target.id}`,
      apply: `bun scripts/cli.ts platform-infra observability apply --target ${target.id} --confirm`,
      validate: `bun scripts/cli.ts platform-infra observability validate --target ${target.id}`,
    },
  };
}

async function validate(config: UniDeskConfig, options: CommonOptions): Promise<Record<string, unknown>> {
  const observability = readObservabilityConfig();
  const target = resolveTarget(observability, options.targetId);
  const result = await capture(config, target.route, ["sh"], statusScript(observability, target, options.full, true));
  const parsed = parseJsonOutput(result.stdout);
  const summary = parsed === null ? null : statusSummary(parsed);
  const ready = summary?.ready === true;
  const validationTrace = parsed !== null && typeof parsed.validationTrace === "object" && parsed.validationTrace !== null && !Array.isArray(parsed.validationTrace)
    ? parsed.validationTrace as Record<string, unknown>
    : null;
  const validationTraceId = typeof validationTrace?.traceId === "string" ? validationTrace.traceId : null;
  const validationTraceOk = validationTrace?.ok === true;
  return {
    ok: result.exitCode === 0 && ready && validationTraceOk,
    action: "platform-infra-observability-validate",
    mutation: false,
    target: targetSummary(target),
    summary,
    validation: {
      readiness: ready ? "passed" : "failed",
      testTrace: validationTrace ?? "not-generated-runtime-not-ready",
      traceQuery: validationTraceId !== null
        ? `bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id ${validationTraceId}`
        : "blocked-until-validation-trace-generated",
      metricsBoundary: "Prometheus/RUM remains outside this trace readiness check.",
    },
    remote: options.raw ? parsed : compactStatus(parsed, options.full) ?? compactCapture(result, { full: true }),
  };
}

async function trace(config: UniDeskConfig, options: TraceOptions): Promise<Record<string, unknown> | RenderedCliResult> {
  if (options.traceId === null) throw new Error("observability trace requires --trace-id <traceId>");
  const observability = readObservabilityConfig();
  const target = resolveTarget(observability, options.targetId);
  const tracePath = observability.probes.traceQueryPathTemplate.replaceAll("{{traceId}}", encodeURIComponent(options.traceId));
  const result = await capture(config, target.route, ["sh"], traceScript(observability, target, tracePath, options));
  const parsed = parseJsonOutput(result.stdout);
  const ok = result.exitCode === 0 && parsed?.ok === true;
  if (!options.full && !options.raw) {
    return renderTraceTable({
      ok,
      target,
      options,
      tracePath,
      query: {
        backend: observability.traceBackend.type,
        service: observability.traceBackend.serviceName,
        path: tracePath,
      },
      result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : asPlainRecord(redactSensitiveUnknown(parsed)) ?? {},
    });
  }
  return {
    ok,
    action: "platform-infra-observability-trace",
    mutation: false,
    target: targetSummary(target),
    traceId: options.traceId,
    query: {
      backend: observability.traceBackend.type,
      service: observability.traceBackend.serviceName,
      path: tracePath,
    },
    result: redactSensitiveUnknown(parsed),
  };
}

async function search(config: UniDeskConfig, options: SearchOptions): Promise<Record<string, unknown> | RenderedCliResult> {
  const observability = readObservabilityConfig();
  const target = resolveTarget(observability, options.targetId);
  const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null;
  const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080);
  const effectiveTempoQuery = options.query ?? (businessTraceId === null ? inferSearchTempoQuery(options) : `{ .traceId = "${businessTraceId}" }`);
  const effectiveOptions: SearchOptions = {
    ...options,
    query: effectiveTempoQuery,
    lookbackMinutes: effectiveLookbackMinutes,
  };
  const endMillis = options.endAt === null ? Date.now() : Date.parse(options.endAt);
  if (!Number.isFinite(endMillis)) throw new Error("--end-at must be an ISO timestamp parseable by Date.parse");
  const endSeconds = Math.floor(endMillis / 1000);
  const startSeconds = endSeconds - (effectiveLookbackMinutes * 60);
  const params = new URLSearchParams({
    start: String(startSeconds),
    end: String(endSeconds),
    limit: String(options.candidateLimit),
  });
  if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery);
  const searchPath = `/api/search?${params.toString()}`;
  const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions));
  const parsed = parseJsonOutput(result.stdout);
  const ok = result.exitCode === 0 && parsed?.ok === true;
  const query = {
    backend: observability.traceBackend.type,
    service: observability.traceBackend.serviceName,
    path: searchPath,
    grep: options.grep,
    tempoQuery: effectiveTempoQuery,
    explicitTempoQuery: options.query,
    pathFilter: options.path,
    statusFilter: options.status,
    lookbackMinutes: effectiveLookbackMinutes,
    requestedLookbackMinutes: options.lookbackMinutes,
    endAt: new Date(endSeconds * 1000).toISOString(),
    businessTraceId,
    mode: businessTraceId === null ? "candidate-grep" : "business-trace-exact",
    candidateLimit: options.candidateLimit,
    limit: options.limit,
  };
  if (!options.full && !options.raw) {
    return renderSearchTable({
      ok,
      target,
      options,
      query,
      result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : compactSearchResult(parsed),
    });
  }
  const exposedResult = parsed === null
    ? compactCapture(result, { full: true })
    : redactSensitiveUnknown(parsed);
  return {
    ok,
    action: "platform-infra-observability-search",
    mutation: false,
    target: targetSummary(target),
    query,
    result: exposedResult,
  };
}

function inferSearchTempoQuery(options: SearchOptions): string | null {
  const filters: string[] = [];
  if (options.path !== null) filters.push(`.http.route = ${JSON.stringify(options.path)}`);
  if (options.status !== null) filters.push(`.http.response.status_code = ${options.status}`);
  return filters.length > 0 ? `{ ${filters.join(" && ")} }` : null;
}

function businessTraceIdFromSearchText(value: string | null): string | null {
  const match = value?.match(/\btrc_[A-Za-z0-9_-]+\b/u);
  return match?.[0] ?? null;
}

async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise<Record<string, unknown> | RenderedCliResult> {
  const observability = readObservabilityConfig();
  const target = resolveTarget(observability, options.targetId);
  const endSeconds = Math.floor(Date.now() / 1000);
  const startSeconds = endSeconds - (options.lookbackMinutes * 60);
  let searchPath: string | null = null;
  if (options.traceId === null && options.businessTraceId !== null) {
    const params = new URLSearchParams({
      start: String(startSeconds),
      end: String(endSeconds),
      limit: String(options.candidateLimit),
      q: `{ .traceId = "${options.businessTraceId}" }`,
    });
    searchPath = `/api/search?${params.toString()}`;
  }
  const result = await capture(config, target.route, ["sh"], diagnoseCodeAgentScript(observability, target, searchPath, options));
  const parsed = parseJsonOutput(result.stdout);
  const ok = result.exitCode === 0 && parsed?.ok === true;
  const query = {
    backend: observability.traceBackend.type,
    service: observability.traceBackend.serviceName,
    businessTraceId: options.businessTraceId,
    traceId: options.traceId,
    path: searchPath,
    lookbackMinutes: options.lookbackMinutes,
    candidateLimit: options.candidateLimit,
    limit: options.limit,
  };
  if (!options.full && !options.raw) {
    return renderDiagnoseCodeAgentTable({
      ok,
      target,
      options,
      query,
      result: parsed === null ? { ok: false, remote: compactCapture(result, { full: false }) } : compactDiagnoseCodeAgentResult(parsed),
    });
  }
  const exposedResult = parsed === null
    ? compactCapture(result, { full: true })
    : options.raw
      ? redactSensitiveUnknown(parsed)
      : {
          ...compactDiagnoseCodeAgentResult(parsed),
          outputMode: "bounded-compact-json",
          rawDisclosure: "Use --raw only for Tempo/API envelope or CLI parser debugging.",
        };
  return {
    ok,
    action: "platform-infra-observability-diagnose-code-agent",
    mutation: false,
    target: targetSummary(target),
    query,
    result: exposedResult,
  };
}

function renderTraceTable(input: {
  ok: boolean;
  target: ObservabilityTarget;
  options: TraceOptions;
  tracePath: string;
  query: Record<string, unknown>;
  result: Record<string, unknown>;
}): RenderedCliResult {
  const spanSource = input.result.grep === null || input.result.grep === undefined
    ? [...asArray(input.result.errorSpans), ...asArray(input.result.spans)]
    : asArray(input.result.spans);
  const spanRows = uniqueSpanRecords(spanSource)
    .sort((left, right) => traceSpanImportanceScore(right) - traceSpanImportanceScore(left))
    .slice(0, 10)
    .map((span) => {
      const attrs = asPlainRecord(span.attributes);
      const durationMs = traceSpanDurationMs(span, attrs);
      return [
        shortenEnd(textValue(span.name), 40),
        shortenEnd(textValue(span.service), 22),
        shortenEnd(spanColumnAttr(attrs, ["http.route", "workbench.ui.route"]), 34),
        spanColumnAttr(attrs, ["http.response.status_code", "http.status_code", "http.response.status_class"]),
        numberValue(durationMs),
        spanColumnAttr(attrs, ["workbench.ui.resource_duration_ms"]),
        spanColumnAttr(attrs, ["workbench.ui.resource_fetch_to_request_ms"]),
        spanColumnAttr(attrs, ["workbench.ui.resource_request_wait_ms"]),
        spanColumnAttr(attrs, ["workbench.ui.resource_response_transfer_ms"]),
        shortenEnd(spanColumnAttr(attrs, ["workbench.ui.resource_next_hop_protocol"]), 14),
        spanDetail(span, 220),
      ];
    });
  const countRows = asArray(input.result.spanNameCounts).slice(0, 8).map((item) => {
    const row = asPlainRecord(item) ?? {};
    return [shortenEnd(textValue(row.name), 64), textValue(row.count)];
  });
  const identity = asPlainRecord(input.result.identity) ?? {};
  const identityRows = ["runId", "commandId", "runnerJobId", "runnerId", "backendProfile", "sourceCommit", "jobName", "podName"]
    .map((key) => [key, joinValues(identity[key], 88)])
    .filter((row) => row[1] !== "-");
  const next = asPlainRecord(input.result.next);
  const lines = [
    `platform-infra observability trace (${input.ok ? "ok" : "not-ok"})`,
    "",
    formatTable(["TRACE", "SPANS", "ERR", "MATCH", "SERVICES", "BUSINESS_TRACE"], [[
      shortenMiddle(input.options.traceId ?? "-", 20),
      textValue(input.result.spanCount),
      textValue(input.result.errorSpanCount),
      textValue(input.result.matchedSpanCount),
      joinValues(input.result.services, 44),
      joinValues(input.result.businessTraceIds, 34),
    ]]),
    "",
    "Identity:",
    formatTable(["FIELD", "VALUE"], identityRows.length > 0 ? identityRows : [["-", "-"]]),
    "",
    "Key spans:",
    formatTable(["NAME", "SERVICE", "ROUTE", "STATUS", "DUR_MS", "RES_MS", "FETCH_REQ", "REQ_WAIT", "RESP_XFER", "PROTO", "DETAIL"], spanRows.length > 0 ? spanRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
    "",
    "Span name counts:",
    formatTable(["NAME", "COUNT"], countRows.length > 0 ? countRows : [["-", "-"]]),
    "",
    "Summary:",
    `  target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)} path=${input.tracePath}`,
    `  grep=${textValue(input.result.grep)} limit=${textValue(input.result.limit)} traceFound=${textValue(input.result.traceFound)} parseOk=${textValue(input.result.parseOk)}`,
    "",
    "Next:",
  ];
  const nextCommands = [
    textValue(next?.fullSummary),
    textValue(next?.grepProviderStream),
    textValue(next?.raw),
  ].filter((item) => item !== "-");
  if (nextCommands.length > 0) {
    for (const command of nextCommands.slice(0, 3)) lines.push(`  ${command}`);
  } else {
    lines.push(`  ${buildTraceCommand(input.target, input.options, true)}`);
  }
  lines.push("", "Disclosure:");
  lines.push("  default view is a bounded table; use --full for structured span summary JSON or --raw for backend response.");
  return {
    ok: input.ok,
    command: "platform-infra observability trace",
    contentType: "text/plain",
    renderedText: lines.join("\n"),
  };
}

function renderSearchTable(input: {
  ok: boolean;
  target: ObservabilityTarget;
  options: SearchOptions;
  query: Record<string, unknown>;
  result: Record<string, unknown>;
}): RenderedCliResult {
  const traces = asArray(input.result.traces).map((item) => asPlainRecord(item) ?? {});
  const requestedLimit = numberFromUnknown(input.options.limit);
  const rowLimit = Math.max(1, Math.min(Number.isFinite(requestedLimit) ? Math.round(requestedLimit) : 12, 40));
  const rows = traces.slice(0, rowLimit).map((trace) => {
    const meta = asPlainRecord(trace.meta) ?? {};
    return [
      shortenMiddle(textValue(trace.traceId), 32),
      shortenEnd(textValue(trace.startAt ?? isoFromUnixNano(meta.startTimeUnixNano)), 20),
      numberValue(trace.durationMs ?? meta.durationMs),
      searchTraceStatus(trace),
      numberValue(trace.spanCount),
      numberValue(trace.errorSpanCount),
      numberValue(trace.matchedSpanCount),
      joinValues(trace.services, 38),
    ];
  });
  const lines = [
    `platform-infra observability search (${input.ok ? "ok" : "not-ok"})`,
    "",
    formatTable(["TRACE", "START", "DUR_MS", "STATUS", "SPANS", "ERR", "MATCH", "SERVICES"], rows.length > 0 ? rows : [["-", "-", "-", "no-match", "-", "-", "-", "-"]]),
    "",
    "Summary:",
    `  target=${input.target.id} backend=${textValue(input.query.backend)} service=${textValue(input.query.service)}`,
    `  grep=${textValue(input.query.grep)} path=${textValue(input.query.pathFilter)} status=${textValue(input.query.statusFilter)} tempoQuery=${shortenMiddle(textValue(input.query.tempoQuery), 80)}`,
    `  mode=${textValue(input.query.mode)} businessTraceId=${textValue(input.query.businessTraceId)} lookbackMinutes=${textValue(input.query.lookbackMinutes)} requestedLookbackMinutes=${textValue(input.query.requestedLookbackMinutes)} endAt=${textValue(input.query.endAt)}`,
    `  candidateLimit=${textValue(input.query.candidateLimit)} limit=${textValue(input.query.limit)}`,
    `  candidates=${textValue(input.result.candidateTraceCount)} scanned=${textValue(input.result.scannedTraceCount)} matched=${textValue(input.result.matchedTraceCount)} stopped=${textValue(input.result.scanStopped)}`,
  ];
  const firstTraceId = traces.length > 0 ? textValue(traces[0].traceId) : "";
  lines.push("", "Next:");
  if (firstTraceId.length > 0 && firstTraceId !== "-") {
    lines.push(`  bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${firstTraceId}`);
  }
  lines.push(`  ${buildSearchCommand(input.target, input.options, true)}`);
  lines.push("", "Disclosure:");
  lines.push("  default view is a bounded table; use --full for structured diagnosis JSON or trace --trace-id for one trace.");
  return {
    ok: input.ok,
    command: "platform-infra observability search",
    contentType: "text/plain",
    renderedText: lines.join("\n"),
  };
}

function renderDiagnoseCodeAgentTable(input: {
  ok: boolean;
  target: ObservabilityTarget;
  options: DiagnoseCodeAgentOptions;
  query: Record<string, unknown>;
  result: Record<string, unknown>;
}): RenderedCliResult {
  const mapping = asPlainRecord(input.result.mapping);
  const identity = asPlainRecord(input.result.identity);
  const agentrun = asPlainRecord(input.result.agentrun);
  const projectionLag = asPlainRecord(input.result.projectionLag);
  const summary = asPlainRecord(input.result.summary);
  const evidence = asPlainRecord(input.result.evidence);
  const rootCauses = asArray(input.result.rootCauseCandidates).map((item) => asPlainRecord(item) ?? {});
  const http = asPlainRecord(input.result.http);
  const services = joinValues(input.result.services, 54);
  const traceId = textValue(mapping?.otelTraceId ?? input.query.traceId);
  const rootCause = textValue(summary?.rootCause ?? rootCauses[0]?.code);
  const rows = [[
    shortenMiddle(traceId, 20),
    shortenEnd(rootCause, 34),
    textValue(agentrun?.terminalStatus),
    textValue(agentrun?.runnerProviderClassification),
    textValue(projectionLag?.status),
    textValue(evidence?.errorSpanCount),
    services,
  ]];
  const rootRows = rootCauses.slice(0, 5).map((candidate) => [
    shortenEnd(textValue(candidate.code), 40),
    textValue(candidate.confidence),
    shortenEnd(textValue(candidate.summary ?? candidate.label), 80),
  ]);
  const httpRows = httpTableRows(http);
  const lines = [
    `platform-infra observability diagnose-code-agent (${input.ok ? "ok" : "not-ok"})`,
    "",
    formatTable(["TRACE", "ROOT_CAUSE", "TERMINAL", "RUNNER", "PROJECTION", "ERR", "SERVICES"], rows),
    "",
    "Identity:",
    `  businessTraceId=${textValue(mapping?.businessTraceId ?? input.query.businessTraceId)} otelTraceId=${traceId}`,
    `  runId=${textValue(identity?.runId)} commandId=${textValue(identity?.commandId)} runnerJobId=${textValue(identity?.runnerJobId)} runnerId=${textValue(identity?.runnerId)}`,
    `  backendProfile=${textValue(identity?.backendProfile)} sourceCommit=${shortenMiddle(textValue(identity?.sourceCommit), 20)}`,
    "",
    "Root causes:",
    formatTable(["CODE", "CONF", "SUMMARY"], rootRows.length > 0 ? rootRows : [["-", "-", "-"]]),
    "",
    "HTTP:",
    formatTable(["METHOD", "ROUTE", "STATUS", "COUNT"], httpRows.length > 0 ? httpRows : [["-", "-", "-", "-"]]),
    "",
    "Summary:",
    `  target=${input.target.id} spanCount=${textValue(input.result.spanCount)} servicePath=${joinValues(input.result.servicePath, 60)}`,
    `  agentrun=${textValue(agentrun?.terminalStatus)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`,
    `  readModel=${shortenEnd(JSON.stringify(input.result.hwlabReadModel ?? null), 140)}`,
  ];
  const next = asPlainRecord(input.result.next);
  lines.push("", "Next:");
  const nextCommands = [
    textValue(next?.diagnoseFull),
    textValue(next?.traceSummary),
    textValue(next?.traceReads),
  ].filter((item) => item !== "-");
  if (nextCommands.length > 0) {
    for (const command of nextCommands.slice(0, 3)) lines.push(`  ${command}`);
  } else {
    lines.push(`  ${buildDiagnoseCommand(input.target, input.options, true)}`);
    if (traceId.length > 0 && traceId !== "-") lines.push(`  bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${traceId}`);
  }
  lines.push("", "Disclosure:");
  lines.push("  default view is a bounded table; use --full for structured diagnosis JSON or trace --grep for span-level drill-down.");
  return {
    ok: input.ok,
    command: "platform-infra observability diagnose-code-agent",
    contentType: "text/plain",
    renderedText: lines.join("\n"),
  };
}

function traceSpanDurationMs(span: Record<string, unknown>, attrs: Record<string, unknown> | null): number {
  const topLevel = numberFromUnknown(span.durationMs);
  if (Number.isFinite(topLevel)) return topLevel;
  if (attrs === null) return Number.NaN;
  for (const key of ["workbench.ui.value_ms", "workbench.ui.resource_duration_ms", "durationMs"]) {
    const value = numberFromUnknown(attrs[key]);
    if (Number.isFinite(value)) return value;
  }
  return Number.NaN;
}

function traceSpanImportanceScore(span: Record<string, unknown>): number {
  const attrs = asPlainRecord(span.attributes);
  const statusCode = textValue(span.statusCode);
  const name = textValue(span.name).toLowerCase();
  const hasFailure = spanColumnAttr(attrs, ["failureKind", "errorSummary", "error.message", "exception.message"]) !== "-";
  const terminalStatus = spanColumnAttr(attrs, ["terminalStatus"]);
  const problemScore = statusCode === "STATUS_CODE_ERROR" || statusCode === "2" || name.includes("error") || hasFailure || terminalStatus === "failed" || terminalStatus === "blocked" ? 1_000_000_000 : 0;
  const durationMs = traceSpanDurationMs(span, attrs);
  const durationScore = Number.isFinite(durationMs) ? durationMs : 0;
  return problemScore + durationScore;
}

function searchTraceStatus(trace: Record<string, unknown>): string {
  if (trace.queryOk === false) return "query-bad";
  if (trace.parseOk === false) return "parse-bad";
  const errorCount = numberFromUnknown(trace.errorSpanCount);
  if (errorCount > 0) return "error";
  const matchedCount = numberFromUnknown(trace.matchedSpanCount);
  if (matchedCount > 0 || trace.rawMatched === true) return "matched";
  return "ok";
}

function uniqueSpanRecords(values: unknown[]): Record<string, unknown>[] {
  const rows: Record<string, unknown>[] = [];
  const seen = new Set<string>();
  for (const value of values) {
    const span = asPlainRecord(value) ?? {};
    const key = `${textValue(span.name)}\n${textValue(span.service)}\n${spanDetail(span, 200)}`;
    if (seen.has(key)) continue;
    seen.add(key);
    rows.push(span);
  }
  return rows;
}

function spanColumnAttr(attrs: Record<string, unknown> | null, keys: string[]): string {
  if (attrs === null) return "-";
  for (const key of keys) {
    const value = attrs[key];
    const text = textValue(value);
    if (text !== "-") return text;
  }
  return "-";
}

function spanDetail(span: Record<string, unknown>, maxLength: number): string {
  const attrs = asPlainRecord(span.attributes);
  const parts = [
    valuePart("durationMs", span.durationMs),
    attrPart(attrs, "failureKind"),
    attrPart(attrs, "errorSummary"),
    attrPart(attrs, "error.message"),
    attrPart(attrs, "exception.message"),
    attrPart(attrs, "terminalStatus"),
    attrPart(attrs, "status"),
    attrPart(attrs, "toolName"),
    attrPart(attrs, "exitCode"),
    attrPart(attrs, "outputSummary"),
    attrPart(attrs, "outputBytes"),
    attrPart(attrs, "durationMs"),
    attrPart(attrs, "cwd"),
    attrPart(attrs, "command"),
    attrPart(attrs, "waitingFor"),
    attrPart(attrs, "lastEventLabel"),
    attrPart(attrs, "idleMs"),
    attrPart(attrs, "idleSeconds"),
    attrPart(attrs, "commandFingerprint"),
    attrPart(attrs, "http.route"),
    attrPart(attrs, "http.response.status_code"),
    attrPart(attrs, "http.response.status_class"),
    attrPart(attrs, "workbench.ui.event_type"),
    attrPart(attrs, "workbench.ui.scope"),
    attrPart(attrs, "workbench.ui.state"),
    attrPart(attrs, "workbench.ui.reason"),
    attrPart(attrs, "workbench.ui.outcome"),
    attrPart(attrs, "workbench.ui.value_ms"),
    attrPart(attrs, "workbench.ui.activity_idle_ms"),
    attrPart(attrs, "workbench.ui.activity_waiting_for"),
    attrPart(attrs, "workbench.ui.activity_last_event_label"),
    attrPart(attrs, "workbench.ui.resource_duration_ms"),
    attrPart(attrs, "workbench.ui.resource_fetch_to_request_ms"),
    attrPart(attrs, "workbench.ui.resource_request_wait_ms"),
    attrPart(attrs, "workbench.ui.resource_response_transfer_ms"),
    attrPart(attrs, "workbench.ui.resource_next_hop_protocol"),
    attrPart(attrs, "workbench.ui.resource_server_timing"),
    attrPart(attrs, "hwlab.http.stage"),
    attrPart(attrs, "hwlab.http.phase"),
    attrPart(attrs, "hwlab.http.phase.outcome"),
    attrPart(attrs, "hwlab.live_builds.service_id"),
    attrPart(attrs, "hwlab.live_builds.service_kind"),
    attrPart(attrs, "hwlab.live_builds.external"),
    attrPart(attrs, "hwlab.live_builds.deploy_manifest_status"),
    attrPart(attrs, "hwlab.live_builds.artifact_catalog_status"),
    attrPart(attrs, "db.system"),
    attrPart(attrs, "db.operation.name"),
    attrPart(attrs, "db.sql.table"),
    attrPart(attrs, "db.query.arg_count"),
    attrPart(attrs, "db.index.expected"),
    attrPart(attrs, "db.pool.max_open"),
    attrPart(attrs, "db.pool.open_connections"),
    attrPart(attrs, "db.pool.in_use"),
    attrPart(attrs, "db.pool.idle"),
    attrPart(attrs, "db.pool.wait_count"),
    attrPart(attrs, "db.pool.wait_duration_ms"),
    attrPart(attrs, "error.code"),
    attrPart(attrs, "error.category"),
    attrPart(attrs, "error.layer"),
  ].filter((item) => item !== null) as string[];
  if (parts.length === 0) return "-";
  return shortenEnd(parts.join(" "), maxLength);
}

function valuePart(key: string, value: unknown): string | null {
  if (value === null || value === undefined || value === "") return null;
  return `${key}=${textValue(value)}`;
}

function attrPart(attrs: Record<string, unknown> | null, key: string): string | null {
  if (attrs === null) return null;
  const value = attrs[key];
  if (value === null || value === undefined || value === "") return null;
  return `${key}=${textValue(value)}`;
}

function httpTableRows(http: Record<string, unknown> | null): string[][] {
  if (http === null) return [];
  const problemRows = asArray(http.problemCounts).map((item) => {
    const row = asPlainRecord(item) ?? {};
    return [
      textValue(row.method),
      shortenEnd(textValue(row.route ?? row.path), 44),
      textValue(row.status ?? row.statusCode),
      textValue(row.count),
    ];
  });
  const statusRows = asArray(http.statusCounts).map((item) => {
    const row = asPlainRecord(item) ?? {};
    return [
      textValue(row.method),
      shortenEnd(textValue(row.route ?? row.path), 44),
      textValue(row.status ?? row.statusCode),
      textValue(row.count),
    ];
  });
  return [...problemRows, ...statusRows].slice(0, 8);
}

function buildTraceCommand(target: ObservabilityTarget, options: TraceOptions, full: boolean): string {
  const parts = ["bun", "scripts/cli.ts", "platform-infra", "observability", "trace", "--target", target.id];
  if (options.traceId !== null) parts.push("--trace-id", options.traceId);
  if (options.grep !== null) parts.push("--grep", options.grep);
  parts.push("--limit", String(options.limit));
  if (full) parts.push("--full");
  return parts.map(cliArg).join(" ");
}

function buildSearchCommand(target: ObservabilityTarget, options: SearchOptions, full: boolean): string {
  const parts = ["bun", "scripts/cli.ts", "platform-infra", "observability", "search", "--target", target.id];
  if (options.grep !== null) parts.push("--grep", options.grep);
  if (options.query !== null) parts.push("--query", options.query);
  if (options.path !== null) parts.push("--path", options.path);
  if (options.status !== null) parts.push("--status", String(options.status));
  if (options.endAt !== null) parts.push("--end-at", options.endAt);
  parts.push("--lookback-minutes", String(options.lookbackMinutes), "--candidate-limit", String(options.candidateLimit), "--limit", String(options.limit));
  if (full) parts.push("--full");
  return parts.map(cliArg).join(" ");
}

function buildDiagnoseCommand(target: ObservabilityTarget, options: DiagnoseCodeAgentOptions, full: boolean): string {
  const parts = ["bun", "scripts/cli.ts", "platform-infra", "observability", "diagnose-code-agent", "--target", target.id];
  if (options.businessTraceId !== null) parts.push("--business-trace-id", options.businessTraceId);
  if (options.traceId !== null) parts.push("--trace-id", options.traceId);
  parts.push("--lookback-minutes", String(options.lookbackMinutes), "--candidate-limit", String(options.candidateLimit), "--limit", String(options.limit));
  if (full) parts.push("--full");
  return parts.map(cliArg).join(" ");
}

function cliArg(value: string): string {
  return /^[A-Za-z0-9_./:=@+-]+$/u.test(value) ? value : shQuote(value);
}

function formatTable(headers: string[], rows: string[][]): string {
  const normalizedRows = rows.map((row) => headers.map((_, index) => row[index] ?? ""));
  const widths = headers.map((header, index) => Math.max(header.length, ...normalizedRows.map((row) => row[index].length)));
  return [
    headers.map((header, index) => header.padEnd(widths[index])).join("  ").trimEnd(),
    ...normalizedRows.map((row) => row.map((cell, index) => cell.padEnd(widths[index])).join("  ").trimEnd()),
  ].join("\n");
}

function textValue(value: unknown): string {
  if (value === null || value === undefined || value === "") return "-";
  if (typeof value === "string") return value;
  if (typeof value === "number" || typeof value === "boolean") return String(value);
  return JSON.stringify(value);
}

function numberValue(value: unknown): string {
  const number = numberFromUnknown(value);
  return Number.isFinite(number) ? String(number) : "-";
}

function numberFromUnknown(value: unknown): number {
  if (typeof value === "number" && Number.isFinite(value)) return value;
  if (typeof value === "string" && value.trim().length > 0) {
    const parsed = Number(value);
    if (Number.isFinite(parsed)) return parsed;
  }
  return Number.NaN;
}

function isoFromUnixNano(value: unknown): string {
  if (value === null || value === undefined || value === "") return "-";
  try {
    const text = String(value);
    if (!/^\d+$/u.test(text)) return "-";
    const millis = Number(BigInt(text) / 1000000n);
    if (!Number.isFinite(millis) || millis <= 0) return "-";
    return new Date(millis).toISOString().replace(/\.000Z$/u, "Z");
  } catch {
    return "-";
  }
}

function joinValues(value: unknown, maxLength: number): string {
  const joined = asArray(value).map(textValue).filter((item) => item !== "-").join(",");
  return shortenEnd(joined.length > 0 ? joined : textValue(value), maxLength);
}

function shortenEnd(value: string, maxLength: number): string {
  if (value.length <= maxLength) return value;
  if (maxLength <= 1) return value.slice(0, maxLength);
  return `${value.slice(0, maxLength - 1)}~`;
}

function shortenMiddle(value: string, maxLength: number): string {
  if (value.length <= maxLength) return value;
  if (maxLength <= 3) return value.slice(0, maxLength);
  const left = Math.ceil((maxLength - 1) / 2);
  const right = Math.floor((maxLength - 1) / 2);
  return `${value.slice(0, left)}~${value.slice(value.length - right)}`;
}

function compactSearchResult(value: unknown): Record<string, unknown> {
  const source = asPlainRecord(value) ?? {};
  const traces = asArray(source.traces).map(compactSearchTrace);
  return {
    ok: source.ok === true,
    searchPath: source.searchPath ?? null,
    grep: source.grep ?? null,
    tempoQuery: source.tempoQuery ?? null,
    pathFilter: source.pathFilter ?? null,
    statusFilter: source.statusFilter ?? null,
    limit: source.limit ?? null,
    candidateLimit: source.candidateLimit ?? null,
    searchParseOk: source.searchParseOk ?? null,
    candidateTraceCount: source.candidateTraceCount ?? null,
    scannedTraceCount: source.scannedTraceCount ?? null,
    matchedTraceCount: source.matchedTraceCount ?? null,
    scanStopped: source.scanStopped ?? null,
    matchingActive: source.matchingActive ?? null,
    traces,
    truncated: source.truncated ?? null,
    next: source.next ?? null,
    searchStderrTail: source.searchStderrTail ?? null,
    disclosure: {
      defaultView: "compact trace rows only; span arrays are omitted to keep stdout below config/unidesk-cli.yaml output.maxStdoutBytes",
      expand: "rerun with --full or query a specific trace id with platform-infra observability trace",
    },
  };
}

function compactSearchTrace(value: unknown): Record<string, unknown> {
  const trace = asPlainRecord(value) ?? {};
  const meta = compactMetaRecord(trace.meta);
  const startTimeUnixNano = trace.startTimeUnixNano ?? meta?.startTimeUnixNano ?? null;
  return {
    traceId: trace.traceId ?? null,
    startAt: isoFromUnixNano(startTimeUnixNano),
    durationMs: trace.durationMs ?? meta?.durationMs ?? null,
    traceCommand: trace.traceCommand ?? null,
    grepCommand: trace.grepCommand ?? null,
    meta,
    queryOk: trace.queryOk ?? null,
    parseOk: trace.parseOk ?? null,
    rawMatched: trace.rawMatched ?? null,
    matchingActive: trace.matchingActive ?? null,
    bodyBytes: trace.bodyBytes ?? null,
    spanCount: trace.spanCount ?? null,
    serviceCount: trace.serviceCount ?? null,
    services: trace.services ?? null,
    businessTraceIds: trace.businessTraceIds ?? null,
    errorSpanCount: trace.errorSpanCount ?? null,
    matchedSpanCount: trace.matchedSpanCount ?? null,
    spanNameCounts: compactNameCounts(trace.spanNameCounts, 6),
    errorSpanNames: compactSpanNames(trace.errorSpans, 5),
    matchedSpanNames: compactSpanNames(trace.matchedSpans, 8),
    stderrTail: trace.stderrTail ?? null,
  };
}

function compactDiagnoseCodeAgentResult(value: unknown): Record<string, unknown> {
  const source = asPlainRecord(value) ?? {};
  const mapping = asPlainRecord(source.mapping);
  const evidence = asPlainRecord(source.evidence);
  return {
    ok: source.ok === true,
    mapping: mapping === null ? null : {
      mode: mapping.mode ?? null,
      businessTraceId: mapping.businessTraceId ?? null,
      otelTraceId: mapping.otelTraceId ?? null,
      searchOk: mapping.searchOk ?? null,
      searchParseOk: mapping.searchParseOk ?? null,
      candidateTraceCount: mapping.candidateTraceCount ?? null,
      scannedCandidateCount: mapping.scannedCandidateCount ?? null,
      candidateSelectionMode: mapping.candidateSelectionMode ?? null,
      selectedScore: mapping.selectedScore ?? null,
      selectedQuality: mapping.selectedQuality ?? null,
      selectedLowConfidence: mapping.selectedLowConfidence ?? null,
      selectedRejectedReason: mapping.selectedRejectedReason ?? null,
      selectedReasons: limitArray(mapping.selectedReasons, 4),
      selectedMeta: compactMetaRecord(mapping.selectedMeta),
      bestCandidate: asArray(mapping.candidatePreview).slice(0, 1).map(compactDiagnoseCandidate)[0] ?? null,
      searchStderrTail: mapping.searchStderrTail ?? null,
    },
    tracePath: source.tracePath ?? null,
    bodyBytes: source.bodyBytes ?? null,
    traceParseOk: source.traceParseOk ?? null,
    spanCount: source.spanCount ?? null,
    services: source.services ?? null,
    servicePath: source.servicePath ?? null,
    businessTraceIds: source.businessTraceIds ?? null,
    identity: compactDiagnoseIdentity(source.identity),
    agentrun: compactDiagnoseAgentRun(source.agentrun),
    hwlabReadModel: source.hwlabReadModel ?? null,
    http: compactDiagnoseHttp(source.http),
    projectionLag: source.projectionLag ?? null,
    summary: source.summary ?? null,
    rootCauseCandidates: asArray(source.rootCauseCandidates).slice(0, 3).map(compactRootCauseCandidate),
    spanNameCounts: compactNameCounts(source.spanNameCounts, 6),
    evidence: evidence === null ? null : {
      httpProblemSpanCount: evidence.httpProblemSpanCount ?? null,
      terminalSpanCount: evidence.terminalSpanCount ?? null,
      turnStatusReadSpanCount: evidence.turnStatusReadSpanCount ?? null,
      projectionSpanCount: evidence.projectionSpanCount ?? null,
      errorSpanCount: evidence.errorSpanCount ?? null,
      errorSpanSampleNames: evidence.errorSpanSampleNames ?? null,
      idleWarningSpanCount: evidence.idleWarningSpanCount ?? null,
      idleWarningSpanTail: compactSpanList(evidence.idleWarningSpanTail, 1),
    },
    next: compactDiagnoseNext(source.next),
    stderrTail: source.stderrTail ?? null,
    disclosure: {
      defaultView: "compact diagnosis; --full is bounded JSON, not a span dump",
      expand: "use trace --grep <span> --full for span evidence; use diagnose-code-agent --raw only for parser/envelope debugging",
    },
  };
}

function compactDiagnoseIdentity(value: unknown): Record<string, unknown> | null {
  return compactRecord(value, [
    "runId",
    "commandId",
    "sessionId",
    "turnId",
    "runnerJobId",
    "runnerId",
    "backendProfile",
    "sourceCommit",
    "jobName",
    "namespace",
  ]);
}

function compactDiagnoseHttp(value: unknown): Record<string, unknown> | null {
  const http = asPlainRecord(value);
  if (http === null) return null;
  return {
    problemCounts: http.problemCounts ?? null,
    problemSpanCount: http.problemSpanCount ?? null,
    statusCounts: asArray(http.statusCounts).slice(0, 4),
  };
}

function compactDiagnoseAgentRun(value: unknown): Record<string, unknown> | null {
  const agentrun = asPlainRecord(value);
  if (agentrun === null) return null;
  return {
    terminalStatus: agentrun.terminalStatus ?? null,
    terminalSource: agentrun.terminalSource ?? null,
    latestSeq: agentrun.latestSeq ?? null,
    terminalEventType: agentrun.terminalEventType ?? null,
    runnerProviderClassification: agentrun.runnerProviderClassification ?? null,
    authority: compactAgentRunAuthority(agentrun.authority),
  };
}

function compactAgentRunAuthority(value: unknown): Record<string, unknown> | null {
  const authority = asPlainRecord(value);
  if (authority === null) return null;
  const result: Record<string, unknown> = {
    queried: authority.queried ?? null,
    namespace: authority.namespace ?? null,
    ok: authority.ok ?? null,
    commandOk: authority.commandOk ?? null,
    resultOk: authority.resultOk ?? null,
    eventsOk: authority.eventsOk ?? null,
    commandState: authority.commandState ?? null,
    commandStatus: authority.commandStatus ?? null,
    resultTerminalStatus: authority.resultTerminalStatus ?? null,
    resultStatus: authority.resultStatus ?? null,
    terminalStatus: authority.terminalStatus ?? null,
    terminalSource: authority.terminalSource ?? null,
    terminalEventSeq: authority.terminalEventSeq ?? null,
    terminalEventType: authority.terminalEventType ?? null,
    latestSeq: authority.latestSeq ?? null,
    eventCount: authority.eventCount ?? null,
    fallback: authority.fallback ?? null,
    error: authority.error ?? null,
    warnings: limitArray(authority.warnings, 3),
  };
  const ok = authority.ok === true && authority.commandOk === true && authority.resultOk === true && authority.eventsOk === true;
  if (!ok) {
    result.attempts = compactAgentRunAuthorityAttempts(authority.attempts);
  }
  return result;
}

function compactAgentRunAuthorityAttempts(value: unknown): unknown[] {
  return asArray(value).slice(0, 3).map((item) => {
    const attempt = asPlainRecord(item) ?? {};
    return {
      name: attempt.name ?? attempt.section ?? attempt.kind ?? null,
      ok: attempt.ok ?? null,
      status: attempt.status ?? null,
      exitCode: attempt.exitCode ?? null,
      url: attempt.url ?? null,
      error: shortenEnd(textValue(attempt.error), 240),
      stderrTail: shortenEnd(textValue(attempt.stderrTail ?? attempt.stderr), 500),
    };
  });
}

function compactDiagnoseCandidate(value: unknown): Record<string, unknown> {
  const candidate = asPlainRecord(value) ?? {};
  return {
    traceId: candidate.traceId ?? null,
    score: candidate.score ?? null,
    reasons: limitArray(candidate.reasons, 3),
    parseOk: candidate.parseOk ?? null,
    queryOk: candidate.queryOk ?? null,
    spanCount: candidate.spanCount ?? null,
    services: candidate.services ?? null,
    servicePath: candidate.servicePath ?? null,
    identity: compactDiagnoseIdentity(candidate.identity),
    terminalStatus: candidate.terminalStatus ?? null,
    errorSpanCount: candidate.errorSpanCount ?? null,
    candidateQuality: candidate.candidateQuality ?? null,
    lowConfidence: candidate.lowConfidence ?? null,
    spanNamePreview: limitArray(candidate.spanNamePreview, 4),
  };
}

function compactDiagnoseNext(value: unknown): Record<string, unknown> | null {
  const next = asPlainRecord(value);
  if (next === null) return null;
  return {
    diagnoseFull: next.diagnoseFull ?? null,
    traceSummary: next.traceSummary ?? null,
    traceReads: next.traceReads ?? null,
  };
}

function compactRootCauseCandidate(value: unknown): Record<string, unknown> {
  const candidate = asPlainRecord(value) ?? {};
  return {
    code: candidate.code ?? null,
    label: candidate.label ?? null,
    confidence: candidate.confidence ?? null,
    summary: candidate.summary ?? null,
    evidenceCount: Array.isArray(candidate.evidence) ? candidate.evidence.length : candidate.evidence === undefined || candidate.evidence === null ? 0 : 1,
  };
}

function compactSpanList(value: unknown, limit: number): unknown[] {
  return asArray(value).slice(0, limit).map((item) => {
    const span = asPlainRecord(item) ?? {};
    const attrs = asPlainRecord(span.attributes) ?? {};
    return {
      name: span.name ?? null,
      service: span.service ?? null,
      attributes: compactRecord(attrs, ["failureKind", "terminalStatus", "status", "eventType", "idleMs", "waitingFor", "lastEventLabel", "http.route", "http.status_code", "http.response.status_code", "workbench.session_id", "workbench.trace_id", "workbench.turn_id", "workbench.read_model.route", "workbench.read_model.count", "workbench.read_model.family", "workbench.read_model.status", "workbench.read_model.reason", "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome", "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind", "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status", "hwlab.live_builds.artifact_catalog_status"]),
    };
  });
}

function compactSpanNames(value: unknown, limit: number): string[] {
  return asArray(value).slice(0, limit).map((item) => {
    const span = asPlainRecord(item);
    return String(span?.name ?? "<unknown>");
  });
}

function compactNameCounts(value: unknown, limit: number): unknown[] {
  return asArray(value).slice(0, limit).map((item) => {
    const record = asPlainRecord(item) ?? {};
    return { name: record.name ?? null, count: record.count ?? null };
  });
}

function compactMetaRecord(value: unknown): Record<string, unknown> | null {
  return compactRecord(value, ["traceID", "traceId", "rootServiceName", "rootTraceName", "startTimeUnixNano", "durationMs"]);
}

function compactRecord(value: unknown, keys: string[]): Record<string, unknown> | null {
  const source = asPlainRecord(value);
  if (source === null) return null;
  const result: Record<string, unknown> = {};
  for (const key of keys) {
    if (Object.prototype.hasOwnProperty.call(source, key)) result[key] = source[key];
  }
  return result;
}

function asPlainRecord(value: unknown): Record<string, unknown> | null {
  if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
  return value as Record<string, unknown>;
}

function asArray(value: unknown): unknown[] {
  return Array.isArray(value) ? value : [];
}

function limitArray(value: unknown, limit: number): unknown[] {
  return asArray(value).slice(0, limit);
}

function renderManifest(observability: ObservabilityConfig, target: ObservabilityTarget): string {
  const collectorImage = imageReference(observability.images.collector);
  const tempoImage = imageReference(observability.images.tempo);
  return [
    target.createNamespace ? namespaceManifest(target) : "",
    allowAllNetworkPolicy(target),
    collectorConfigMap(observability, target),
    collectorDeployment(observability, target, collectorImage),
    collectorService(observability, target),
    tempoConfigMap(observability, target),
    tempoDeployment(observability, target, tempoImage),
    tempoService(observability, target),
  ].filter((item) => item.trim().length > 0).join("\n---\n");
}

function namespaceManifest(target: ObservabilityTarget): string {
  return `apiVersion: v1
kind: Namespace
metadata:
  name: ${target.namespace}
  labels:
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
    unidesk.ai/runtime-node: ${target.id}
`;
}

function allowAllNetworkPolicy(target: ObservabilityTarget): string {
  return `apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-all
  namespace: ${target.namespace}
  labels:
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
spec:
  podSelector: {}
  policyTypes:
    - Ingress
    - Egress
  ingress:
    - {}
  egress:
    - {}
`;
}

function collectorConfigMap(observability: ObservabilityConfig, target: ObservabilityTarget): string {
  const config = `receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:${observability.collector.otlp.grpcPort}
      http:
        endpoint: 0.0.0.0:${observability.collector.otlp.httpPort}
processors:
  batch: {}
exporters:
  otlp/tempo:
    endpoint: ${observability.traceBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.traceBackend.otlp.grpcPort}
    tls:
      insecure: true
extensions:
  health_check:
    endpoint: 0.0.0.0:${observability.collector.healthPort}
service:
  extensions: [health_check]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp/tempo]
`;
  return `apiVersion: v1
kind: ConfigMap
metadata:
  name: ${observability.collector.configMapName}
  namespace: ${target.namespace}
  labels:
    app.kubernetes.io/name: ${observability.collector.deploymentName}
    app.kubernetes.io/component: tracing
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
  annotations:
    unidesk.ai/spec: "${observability.metadata.spec}"
data:
  collector.yaml: |
${indent(config, 4)}
`;
}

function collectorDeployment(observability: ObservabilityConfig, target: ObservabilityTarget, image: string): string {
  return `apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${observability.collector.deploymentName}
  namespace: ${target.namespace}
  labels:
    app.kubernetes.io/name: ${observability.collector.deploymentName}
    app.kubernetes.io/component: tracing
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
spec:
  replicas: ${observability.collector.replicas}
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ${observability.collector.deploymentName}
      app.kubernetes.io/component: tracing
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ${observability.collector.deploymentName}
        app.kubernetes.io/component: tracing
        app.kubernetes.io/part-of: platform-infra
      annotations:
        unidesk.ai/spec: "${observability.metadata.spec}"
    spec:
      containers:
        - name: collector
          image: ${image}
          imagePullPolicy: ${observability.images.collector.pullPolicy}
          args:
            - --config=/etc/otelcol/collector.yaml
          ports:
            - name: otlp-grpc
              containerPort: ${observability.collector.otlp.grpcPort}
            - name: otlp-http
              containerPort: ${observability.collector.otlp.httpPort}
            - name: health
              containerPort: ${observability.collector.healthPort}
          readinessProbe:
            httpGet:
              path: /
              port: health
          volumeMounts:
            - name: config
              mountPath: /etc/otelcol/collector.yaml
              subPath: collector.yaml
              readOnly: true
      volumes:
        - name: config
          configMap:
            name: ${observability.collector.configMapName}
`;
}

function collectorService(observability: ObservabilityConfig, target: ObservabilityTarget): string {
  return `apiVersion: v1
kind: Service
metadata:
  name: ${observability.collector.serviceName}
  namespace: ${target.namespace}
  labels:
    app.kubernetes.io/name: ${observability.collector.deploymentName}
    app.kubernetes.io/component: tracing
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: ${observability.collector.deploymentName}
    app.kubernetes.io/component: tracing
  ports:
    - name: otlp-grpc
      port: ${observability.collector.otlp.grpcPort}
      targetPort: otlp-grpc
    - name: otlp-http
      port: ${observability.collector.otlp.httpPort}
      targetPort: otlp-http
    - name: health
      port: ${observability.collector.healthPort}
      targetPort: health
`;
}

function tempoConfigMap(observability: ObservabilityConfig, _target: ObservabilityTarget): string {
  const config = `server:
  http_listen_port: ${observability.traceBackend.httpPort}
distributor:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:${observability.traceBackend.otlp.grpcPort}
        http:
          endpoint: 0.0.0.0:${observability.traceBackend.otlp.httpPort}
ingester:
  trace_idle_period: 10s
  max_block_duration: 5m
compactor:
  compaction:
    block_retention: ${observability.traceBackend.storage.retention}
storage:
  trace:
    backend: local
    wal:
      path: /var/tempo/wal
    local:
      path: /var/tempo/traces
`;
  return `apiVersion: v1
kind: ConfigMap
metadata:
  name: ${observability.traceBackend.configMapName}
  namespace: ${_target.namespace}
  labels:
    app.kubernetes.io/name: ${observability.traceBackend.deploymentName}
    app.kubernetes.io/component: trace-backend
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
  annotations:
    unidesk.ai/spec: "${observability.metadata.spec}"
data:
  tempo.yaml: |
${indent(config, 4)}
`;
}

function tempoDeployment(observability: ObservabilityConfig, target: ObservabilityTarget, image: string): string {
  return `apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${observability.traceBackend.deploymentName}
  namespace: ${target.namespace}
  labels:
    app.kubernetes.io/name: ${observability.traceBackend.deploymentName}
    app.kubernetes.io/component: trace-backend
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
spec:
  replicas: ${observability.traceBackend.replicas}
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ${observability.traceBackend.deploymentName}
      app.kubernetes.io/component: trace-backend
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ${observability.traceBackend.deploymentName}
        app.kubernetes.io/component: trace-backend
        app.kubernetes.io/part-of: platform-infra
      annotations:
        unidesk.ai/spec: "${observability.metadata.spec}"
    spec:
      containers:
        - name: tempo
          image: ${image}
          imagePullPolicy: ${observability.images.tempo.pullPolicy}
          args:
            - -config.file=/etc/tempo/tempo.yaml
          ports:
            - name: http
              containerPort: ${observability.traceBackend.httpPort}
            - name: otlp-grpc
              containerPort: ${observability.traceBackend.otlp.grpcPort}
            - name: otlp-http
              containerPort: ${observability.traceBackend.otlp.httpPort}
          readinessProbe:
            httpGet:
              path: ${observability.probes.readinessPath}
              port: http
          volumeMounts:
            - name: config
              mountPath: /etc/tempo/tempo.yaml
              subPath: tempo.yaml
              readOnly: true
            - name: data
              mountPath: /var/tempo
      volumes:
        - name: config
          configMap:
            name: ${observability.traceBackend.configMapName}
        - name: data
          emptyDir: {}
`;
}

function tempoService(observability: ObservabilityConfig, target: ObservabilityTarget): string {
  return `apiVersion: v1
kind: Service
metadata:
  name: ${observability.traceBackend.serviceName}
  namespace: ${target.namespace}
  labels:
    app.kubernetes.io/name: ${observability.traceBackend.deploymentName}
    app.kubernetes.io/component: trace-backend
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: ${observability.traceBackend.deploymentName}
    app.kubernetes.io/component: trace-backend
  ports:
    - name: http
      port: ${observability.traceBackend.httpPort}
      targetPort: http
    - name: otlp-grpc
      port: ${observability.traceBackend.otlp.grpcPort}
      targetPort: otlp-grpc
    - name: otlp-http
      port: ${observability.traceBackend.otlp.httpPort}
      targetPort: otlp-http
`;
}

function applyScript(params: {
  yaml: string;
  target: ObservabilityTarget;
  dryRun: boolean;
  wait: boolean;
  collectorDeploymentName: string;
  backendDeploymentName: string;
}): string {
  const encoded = Buffer.from(params.yaml, "utf8").toString("base64");
  const dryRunArg = params.dryRun ? "--dry-run=server" : "";
  const waitDisposition = params.wait ? "skipped-short-connection-use-status" : "skipped";
  return `
set -u
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
manifest="$tmp/platform-infra-observability.yaml"
printf '%s' '${encoded}' | base64 -d > "$manifest"
kubectl apply --server-side --field-manager=${fieldManager} ${dryRunArg} -f "$manifest" >"$tmp/apply.out" 2>"$tmp/apply.err"
apply_rc=$?
collector_rollout_rc=0
backend_rollout_rc=0
wait_disposition=${waitDisposition}
python3 - "$apply_rc" "$collector_rollout_rc" "$backend_rollout_rc" "$wait_disposition" "$tmp/apply.out" "$tmp/apply.err" "$tmp/collector-rollout.out" "$tmp/collector-rollout.err" "$tmp/backend-rollout.out" "$tmp/backend-rollout.err" <<'PY'
import json, os, sys
def text(path, limit=6000):
    try:
        return open(path, encoding="utf-8", errors="replace").read()[-limit:]
    except FileNotFoundError:
        return ""
apply_rc = int(sys.argv[1])
collector_rc = int(sys.argv[2])
backend_rc = int(sys.argv[3])
payload = {
  "ok": apply_rc == 0 and collector_rc == 0 and backend_rc == 0,
  "target": "${params.target.id}",
  "namespace": "${params.target.namespace}",
  "dryRun": ${params.dryRun ? "True" : "False"},
  "apply": {"exitCode": apply_rc, "stdout": text(sys.argv[5]), "stderr": text(sys.argv[6])},
  "rollout": {
    "disposition": sys.argv[4],
    "collector": {"exitCode": collector_rc, "stdout": text(sys.argv[7]), "stderr": text(sys.argv[8])},
    "backend": {"exitCode": backend_rc, "stdout": text(sys.argv[9]), "stderr": text(sys.argv[10])},
  },
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
sys.exit(0 if payload["ok"] else 1)
PY
`;
}

function statusScript(observability: ObservabilityConfig, target: ObservabilityTarget, full: boolean, generateValidationTrace = false): string {
  const endpointsJson = JSON.stringify(observability.probes.statusEndpoints);
  return `
set -u
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
capture_json() {
  name="$1"
  shift
  "$@" >"$tmp/$name.json" 2>"$tmp/$name.err"
  echo $? >"$tmp/$name.rc"
}
capture_raw() {
  name="$1"
  shift
  "$@" >"$tmp/$name.out" 2>"$tmp/$name.err"
  echo $? >"$tmp/$name.rc"
}
capture_json namespace kubectl get namespace ${shQuote(target.namespace)} -o json
capture_json deployments kubectl -n ${shQuote(target.namespace)} get deployment ${shQuote(observability.collector.deploymentName)} ${shQuote(observability.traceBackend.deploymentName)} -o json
capture_json services kubectl -n ${shQuote(target.namespace)} get service ${shQuote(observability.collector.serviceName)} ${shQuote(observability.traceBackend.serviceName)} -o json
capture_json pods kubectl -n ${shQuote(target.namespace)} get pods -l ${shQuote(`app.kubernetes.io/name in (${observability.collector.deploymentName},${observability.traceBackend.deploymentName})`)} -o json
capture_json events kubectl -n ${shQuote(target.namespace)} get events -o json
python3 - "$tmp" '${endpointsJson.replaceAll("'", "'\"'\"'")}' <<'PY'
import json, random, socket, subprocess, sys, time, urllib.error, urllib.request
tmp = sys.argv[1]
endpoints = json.loads(sys.argv[2])
namespace = "${target.namespace}"
def read(path, binary=False, limit=8000):
    try:
        mode = "rb" if binary else "r"
        with open(path, mode, encoding=None if binary else "utf-8", errors=None if binary else "replace") as fh:
            data = fh.read()
        if binary:
            data = data.decode("utf-8", errors="replace")
        return data[-limit:]
    except FileNotFoundError:
        return ""
def rc(name):
    try:
        return int(read(f"{tmp}/{name}.rc").strip() or "1")
    except ValueError:
        return 1
def parsed_json(name):
    raw = read(f"{tmp}/{name}.json", limit=1000000)
    try:
        return json.loads(raw) if raw else None
    except Exception:
        return None
def compact_section(name):
    return {
        "exitCode": rc(name),
        "stderrTail": read(f"{tmp}/{name}.err", limit=2000),
        "json": parsed_json(name),
    }
probe_results = []
for ep in endpoints:
    path = ep.get("path") or "/"
    if not path.startswith("/"):
        path = "/" + path
    proxy_path = f"/api/v1/namespaces/{namespace}/services/http:{ep['service']}:{ep['portName']}/proxy{path}"
    proc = subprocess.run(["kubectl", "get", "--raw", proxy_path], text=True, capture_output=True, timeout=20)
    probe_results.append({
        "name": ep["name"],
        "service": ep["service"],
        "portName": ep["portName"],
        "path": path,
        "exitCode": proc.returncode,
        "stdoutTail": proc.stdout[-1000:],
        "stderrTail": proc.stderr[-1000:],
        "ok": proc.returncode == 0,
    })
def free_port():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(("127.0.0.1", 0))
    port = sock.getsockname()[1]
    sock.close()
    return port
def parse_body(raw):
    try:
        return json.loads(raw) if raw else None
    except Exception:
        return raw[-2000:]
def http_request(method, url, body=None, timeout=10):
    data = None
    headers = {}
    if body is not None:
        data = json.dumps(body).encode("utf-8")
        headers["Content-Type"] = "application/json"
    req = urllib.request.Request(url, data=data, headers=headers, method=method)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read(200000).decode("utf-8", errors="replace")
            status = int(getattr(resp, "status", 0) or 0)
            return {"ok": 200 <= status < 300, "status": status, "body": parse_body(raw)}
    except urllib.error.HTTPError as exc:
        raw = exc.read(200000).decode("utf-8", errors="replace")
        return {"ok": False, "status": exc.code, "body": parse_body(raw)}
    except Exception as exc:
        return {"ok": False, "error": type(exc).__name__ + ": " + str(exc)}
def wait_proxy(base_url):
    last = None
    for _ in range(30):
        last = http_request("GET", base_url + "/version", timeout=1)
        if last.get("ok") is True:
            return last
        time.sleep(0.2)
    return last or {"ok": False, "error": "kubectl proxy did not answer"}
def compact_http(result):
    if not isinstance(result, dict):
        return {"ok": False, "error": "missing-result"}
    compact = {"ok": result.get("ok") is True}
    if "status" in result:
        compact["status"] = result.get("status")
    if "error" in result:
        compact["error"] = result.get("error")
    body = result.get("body")
    if isinstance(body, dict):
        compact["bodyKeys"] = sorted(list(body.keys()))[:12]
    elif isinstance(body, str) and body:
        compact["bodyTail"] = body[-1000:]
    return compact
def validation_trace_payload(trace_id, span_id):
    now = time.time_ns()
    return {
        "resourceSpans": [{
            "resource": {"attributes": [
                {"key": "service.name", "value": {"stringValue": "unidesk-observability-validate"}},
                {"key": "deployment.environment", "value": {"stringValue": "${target.id}"}},
                {"key": "unidesk.node", "value": {"stringValue": "${target.id}"}},
                {"key": "k8s.namespace.name", "value": {"stringValue": namespace}},
            ]},
            "scopeSpans": [{
                "scope": {"name": "unidesk-cli-platform-infra-observability"},
                "spans": [{
                    "traceId": trace_id,
                    "spanId": span_id,
                    "name": "unidesk.observability.validate",
                    "kind": 1,
                    "startTimeUnixNano": str(now),
                    "endTimeUnixNano": str(now + 1000000),
                    "attributes": [
                        {"key": "unidesk.issue", "value": {"stringValue": "489"}},
                        {"key": "unidesk.spec", "value": {"stringValue": "${observability.metadata.spec}"}},
                        {"key": "unidesk.validation", "value": {"stringValue": "platform-infra-observability"}},
                    ],
                    "status": {"code": 1},
                }],
            }],
        }],
    }
def generate_test_trace():
    trace_id = ("%032x" % random.getrandbits(128))
    span_id = ("%016x" % random.getrandbits(64))
    port = free_port()
    base_url = "http://127.0.0.1:%s" % port
    stdout_path = f"{tmp}/kubectl-proxy.out"
    stderr_path = f"{tmp}/kubectl-proxy.err"
    with open(stdout_path, "w", encoding="utf-8") as stdout, open(stderr_path, "w", encoding="utf-8") as stderr:
        proc = subprocess.Popen(
            ["kubectl", "proxy", "--address=127.0.0.1", "--port", str(port), "--accept-hosts=^127\\\\.0\\\\.0\\\\.1$"],
            stdout=stdout,
            stderr=stderr,
            text=True,
        )
    try:
        proxy_ready = wait_proxy(base_url)
        if proxy_ready.get("ok") is not True:
            return {
                "ok": False,
                "stage": "proxy-start",
                "traceId": trace_id,
                "proxy": compact_http(proxy_ready),
                "proxyStderrTail": read(stderr_path, limit=2000),
            }
        collector_url = base_url + "/api/v1/namespaces/%s/services/http:${observability.collector.serviceName}:otlp-http/proxy/v1/traces" % namespace
        tempo_url = base_url + "/api/v1/namespaces/%s/services/http:${observability.traceBackend.serviceName}:http/proxy/api/traces/%s" % (namespace, trace_id)
        ingest = http_request("POST", collector_url, validation_trace_payload(trace_id, span_id), timeout=15)
        query = None
        if ingest.get("ok") is True:
            for _ in range(20):
                query = http_request("GET", tempo_url, timeout=10)
                if query.get("ok") is True:
                    break
                time.sleep(1)
        return {
            "ok": ingest.get("ok") is True and isinstance(query, dict) and query.get("ok") is True,
            "stage": "query" if ingest.get("ok") is True else "ingest",
            "traceId": trace_id,
            "spanId": span_id,
            "collectorEndpoint": "${observability.collector.serviceName}:otlp-http",
            "backendEndpoint": "${observability.traceBackend.serviceName}:http",
            "ingest": compact_http(ingest),
            "query": compact_http(query),
            "valuesPrinted": False,
        }
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=3)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait(timeout=3)
runtime_ready = rc("namespace") == 0 and rc("deployments") == 0 and rc("services") == 0 and all(item["ok"] for item in probe_results)
validation_trace = generate_test_trace() if (${generateValidationTrace ? "True" : "False"} and runtime_ready) else None
payload = {
    "ok": runtime_ready and (validation_trace is None or validation_trace.get("ok") is True),
    "target": "${target.id}",
    "namespace": namespace,
    "sections": {
        "namespace": compact_section("namespace"),
        "deployments": compact_section("deployments"),
        "services": compact_section("services"),
        "pods": compact_section("pods"),
        "events": compact_section("events"),
    },
    "probes": probe_results,
    "validationTrace": validation_trace,
}
print(json.dumps(payload, ensure_ascii=False, indent=2 if ${full ? "True" : "False"} else None))
sys.exit(0 if payload["ok"] else 1)
PY
`;
}

function traceScript(observability: ObservabilityConfig, target: ObservabilityTarget, tracePath: string, options: TraceOptions): string {
  const proxyPath = `/api/v1/namespaces/${target.namespace}/services/http:${observability.traceBackend.serviceName}:http/proxy${tracePath}`;
  const grepLiteral = options.grep === null ? "None" : JSON.stringify(options.grep);
  return `
set -u
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
kubectl get --raw ${shQuote(proxyPath)} >"$tmp/trace.out" 2>"$tmp/trace.err"
rc=$?
python3 - "$rc" "$tmp/trace.out" "$tmp/trace.err" <<'PY'
import collections, json, sys
FULL = ${options.full ? "True" : "False"}
RAW = ${options.raw ? "True" : "False"}
GREP = ${grepLiteral}
LIMIT = ${options.limit}
IMPORTANT_ATTRS = [
    "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
    "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
    "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
    "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
    "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
    "lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
    "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
    "terminalStatus", "phase", "message", "http.route", "http.status_code",
    "http.response.status_code", "http.method", "http.request.method",
    "http.response.status_class", "workbench.ui.event_type",
    "workbench.ui.scope", "workbench.ui.state", "workbench.ui.reason",
    "workbench.ui.route", "workbench.ui.outcome", "workbench.ui.value_ms",
    "workbench.ui.visibility", "workbench.ui.event_name",
    "workbench.ui.activity_idle_ms", "workbench.ui.activity_waiting_for",
    "workbench.ui.activity_last_event_label",
    "workbench.ui.resource_duration_ms",
    "workbench.ui.resource_fetch_to_request_ms",
    "workbench.ui.resource_request_wait_ms",
    "workbench.ui.resource_response_transfer_ms",
    "workbench.ui.resource_transfer_size",
    "workbench.ui.resource_encoded_body_size",
    "workbench.ui.resource_decoded_body_size",
    "workbench.ui.resource_next_hop_protocol",
    "workbench.ui.resource_server_timing",
    "workbench.session_id", "workbench.trace_id", "workbench.turn_id",
    "workbench.read_model.route", "workbench.read_model.count",
    "workbench.read_model.family", "workbench.read_model.status",
    "workbench.read_model.reason",
    "db.system", "db.operation.name", "db.sql.table", "db.query.arg_count",
    "db.index.expected", "db.pool.max_open", "db.pool.open_connections",
    "db.pool.in_use", "db.pool.idle", "db.pool.wait_count",
    "db.pool.wait_duration_ms", "db.pool.max_idle_closed",
    "db.pool.max_lifetime_closed",
    "hwlab.runtime.stage", "hwlab.http.stage", "hwlab.http.phase",
    "hwlab.http.phase.outcome", "hwlab.live_builds.service_id",
    "hwlab.live_builds.service_kind", "hwlab.live_builds.external",
    "hwlab.live_builds.deploy_manifest_status",
    "hwlab.live_builds.artifact_catalog_status",
    "error.code", "error.category", "error.layer",
    "stage", "causeStage", "causeCode",
    "eventType", "runnerId", "attemptId", "backendProfile",
    "sourceCommit", "jobName", "podName", "logPath",
    "toolName", "type", "itemType", "itemId", "status", "exitCode",
    "durationMs", "cwd", "processId", "command", "commandFingerprint",
    "outputSummary", "outputBytes", "outputTruncated", "storageKind",
    "storagePvcName", "storagePvcPhase", "resumeMode", "rolloutId",
    "returnedEvents", "sinceSeq", "afterSeq", "limit", "fromSeq", "toSeq",
    "totalEvents", "hasMore", "fullTraceLoaded", "rawEventCount",
    "maxSeq", "traceLastSeq", "endSeq", "commandFiltered",
    "error", "error.message", "exception.type", "exception.message",
    "valuesPrinted",
]
IMPORTANT_NAMES = {
    "durable_admission", "billing_preflight", "agentrun_dispatch",
    "projection_write", "trace_events_read", "turn_status_read",
    "run_created", "runner_job_created", "runner_job_status",
    "command_result", "projection_sync",
}
def text(path, limit=12000):
    try:
        return open(path, encoding="utf-8", errors="replace").read()[-limit:]
    except FileNotFoundError:
        return ""
def read_all(path):
    try:
        return open(path, encoding="utf-8", errors="replace").read()
    except FileNotFoundError:
        return ""
def attr_value(value):
    if not isinstance(value, dict):
        return value
    inner = value.get("value", value)
    if not isinstance(inner, dict):
        return inner
    for key in ("stringValue", "intValue", "doubleValue", "boolValue"):
        if key in inner:
            return inner.get(key)
    if "arrayValue" in inner:
        return "<array>"
    if "kvlistValue" in inner:
        return "<kvlist>"
    if "bytesValue" in inner:
        return "<bytes>"
    return inner
def attrs_to_dict(attrs):
    if not isinstance(attrs, list):
        return {}
    output = {}
    for item in attrs:
        if not isinstance(item, dict):
            continue
        key = item.get("key")
        if isinstance(key, str):
            output[key] = attr_value(item.get("value"))
    return output
def nanos_to_ms(start, end):
    try:
        return round((int(end) - int(start)) / 1000000, 3)
    except Exception:
        return None
def span_status_code(status):
    if not isinstance(status, dict):
        return None
    return status.get("code")
def selected_attrs(attrs):
    output = {}
    for key in IMPORTANT_ATTRS:
        if key in attrs:
            output[key] = attrs[key]
    return output
def is_error_span(span, attrs):
    status = span.get("status") if isinstance(span, dict) else {}
    code = span_status_code(status)
    name = str(span.get("name", "")).lower() if isinstance(span, dict) else ""
    failure = str(attrs.get("failureKind", "")).strip()
    terminal = str(attrs.get("terminalStatus", "")).strip()
    return code in ("STATUS_CODE_ERROR", 2) or "error" in name or bool(failure) or terminal in ("failed", "blocked")
def compact_span(span, service, resource_attrs, scope_name):
    attrs = attrs_to_dict(span.get("attributes"))
    status = span.get("status") if isinstance(span.get("status"), dict) else {}
    item = {
        "name": span.get("name"),
        "service": service,
        "scope": scope_name,
        "statusCode": span_status_code(status),
        "statusMessage": status.get("message"),
        "durationMs": nanos_to_ms(span.get("startTimeUnixNano"), span.get("endTimeUnixNano")),
        "attributes": selected_attrs(attrs),
    }
    if "deployment.environment" in resource_attrs:
        item["environment"] = resource_attrs.get("deployment.environment")
    if "git.commit" in resource_attrs:
        item["gitCommit"] = resource_attrs.get("git.commit")
    return item
def batches_from_body(body):
    if not isinstance(body, dict):
        return []
    for key in ("batches", "resourceSpans"):
        value = body.get(key)
        if isinstance(value, list):
            return value
    return []
def scope_spans_from_batch(batch):
    if not isinstance(batch, dict):
        return []
    for key in ("scopeSpans", "instrumentationLibrarySpans"):
        value = batch.get(key)
        if isinstance(value, list):
            return value
    return []
def grep_matches(item):
    if GREP is None:
        return False
    return GREP.lower() in json.dumps(item, ensure_ascii=False, sort_keys=True).lower()
def key_signature(item):
    attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
    return json.dumps({
        "name": item.get("name"),
        "service": item.get("service"),
        "statusCode": item.get("statusCode"),
        "runId": attrs.get("runId"),
        "commandId": attrs.get("commandId"),
        "failureKind": attrs.get("failureKind"),
        "terminalStatus": attrs.get("terminalStatus"),
        "phase": attrs.get("phase"),
        "http.route": attrs.get("http.route"),
    }, ensure_ascii=False, sort_keys=True)
body = read_all(sys.argv[2])
body_bytes = len(body.encode("utf-8"))
try:
    parsed = json.loads(body) if body else None
except Exception:
    parsed = body[-12000:]
if not isinstance(parsed, dict):
    payload = {
      "ok": int(sys.argv[1]) == 0,
      "path": "${tracePath}",
      "proxyPath": "${proxyPath}",
      "bodyBytes": body_bytes,
      "parseOk": False,
      "bodyTail": text(sys.argv[2], 12000),
      "stderrTail": text(sys.argv[3], 4000),
    }
    print(json.dumps(payload, ensure_ascii=False, indent=2))
    sys.exit(0 if payload["ok"] else 1)
spans = []
services = set()
business_trace_ids = set()
identity_values = collections.defaultdict(set)
name_counts = collections.Counter()
error_spans = []
matched_spans = []
key_spans = []
seen_key_spans = set()
for batch in batches_from_body(parsed):
    resource = batch.get("resource") if isinstance(batch.get("resource"), dict) else {}
    resource_attrs = attrs_to_dict(resource.get("attributes"))
    service = resource_attrs.get("service.name") or "<unknown-service>"
    services.add(service)
    for scope_span in scope_spans_from_batch(batch):
        scope = scope_span.get("scope") if isinstance(scope_span.get("scope"), dict) else {}
        scope_name = scope.get("name")
        raw_spans = scope_span.get("spans") if isinstance(scope_span.get("spans"), list) else []
        for span in raw_spans:
            if not isinstance(span, dict):
                continue
            item = compact_span(span, service, resource_attrs, scope_name)
            attrs = item.get("attributes", {})
            if isinstance(attrs.get("traceId"), str):
                business_trace_ids.add(attrs.get("traceId"))
            if isinstance(attrs, dict):
                for key in ("runId", "commandId", "runnerJobId", "runnerId", "backendProfile", "sourceCommit", "jobName", "podName"):
                    value = attrs.get(key)
                    if value not in (None, ""):
                        identity_values[key].add(str(value))
            name = str(item.get("name") or "")
            name_counts[name] += 1
            spans.append(item)
            error = is_error_span(span, attrs if isinstance(attrs, dict) else {})
            matched = grep_matches(item)
            if error:
                error_spans.append(item)
            if matched:
                matched_spans.append(item)
            if error or name in IMPORTANT_NAMES or name.startswith("runner_error.") or name.startswith("runner_terminal.") or name.startswith("workbench.api_request."):
                signature = key_signature(item)
                if signature not in seen_key_spans:
                    seen_key_spans.add(signature)
                    key_spans.append(item)
display_source = matched_spans if GREP is not None else key_spans
payload = {
  "ok": int(sys.argv[1]) == 0,
  "path": "${tracePath}",
  "proxyPath": "${proxyPath}",
  "bodyBytes": body_bytes,
  "parseOk": True,
  "traceFound": len(spans) > 0,
  "spanCount": len(spans),
  "serviceCount": len(services),
  "services": sorted(services),
  "businessTraceIds": sorted(business_trace_ids)[:20],
  "identity": {key: sorted(values)[:8] for key, values in identity_values.items() if values},
  "errorSpanCount": len(error_spans),
  "matchedSpanCount": len(matched_spans) if GREP is not None else None,
  "grep": GREP,
  "limit": LIMIT,
  "spanNameCounts": [{"name": name, "count": count} for name, count in name_counts.most_common(20)],
  "errorSpans": error_spans[:LIMIT],
  "spans": (spans if FULL and GREP is None else display_source)[:LIMIT],
  "truncated": {
    "errorSpans": len(error_spans) > LIMIT,
    "spans": (len(spans) if FULL and GREP is None else len(display_source)) > LIMIT,
  },
  "next": {
    "fullSummary": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id ${options.traceId ?? "<traceId>"} --full --limit 200",
    "grepProviderStream": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id ${options.traceId ?? "<traceId>"} --grep provider-stream-disconnected --limit 40",
    "raw": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id ${options.traceId ?? "<traceId>"} --raw",
  },
  "stderrTail": text(sys.argv[3], 4000),
}
if RAW:
    payload["rawBody"] = parsed
print(json.dumps(payload, ensure_ascii=False, indent=2))
sys.exit(0 if payload["ok"] else 1)
PY
`;
}

function searchScript(observability: ObservabilityConfig, target: ObservabilityTarget, searchPath: string, options: SearchOptions): string {
  const proxyPrefix = `/api/v1/namespaces/${target.namespace}/services/http:${observability.traceBackend.serviceName}:http/proxy`;
  const searchProxyPath = `${proxyPrefix}${searchPath}`;
  const grepLiteral = options.grep === null ? "None" : JSON.stringify(options.grep);
  const effectiveQuery = inferSearchTempoQuery(options);
  const queryLiteral = effectiveQuery === null ? "None" : JSON.stringify(effectiveQuery);
  const pathLiteral = options.path === null ? "None" : JSON.stringify(options.path);
  const statusLiteral = options.status === null ? "None" : String(options.status);
  return `
set -u
python3 - <<'PY'
import collections, json, re, subprocess, time

SEARCH_PROXY_PATH = ${JSON.stringify(searchProxyPath)}
TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)}
TRACE_PATH_TEMPLATE = ${JSON.stringify(observability.probes.traceQueryPathTemplate)}
FULL = ${options.full ? "True" : "False"}
RAW = ${options.raw ? "True" : "False"}
GREP = ${grepLiteral}
QUERY = ${queryLiteral}
PATH_FILTER = ${pathLiteral}
STATUS_FILTER = ${statusLiteral}
LIMIT = ${options.limit}
CANDIDATE_LIMIT = ${options.candidateLimit}
DEADLINE = time.time() + 50
BUSINESS_TRACE_GREP = bool(GREP is not None and re.search(r"\\btrc_[A-Za-z0-9_-]+\\b", GREP))
IMPORTANT_ATTRS = [
    "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
    "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
    "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
    "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
    "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
    "lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
    "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
    "terminalStatus", "phase", "message", "http.route", "http.status_code",
    "http.response.status_code", "http.method", "http.request.method",
    "http.response.status_class", "workbench.ui.event_type",
    "workbench.ui.scope", "workbench.ui.state", "workbench.ui.reason",
    "workbench.ui.route", "workbench.ui.outcome", "workbench.ui.value_ms",
    "workbench.ui.visibility", "workbench.ui.event_name",
    "workbench.ui.activity_idle_ms", "workbench.ui.activity_waiting_for",
    "workbench.ui.activity_last_event_label",
    "workbench.ui.resource_duration_ms",
    "workbench.ui.resource_fetch_to_request_ms",
    "workbench.ui.resource_request_wait_ms",
    "workbench.ui.resource_response_transfer_ms",
    "workbench.ui.resource_transfer_size",
    "workbench.ui.resource_encoded_body_size",
    "workbench.ui.resource_decoded_body_size",
    "workbench.ui.resource_next_hop_protocol",
    "workbench.ui.resource_server_timing",
    "workbench.session_id", "workbench.trace_id", "workbench.turn_id",
    "workbench.read_model.route", "workbench.read_model.count",
    "workbench.read_model.family", "workbench.read_model.status",
    "workbench.read_model.reason",
    "http.target", "http.url", "url.path",
    "db.system", "db.operation.name", "db.sql.table", "db.query.arg_count",
    "db.index.expected", "db.pool.max_open", "db.pool.open_connections",
    "db.pool.in_use", "db.pool.idle", "db.pool.wait_count",
    "db.pool.wait_duration_ms", "db.pool.max_idle_closed",
    "db.pool.max_lifetime_closed",
    "hwlab.runtime.stage", "hwlab.http.stage", "hwlab.http.phase",
    "hwlab.http.phase.outcome", "hwlab.live_builds.service_id",
    "hwlab.live_builds.service_kind", "hwlab.live_builds.external",
    "hwlab.live_builds.deploy_manifest_status",
    "hwlab.live_builds.artifact_catalog_status",
    "error.code", "error.category", "error.layer",
    "stage", "causeStage", "causeCode",
    "eventType", "runnerId", "attemptId", "backendProfile",
    "sourceCommit", "jobName", "podName", "logPath",
    "toolName", "type", "itemType", "itemId", "status", "exitCode",
    "durationMs", "cwd", "processId", "command", "commandFingerprint",
    "outputSummary", "outputBytes", "outputTruncated", "storageKind",
    "storagePvcName", "storagePvcPhase", "resumeMode", "rolloutId",
    "returnedEvents", "sinceSeq", "afterSeq", "limit", "fromSeq", "toSeq",
    "totalEvents", "hasMore", "fullTraceLoaded", "rawEventCount",
    "maxSeq", "traceLastSeq", "endSeq", "commandFiltered",
    "error", "error.message", "exception.type", "exception.message",
    "valuesPrinted",
]

def run_kubectl(proxy_path, timeout=10):
    try:
        proc = subprocess.run(
            ["kubectl", "get", "--raw", proxy_path],
            capture_output=True,
            text=True,
            timeout=timeout,
        )
        return proc.returncode, proc.stdout, proc.stderr
    except subprocess.TimeoutExpired as exc:
        return 124, exc.stdout or "", "timeout while querying tempo"
    except Exception as exc:
        return 125, "", str(exc)

def parse_json(body):
    try:
        return json.loads(body) if body else None
    except Exception:
        return None

def attr_value(value):
    if not isinstance(value, dict):
        return value
    inner = value.get("value", value)
    if not isinstance(inner, dict):
        return inner
    for key in ("stringValue", "intValue", "doubleValue", "boolValue"):
        if key in inner:
            return inner.get(key)
    if "arrayValue" in inner:
        return "<array>"
    if "kvlistValue" in inner:
        return "<kvlist>"
    if "bytesValue" in inner:
        return "<bytes>"
    return inner

def attrs_to_dict(attrs):
    if not isinstance(attrs, list):
        return {}
    output = {}
    for item in attrs:
        if not isinstance(item, dict):
            continue
        key = item.get("key")
        if isinstance(key, str):
            output[key] = attr_value(item.get("value"))
    return output

def selected_attrs(attrs):
    output = {}
    for key in IMPORTANT_ATTRS:
        if key in attrs:
            output[key] = attrs[key]
    return output

def nanos_to_ms(start, end):
    try:
        return round((int(end) - int(start)) / 1000000, 3)
    except Exception:
        return None

def span_status_code(status):
    if not isinstance(status, dict):
        return None
    return status.get("code")

def to_int(value):
    if value is None:
        return None
    if isinstance(value, bool):
        return 1 if value else 0
    if isinstance(value, int):
        return value
    if isinstance(value, float) and value.is_integer():
        return int(value)
    if isinstance(value, str):
        try:
            return int(value.strip())
        except Exception:
            return None
    return None

def is_error_span(span, attrs):
    status = span.get("status") if isinstance(span, dict) else {}
    code = span_status_code(status)
    name = str(span.get("name", "")).lower() if isinstance(span, dict) else ""
    failure = str(attrs.get("failureKind", "")).strip()
    terminal = str(attrs.get("terminalStatus", "")).strip()
    http_status = to_int(attrs.get("http.response.status_code"))
    if http_status is None:
        http_status = to_int(attrs.get("http.status_code"))
    return code in ("STATUS_CODE_ERROR", 2) or "error" in name or bool(failure) or terminal in ("failed", "blocked") or (http_status is not None and http_status >= 500)

def batches_from_body(body):
    if not isinstance(body, dict):
        return []
    for key in ("batches", "resourceSpans"):
        value = body.get(key)
        if isinstance(value, list):
            return value
    return []

def scope_spans_from_batch(batch):
    if not isinstance(batch, dict):
        return []
    for key in ("scopeSpans", "instrumentationLibrarySpans"):
        value = batch.get(key)
        if isinstance(value, list):
            return value
    return []

def compact_span(span, service, resource_attrs, scope_name):
    attrs = attrs_to_dict(span.get("attributes"))
    status = span.get("status") if isinstance(span.get("status"), dict) else {}
    item = {
        "name": span.get("name"),
        "service": service,
        "scope": scope_name,
        "statusCode": span_status_code(status),
        "statusMessage": status.get("message"),
        "durationMs": nanos_to_ms(span.get("startTimeUnixNano"), span.get("endTimeUnixNano")),
        "attributes": selected_attrs(attrs),
    }
    if "deployment.environment" in resource_attrs:
        item["environment"] = resource_attrs.get("deployment.environment")
    if "git.commit" in resource_attrs:
        item["gitCommit"] = resource_attrs.get("git.commit")
    return item

def grep_matches_text(text):
    return GREP is not None and GREP.lower() in text.lower()

def grep_matches_item(item):
    return GREP is not None and grep_matches_text(json.dumps(item, ensure_ascii=False, sort_keys=True))

def span_matches_filters(item):
    attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
    if PATH_FILTER is not None:
        candidates = [
            attrs.get("http.route"),
            attrs.get("http.target"),
            attrs.get("http.url"),
            attrs.get("url.path"),
            item.get("name"),
        ]
        if not any(isinstance(value, str) and (value == PATH_FILTER or value.startswith(PATH_FILTER + "?")) for value in candidates):
            return False
    if STATUS_FILTER is not None:
        status_value = to_int(attrs.get("http.response.status_code"))
        if status_value is None:
            status_value = to_int(attrs.get("http.status_code"))
        if status_value != STATUS_FILTER:
            return False
    return True

def extract_traces(search_body):
    if not isinstance(search_body, dict):
        return []
    value = search_body.get("traces")
    if isinstance(value, list):
        return value
    value = search_body.get("data")
    if isinstance(value, list):
        return value
    return []

def trace_id_from_meta(meta):
    if not isinstance(meta, dict):
        return None
    for key in ("traceID", "traceId", "trace_id"):
        value = meta.get(key)
        normalized = normalize_trace_id(value)
        if normalized is not None:
            return normalized
    return None

def normalize_trace_id(value):
    if not isinstance(value, str):
        return None
    text = value.strip().lower()
    if text.startswith("0x"):
        text = text[2:]
    if not text or len(text) > 32:
        return None
    if any(ch not in "0123456789abcdef" for ch in text):
        return None
    return text.rjust(32, "0")

def compact_meta(meta):
    if not isinstance(meta, dict):
        return {}
    output = {}
    for key in ("traceID", "traceId", "rootServiceName", "rootTraceName", "startTimeUnixNano", "durationMs", "spanSet", "spanSets"):
        if key in meta:
            output[key] = meta[key]
    return output

def trace_summary(trace_id, meta, body, rc, stderr):
    parsed = parse_json(body)
    raw_matched = grep_matches_text(body)
    matching_active = GREP is not None or PATH_FILTER is not None or STATUS_FILTER is not None
    if not isinstance(parsed, dict):
        return {
            "traceId": trace_id,
            "traceCommand": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % trace_id,
            "meta": compact_meta(meta),
            "queryOk": rc == 0,
            "parseOk": False,
            "rawMatched": raw_matched,
            "matchingActive": matching_active,
            "bodyBytes": len(body.encode("utf-8")),
            "stderrTail": (stderr or "")[-1000:],
        }
    services = set()
    business_trace_ids = set()
    name_counts = collections.Counter()
    spans = []
    error_spans = []
    matched_spans = []
    for batch in batches_from_body(parsed):
        resource = batch.get("resource") if isinstance(batch.get("resource"), dict) else {}
        resource_attrs = attrs_to_dict(resource.get("attributes"))
        service = resource_attrs.get("service.name") or "<unknown-service>"
        services.add(service)
        for scope_span in scope_spans_from_batch(batch):
            scope = scope_span.get("scope") if isinstance(scope_span.get("scope"), dict) else {}
            scope_name = scope.get("name")
            raw_spans = scope_span.get("spans") if isinstance(scope_span.get("spans"), list) else []
            for span in raw_spans:
                if not isinstance(span, dict):
                    continue
                item = compact_span(span, service, resource_attrs, scope_name)
                attrs = item.get("attributes", {})
                if isinstance(attrs, dict) and isinstance(attrs.get("traceId"), str):
                    business_trace_ids.add(attrs.get("traceId"))
                name = str(item.get("name") or "")
                name_counts[name] += 1
                spans.append(item)
                if is_error_span(span, attrs if isinstance(attrs, dict) else {}):
                    error_spans.append(item)
                if span_matches_filters(item) and (GREP is None or grep_matches_item(item)):
                    matched_spans.append(item)
    return {
        "traceId": trace_id,
        "traceCommand": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % trace_id,
        "grepCommand": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep %s --limit 80" % (trace_id, json.dumps(GREP) if GREP is not None else "<text>"),
        "meta": compact_meta(meta),
        "queryOk": rc == 0,
        "parseOk": True,
        "rawMatched": raw_matched,
        "matchingActive": matching_active,
        "bodyBytes": len(body.encode("utf-8")),
        "spanCount": len(spans),
        "serviceCount": len(services),
        "services": sorted(services),
        "businessTraceIds": sorted(business_trace_ids)[:20],
        "errorSpanCount": len(error_spans),
        "matchedSpanCount": len(matched_spans) if matching_active else None,
        "spanNameCounts": [{"name": name, "count": count} for name, count in name_counts.most_common(10)],
        "errorSpans": error_spans[:LIMIT] if FULL else error_spans[: min(3, LIMIT)],
        "matchedSpans": matched_spans[:LIMIT],
        "stderrTail": (stderr or "")[-1000:],
    }

def rich_trace_rank(summary):
    services = set(summary.get("services") or [])
    service_score = 0
    for service in ("agentrun-runner", "agentrun-manager", "hwlab-cloud-api"):
        if service in services:
            service_score += 1
    return (
        service_score,
        int(summary.get("spanCount") or 0),
        int(summary.get("errorSpanCount") or 0),
        int(summary.get("matchedSpanCount") or 0),
    )

search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15)
search_parsed = parse_json(search_body)
search_parse_ok = isinstance(search_parsed, dict)
trace_metas = extract_traces(search_parsed)
candidate_trace_ids = []
seen = set()
for meta in trace_metas:
    trace_id = trace_id_from_meta(meta)
    if trace_id is None or trace_id in seen:
        continue
    seen.add(trace_id)
    candidate_trace_ids.append((trace_id, meta))
    if len(candidate_trace_ids) >= CANDIDATE_LIMIT:
        break

matched = []
scanned = []
scan_stopped = None
for trace_id, meta in candidate_trace_ids:
    if time.time() > DEADLINE:
        scan_stopped = "deadline"
        break
    trace_path = TRACE_PATH_TEMPLATE.replace("{{traceId}}", trace_id)
    trace_rc, trace_body, trace_err = run_kubectl(TRACE_PROXY_PREFIX + trace_path, timeout=8)
    summary = trace_summary(trace_id, meta, trace_body, trace_rc, trace_err)
    scanned.append(summary)
    if summary.get("matchingActive") is not True or summary.get("rawMatched") is True or (summary.get("matchedSpanCount") or 0) > 0:
        matched.append(summary)
    if len(matched) >= LIMIT and GREP is not None and not BUSINESS_TRACE_GREP:
        break
if BUSINESS_TRACE_GREP:
    matched.sort(key=rich_trace_rank, reverse=True)

payload = {
    "ok": search_rc == 0 and search_parse_ok,
    "searchPath": "${searchPath}",
    "searchProxyPath": SEARCH_PROXY_PATH,
    "grep": GREP,
    "tempoQuery": QUERY,
    "pathFilter": PATH_FILTER,
    "statusFilter": STATUS_FILTER,
    "businessTraceSearch": BUSINESS_TRACE_GREP,
    "limit": LIMIT,
    "candidateLimit": CANDIDATE_LIMIT,
    "searchParseOk": search_parse_ok,
    "candidateTraceCount": len(candidate_trace_ids),
    "scannedTraceCount": len(scanned),
    "matchedTraceCount": len(matched),
    "scanStopped": scan_stopped,
    "matchingActive": GREP is not None or PATH_FILTER is not None or STATUS_FILTER is not None,
    "traces": matched[:LIMIT] if (GREP is not None or PATH_FILTER is not None or STATUS_FILTER is not None) else scanned[:LIMIT],
    "truncated": {
        "candidateTraces": len(trace_metas) > len(candidate_trace_ids),
        "matchedTraces": len(matched) > LIMIT,
    },
    "next": {
        "expandWindow": "bun scripts/cli.ts platform-infra observability search --target ${target.id} --grep <text> --lookback-minutes 1440 --candidate-limit 200 --limit 40",
        "pathStatus": "bun scripts/cli.ts platform-infra observability search --target ${target.id} --path /v1/workbench/sessions --status 502 --lookback-minutes 1440 --candidate-limit 200 --limit 40",
        "traceDetail": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id <traceId> --grep <text> --limit 80",
        "raw": "bun scripts/cli.ts platform-infra observability search --target ${target.id} --grep <text> --raw",
    },
    "searchStderrTail": (search_err or "")[-2000:],
}
if RAW:
    payload["rawSearchBody"] = search_parsed if search_parse_ok else search_body[-12000:]
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 1)
PY
`;
}

function diagnoseCodeAgentScript(observability: ObservabilityConfig, target: ObservabilityTarget, searchPath: string | null, options: DiagnoseCodeAgentOptions): string {
  const proxyPrefix = `/api/v1/namespaces/${target.namespace}/services/http:${observability.traceBackend.serviceName}:http/proxy`;
  const searchProxyPath = searchPath === null ? null : `${proxyPrefix}${searchPath}`;
  const traceIdLiteral = options.traceId === null ? "None" : JSON.stringify(options.traceId);
  const businessTraceIdLiteral = options.businessTraceId === null ? "None" : JSON.stringify(options.businessTraceId);
  const searchPathLiteral = searchPath === null ? "None" : JSON.stringify(searchPath);
  const searchProxyPathLiteral = searchProxyPath === null ? "None" : JSON.stringify(searchProxyPath);
  const businessTraceIdForNext = options.businessTraceId ?? "<trc_...>";
  return `
set -u
python3 - <<'PY'
import collections, json, re, subprocess, time, urllib.parse

BUSINESS_TRACE_ID = ${businessTraceIdLiteral}
TRACE_ID = ${traceIdLiteral}
TARGET_ID = ${JSON.stringify(target.id)}
SEARCH_PATH = ${searchPathLiteral}
SEARCH_PROXY_PATH = ${searchProxyPathLiteral}
TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)}
TRACE_PATH_TEMPLATE = ${JSON.stringify(observability.probes.traceQueryPathTemplate)}
FULL = ${options.full ? "True" : "False"}
RAW = ${options.raw ? "True" : "False"}
LIMIT = ${options.limit}
CANDIDATE_LIMIT = ${options.candidateLimit}
DEADLINE = time.time() + 50
IMPORTANT_ATTRS = [
    "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
    "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
    "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
    "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
    "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
    "lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
    "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
    "terminalStatus", "phase", "message", "http.route", "http.status_code",
    "http.response.status_code", "http.method", "http.request.method",
    "http.response.status_class", "workbench.ui.event_type",
    "workbench.ui.scope", "workbench.ui.state", "workbench.ui.reason",
    "workbench.ui.route", "workbench.ui.outcome", "workbench.ui.value_ms",
    "workbench.ui.visibility", "workbench.ui.event_name",
    "workbench.ui.activity_idle_ms", "workbench.ui.activity_waiting_for",
    "workbench.ui.activity_last_event_label",
    "workbench.ui.resource_duration_ms",
    "workbench.ui.resource_fetch_to_request_ms",
    "workbench.ui.resource_request_wait_ms",
    "workbench.ui.resource_response_transfer_ms",
    "workbench.ui.resource_transfer_size",
    "workbench.ui.resource_encoded_body_size",
    "workbench.ui.resource_decoded_body_size",
    "workbench.ui.resource_next_hop_protocol",
    "workbench.ui.resource_server_timing",
    "workbench.session_id", "workbench.trace_id", "workbench.turn_id",
    "workbench.read_model.route", "workbench.read_model.count",
    "workbench.read_model.family", "workbench.read_model.status",
    "workbench.read_model.reason",
    "db.system", "db.operation.name", "db.sql.table", "db.query.arg_count",
    "db.index.expected", "db.pool.max_open", "db.pool.open_connections",
    "db.pool.in_use", "db.pool.idle", "db.pool.wait_count",
    "db.pool.wait_duration_ms", "db.pool.max_idle_closed",
    "db.pool.max_lifetime_closed",
    "hwlab.runtime.stage", "hwlab.http.stage", "hwlab.http.phase",
    "hwlab.http.phase.outcome", "hwlab.live_builds.service_id",
    "hwlab.live_builds.service_kind", "hwlab.live_builds.external",
    "hwlab.live_builds.deploy_manifest_status",
    "hwlab.live_builds.artifact_catalog_status",
    "error.code", "error.category", "error.layer",
    "stage", "causeStage", "causeCode",
    "eventType", "runnerId", "attemptId", "backendProfile",
    "sourceCommit", "jobName", "podName", "logPath",
    "toolName", "type", "itemType", "itemId", "status", "exitCode",
    "durationMs", "cwd", "processId", "command", "commandFingerprint",
    "outputSummary", "outputBytes", "outputTruncated", "storageKind",
    "storagePvcName", "storagePvcPhase", "resumeMode", "rolloutId",
    "returnedEvents", "sinceSeq", "afterSeq", "limit", "fromSeq", "toSeq",
    "totalEvents", "hasMore", "fullTraceLoaded", "rawEventCount",
    "maxSeq", "traceLastSeq", "endSeq", "commandFiltered",
    "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq",
    "lastSeq", "sourceEventCount", "projectedSeq", "lastProjectedSeq",
    "status", "turnStatus", "error", "error.message", "exception.type",
    "exception.message", "valuesPrinted",
]
IDENTITY_KEYS = [
    "runId", "commandId", "sessionId", "turnId", "threadId",
    "runnerJobId", "runnerId", "attemptId", "backendProfile",
    "sourceCommit", "jobName", "podName",
]

def run_kubectl(proxy_path, timeout=15):
    try:
        proc = subprocess.run(
            ["kubectl", "get", "--raw", proxy_path],
            capture_output=True,
            text=True,
            timeout=timeout,
        )
        return proc.returncode, proc.stdout, proc.stderr
    except subprocess.TimeoutExpired as exc:
        return 124, exc.stdout or "", "timeout while querying tempo"
    except Exception as exc:
        return 125, "", str(exc)

def run_kubectl_json(proxy_path, timeout=15):
    rc, body, stderr = run_kubectl(proxy_path, timeout=timeout)
    parsed = parse_json(body)
    return {
        "ok": rc == 0 and isinstance(parsed, dict),
        "rc": rc,
        "json": parsed if isinstance(parsed, dict) else None,
        "stderrTail": (stderr or "")[-1000:],
        "bodyBytes": len((body or "").encode("utf-8")),
    }

def first_present(*values):
    for value in values:
        if value not in (None, ""):
            return value
    return None

def shell_quote(value):
    return "'" + str(value).replace("'", "'\\''") + "'"

def unwrap_payload(value):
    if not isinstance(value, dict):
        return {}
    data = value.get("data")
    if isinstance(data, dict):
        return data
    result = value.get("result")
    if isinstance(result, dict):
        return result
    return value

def guess_agentrun_namespace(identity):
    if isinstance(identity, dict):
        for key in ("jobName", "podName", "runnerJobId"):
            text = str(identity.get(key) or "")
            match = re.search(r"\\bagentrun-v[0-9]+\\b", text)
            if match:
                return match.group(0)
    if str(TARGET_ID).upper() == "D601":
        return "agentrun-v02"
    return "agentrun-v01"

def agentrun_service_proxy_paths(namespace, api_path):
    return [
        "/api/v1/namespaces/%s/services/http:agentrun-mgr:http/proxy%s" % (namespace, api_path),
        "/api/v1/namespaces/%s/services/http:agentrun-mgr:8080/proxy%s" % (namespace, api_path),
    ]

def run_agentrun_mgr_json(namespace, api_path, timeout=12):
    url = "http://127.0.0.1:8080%s" % api_path
    script_parts = []
    script_parts.append("url=" + shell_quote(url) + "; ")
    script_parts.append("api_key=\\\"\${AGENTRUN_API_KEY:-}\\\"; ")
    script_parts.append("if [ -z \\\"$api_key\\\" ]; then api_key=\\\"\${HWLAB_API_KEY:-}\\\"; fi; ")
    script_parts.append("if command -v wget >/dev/null 2>&1; then ")
    script_parts.append("wget -qO- --header=\\\"Authorization: Bearer $api_key\\\" \\\"$url\\\"; ")
    script_parts.append("else ")
    script_parts.append("curl -fsS -H \\\"Authorization: Bearer $api_key\\\" \\\"$url\\\"; ")
    script_parts.append("fi")
    script = "".join(script_parts)
    try:
        proc = subprocess.run(
            ["kubectl", "-n", namespace, "exec", "deploy/agentrun-mgr", "--", "sh", "-lc", script],
            capture_output=True,
            text=True,
            timeout=timeout,
        )
        parsed = parse_json(proc.stdout)
        return {
            "ok": proc.returncode == 0 and isinstance(parsed, dict),
            "rc": proc.returncode,
            "json": parsed if isinstance(parsed, dict) else None,
            "stderrTail": (proc.stderr or "")[-1000:],
            "bodyBytes": len((proc.stdout or "").encode("utf-8")),
            "method": "manager-exec-localhost",
        }
    except subprocess.TimeoutExpired as exc:
        return {"ok": False, "rc": 124, "json": None, "stderrTail": "timeout while querying agentrun manager", "bodyBytes": len((exc.stdout or "").encode("utf-8")), "method": "manager-exec-localhost"}
    except Exception as exc:
        return {"ok": False, "rc": 125, "json": None, "stderrTail": str(exc), "bodyBytes": 0, "method": "manager-exec-localhost"}

def get_agentrun_json(namespace, api_path, timeout=10):
    attempts = []
    exec_response = run_agentrun_mgr_json(namespace, api_path, timeout=timeout)
    attempts.append(exec_response)
    if exec_response.get("ok"):
        return exec_response, attempts
    for proxy_path in agentrun_service_proxy_paths(namespace, api_path):
        response = run_kubectl_json(proxy_path, timeout=timeout)
        response["proxyPath"] = proxy_path
        response["method"] = "service-proxy"
        attempts.append(response)
        if response.get("ok"):
            return response, attempts
    return attempts[-1] if attempts else {"ok": False, "json": None}, attempts

def event_items(value):
    if not isinstance(value, dict):
        return []
    for key in ("items", "events", "data"):
        items = value.get(key)
        if isinstance(items, list):
            return [item for item in items if isinstance(item, dict)]
    return []

def terminal_from_events(events):
    latest = None
    for item in events:
        payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
        event_type = item.get("type") or item.get("eventType") or payload.get("type")
        status = first_present(payload.get("terminalStatus"), payload.get("status"), item.get("terminalStatus"), item.get("status"))
        if event_type == "terminal_status" or status in ("completed", "failed", "blocked", "cancelled", "canceled"):
            latest = {
                "seq": item.get("seq"),
                "type": event_type,
                "status": status,
                "failureKind": first_present(payload.get("failureKind"), item.get("failureKind")),
            }
    return latest

def classify_terminal_status(status, spans):
    if status in ("completed", "succeeded", "success"):
        return "completed"
    if status in ("failed", "error", "timeout", "cancelled", "canceled"):
        return "failed"
    if status == "blocked":
        return "blocked"
    if any(str(item.get("service") or "") == "agentrun-runner" for item in spans):
        return "running"
    return "unknown"

def agentrun_authority_summary(identity):
    run_id = identity.get("runId") if isinstance(identity, dict) else None
    command_id = identity.get("commandId") if isinstance(identity, dict) else None
    if not run_id or not command_id:
        return {
            "queried": False,
            "reason": "missing-run-or-command-id",
        }
    namespace = guess_agentrun_namespace(identity)
    run_q = urllib.parse.quote(str(run_id), safe="")
    command_q = urllib.parse.quote(str(command_id), safe="")
    command_response, command_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s" % (run_q, command_q), timeout=8)
    result_response, result_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s/result" % (run_q, command_q), timeout=12)
    events_response, events_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/events?afterSeq=0&limit=1000" % run_q, timeout=12)
    command_payload = unwrap_payload(command_response.get("json"))
    result_payload = unwrap_payload(result_response.get("json"))
    events = event_items(events_response.get("json"))
    terminal_event = terminal_from_events(events)
    command_state = first_present(command_payload.get("state"), command_payload.get("status"))
    result_status = first_present(
        result_payload.get("terminalStatus"),
        result_payload.get("terminal_status"),
        result_payload.get("state"),
        result_payload.get("status"),
    )
    terminal_status = first_present(
        terminal_event.get("status") if isinstance(terminal_event, dict) else None,
        result_status,
        command_state if command_state in ("completed", "failed", "blocked", "cancelled", "canceled") else None,
    )
    return {
        "queried": True,
        "namespace": namespace,
        "ok": bool(command_response.get("ok") or result_response.get("ok") or events_response.get("ok")),
        "commandOk": command_response.get("ok"),
        "resultOk": result_response.get("ok"),
        "eventsOk": events_response.get("ok"),
        "commandState": command_state,
        "terminalStatus": terminal_status,
        "resultTerminalStatus": result_status,
        "resultOkValue": result_payload.get("ok"),
        "authority": result_payload.get("authority"),
        "fallback": result_payload.get("fallback"),
        "failureKind": first_present(
            result_payload.get("failureKind"),
            result_payload.get("terminalFailureKind"),
            terminal_event.get("failureKind") if isinstance(terminal_event, dict) else None,
        ),
        "terminalEventType": terminal_event.get("type") if isinstance(terminal_event, dict) else None,
        "terminalEventSeq": terminal_event.get("seq") if isinstance(terminal_event, dict) else None,
        "eventCount": len(events),
        "attempts": {
            "command": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in command_attempts],
            "result": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in result_attempts],
            "events": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in events_attempts],
        },
    }

def parse_json(body):
    try:
        return json.loads(body) if body else None
    except Exception:
        return None

def attr_value(value):
    if not isinstance(value, dict):
        return value
    inner = value.get("value", value)
    if not isinstance(inner, dict):
        return inner
    for key in ("stringValue", "intValue", "doubleValue", "boolValue"):
        if key in inner:
            return inner.get(key)
    if "arrayValue" in inner:
        return "<array>"
    if "kvlistValue" in inner:
        return "<kvlist>"
    if "bytesValue" in inner:
        return "<bytes>"
    return inner

def attrs_to_dict(attrs):
    if not isinstance(attrs, list):
        return {}
    output = {}
    for item in attrs:
        if not isinstance(item, dict):
            continue
        key = item.get("key")
        if isinstance(key, str):
            output[key] = attr_value(item.get("value"))
    return output

def selected_attrs(attrs):
    output = {}
    for key in IMPORTANT_ATTRS:
        if key in attrs:
            output[key] = attrs[key]
    return output

def to_int(value):
    if value is None:
        return None
    if isinstance(value, bool):
        return 1 if value else 0
    if isinstance(value, int):
        return value
    if isinstance(value, float) and value.is_integer():
        return int(value)
    if isinstance(value, str):
        try:
            return int(value.strip())
        except Exception:
            return None
    return None

def to_bool(value):
    if isinstance(value, bool):
        return value
    if isinstance(value, str):
        lowered = value.strip().lower()
        if lowered in ("true", "1", "yes"):
            return True
        if lowered in ("false", "0", "no"):
            return False
    return None

def nanos_to_ms(start, end):
    try:
        return round((int(end) - int(start)) / 1000000, 3)
    except Exception:
        return None

def span_status_code(status):
    if not isinstance(status, dict):
        return None
    return status.get("code")

def is_error_span(span, attrs):
    status = span.get("status") if isinstance(span, dict) else {}
    code = span_status_code(status)
    name = str(span.get("name", "")).lower() if isinstance(span, dict) else ""
    failure = str(attrs.get("failureKind", "")).strip()
    terminal = str(attrs.get("terminalStatus", "")).strip()
    return code in ("STATUS_CODE_ERROR", 2) or "error" in name or bool(failure) or terminal in ("failed", "blocked")

def batches_from_body(body):
    if not isinstance(body, dict):
        return []
    for key in ("batches", "resourceSpans"):
        value = body.get(key)
        if isinstance(value, list):
            return value
    return []

def scope_spans_from_batch(batch):
    if not isinstance(batch, dict):
        return []
    for key in ("scopeSpans", "instrumentationLibrarySpans"):
        value = batch.get(key)
        if isinstance(value, list):
            return value
    return []

def extract_traces(search_body):
    if not isinstance(search_body, dict):
        return []
    value = search_body.get("traces")
    if isinstance(value, list):
        return value
    value = search_body.get("data")
    if isinstance(value, list):
        return value
    return []

def trace_id_from_meta(meta):
    if not isinstance(meta, dict):
        return None
    for key in ("traceID", "traceId", "trace_id"):
        value = meta.get(key)
        normalized = normalize_trace_id(value)
        if normalized is not None:
            return normalized
    return None

def normalize_trace_id(value):
    if not isinstance(value, str):
        return None
    text = value.strip().lower()
    if text.startswith("0x"):
        text = text[2:]
    if not text or len(text) > 32:
        return None
    if any(ch not in "0123456789abcdef" for ch in text):
        return None
    return text.rjust(32, "0")

def compact_meta(meta):
    if not isinstance(meta, dict):
        return {}
    output = {}
    for key in ("traceID", "traceId", "rootServiceName", "rootTraceName", "startTimeUnixNano", "durationMs"):
        if key in meta:
            output[key] = meta[key]
    return output

def compact_span(span, service, resource_attrs, scope_name, index):
    raw_attrs = attrs_to_dict(span.get("attributes"))
    attrs = selected_attrs(raw_attrs)
    status = span.get("status") if isinstance(span.get("status"), dict) else {}
    item = {
        "name": span.get("name"),
        "service": service,
        "scope": scope_name,
        "statusCode": span_status_code(status),
        "statusMessage": status.get("message"),
        "durationMs": nanos_to_ms(span.get("startTimeUnixNano"), span.get("endTimeUnixNano")),
        "attributes": attrs,
        "_index": index,
        "_start": to_int(span.get("startTimeUnixNano")) or index,
    }
    if "deployment.environment" in resource_attrs:
        item["environment"] = resource_attrs.get("deployment.environment")
    if "git.commit" in resource_attrs:
        item["gitCommit"] = resource_attrs.get("git.commit")
    return item, raw_attrs

def public_span(item):
    if not isinstance(item, dict):
        return item
    return {key: value for key, value in item.items() if not key.startswith("_")}

def tiny_span(item):
    if not isinstance(item, dict):
        return item
    attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
    keep_attrs = {}
    for key in (
        "http.route", "http.status_code", "http.method", "status",
        "terminalStatus", "eventType", "returnedEvents", "sinceSeq",
        "fromSeq", "toSeq", "totalEvents", "hasMore", "fullTraceLoaded",
        "afterSeq", "rawEventCount", "failureKind", "willRetry",
        "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
        "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
        "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
        "lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
        "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
        "toolName", "type", "itemType", "itemId", "status", "exitCode",
        "durationMs", "cwd", "processId", "command", "commandFingerprint",
        "outputSummary", "outputBytes", "outputTruncated",
        "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome",
        "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind",
        "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status",
        "hwlab.live_builds.artifact_catalog_status",
    ):
        if key in attrs:
            keep_attrs[key] = attrs[key]
    return {
        "name": item.get("name"),
        "service": item.get("service"),
        "attributes": keep_attrs,
    }

def tiny_terminal(terminal):
    if not isinstance(terminal, dict):
        return terminal
    return {
        "status": terminal.get("status"),
        "eventType": terminal.get("eventType"),
        "span": tiny_span(terminal.get("span")),
    }

def identity_from_spans(spans):
    identity = {}
    for key in IDENTITY_KEYS:
        preferred = None
        fallback = None
        for item in spans:
            attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
            value = attrs.get(key)
            if value in (None, ""):
                continue
            if fallback is None:
                fallback = value
            service = str(item.get("service") or "")
            if service.startswith("agentrun"):
                preferred = value
                break
        if preferred is not None or fallback is not None:
            identity[key] = preferred if preferred is not None else fallback
    return identity

def flatten_trace(trace_body):
    services = set()
    business_trace_ids = set()
    name_counts = collections.Counter()
    spans = []
    error_spans = []
    raw_index = 0
    for batch in batches_from_body(trace_body):
        resource = batch.get("resource") if isinstance(batch.get("resource"), dict) else {}
        resource_attrs = attrs_to_dict(resource.get("attributes"))
        service = resource_attrs.get("service.name") or "<unknown-service>"
        services.add(service)
        for scope_span in scope_spans_from_batch(batch):
            scope = scope_span.get("scope") if isinstance(scope_span.get("scope"), dict) else {}
            scope_name = scope.get("name")
            raw_spans = scope_span.get("spans") if isinstance(scope_span.get("spans"), list) else []
            for span in raw_spans:
                if not isinstance(span, dict):
                    continue
                item, raw_attrs = compact_span(span, service, resource_attrs, scope_name, raw_index)
                raw_index += 1
                attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
                if isinstance(attrs.get("traceId"), str):
                    business_trace_ids.add(attrs.get("traceId"))
                name = str(item.get("name") or "")
                name_counts[name] += 1
                spans.append(item)
                if is_error_span(span, raw_attrs):
                    error_spans.append(item)
    spans.sort(key=lambda item: (item.get("_start", 0), item.get("_index", 0)))
    error_spans.sort(key=lambda item: (item.get("_start", 0), item.get("_index", 0)))
    return {
        "spans": spans,
        "services": sorted(services),
        "businessTraceIds": sorted(business_trace_ids),
        "spanNameCounts": [{"name": name, "count": count} for name, count in name_counts.most_common(20)],
        "errorSpans": error_spans,
    }

def raw_trace_shape(trace_body):
    batches = batches_from_body(trace_body)
    scope_count = 0
    span_count = 0
    first_resource_attrs = {}
    first_scope_keys = []
    first_span_keys = []
    first_span_attr_keys = []
    for batch_index, batch in enumerate(batches):
        resource = batch.get("resource") if isinstance(batch.get("resource"), dict) else {}
        resource_attrs = attrs_to_dict(resource.get("attributes"))
        if batch_index == 0:
            for key in ("service.name", "deployment.environment", "git.commit"):
                if key in resource_attrs:
                    first_resource_attrs[key] = resource_attrs[key]
        for scope_span in scope_spans_from_batch(batch):
            scope_count += 1
            if not first_scope_keys and isinstance(scope_span, dict):
                first_scope_keys = sorted(scope_span.keys())
            raw_spans = scope_span.get("spans") if isinstance(scope_span.get("spans"), list) else []
            span_count += len(raw_spans)
            if not first_span_keys and raw_spans and isinstance(raw_spans[0], dict):
                first_span_keys = sorted(raw_spans[0].keys())
                first_span_attr_keys = sorted(attrs_to_dict(raw_spans[0].get("attributes")).keys())[:50]
    return {
        "mode": "bounded-raw-shape",
        "note": "Full Tempo body is intentionally not printed to avoid secrets/raw transcript and SSH 60s output timeouts.",
        "topLevelKeys": sorted(trace_body.keys()) if isinstance(trace_body, dict) else [],
        "resourceSpanCount": len(batches),
        "scopeSpanCount": scope_count,
        "spanCount": span_count,
        "firstResourceAttributes": first_resource_attrs,
        "firstScopeKeys": first_scope_keys,
        "firstSpanKeys": first_span_keys,
        "firstSpanAttributeKeys": first_span_attr_keys,
    }

def http_status_summary(spans):
    grouped = collections.Counter()
    problem_spans = []
    for item in spans:
        attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
        code = to_int(attrs.get("http.status_code"))
        if code is None:
            continue
        method = attrs.get("http.method") or ""
        route = attrs.get("http.route") or item.get("name") or "<unknown-route>"
        grouped[(str(method), str(route), code)] += 1
        if code in (401, 403) or code >= 500:
            problem_spans.append(item)
    status_counts = [
        {"method": method or None, "route": route, "status": status, "count": count}
        for (method, route, status), count in grouped.most_common()
    ]
    problem_counts = [
        {"method": method or None, "route": route, "status": status, "count": count}
        for (method, route, status), count in grouped.most_common()
        if status in (401, 403) or status >= 500
    ]
    return {
        "statusCounts": status_counts[:20],
        "problemCounts": problem_counts[:20],
        "problemSpans": problem_spans,
        "actorForbidden": any(to_int((item.get("attributes") or {}).get("http.status_code")) == 403 and item.get("service") == "hwlab-cloud-api" for item in problem_spans),
    }

def agentrun_summary(spans, authority=None):
    terminal_spans = []
    latest_seq = None
    failure_kind = None
    failure_message = None
    terminal_category = None
    seq_keys = ("maxSeq", "traceLastSeq", "endSeq", "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq", "lastSeq")
    for item in spans:
        service = str(item.get("service") or "")
        name = str(item.get("name") or "")
        attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
        if service.startswith("agentrun"):
            for key in seq_keys:
                value = to_int(attrs.get(key))
                if value is not None:
                    latest_seq = value if latest_seq is None else max(latest_seq, value)
        terminal_status = attrs.get("terminalStatus")
        if terminal_status or name.startswith("runner_terminal."):
            derived = terminal_status
            if not derived and name.startswith("runner_terminal."):
                derived = name.split(".", 1)[1]
            if attrs.get("failureKind") not in (None, ""):
                failure_kind = attrs.get("failureKind")
            if attrs.get("failureMessage") not in (None, ""):
                failure_message = attrs.get("failureMessage")
            if attrs.get("terminalCategory") not in (None, ""):
                terminal_category = attrs.get("terminalCategory")
            terminal_spans.append({
                "status": derived,
                "eventType": attrs.get("eventType"),
                "failureKind": attrs.get("failureKind"),
                "terminalCategory": attrs.get("terminalCategory"),
                "span": public_span(item),
            })
    terminal_status = terminal_spans[-1]["status"] if terminal_spans else None
    terminal_event_type = None
    for terminal in reversed(terminal_spans):
        if terminal.get("eventType"):
            terminal_event_type = terminal.get("eventType")
            break
    terminal_source = "otel-span" if terminal_status not in (None, "") else None
    if isinstance(authority, dict) and authority.get("terminalStatus") not in (None, ""):
        terminal_status = authority.get("terminalStatus")
        terminal_event_type = authority.get("terminalEventType") or terminal_event_type
        failure_kind = authority.get("failureKind") or failure_kind
        terminal_source = "agentrun-authority"
    runner_classification = classify_terminal_status(terminal_status, spans)
    return {
        "terminalStatus": terminal_status,
        "failureKind": failure_kind,
        "failureMessage": failure_message,
        "terminalCategory": terminal_category,
        "latestSeq": latest_seq,
        "terminalEventType": terminal_event_type,
        "terminalSource": terminal_source,
        "authority": authority if isinstance(authority, dict) else None,
        "terminalSpans": terminal_spans,
        "runnerProviderClassification": runner_classification,
    }

def hwlab_read_model_summary(spans):
    trace_read_spans = []
    turn_read_spans = []
    projection_spans = []
    source_event_count = None
    requested_since_seq = None
    latest_projected_seq = None
    turn_statuses = []
    turn_status_counts = collections.Counter()
    for item in spans:
        if item.get("service") != "hwlab-cloud-api":
            continue
        name = str(item.get("name") or "")
        attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
        if name == "trace_events_read":
            trace_read_spans.append(public_span(item))
            for key in ("totalEvents", "sourceEventCount"):
                value = to_int(attrs.get(key))
                if value is not None:
                    source_event_count = value if source_event_count is None else max(source_event_count, value)
            for key in ("sinceSeq", "fromSeq"):
                value = to_int(attrs.get(key))
                if value is not None:
                    requested_since_seq = value if requested_since_seq is None else max(requested_since_seq, value)
        if name == "turn_status_read":
            turn_read_spans.append(public_span(item))
            if attrs.get("turnStatus") is not None or attrs.get("status") is not None:
                status_value = attrs.get("turnStatus") if attrs.get("turnStatus") is not None else attrs.get("status")
                turn_statuses.append(status_value)
                turn_status_counts[str(status_value)] += 1
        if name == "projection_sync":
            projection_spans.append(public_span(item))
            after_seq = to_int(attrs.get("afterSeq"))
            raw_count = to_int(attrs.get("rawEventCount"))
            projected = None
            for key in ("maxSeq", "traceLastSeq", "endSeq", "projectedSeq", "lastProjectedSeq"):
                value = to_int(attrs.get(key))
                if value is not None:
                    projected = value if projected is None else max(projected, value)
            if after_seq is not None:
                projected = after_seq + (raw_count or 0)
            if projected is not None:
                latest_projected_seq = projected if latest_projected_seq is None else max(latest_projected_seq, projected)
                source_event_count = projected if source_event_count is None else max(source_event_count, projected)
    has_more_values = []
    full_loaded_values = []
    for item in trace_read_spans:
        attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
        if "hasMore" in attrs:
            has_more_values.append(to_bool(attrs.get("hasMore")))
        if "fullTraceLoaded" in attrs:
            full_loaded_values.append(to_bool(attrs.get("fullTraceLoaded")))
    return {
        "sourceEventCount": source_event_count,
        "requestedSinceSeq": requested_since_seq,
        "latestProjectedSeq": latest_projected_seq,
        "traceStatus": {
            "readCount": len(trace_read_spans),
            "anyHasMore": any(value is True for value in has_more_values),
            "anyFullTraceLoaded": any(value is True for value in full_loaded_values),
        },
        "turnStatus": turn_statuses[-1] if turn_statuses else None,
        "turnStatusCounts": [{"status": status, "count": count} for status, count in turn_status_counts.most_common()],
        "traceEventReadSpans": trace_read_spans,
        "turnStatusReadSpans": turn_read_spans,
        "projectionSpans": projection_spans,
    }

def projection_lag_summary(agentrun, read_model):
    source_event_count = read_model.get("sourceEventCount")
    requested_since_seq = read_model.get("requestedSinceSeq")
    latest_projected_seq = read_model.get("latestProjectedSeq")
    agent_latest_seq = agentrun.get("latestSeq")
    reasons = []
    status = "none"
    if requested_since_seq is not None and source_event_count is not None and requested_since_seq > source_event_count:
        status = "confirmed"
        reasons.append("requested sinceSeq %s is greater than HWLAB sourceEventCount %s" % (requested_since_seq, source_event_count))
    if latest_projected_seq is not None and requested_since_seq is not None and latest_projected_seq < requested_since_seq:
        status = "confirmed"
        reasons.append("latestProjectedSeq %s is behind requested sinceSeq %s" % (latest_projected_seq, requested_since_seq))
    if agent_latest_seq is not None and source_event_count is not None and agent_latest_seq > source_event_count:
        status = "confirmed"
        reasons.append("AgentRun latestSeq %s is greater than HWLAB sourceEventCount %s" % (agent_latest_seq, source_event_count))
    if status == "none" and agentrun.get("terminalStatus") == "completed" and source_event_count is not None and read_model.get("traceStatus", {}).get("readCount", 0) > 0:
        status = "suspected"
        reasons.append("AgentRun completed while HWLAB read model still needs endpoint/status confirmation")
    return {
        "status": status,
        "reasons": reasons,
    }

def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error_spans, idle_warning_spans):
    candidates = []
    if http_summary.get("actorForbidden"):
        candidates.append({
            "code": "actor_forbidden",
            "label": "actor forbidden",
            "confidence": 0.98,
            "summary": "actor forbidden: HWLAB Code Agent read endpoint returned HTTP 403, likely actor/owner mismatch for this trace or turn.",
            "evidence": http_summary.get("problemCounts", [])[:5],
        })
    if lag_summary.get("status") == "confirmed":
        candidates.append({
            "code": "hwlab_projection_stale",
            "label": "HWLAB projection stale",
            "confidence": 0.94,
            "summary": "HWLAB projection stale: read model sequence is behind the caller/requested event window.",
            "evidence": {
                "sourceEventCount": read_model.get("sourceEventCount"),
                "requestedSinceSeq": read_model.get("requestedSinceSeq"),
                "latestProjectedSeq": read_model.get("latestProjectedSeq"),
                "reasons": lag_summary.get("reasons", []),
            },
        })
    elif lag_summary.get("status") == "suspected":
        candidates.append({
            "code": "hwlab_projection_lag_suspected",
            "label": "HWLAB projection stale",
            "confidence": 0.64,
            "summary": "HWLAB projection stale is suspected because AgentRun is terminal but HWLAB read-model evidence is incomplete.",
            "evidence": lag_summary.get("reasons", []),
        })
    if agentrun.get("terminalStatus") in ("failed", "error", "timeout", "blocked", "cancelled"):
        candidates.append({
            "code": "agentrun_terminal_failed",
            "label": "AgentRun terminal failed",
            "confidence": 0.9,
            "summary": "AgentRun reached a non-success terminal state; inspect result/events/runnerjob before retrying or continuing the session.",
            "evidence": {
                "terminalStatus": agentrun.get("terminalStatus"),
                "failureKind": agentrun.get("failureKind"),
                "terminalCategory": agentrun.get("terminalCategory"),
                "runnerProviderClassification": agentrun.get("runnerProviderClassification"),
            },
        })
    if agentrun.get("terminalStatus") == "completed":
        candidates.append({
            "code": "agentrun_completed_not_provider_blocked",
            "label": "AgentRun completed",
            "confidence": 0.88,
            "summary": "AgentRun completed: runner/provider reached terminal completed, so this trace is not explained by an active runner hang.",
            "evidence": {
                "terminalStatus": agentrun.get("terminalStatus"),
                "terminalEventType": agentrun.get("terminalEventType"),
                "runnerProviderClassification": agentrun.get("runnerProviderClassification"),
            },
        })
    if error_spans and agentrun.get("terminalStatus") == "completed":
        candidates.append({
            "code": "tool_call_failures_recovered",
            "label": "tool call failures recovered",
            "confidence": 0.42,
            "summary": "tool call failure spans exist, but AgentRun still completed; treat them as secondary unless correlated with user-visible failure.",
            "evidence": {
                "errorSpanCount": len(error_spans),
                "sampleNames": sorted(set(str(item.get("name") or "") for item in error_spans))[:5],
            },
        })
    if idle_warning_spans and agentrun.get("terminalStatus") in (None, ""):
        candidates.append({
            "code": "agentrun_runner_idle_warning_active",
            "label": "AgentRun runner idle warning active",
            "confidence": 0.72,
            "summary": "AgentRun runner is still emitting idle warnings and no runner terminal span is present; inspect waitingFor/lastEventLabel before treating the Workbench UI as terminal.",
            "evidence": [tiny_span(item) for item in idle_warning_spans[-3:]],
        })
    candidates.sort(key=lambda item: item.get("confidence", 0), reverse=True)
    return candidates

def candidate_score(trace_id, meta, trace_body, trace_rc, trace_err):
    parsed = parse_json(trace_body)
    meta_root_service = str(meta.get("rootServiceName") or "") if isinstance(meta, dict) else ""
    meta_root_name = str(meta.get("rootTraceName") or "") if isinstance(meta, dict) else ""
    if not isinstance(parsed, dict):
        return {
            "traceId": trace_id,
            "score": -100,
            "reasons": ["trace parse failed"],
            "parseOk": False,
            "queryOk": trace_rc == 0,
            "meta": compact_meta(meta),
            "spanCount": 0,
            "services": [],
            "servicePath": {
                "hwlab-cloud-api": "unknown",
                "agentrun-manager": "unknown",
                "agentrun-runner": "unknown",
                "complete": False,
            },
            "rootTraceName": meta_root_name or None,
            "rootServiceName": meta_root_service or None,
            "stderrTail": (trace_err or "")[-1000:],
        }, None
    flat = flatten_trace(parsed)
    spans = flat["spans"]
    services = set(flat["services"])
    names = [str(item.get("name") or "") for item in spans]
    lowered_names = [name.lower() for name in names]
    identity = identity_from_spans(spans)
    agentrun = agentrun_summary(spans)
    error_spans = flat["errorSpans"]
    score = 0
    reasons = []
    def add(points, reason):
        nonlocal score
        score += points
        reasons.append("%+d %s" % (points, reason))
    if "agentrun-runner" in services:
        add(120, "contains agentrun-runner")
    if "agentrun-manager" in services:
        add(90, "contains agentrun-manager")
    if "hwlab-cloud-api" in services:
        add(20, "contains hwlab-cloud-api")
    span_points = min(len(spans), 80)
    if span_points:
        add(span_points, "span count %s" % len(spans))
    for key, points in (("runId", 50), ("commandId", 50), ("runnerJobId", 35), ("runnerId", 30), ("attemptId", 20), ("backendProfile", 15)):
        if identity.get(key) not in (None, ""):
            add(points, "identity %s" % key)
    if any("codex_stdio" in name for name in lowered_names):
        add(45, "codex stdio span")
    if any("runner" in name for name in lowered_names):
        add(30, "runner span")
    if any("provider" in name for name in lowered_names):
        add(25, "provider span")
    if any("tool_call" in name or "command" in name for name in lowered_names):
        add(20, "tool/command span")
    if any("turn_completed" in name or "runner_terminal" in name or "command_result" in name for name in lowered_names):
        add(35, "terminal/result span")
    if error_spans:
        add(40 + min(len(error_spans), 20), "error span count %s" % len(error_spans))
    if agentrun.get("terminalStatus") not in (None, ""):
        add(25, "terminal status %s" % agentrun.get("terminalStatus"))
    is_workbench_get_single_span = (
        len(spans) <= 1
        and services.issubset({"hwlab-cloud-api"})
        and meta_root_name.startswith("GET /v1/workbench/")
    )
    if is_workbench_get_single_span:
        add(-80, "single-span Workbench GET/SSE trace")
    candidate_quality = "low-confidence-workbench-single-span" if is_workbench_get_single_span else "normal"
    service_path = {
        service: ("reached" if service in services else "missing")
        for service in ("hwlab-cloud-api", "agentrun-manager", "agentrun-runner")
    }
    service_path["complete"] = all(service in services for service in ("hwlab-cloud-api", "agentrun-manager", "agentrun-runner"))
    return {
        "traceId": trace_id,
        "score": score,
        "reasons": reasons[:12],
        "parseOk": True,
        "queryOk": trace_rc == 0,
        "meta": compact_meta(meta),
        "spanCount": len(spans),
        "services": sorted(services),
        "servicePath": service_path,
        "identity": {key: identity.get(key) for key in ("runId", "commandId", "runnerJobId", "runnerId", "backendProfile") if identity.get(key) not in (None, "")},
        "terminalStatus": agentrun.get("terminalStatus"),
        "errorSpanCount": len(error_spans),
        "candidateQuality": candidate_quality,
        "lowConfidence": candidate_quality != "normal",
        "rootTraceName": meta_root_name or None,
        "rootServiceName": meta_root_service or None,
        "spanNamePreview": names[:8],
        "traceCommand": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % trace_id,
    }, parsed

def resolve_trace():
    if TRACE_ID is not None:
        return TRACE_ID, {
            "mode": "trace-id",
            "businessTraceId": BUSINESS_TRACE_ID,
            "otelTraceId": TRACE_ID,
            "searchPath": None,
            "searchOk": None,
            "searchParseOk": None,
            "candidateTraceCount": None,
            "selectedMeta": None,
        }, None
    if SEARCH_PROXY_PATH is None:
        return None, None, "missing trace id and search path"
    search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15)
    search_parsed = parse_json(search_body)
    metas = extract_traces(search_parsed)
    candidate_summaries = []
    selected = None
    selected_trace_id = None
    selected_score = None
    seen = set()
    scanned = 0
    for meta in metas:
        trace_id = trace_id_from_meta(meta)
        if not trace_id or trace_id in seen:
            continue
        seen.add(trace_id)
        if scanned >= CANDIDATE_LIMIT or time.time() > DEADLINE:
            break
        scanned += 1
        trace_path = TRACE_PATH_TEMPLATE.replace("{{traceId}}", trace_id)
        trace_rc, trace_body, trace_err = run_kubectl(TRACE_PROXY_PREFIX + trace_path, timeout=8)
        summary, _parsed = candidate_score(trace_id, meta, trace_body, trace_rc, trace_err)
        candidate_summaries.append(summary)
        score = summary.get("score")
        if isinstance(score, (int, float)) and (selected_score is None or score > selected_score):
            selected = meta
            selected_trace_id = trace_id
            selected_score = score
    if selected_trace_id is None:
        for meta in metas:
            trace_id = trace_id_from_meta(meta)
            if trace_id:
                selected = meta
                selected_trace_id = trace_id
                selected_score = None
                break
    candidate_summaries.sort(key=lambda item: item.get("score", -999999), reverse=True)
    selected_summary = next((item for item in candidate_summaries if item.get("traceId") == selected_trace_id), None)
    selected_quality = selected_summary.get("candidateQuality") if isinstance(selected_summary, dict) else None
    selected_low_confidence = bool(selected_summary.get("lowConfidence")) if isinstance(selected_summary, dict) else False
    selected_service_path = selected_summary.get("servicePath") if isinstance(selected_summary, dict) else None
    selected_complete = bool(selected_service_path.get("complete")) if isinstance(selected_service_path, dict) else False
    selected_rejected_reason = None
    if selected_low_confidence and not selected_complete:
        selected_rejected_reason = "best candidate is only a single-span Workbench GET/SSE trace; no high-confidence Code Agent trace was found in the scanned window"
    mapping = {
        "mode": "business-trace-id",
        "businessTraceId": BUSINESS_TRACE_ID,
        "otelTraceId": selected_trace_id,
        "searchPath": SEARCH_PATH,
        "searchProxyPath": SEARCH_PROXY_PATH,
        "searchOk": search_rc == 0,
        "searchParseOk": isinstance(search_parsed, dict),
        "candidateTraceCount": len(metas),
        "scannedCandidateCount": scanned,
        "candidateSelectionMode": "scored-service-and-span-content",
        "selectedScore": selected_score,
        "selectedQuality": selected_quality,
        "selectedLowConfidence": selected_low_confidence,
        "selectedRejectedReason": selected_rejected_reason,
        "selectedReasons": selected_summary.get("reasons", []) if isinstance(selected_summary, dict) else [],
        "selectedMeta": compact_meta(selected),
        "candidatePreview": candidate_summaries[:8],
        "searchStderrTail": (search_err or "")[-2000:],
    }
    if RAW:
        mapping["rawSearchBody"] = search_parsed if isinstance(search_parsed, dict) else search_body[-12000:]
    if selected_trace_id is None:
        return None, mapping, "no OTel trace found for business trace"
    if selected_rejected_reason is not None:
        return None, mapping, selected_rejected_reason
    return selected_trace_id, mapping, None

resolved_trace_id, mapping, resolve_error = resolve_trace()
if resolved_trace_id is None:
    payload = {
        "ok": False,
        "mapping": mapping,
        "error": resolve_error,
        "summary": {
            "rootCause": "otel trace not found",
            "facts": [],
        },
        "next": {
            "expandWindow": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --lookback-minutes 10080 --candidate-limit 500 --full",
            "search": "bun scripts/cli.ts platform-infra observability search --target ${target.id} --query '{ .traceId = \\"${businessTraceIdForNext}\\" }' --lookback-minutes 10080 --candidate-limit 500 --limit 10",
        },
    }
    print(json.dumps(payload, ensure_ascii=False, indent=2))
    raise SystemExit(1)

trace_path = TRACE_PATH_TEMPLATE.replace("{{traceId}}", resolved_trace_id)
trace_proxy_path = TRACE_PROXY_PREFIX + trace_path
trace_rc, trace_body, trace_err = run_kubectl(trace_proxy_path, timeout=25)
trace_parsed = parse_json(trace_body)
if not isinstance(trace_parsed, dict):
    payload = {
        "ok": False,
        "mapping": mapping,
        "tracePath": trace_path,
        "traceProxyPath": trace_proxy_path,
        "traceQueryOk": trace_rc == 0,
        "traceParseOk": False,
        "bodyBytes": len(trace_body.encode("utf-8")),
        "bodyTail": trace_body[-12000:],
        "stderrTail": (trace_err or "")[-4000:],
    }
    print(json.dumps(payload, ensure_ascii=False, indent=2))
    raise SystemExit(1)

flat = flatten_trace(trace_parsed)
spans = flat["spans"]
services = flat["services"]
business_trace_ids = flat["businessTraceIds"]
error_spans = flat["errorSpans"]
idle_warning_spans = [item for item in spans if str(item.get("name") or "") == "codex_stdio.idle_warning"]
http_summary = http_status_summary(spans)
read_model = hwlab_read_model_summary(spans)
identity = identity_from_spans(spans)
agentrun_authority = agentrun_authority_summary(identity)
agentrun = agentrun_summary(spans, agentrun_authority)
lag = projection_lag_summary(agentrun, read_model)
candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans, idle_warning_spans)
expected_services = ["hwlab-cloud-api", "agentrun-manager", "agentrun-runner"]
service_path = {
    service: ("reached" if service in services else "missing")
    for service in expected_services
}
service_path["complete"] = all(service in services for service in expected_services)
facts = []
if http_summary.get("actorForbidden"):
    facts.append("actor forbidden")
terminal_status = agentrun.get("terminalStatus")
if terminal_status in ("failed", "error", "timeout", "blocked", "cancelled"):
    failure_kind = agentrun.get("failureKind")
    if failure_kind:
        facts.append(f"AgentRun terminal failed ({failure_kind})")
    else:
        facts.append("AgentRun terminal failed")
if terminal_status == "completed":
    facts.append("AgentRun completed")
if lag.get("status") in ("confirmed", "suspected"):
    facts.append("HWLAB projection stale")
if idle_warning_spans and terminal_status in (None, ""):
    facts.append("AgentRun runner idle warnings active")
if not facts:
    facts.append("no dominant Code Agent root cause classified")
summary = {
    "rootCause": "; ".join(facts),
    "facts": facts,
    "classification": {
        "projectionLag": lag.get("status"),
        "runnerProvider": agentrun.get("runnerProviderClassification"),
        "actorForbidden": http_summary.get("actorForbidden"),
        "terminalStatus": terminal_status,
        "failureKind": agentrun.get("failureKind"),
    },
}
evidence = {
    "httpProblemSpanCount": len(http_summary.get("problemSpans", [])),
    "httpProblemSpanSamples": [tiny_span(item) for item in http_summary.get("problemSpans", [])[:3]],
    "terminalSpanCount": len(agentrun.get("terminalSpans", [])),
    "terminalSpanSamples": [tiny_terminal(item) for item in agentrun.get("terminalSpans", [])[:3]],
    "traceEventReadSpans": [tiny_span(item) for item in read_model.get("traceEventReadSpans", [])[:3]],
    "turnStatusReadSpanCount": len(read_model.get("turnStatusReadSpans", [])),
    "turnStatusReadSpanSamples": [tiny_span(item) for item in read_model.get("turnStatusReadSpans", [])[:3]],
    "projectionSpanCount": len(read_model.get("projectionSpans", [])),
    "projectionSpanTail": [tiny_span(item) for item in read_model.get("projectionSpans", [])[-3:]],
    "errorSpanCount": len(error_spans),
    "errorSpanSampleNames": sorted(set(str(item.get("name") or "") for item in error_spans))[:5],
    "idleWarningSpanCount": len(idle_warning_spans),
    "idleWarningSpanTail": [tiny_span(item) for item in idle_warning_spans[-3:]],
}
payload = {
    "ok": trace_rc == 0 and len(spans) > 0,
    "mapping": mapping,
    "tracePath": trace_path,
    "traceProxyPath": trace_proxy_path,
    "bodyBytes": len(trace_body.encode("utf-8")),
    "traceParseOk": True,
    "spanCount": len(spans),
    "services": services,
    "servicePath": service_path,
    "businessTraceIds": business_trace_ids[:20],
    "identity": identity,
    "agentrun": {
        "terminalStatus": agentrun.get("terminalStatus"),
        "terminalSource": agentrun.get("terminalSource"),
        "latestSeq": agentrun.get("latestSeq"),
        "terminalEventType": agentrun.get("terminalEventType"),
        "runnerProviderClassification": agentrun.get("runnerProviderClassification"),
        "authority": agentrun.get("authority"),
    },
    "hwlabReadModel": {
        "sourceEventCount": read_model.get("sourceEventCount"),
        "requestedSinceSeq": read_model.get("requestedSinceSeq"),
        "latestProjectedSeq": read_model.get("latestProjectedSeq"),
        "traceStatus": read_model.get("traceStatus"),
        "turnStatus": read_model.get("turnStatus"),
        "turnStatusCounts": read_model.get("turnStatusCounts"),
    },
    "http": {
        "statusCounts": http_summary.get("statusCounts", [])[:20],
        "problemCounts": http_summary.get("problemCounts", [])[:20],
        "actorForbidden": http_summary.get("actorForbidden"),
    },
    "projectionLag": lag,
    "summary": summary,
    "rootCauseCandidates": candidates,
    "spanNameCounts": flat["spanNameCounts"][:12],
    "evidence": evidence,
    "next": {
        "diagnoseFull": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --full",
        "traceSummary": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % resolved_trace_id,
        "traceReads": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep trace_events_read --limit 20 --full" % resolved_trace_id,
        "turnStatus": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep turn_status_read --limit 20 --full" % resolved_trace_id,
        "projection": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep projection_sync --limit 120 --full" % resolved_trace_id,
        "runnerTerminal": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep runner_terminal --limit 20 --full" % resolved_trace_id,
        "raw": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --trace-id %s --raw" % resolved_trace_id,
    },
    "stderrTail": (trace_err or "")[-4000:],
}
if FULL:
    payload["full"] = {
        "traceEventReadSpans": read_model.get("traceEventReadSpans", []),
        "turnStatusReadSpans": read_model.get("turnStatusReadSpans", []),
        "projectionSpans": read_model.get("projectionSpans", [])[:LIMIT],
        "terminalSpans": agentrun.get("terminalSpans", [])[:LIMIT],
        "errorSpans": [public_span(item) for item in error_spans[:LIMIT]],
        "httpProblemSpans": [public_span(item) for item in http_summary.get("problemSpans", [])[:LIMIT]],
        "selectedSpans": [public_span(item) for item in spans if item.get("name") in ("trace_events_read", "turn_status_read", "projection_sync", "command_result")][:LIMIT],
    }
if RAW:
    payload["raw"] = raw_trace_shape(trace_parsed)
print(json.dumps(payload, ensure_ascii=False, indent=2 if FULL or RAW else None))
raise SystemExit(0 if payload["ok"] else 1)
PY
`;
}

function configSummary(observability: ObservabilityConfig, target: ObservabilityTarget): Record<string, unknown> {
  return {
    configPath: configLabel,
    spec: observability.metadata.spec,
    target: targetSummary(target),
    images: {
      collector: imageReference(observability.images.collector),
      traceBackend: imageReference(observability.images.tempo),
    },
    collector: observability.collector,
    traceBackend: observability.traceBackend,
    sampling: observability.sampling,
  };
}

function targetSummary(target: ObservabilityTarget): Record<string, unknown> {
  return {
    id: target.id,
    route: target.route,
    namespace: target.namespace,
    role: target.role,
    createNamespace: target.createNamespace,
  };
}

function policyChecks(yaml: string, target: ObservabilityTarget): Array<Record<string, unknown>> {
  return [
    { name: "yaml-source-of-truth", ok: true, detail: "All concrete images, routes, namespace, ports, retention and sampling values are read from config/platform-infra/observability.yaml." },
    { name: "clusterip-only", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(yaml), detail: "Collector and trace backend stay ClusterIP-only." },
    { name: "no-ingress", ok: !/^\s*kind:\s*Ingress\s*$/mu.test(yaml), detail: "No public ingress is rendered for the first tracing backend." },
    { name: "no-host-network", ok: !/^\s*hostNetwork:\s*true\s*$/mu.test(yaml), detail: "Pods must not use host network." },
    { name: "allow-all-network-policy", ok: yaml.includes("kind: NetworkPolicy") && yaml.includes("name: allow-all") && yaml.includes(`namespace: ${target.namespace}`), detail: `NetworkPolicy/allow-all is rendered in ${target.namespace}.` },
  ];
}

function statusSummary(payload: Record<string, unknown>): Record<string, unknown> {
  const sections = asRecord(payload.sections, "status.sections");
  const deployments = objectList(sectionJson(sections, "deployments"));
  const services = objectList(sectionJson(sections, "services"));
  const pods = objectList(sectionJson(sections, "pods"));
  const events = objectList(sectionJson(sections, "events"));
  const podEvents = latestPodEvents(events);
  const probes = Array.isArray(payload.probes) ? payload.probes as Array<Record<string, unknown>> : [];
  const readyDeployments = deployments.map((item) => deploymentSummary(item));
  return {
    ready: payload.ok === true,
    namespace: payload.namespace,
    deployments: readyDeployments,
    services: services.map((item) => metadataName(item)),
    pods: pods.map((item) => podSummary(item, podEvents)),
    probes: probes.map((item) => ({
      name: item.name,
      ok: item.ok === true,
      service: item.service,
      path: item.path,
      stderrTail: item.ok === true ? "" : item.stderrTail,
    })),
  };
}

function compactStatus(payload: Record<string, unknown> | null, full: boolean): Record<string, unknown> | null {
  if (payload === null) return null;
  if (full) return redactSensitiveUnknown(payload) as Record<string, unknown>;
  return statusSummary(payload);
}

function sectionJson(sections: Record<string, unknown>, name: string): unknown {
  const section = asRecord(sections[name], `sections.${name}`);
  return section.json;
}

function objectList(value: unknown): Record<string, unknown>[] {
  if (typeof value !== "object" || value === null) return [];
  const items = (value as Record<string, unknown>).items;
  if (!Array.isArray(items)) return [];
  return items.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item));
}

function deploymentSummary(item: Record<string, unknown>): Record<string, unknown> {
  const spec = item.spec as Record<string, unknown> | undefined;
  const status = item.status as Record<string, unknown> | undefined;
  const replicas = typeof spec?.replicas === "number" ? spec.replicas : null;
  const available = typeof status?.availableReplicas === "number" ? status.availableReplicas : 0;
  return {
    name: metadataName(item),
    replicas,
    availableReplicas: available,
    ready: replicas !== null && available >= replicas,
  };
}

function podSummary(item: Record<string, unknown>, podEvents: Map<string, Record<string, unknown>>): Record<string, unknown> {
  const status = item.status as Record<string, unknown> | undefined;
  const waiting = firstWaitingContainer(status);
  const scheduled = conditionSummary(status, "PodScheduled");
  const ready = conditionSummary(status, "Ready");
  const name = metadataName(item);
  const showEvent = status?.phase !== "Running" || waiting !== null || scheduled !== null || ready !== null;
  const latestEvent = showEvent && name !== null ? podEvents.get(name) ?? null : null;
  return {
    name,
    phase: status?.phase ?? null,
    reason: waiting?.reason ?? scheduled?.reason ?? ready?.reason ?? null,
    message: waiting?.message ?? scheduled?.message ?? ready?.message ?? null,
    latestEvent: latestEvent === null ? null : {
      reason: stringValue(latestEvent.reason),
      message: stringValue(latestEvent.message),
      type: stringValue(latestEvent.type),
      count: typeof latestEvent.count === "number" ? latestEvent.count : null,
      timestamp: eventTimestamp(latestEvent),
    },
  };
}

function latestPodEvents(events: Record<string, unknown>[]): Map<string, Record<string, unknown>> {
  const byPod = new Map<string, Record<string, unknown>>();
  for (const item of events) {
    const involved = item.involvedObject;
    if (typeof involved !== "object" || involved === null || Array.isArray(involved)) continue;
    const podName = (involved as Record<string, unknown>).name;
    const kind = (involved as Record<string, unknown>).kind;
    if (kind !== "Pod" || typeof podName !== "string") continue;
    const current = byPod.get(podName);
    if (current === undefined || eventTimestamp(item) > eventTimestamp(current)) byPod.set(podName, item);
  }
  return byPod;
}

function eventTimestamp(item: Record<string, unknown>): string | null {
  const candidate = stringValue(item.eventTime)
    ?? stringValue(item.lastTimestamp)
    ?? stringValue(item.deprecatedLastTimestamp)
    ?? (typeof item.metadata === "object" && item.metadata !== null && !Array.isArray(item.metadata)
      ? stringValue((item.metadata as Record<string, unknown>).creationTimestamp)
      : null);
  return candidate;
}

function stringValue(value: unknown): string | null {
  return typeof value === "string" && value.length > 0 ? value : null;
}

function firstWaitingContainer(status: Record<string, unknown> | undefined): { reason: string | null; message: string | null } | null {
  const statuses = Array.isArray(status?.containerStatuses) ? status.containerStatuses : [];
  for (const item of statuses) {
    if (typeof item !== "object" || item === null || Array.isArray(item)) continue;
    const state = (item as Record<string, unknown>).state;
    if (typeof state !== "object" || state === null || Array.isArray(state)) continue;
    const waiting = (state as Record<string, unknown>).waiting;
    if (typeof waiting !== "object" || waiting === null || Array.isArray(waiting)) continue;
    const record = waiting as Record<string, unknown>;
    return {
      reason: typeof record.reason === "string" ? record.reason : null,
      message: typeof record.message === "string" ? record.message : null,
    };
  }
  return null;
}

function conditionSummary(status: Record<string, unknown> | undefined, type: string): { reason: string | null; message: string | null } | null {
  const conditions = Array.isArray(status?.conditions) ? status.conditions : [];
  for (const item of conditions) {
    if (typeof item !== "object" || item === null || Array.isArray(item)) continue;
    const record = item as Record<string, unknown>;
    if (record.type !== type || record.status === "True") continue;
    return {
      reason: typeof record.reason === "string" ? record.reason : null,
      message: typeof record.message === "string" ? record.message : null,
    };
  }
  return null;
}

function metadataName(item: Record<string, unknown>): string | null {
  const metadata = item.metadata as Record<string, unknown> | undefined;
  return typeof metadata?.name === "string" ? metadata.name : null;
}

function manifestObjectSummary(yaml: string): Array<Record<string, string>> {
  const docs = yaml.split(/^---$/mu);
  return docs.map((doc) => {
    const kind = doc.match(/^\s*kind:\s*(.+)$/mu)?.[1]?.trim() ?? "unknown";
    const name = doc.match(/^\s*name:\s*(.+)$/mu)?.[1]?.trim() ?? "unknown";
    return { kind, name };
  });
}

function imageReference(image: ImageSpec): string {
  return `${image.repository}:${image.tag}`;
}

function indent(value: string, spaces: number): string {
  const prefix = " ".repeat(spaces);
  return value.split("\n").map((line) => `${prefix}${line}`).join("\n");
}