Merge remote-tracking branch 'origin/master' into feat/sub2api-runtime-batch-reconcile

2026-07-14 12:50:25 +02:00
parent 551059a241 0dfcf3fdeb
commit 67f98e8620
81 changed files with 6136 additions and 96 deletions
@@ -0,0 +1,620 @@
+import { createHash } from "node:crypto";
+
+import type { UniDeskConfig } from "../config";
+import { CliInputError, type RenderedCliResult } from "../output";
+
+import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote";
+import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target";
+
+type FaultLevel = "P0" | "P1" | "P2";
+
+interface FaultOptions {
+  level: FaultLevel | null;
+  group: string | null;
+  account: string | null;
+  model: string | null;
+  stream: "sync" | "stream" | null;
+  endpoint: string | null;
+  requestId: string | null;
+  pageToken: string | null;
+  targetId: string;
+  json: boolean;
+}
+
+const PAGE_SIZE = 20;
+
+export async function codexPoolFaults(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
+  if (args.includes("--help")) return renderFaultHelp();
+  const options = parseFaultOptions(args);
+  const target = codexPoolRuntimeTarget(options.targetId);
+  const scope = faultScope(options);
+  const offset = decodePageToken(options.pageToken, scope);
+  const payload = {
+    filters: {
+      level: options.level,
+      group: options.group,
+      account: options.account,
+      model: options.model,
+      stream: options.stream,
+      endpoint: options.endpoint,
+      requestId: options.requestId,
+    },
+    offset,
+    pageSize: PAGE_SIZE,
+  };
+  const result = await runRemoteCodexPoolScript(config, "faults", faultScript(payload, target), target);
+  const parsed = parseJsonOutput(result.stdout);
+  const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
+  const data = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
+  const pagination = data && typeof data.pagination === "object" && data.pagination !== null
+    ? data.pagination as Record<string, unknown>
+    : null;
+  if (pagination?.hasMore === true) pagination.nextPageToken = encodePageToken(offset + PAGE_SIZE, scope);
+  if (pagination) {
+    pagination.pageSize = PAGE_SIZE;
+    pagination.pageToken = options.pageToken;
+  }
+  const response = {
+    ok,
+    action: "platform-infra-sub2api-codex-pool-faults",
+    target: {
+      id: target.id,
+      route: target.route,
+      runtimeMode: target.runtimeMode,
+      endpoint: target.serviceDns,
+    },
+    source: {
+      kind: "sub2api-native-admin-ops",
+      versionBoundary: "Sub2API native admin Ops facts with explicit, version-neutral UniDesk CLI projection",
+      mutation: false,
+    },
+    filters: payload.filters,
+    faults: data,
+    remote: compactCapture(result, { full: result.exitCode !== 0 || data === null }),
+    valuesPrinted: false,
+  };
+  if (options.json) return response;
+  return renderFaults(response);
+}
+
+function renderFaultHelp(): RenderedCliResult {
+  return {
+    ok: true,
+    command: "platform-infra sub2api codex-pool faults --help",
+    renderedText: [
+      "SUB2API CODEX-POOL FAULTS",
+      "Usage:",
+      "  bun scripts/cli.ts platform-infra sub2api codex-pool faults [options]",
+      "Options:",
+      "  --level P0|P1|P2",
+      "  --group <name-or-id>",
+      "  --account <name-or-id>",
+      "  --model <model>",
+      "  --stream sync|stream",
+      "  --endpoint <path>",
+      "  --request-id <stable-id>",
+      "  --page-token <token>",
+      "  --target <id>",
+      "  --json",
+      "Notes:",
+      `  Fixed page size: ${PAGE_SIZE}; --limit is intentionally unsupported.`,
+      "  Default output is a bounded Kubernetes-style table; JSON requires --json.",
+    ].join("\n"),
+    contentType: "text/plain",
+  };
+}
+
+function parseFaultOptions(args: string[]): FaultOptions {
+  let level: FaultLevel | null = null;
+  let group: string | null = null;
+  let account: string | null = null;
+  let model: string | null = null;
+  let stream: "sync" | "stream" | null = null;
+  let endpoint: string | null = null;
+  let requestId: string | null = null;
+  let pageToken: string | null = null;
+  let targetId = defaultCodexPoolRuntimeTargetId();
+  let json = false;
+  const readValue = (index: number, name: string): [string, number] => {
+    const value = args[index + 1];
+    if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
+    return [validateSelector(value, name), index + 1];
+  };
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index]!;
+    if (arg === "--json") json = true;
+    else if (arg === "--level") {
+      const [value, next] = readValue(index, "--level");
+      level = parseLevel(value);
+      index = next;
+    } else if (arg.startsWith("--level=")) level = parseLevel(arg.slice(8));
+    else if (arg === "--group") [group, index] = readValue(index, "--group");
+    else if (arg.startsWith("--group=")) group = validateSelector(arg.slice(8), "--group");
+    else if (arg === "--account") [account, index] = readValue(index, "--account");
+    else if (arg.startsWith("--account=")) account = validateSelector(arg.slice(10), "--account");
+    else if (arg === "--model") [model, index] = readValue(index, "--model");
+    else if (arg.startsWith("--model=")) model = validateSelector(arg.slice(8), "--model");
+    else if (arg === "--stream") {
+      const [value, next] = readValue(index, "--stream");
+      stream = parseStream(value);
+      index = next;
+    } else if (arg.startsWith("--stream=")) stream = parseStream(arg.slice(9));
+    else if (arg === "--endpoint") [endpoint, index] = readValue(index, "--endpoint");
+    else if (arg.startsWith("--endpoint=")) endpoint = validateSelector(arg.slice(11), "--endpoint");
+    else if (arg === "--request-id") [requestId, index] = readValue(index, "--request-id");
+    else if (arg.startsWith("--request-id=")) requestId = validateSelector(arg.slice(13), "--request-id");
+    else if (arg === "--page-token") [pageToken, index] = readValue(index, "--page-token");
+    else if (arg.startsWith("--page-token=")) pageToken = validateSelector(arg.slice(13), "--page-token");
+    else if (arg === "--target") [targetId, index] = readValue(index, "--target");
+    else if (arg.startsWith("--target=")) targetId = validateSelector(arg.slice(9), "--target");
+    else if (arg === "--limit" || arg.startsWith("--limit=")) throw new Error("faults uses a fixed page size; use --page-token for progressive disclosure");
+    else throw new Error(`unsupported faults option: ${arg}`);
+  }
+  if (level !== null && group === null) {
+    throw new CliInputError("--level requires --group; use the default command for the all-groups overview", {
+      code: "missing-required-option",
+      argument: "--group",
+      usage: [
+        "bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01",
+        "bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01 --level P0 --group <name-or-id>",
+      ],
+      hint: "Run without --level for the all-groups overview, then drill down with --level and --group.",
+    });
+  }
+  return { level, group, account, model, stream, endpoint, requestId, pageToken, targetId, json };
+}
+
+function parseLevel(value: string): FaultLevel {
+  const normalized = value.toUpperCase();
+  if (normalized !== "P0" && normalized !== "P1" && normalized !== "P2") throw new Error("--level must be P0, P1, or P2");
+  return normalized;
+}
+
+function parseStream(value: string): "sync" | "stream" {
+  if (value !== "sync" && value !== "stream") throw new Error("--stream must be sync or stream");
+  return value;
+}
+
+function validateSelector(value: string, name: string): string {
+  const normalized = value.trim();
+  if (!normalized || normalized.length > 512 || /[\r\n\0]/u.test(normalized)) throw new Error(`${name} has an unsupported format`);
+  return normalized;
+}
+
+function faultScope(options: FaultOptions): string {
+  return createHash("sha256").update(JSON.stringify({
+    level: options.level,
+    group: options.group,
+    account: options.account,
+    model: options.model,
+    stream: options.stream,
+    endpoint: options.endpoint,
+    requestId: options.requestId,
+    targetId: options.targetId,
+  })).digest("hex").slice(0, 16);
+}
+
+function encodePageToken(offset: number, scope: string): string {
+  return Buffer.from(JSON.stringify({ v: 1, offset, scope }), "utf8").toString("base64url");
+}
+
+function decodePageToken(token: string | null, scope: string): number {
+  if (token === null) return 0;
+  try {
+    const parsed = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as Record<string, unknown>;
+    if (parsed.v !== 1 || parsed.scope !== scope || !Number.isInteger(parsed.offset) || Number(parsed.offset) < 0) throw new Error("invalid");
+    return Number(parsed.offset);
+  } catch {
+    throw new Error("--page-token is invalid or belongs to different filters");
+  }
+}
+
+function renderFaults(response: Record<string, unknown>): RenderedCliResult {
+  const faults = response.faults && typeof response.faults === "object" ? response.faults as Record<string, unknown> : {};
+  const lines = ["PLATFORM-INFRA SUB2API FAULTS"];
+  const window = record(faults.window);
+  lines.push(`WINDOW ${text(window.timeRange)}  SOURCE native-admin-ops  PROJECTION unidesk-cli`);
+  lines.push("");
+  const summary = arrayOfRecords(faults.summary);
+  if (summary.length > 0) {
+    lines.push("GROUPS");
+    lines.push(table(["GROUP", "P0", "CUSTOMER_ERRS", "GROUP_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERRS", "GROUP_UP_ERR%", "ABSORB%"], summary.map((row) => [
+      `${text(row.groupName)} (${text(row.groupId)})`,
+      text(row.p0), text(row.customerErrorCount), percent(row.customerErrorRatePercent),
+      text(row.p1), milliseconds(row.ttftP99Ms),
+      text(row.p2), text(row.upstreamErrorCount), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent),
+    ])));
+  }
+  const details = arrayOfRecords(faults.details);
+  if (details.length > 0) {
+    lines.push("", `DETAILS ${text(faults.level ?? "SUMMARY")}`);
+    const columns = Array.isArray(faults.detailColumns) ? faults.detailColumns.map(String) : [];
+    lines.push(table(columns, details.map((row) => columns.map((column) => text(row[column])))));
+  }
+  const boundary = record(faults.boundary);
+  lines.push("", "BOUNDARY");
+  for (const value of Array.isArray(boundary.notes) ? boundary.notes : []) lines.push(`- ${String(value)}`);
+  const pagination = record(faults.pagination);
+  if (pagination.nextPageToken) lines.push("", `NEXT_PAGE_TOKEN ${text(pagination.nextPageToken)}`);
+  if (faults.next) lines.push("", "NEXT", ...String(faults.next).split("\n").map((line) => `  ${line}`));
+  return {
+    ok: response.ok === true,
+    command: "platform-infra sub2api codex-pool faults",
+    renderedText: lines.join("\n"),
+    contentType: "text/plain",
+    projection: response,
+  };
+}
+
+function record(value: unknown): Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
+function arrayOfRecords(value: unknown): Record<string, unknown>[] {
+  return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
+}
+
+function text(value: unknown): string {
+  if (value === null || value === undefined || value === "") return "-";
+  if (typeof value === "boolean") return value ? "yes" : "no";
+  return String(value).replace(/[\r\n\t]+/gu, " ");
+}
+
+function percent(value: unknown): string {
+  return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : text(value);
+}
+
+function milliseconds(value: unknown): string {
+  return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)}ms` : text(value);
+}
+
+function table(headers: string[], rows: string[][]): string {
+  const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length)));
+  return [headers, ...rows].map((row) => row.map((cell, index) => cell.padEnd(widths[index]!)).join("  ").trimEnd()).join("\n");
+}
+
+function pyJson(value: unknown): string {
+  return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
+}
+
+function faultScript(payload: unknown, target: ReturnType<typeof codexPoolRuntimeTarget>): string {
+  return `
+set -u
+python3 - <<'PY'
+import base64
+import json
+import math
+import subprocess
+import sys
+from urllib.parse import urlencode
+
+RUNTIME_MODE = ${pyJson(target.runtimeMode)}
+NAMESPACE = ${pyJson(target.namespace)}
+HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)}
+APP_SECRET_NAME = ${pyJson(target.appSecretName)}
+PAYLOAD = ${pyJson(payload)}
+APP_POD = None
+
+def run(cmd, input_bytes=None):
+    return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+def docker(args):
+    proc = run(["docker", *args])
+    if proc.returncode == 0:
+        return proc
+    sudo_proc = run(["sudo", "-n", "docker", *args])
+    return sudo_proc if sudo_proc.returncode == 0 else proc
+
+def kubectl(args, input_bytes=None):
+    return run(["kubectl", *args], input_bytes)
+
+def kube_json(args, label):
+    proc = kubectl([*args, "-o", "json"])
+    if proc.returncode != 0:
+        raise RuntimeError(label + " failed")
+    return json.loads(proc.stdout.decode("utf-8"))
+
+if RUNTIME_MODE != "host-docker":
+    pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app=sub2api"], "list pods").get("items") or []
+    ready = [item for item in pods if item.get("status", {}).get("phase") == "Running"]
+    if not ready:
+        raise RuntimeError("sub2api app pod not found")
+    APP_POD = ready[0]["metadata"]["name"]
+
+def config_value(name, key, default=None):
+    if RUNTIME_MODE == "host-docker":
+        proc = docker(["inspect", "sub2api-app", "--format", "{{json .Config.Env}}"])
+        if proc.returncode != 0:
+            return default
+        for item in json.loads(proc.stdout.decode("utf-8")):
+            if item.startswith(key + "="):
+                return item.split("=", 1)[1]
+        return default
+    data = kube_json(["-n", NAMESPACE, "get", "configmap", name], "configmap/" + name).get("data") or {}
+    return data.get(key, default)
+
+def secret_value(name, key):
+    if RUNTIME_MODE == "host-docker":
+        return config_value("", key)
+    data = kube_json(["-n", NAMESPACE, "get", "secret", name], "secret/" + name).get("data") or {}
+    return base64.b64decode(data[key]).decode("utf-8") if key in data else None
+
+def curl_api(method, path, bearer=None, payload=None):
+    body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
+    script = r'''set -eu
+method="$1"; url="$2"; token="\${3:-}"; tmp="$(mktemp)"; trap 'rm -f "$tmp"' EXIT; cat > "$tmp"
+args=""; [ -n "$token" ] && args="Authorization: Bearer $token"
+if [ -n "$args" ] && [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
+elif [ -n "$args" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" "$url"
+elif [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
+else curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"; fi'''
+    base = f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}" if RUNTIME_MODE == "host-docker" else "http://127.0.0.1:8080"
+    cmd = ["sh", "-c", script, "sh", method, base + path, bearer or ""]
+    proc = run(cmd, body) if RUNTIME_MODE == "host-docker" else run(["kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", *cmd], body)
+    output = proc.stdout.decode("utf-8", errors="replace")
+    marker = "\\n__HTTP_CODE__:"
+    pos = output.rfind(marker)
+    status = int(output[pos + len(marker):].strip()[-3:]) if pos >= 0 else 0
+    raw = output[:pos] if pos >= 0 else output
+    try:
+        parsed = json.loads(raw) if raw.strip() else None
+    except json.JSONDecodeError:
+        parsed = None
+    return {"ok": proc.returncode == 0 and 200 <= status < 300, "status": status, "json": parsed, "body": raw[:300]}
+
+def data_of(response, label):
+    parsed = response.get("json")
+    code = parsed.get("code") if isinstance(parsed, dict) else None
+    if response.get("ok") is not True or (code is not None and code != 0):
+        message = parsed.get("message") if isinstance(parsed, dict) else response.get("body")
+        raise RuntimeError(f"{label} failed: http={response.get('status')} message={message}")
+    return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
+
+def login():
+    email = config_value("sub2api-config", "ADMIN_EMAIL", "admin@example.com")
+    password = secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD")
+    if not password:
+        raise RuntimeError("ADMIN_PASSWORD missing")
+    data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login")
+    token = data.get("access_token") or data.get("token") if isinstance(data, dict) else None
+    if not token:
+        raise RuntimeError("admin login response has no token")
+    return token
+
+def get(token, path, params=None, label=None):
+    suffix = "?" + urlencode({key: value for key, value in (params or {}).items() if value is not None}) if params else ""
+    return data_of(curl_api("GET", path + suffix, bearer=token), label or path)
+
+def items_of(data):
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        for key in ("items", "groups", "accounts"):
+            if isinstance(data.get(key), list):
+                return data[key]
+    return []
+
+def total_of(data):
+    return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data))
+
+def all_items(token, path, params, label):
+    page = 1
+    page_size = 500
+    rows = []
+    while True:
+        data = get(token, path, {**params, "page": page, "page_size": page_size}, label)
+        batch = items_of(data)
+        rows.extend(batch)
+        total = total_of(data)
+        if not batch or len(rows) >= total:
+            return rows
+        page += 1
+
+def percent(value):
+    if not isinstance(value, (int, float)):
+        return None
+    return round(value * 100 if value <= 1 else value, 4)
+
+def selector_id(value):
+    return int(value) if isinstance(value, str) and value.isdigit() else None
+
+def scope_filters(group_id=None):
+    result = {"time_range": "24h", "platform": "openai", "group_id": group_id}
+    return result
+
+def matches_detail_filters(item):
+    filters = PAYLOAD["filters"]
+    account = filters.get("account")
+    if account:
+        account_match = str(item.get("account_id") or "") == str(account) or str(item.get("account_name") or "") == str(account)
+        if not account_match:
+            return False
+    if filters.get("model") and str(item.get("requested_model") or item.get("model") or "") != str(filters["model"]):
+        return False
+    if filters.get("stream"):
+        expected_stream = filters["stream"] == "stream"
+        if item.get("stream") is not expected_stream:
+            return False
+    if filters.get("endpoint") and str(item.get("inbound_endpoint") or item.get("request_path") or item.get("path") or "") != str(filters["endpoint"]):
+        return False
+    if filters.get("requestId"):
+        request_id = filters["requestId"]
+        if item.get("request_id") != request_id and item.get("client_request_id") != request_id:
+            return False
+    return True
+
+def has_detail_filters():
+    filters = PAYLOAD["filters"]
+    return any(filters.get(key) for key in ("account", "model", "stream", "endpoint", "requestId"))
+
+DETAIL_ROWS = {}
+
+def error_rows(token, path, group_id, apply_filters):
+    cache_key = (path, group_id, apply_filters)
+    if cache_key not in DETAIL_ROWS:
+        rows = all_items(token, path, {**scope_filters(group_id), "view": "all", "include_detail": 1}, "fault detail scan")
+        DETAIL_ROWS[cache_key] = [item for item in rows if matches_detail_filters(item)] if apply_filters else rows
+    return DETAIL_ROWS[cache_key]
+
+def selected_groups(groups):
+    selector = PAYLOAD["filters"].get("group")
+    if not selector:
+        return groups
+    selected = [item for item in groups if str(item.get("id")) == str(selector) or item.get("name") == selector]
+    if len(selected) != 1:
+        raise RuntimeError("group-not-found-or-ambiguous: " + str(selector))
+    return selected
+
+def threshold_state(value, threshold):
+    if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)):
+        return "unavailable"
+    return "breach" if value >= threshold else "ok"
+
+def availability_by_account(token, group_id):
+    data = get(token, "/api/v1/admin/ops/account-availability", {"platform": "openai", "group_id": group_id}, "account availability")
+    raw = data.get("account") if isinstance(data, dict) else None
+    return raw if isinstance(raw, dict) else {}
+
+def detail_row(item, group_metrics, availability, level, absorbed=None):
+    account_id = item.get("account_id")
+    state = availability.get(str(account_id), {}) if isinstance(availability, dict) else {}
+    endpoint = item.get("inbound_endpoint") or item.get("request_path") or item.get("path")
+    base = {
+        "GROUP": f"{item.get('group_name') or group_metrics.get('groupName')} ({item.get('group_id') or group_metrics.get('groupId')})",
+        "ACCOUNT": f"{item.get('account_name') or '-'} ({account_id or '-'})",
+        "MODEL": item.get("requested_model") or item.get("model") or "-",
+        "MODE": "stream" if item.get("stream") is True else "sync" if item.get("stream") is False else "-",
+        "ENDPOINT": endpoint or "-",
+        "STATUS": item.get("status_code") or "-",
+        "REQUEST_ID": item.get("request_id") or item.get("client_request_id") or "-",
+    }
+    if level == "P0":
+        base.update({"CUSTOMER_ERR%": group_metrics.get("customerErrorRatePercent"), "ABSORBED": False})
+    else:
+        unavailable_reason = str(state.get("unavailable_reason") or state.get("reason") or "").lower()
+        if state.get("is_available") is False and ("temporary" in unavailable_reason or "cooldown" in unavailable_reason):
+            temp_unsched = "active"
+        elif state.get("is_available") is True:
+            temp_unsched = "none"
+        else:
+            temp_unsched = "unavailable"
+        base.update({"ROOT": item.get("error_source") or item.get("message") or "upstream", "TEMP_UNSCHED": temp_unsched, "ABSORBED": absorbed})
+    return base
+
+def execute():
+    token = login()
+    thresholds = get(token, "/api/v1/admin/ops/settings/metric-thresholds", label="metric thresholds")
+    groups = selected_groups(items_of(get(token, "/api/v1/admin/groups/all", {"platform": "openai"}, "list groups")))
+    summaries = []
+    group_data = []
+    for group in groups:
+        group_id = group.get("id")
+        native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview")
+        if has_detail_filters():
+            customer_total = len(error_rows(token, "/api/v1/admin/ops/request-errors", group_id, True))
+            upstream_total = len(error_rows(token, "/api/v1/admin/ops/upstream-errors", group_id, True))
+        else:
+            request_errors = get(token, "/api/v1/admin/ops/request-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors")
+            upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors")
+            customer_total = total_of(request_errors)
+            upstream_total = total_of(upstream_errors)
+        absorbed = max(0, upstream_total - customer_total)
+        absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None
+        customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None)
+        upstream_rate = percent(native.get("upstream_error_rate") if isinstance(native, dict) else None)
+        ttft = native.get("ttft") if isinstance(native, dict) and isinstance(native.get("ttft"), dict) else {}
+        metrics = {
+            "groupId": group_id,
+            "groupName": group.get("name"),
+            "customerErrorCount": customer_total,
+            "customerErrorRatePercent": customer_rate,
+            "upstreamErrorCount": upstream_total,
+            "upstreamErrorRatePercent": upstream_rate,
+            "absorbedCountProjection": absorbed,
+            "absorbedPercent": absorbed_percent,
+            "ttftP99Ms": ttft.get("p99_ms"),
+        }
+        metrics["p0"] = "active" if customer_total > 0 else "clear"
+        metrics["p1"] = threshold_state(metrics["ttftP99Ms"], thresholds.get("ttft_p99_ms_max"))
+        metrics["p2"] = threshold_state(upstream_rate, thresholds.get("upstream_error_rate_percent_max"))
+        summaries.append(metrics)
+        group_data.append((group, metrics))
+    level = PAYLOAD["filters"].get("level")
+    offset = int(PAYLOAD.get("offset") or 0)
+    page = offset // int(PAYLOAD["pageSize"]) + 1
+    details = []
+    detail_columns = []
+    total = 0
+    if level in ("P0", "P2"):
+        for group, metrics in group_data:
+            group_id = group.get("id")
+            endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors"
+            if has_detail_filters():
+                rows = error_rows(token, endpoint, group_id, True)
+                total += len(rows)
+                rows = rows[offset:offset + PAYLOAD["pageSize"]]
+            else:
+                data = get(token, endpoint, {**scope_filters(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details")
+                total += total_of(data)
+                rows = items_of(data)
+            availability = availability_by_account(token, group_id) if level == "P2" else {}
+            customer_request_ids = set()
+            if level == "P2":
+                customer_request_ids = {
+                    item.get("request_id") or item.get("client_request_id")
+                    for item in error_rows(token, "/api/v1/admin/ops/request-errors", group_id, False)
+                    if item.get("request_id") or item.get("client_request_id")
+                }
+            for item in rows:
+                absorbed = None
+                if level == "P2" and item.get("request_id"):
+                    absorbed = item.get("request_id") not in customer_request_ids
+                details.append(detail_row(item, metrics, availability, level, absorbed))
+        detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"]
+    elif level == "P1":
+        total = len(group_data) * 2
+        endpoint_names = ["/v1/responses", "/v1/responses/compact"]
+        for group, metrics in group_data:
+            for endpoint in endpoint_names:
+                if PAYLOAD["filters"].get("endpoint") and PAYLOAD["filters"].get("endpoint") != endpoint:
+                    continue
+                details.append({
+                    "GROUP": f"{metrics.get('groupName')} ({metrics.get('groupId')})",
+                    "ENDPOINT": endpoint,
+                    "SAMPLES": "unavailable",
+                    "P50": "unavailable",
+                    "P95": "unavailable",
+                    "P99": "unavailable",
+                    "MAX": "unavailable",
+                    "OUTLIER_REQUEST_ID": "unavailable",
+                    "BOUNDARY": "native endpoint/request TTFT unavailable; total duration not substituted",
+                })
+        detail_columns = ["GROUP", "ENDPOINT", "SAMPLES", "P50", "P95", "P99", "MAX", "OUTLIER_REQUEST_ID", "BOUNDARY"]
+    return {
+        "ok": True,
+        "level": level,
+        "window": {"timeRange": "24h"},
+        "thresholds": thresholds,
+        "summary": summaries,
+        "details": details[:PAYLOAD["pageSize"]],
+        "detailColumns": detail_columns,
+        "pagination": {"offset": offset, "total": total, "hasMore": offset + PAYLOAD["pageSize"] < total},
+        "boundary": {"notes": [
+            "P0/P2 facts come from native admin ops request-errors, upstream-errors, overview, and account-availability.",
+            "P0/P1/P2 labels and failover/retry absorption are UniDesk CLI projections; Sub2API native severity is not rewritten.",
+            "P1 endpoint-level samples and request-level TTFT are unavailable in the observed native Ops response; total duration is never used as TTFT.",
+            "Absorption summary is a bounded count projection from native upstream/customer totals; P2 detail absorption is correlated by stable request ID.",
+        ]},
+        "next": "Use --level P0|P1|P2 for details; reuse filters with --page-token for the next fixed page; use codex-pool trace --request-id <id> for trace disclosure.",
+        "valuesPrinted": False,
+    }
+
+try:
+    output = execute()
+except Exception as exc:
+    output = {"ok": False, "error": str(exc), "valuesPrinted": False}
+print(json.dumps(output, ensure_ascii=False, indent=2))
+sys.exit(0 if output.get("ok") else 1)
+PY
+`;
+}
@@ -0,0 +1,593 @@
+import type { UniDeskConfig } from "../config";
+import type { RenderedCliResult } from "../output";
+
+import { readCodexPoolConfig } from "./config";
+import { capture, compactCapture, parseJsonOutput } from "./remote";
+import { renderedCliResult, renderTable, shorten, shortIso, textValue } from "./render";
+import { codexPoolRuntimeTarget } from "./runtime-target";
+import type { CodexPoolConfig, CodexPoolRuntimeTarget, FeedbackOptions } from "./types";
+
+function pyJson(value: unknown): string {
+  return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
+}
+
+export async function codexPoolFeedback(config: UniDeskConfig, options: FeedbackOptions): Promise<Record<string, unknown> | RenderedCliResult> {
+  const pool = readCodexPoolConfig();
+  const target = codexPoolRuntimeTarget(options.targetId);
+  const result = await capture(config, target.route, ["sh"], feedbackScript(pool, options, target));
+  const report = parseJsonOutput(result.stdout);
+  const ok = result.exitCode === 0 && report?.ok === true;
+  const remote = compactCapture(result, { full: result.exitCode !== 0 || report === null });
+  if (options.json) {
+    return {
+      ok,
+      action: "platform-infra-sub2api-codex-pool-feedback",
+      target: { id: target.id, route: target.route, runtimeMode: target.runtimeMode },
+      report,
+      remote,
+      valuesPrinted: false,
+    };
+  }
+  return renderedCliResult(ok, "platform-infra sub2api codex-pool feedback", renderFeedbackReport(report, options, remote));
+}
+
+function feedbackScript(pool: CodexPoolConfig, options: FeedbackOptions, target: CodexPoolRuntimeTarget): string {
+  return `
+set -u
+python3 - <<'PY'
+import base64
+import json
+import re
+import subprocess
+from datetime import datetime, timezone, timedelta
+from urllib.parse import quote, urlencode
+
+RUNTIME_MODE = ${pyJson(target.runtimeMode)}
+NAMESPACE = ${pyJson(target.namespace)}
+APP_SECRET_NAME = ${pyJson(target.appSecretName)}
+ADMIN_EMAIL_DEFAULT = ${pyJson(pool.adminEmailDefault)}
+HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)}
+HOST_DOCKER_ENV_PATH = ${pyJson(target.hostDockerEnvPath)}
+USER_SELECTOR = ${pyJson(options.user)}
+WINDOW = ${pyJson(options.window)}
+REQUEST_SELECTOR = ${pyJson(options.requestId)}
+PAGE_TOKEN = ${pyJson(options.pageToken)}
+APP_CONTAINER = "sub2api-app"
+PAGE_SIZE = 10
+
+def run(command, input_bytes=None):
+    return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+def text(value, limit=1000):
+    if isinstance(value, bytes):
+        value = value.decode("utf-8", errors="replace")
+    return str(value or "")[-limit:]
+
+def docker(args):
+    proc = run(["docker", *args])
+    if proc.returncode == 0:
+        return proc
+    fallback = run(["sudo", "-n", "docker", *args])
+    return fallback if fallback.returncode == 0 else proc
+
+def read_host_env():
+    if RUNTIME_MODE != "host-docker":
+        return {}
+    try:
+        with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle:
+            lines = handle.read().splitlines()
+    except PermissionError:
+        proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH])
+        if proc.returncode != 0:
+            raise RuntimeError("read host-docker env failed: " + text(proc.stderr))
+        lines = proc.stdout.decode("utf-8", errors="replace").splitlines()
+    values = {}
+    for line in lines:
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#") or "=" not in stripped:
+            continue
+        key, value = stripped.split("=", 1)
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
+            value = value[1:-1]
+        values[key.strip()] = value
+    return values
+
+def runtime_secret(key):
+    if RUNTIME_MODE == "host-docker":
+        return read_host_env().get(key)
+    proc = run(["kubectl", "-n", NAMESPACE, "get", "secret", APP_SECRET_NAME, "-o", "json"])
+    if proc.returncode != 0:
+        raise RuntimeError("read app secret failed: " + text(proc.stderr))
+    raw = (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
+    return base64.b64decode(raw).decode("utf-8") if raw else None
+
+def runtime_config(key):
+    if RUNTIME_MODE == "host-docker":
+        return read_host_env().get(key)
+    proc = run(["kubectl", "-n", NAMESPACE, "get", "configmap", "sub2api-config", "-o", "json"])
+    if proc.returncode != 0:
+        return None
+    return (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
+
+def parse_curl(proc):
+    stdout = proc.stdout.decode("utf-8", errors="replace")
+    marker = "\\n__HTTP_CODE__:"
+    position = stdout.rfind(marker)
+    if position < 0:
+        return {"ok": False, "httpStatus": 0, "json": None, "message": text(proc.stderr)}
+    body = stdout[:position]
+    try:
+        status = int(stdout[position + len(marker):].strip()[-3:])
+    except Exception:
+        status = 0
+    try:
+        parsed = json.loads(body) if body.strip() else None
+    except Exception:
+        parsed = None
+    message = parsed.get("message") if isinstance(parsed, dict) else text(body)
+    return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "message": message}
+
+def curl_api(method, path, bearer=None, payload=None):
+    body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
+    script = r'''set -eu
+method="$1"
+url="$2"
+token="\${3:-}"
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+cat > "$tmp"
+args=""
+if [ -n "$token" ]; then args="Authorization: Bearer $token"; fi
+if [ "$method" = GET ] && [ ! -s "$tmp" ]; then
+  if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -H "$args" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' "$url"; fi
+else
+  if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "$args" --data-binary @"$tmp" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"; fi
+fi'''
+    if RUNTIME_MODE == "host-docker":
+        proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body)
+    else:
+        proc = run(["kubectl", "-n", NAMESPACE, "exec", "deploy/sub2api", "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""], body)
+    return parse_curl(proc)
+
+def api_data(method, path, token=None, payload=None, label=None):
+    response = curl_api(method, path, token, payload)
+    parsed = response.get("json")
+    code = parsed.get("code") if isinstance(parsed, dict) else None
+    if not response.get("ok") or (code is not None and code != 0):
+        raise RuntimeError(f"{label or path} failed: http={response.get('httpStatus')} message={response.get('message')}")
+    return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
+
+def find_token(value):
+    if isinstance(value, dict):
+        for key in ("access_token", "token"):
+            if isinstance(value.get(key), str) and value.get(key):
+                return value[key]
+        for item in value.values():
+            token = find_token(item)
+            if token:
+                return token
+    return None
+
+def login():
+    email = runtime_config("ADMIN_EMAIL") or ADMIN_EMAIL_DEFAULT
+    password = runtime_secret("ADMIN_PASSWORD")
+    if not password:
+        raise RuntimeError("ADMIN_PASSWORD missing")
+    token = find_token(api_data("POST", "/api/v1/auth/login", payload={"email": email, "password": password}, label="admin login"))
+    if not token:
+        raise RuntimeError("admin login returned no token")
+    return email, token
+
+def items(data):
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        if isinstance(data.get("items"), list):
+            return data["items"]
+        for key in ("users", "api_keys", "keys", "groups", "accounts", "proxies", "logs"):
+            if isinstance(data.get(key), list):
+                return data[key]
+    return []
+
+def page_all(token, path, query=None, page_size=100):
+    query = dict(query or {})
+    collected = []
+    page = 1
+    while True:
+        params = {**query, "page": page, "page_size": page_size}
+        separator = "&" if "?" in path else "?"
+        data = api_data("GET", path + separator + urlencode(params), token, label=path)
+        batch = items(data)
+        collected.extend(batch)
+        total = data.get("total") if isinstance(data, dict) else None
+        if not batch or (isinstance(total, int) and len(collected) >= total) or len(batch) < page_size:
+            break
+        page += 1
+    return collected
+
+def exact_user(token):
+    if USER_SELECTOR.isdigit() and int(USER_SELECTOR) > 0:
+        user = api_data("GET", f"/api/v1/admin/users/{int(USER_SELECTOR)}", token, label="get user")
+        return user if isinstance(user, dict) else None
+    candidates = page_all(token, "/api/v1/admin/users", {"search": USER_SELECTOR}, 100)
+    exact = [user for user in candidates if isinstance(user, dict) and str(user.get("email") or "").lower() == USER_SELECTOR.lower()]
+    if len(exact) > 1:
+        raise RuntimeError("email matched multiple users")
+    return exact[0] if exact else None
+
+def safe_call(errors, name, callback, fallback):
+    try:
+        return callback()
+    except Exception as exc:
+        errors.append({"source": name, "error": redact_text(str(exc))})
+        return fallback
+
+def redact_text(value):
+    value = str(value or "")
+    value = re.sub(r"(?i)(bearer\\s+)[A-Za-z0-9._~+\\-/=]+", r"\\1<redacted>", value)
+    value = re.sub(r"(?i)((?:api[_-]?key|token|password)\\s*[=:]\\s*)[^\\s,;]+", r"\\1<redacted>", value)
+    value = re.sub(r"\\bsk-[A-Za-z0-9_-]{8,}\\b", "sk-<redacted>", value)
+    return value[:500]
+
+def iso_epoch(value):
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        match = re.match(r"^(\\d{4}-\\d\\d-\\d\\d[T ]\\d\\d:\\d\\d:\\d\\d)(?:\\.(\\d+))?(Z|[+-]\\d\\d:?\\d\\d)$", value)
+        if not match:
+            return None
+        base = datetime.strptime(match.group(1).replace(" ", "T"), "%Y-%m-%dT%H:%M:%S")
+        fraction = int((match.group(2) or "0")[:6].ljust(6, "0"))
+        zone = match.group(3)
+        if zone == "Z":
+            offset = timezone.utc
+        else:
+            sign = 1 if zone[0] == "+" else -1
+            digits = zone[1:].replace(":", "")
+            offset = timezone(sign * timedelta(hours=int(digits[:2]), minutes=int(digits[2:])))
+        return base.replace(microsecond=fraction, tzinfo=offset).timestamp()
+    except Exception:
+        return None
+
+def encode_page_token(row):
+    payload = json.dumps({"v": 1, "at": row.get("at"), "requestId": row.get("requestId")}, separators=(",", ":")).encode("utf-8")
+    return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=")
+
+def decode_page_token(value):
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        padded = value + "=" * (-len(value) % 4)
+        parsed = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8"))
+    except Exception:
+        raise RuntimeError("invalid page token")
+    if not isinstance(parsed, dict) or parsed.get("v") != 1 or not parsed.get("at") or not parsed.get("requestId"):
+        raise RuntimeError("invalid page token")
+    return parsed
+
+def dict_by_id(rows):
+    return {str(row.get("id")): row for row in rows if isinstance(row, dict) and row.get("id") is not None}
+
+def keyed_map(value, key):
+    source = value.get(key) if isinstance(value, dict) else None
+    return source if isinstance(source, dict) else {}
+
+def runtime_health():
+    if RUNTIME_MODE == "host-docker":
+        proc = docker(["inspect", APP_CONTAINER, "sub2api-redis"])
+        if proc.returncode != 0:
+            return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
+        rows = []
+        for item in json.loads(proc.stdout.decode("utf-8")):
+            state = item.get("State") or {}
+            health = state.get("Health") or {}
+            rows.append({"name": str(item.get("Name") or "").lstrip("/"), "running": state.get("Running"), "status": health.get("Status") or state.get("Status")})
+        return {"ok": all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
+    proc = run(["kubectl", "-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api", "-o", "json"])
+    if proc.returncode != 0:
+        return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
+    rows = []
+    for item in json.loads(proc.stdout.decode("utf-8")).get("items") or []:
+        status = item.get("status") or {}
+        containers = status.get("containerStatuses") or []
+        rows.append({"name": (item.get("metadata") or {}).get("name"), "running": status.get("phase") == "Running", "status": "ready" if containers and all(entry.get("ready") is True for entry in containers) else status.get("phase")})
+    return {"ok": bool(rows) and all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
+
+def compact_log(row):
+    extra = row.get("extra") if isinstance(row.get("extra"), dict) else {}
+    allow = {}
+    for key in ("phase", "error_owner", "status_code", "duration_ms", "latency_ms", "time_to_first_token_ms", "upstream_status", "path", "stream"):
+        if key in extra:
+            allow[key] = extra.get(key)
+    return {
+        "id": row.get("id"),
+        "at": row.get("created_at"),
+        "level": row.get("level"),
+        "component": row.get("component"),
+        "message": redact_text(row.get("message")),
+        "requestId": row.get("request_id"),
+        "clientRequestId": row.get("client_request_id"),
+        "accountId": row.get("account_id"),
+        "extra": allow,
+    }
+
+def related_logs(request, logs):
+    ids = {str(request.get("request_id") or "")}
+    matched = []
+    changed = True
+    while changed:
+        changed = False
+        for row in logs:
+            request_id = str(row.get("request_id") or "")
+            client_id = str(row.get("client_request_id") or "")
+            if row in matched or (request_id not in ids and client_id not in ids):
+                continue
+            matched.append(row)
+            before = len(ids)
+            ids.update(value for value in (request_id, client_id) if value)
+            changed = changed or len(ids) != before
+    return matched
+
+def classify(request, logs, account, proxy, queued):
+    evidence = " ".join([str(request.get("phase") or ""), str(request.get("message") or ""), *[str(log.get("component") or "") + " " + str(log.get("message") or "") for log in logs]]).lower()
+    if any(marker in evidence for marker in ("client disconnect", "broken pipe", "context canceled", "client canceled")):
+        return "client-connection"
+    if any(marker in evidence for marker in ("proxy", "dial tcp", "dns", "connection refused", "network")) or (proxy and proxy.get("status") not in (None, "active")):
+        return "proxy-egress"
+    if any(marker in evidence for marker in ("upstream", "provider", "first token", "stream")) or str(request.get("phase") or "") == "upstream":
+        return "upstream-first-byte-stream"
+    if queued > 0:
+        return "sub2api-queue"
+    if request.get("kind") == "error" or (isinstance(request.get("status_code"), int) and request.get("status_code") >= 400):
+        return "sub2api-or-upstream-unknown"
+    return "completed"
+
+def main():
+    errors = []
+    admin_email, token = login()
+    user = exact_user(token)
+    if not isinstance(user, dict):
+        return {"ok": False, "mode": "feedback", "error": "user-not-found", "selector": USER_SELECTOR, "valuesPrinted": False}
+    user_id = user.get("id")
+    groups = safe_call(errors, "groups", lambda: items(api_data("GET", "/api/v1/admin/groups/all", token, label="groups")), [])
+    keys = safe_call(errors, "api-keys", lambda: page_all(token, f"/api/v1/admin/users/{user_id}/api-keys", {}, 100), [])
+    requests = safe_call(errors, "ops-requests", lambda: page_all(token, "/api/v1/admin/ops/requests", {"user_id": user_id, "time_range": WINDOW, "kind": "all", "sort": "created_at_desc"}, 100), [])
+    logs = safe_call(errors, "ops-system-logs", lambda: page_all(token, "/api/v1/admin/ops/system-logs", {"user_id": user_id, "time_range": WINDOW}, 200), [])
+    concurrency = safe_call(errors, "ops-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/concurrency", token, label="concurrency"), {})
+    user_concurrency = safe_call(errors, "ops-user-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/user-concurrency", token, label="user concurrency"), {})
+    availability = safe_call(errors, "ops-account-availability", lambda: api_data("GET", "/api/v1/admin/ops/account-availability", token, label="account availability"), {})
+    accounts = safe_call(errors, "accounts", lambda: page_all(token, "/api/v1/admin/accounts", {}, 100), [])
+    proxies = safe_call(errors, "proxies", lambda: page_all(token, "/api/v1/admin/proxies", {}, 100), [])
+    infrastructure = safe_call(errors, "infrastructure", runtime_health, {"ok": False, "mode": RUNTIME_MODE})
+
+    groups_by_id = dict_by_id(groups)
+    keys_by_id = dict_by_id(keys)
+    accounts_by_id = dict_by_id(accounts)
+    proxies_by_id = dict_by_id(proxies)
+    user_current_map = keyed_map(user_concurrency, "user")
+    account_concurrency_map = keyed_map(concurrency, "account")
+    account_availability_map = keyed_map(availability, "account")
+    current_user = user_current_map.get(str(user_id)) or {}
+    user_current = int(current_user.get("current_in_use") or user.get("current_concurrency") or 0)
+    user_waiting = int(current_user.get("waiting_in_queue") or 0)
+
+    compact_keys = []
+    for key in keys:
+        group = groups_by_id.get(str(key.get("group_id"))) or {}
+        compact_keys.append({
+            "id": key.get("id"), "name": key.get("name"), "status": key.get("status"),
+            "groupId": key.get("group_id"), "groupName": group.get("name"),
+            "currentConcurrency": key.get("current_concurrency"), "keyPrinted": False,
+        })
+
+    timeline = []
+    ordered = sorted([row for row in requests if isinstance(row, dict)], key=lambda row: iso_epoch(row.get("created_at")) or 0)
+    previous_epoch = None
+    for row in ordered:
+        matched_logs = related_logs(row, logs)
+        client_id = next((log.get("client_request_id") for log in matched_logs if log.get("client_request_id")), None)
+        account = accounts_by_id.get(str(row.get("account_id"))) or {}
+        proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
+        account_load = account_concurrency_map.get(str(row.get("account_id"))) or {}
+        queued = int(account_load.get("waiting_in_queue") or 0)
+        epoch = iso_epoch(row.get("created_at"))
+        gap = round(epoch - previous_epoch, 1) if epoch is not None and previous_epoch is not None else None
+        previous_epoch = epoch if epoch is not None else previous_epoch
+        timeline.append({
+            "at": row.get("created_at"), "gapSeconds": gap, "kind": row.get("kind"),
+            "requestId": row.get("request_id"), "clientRequestId": client_id,
+            "model": row.get("model"), "statusCode": row.get("status_code"), "durationMs": row.get("duration_ms"),
+            "apiKeyId": row.get("api_key_id"), "apiKeyName": (keys_by_id.get(str(row.get("api_key_id"))) or {}).get("name"),
+            "accountId": row.get("account_id"), "accountName": account.get("name"),
+            "proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"),
+            "attribution": classify(row, matched_logs, account, proxy, queued),
+            "evidence": {"systemLogCount": len(matched_logs), "accountStatus": account.get("status"), "accountSchedulable": account.get("schedulable"), "proxyStatus": proxy.get("status")},
+        })
+
+    all_timeline = timeline
+    max_gap = max([row.get("gapSeconds") for row in all_timeline if isinstance(row.get("gapSeconds"), (int, float))], default=None)
+    next_page_token = None
+    more_available = False
+    if REQUEST_SELECTOR:
+        selected_ids = {REQUEST_SELECTOR}
+        for row in all_timeline:
+            if row.get("requestId") == REQUEST_SELECTOR or row.get("clientRequestId") == REQUEST_SELECTOR:
+                selected_ids.update(value for value in (row.get("requestId"), row.get("clientRequestId")) if isinstance(value, str) and value)
+        timeline = [row for row in all_timeline if any(isinstance(value, str) and value in selected_ids for value in (row.get("requestId"), row.get("clientRequestId")))]
+        detail_logs = [compact_log(row) for row in logs if any(isinstance(value, str) and value in selected_ids for value in (row.get("request_id"), row.get("client_request_id")))]
+    else:
+        detail_logs = []
+        ordered_desc = list(reversed(all_timeline))
+        cursor = decode_page_token(PAGE_TOKEN)
+        start = 0
+        if cursor:
+            start = next((index + 1 for index, row in enumerate(ordered_desc) if row.get("at") == cursor.get("at") and row.get("requestId") == cursor.get("requestId")), -1)
+            if start < 0:
+                raise RuntimeError("page token cursor is no longer present in the selected window")
+        timeline = ordered_desc[start:start + PAGE_SIZE]
+        more_available = start + len(timeline) < len(ordered_desc)
+        if more_available and timeline:
+            next_page_token = encode_page_token(timeline[-1])
+    if not requests and user_waiting > 0:
+        conclusion = "sub2api-queue"
+    elif not requests and user_current > 0:
+        conclusion = "in-flight"
+    elif not requests:
+        conclusion = "client-before-submit-or-no-inbound-evidence"
+    elif timeline:
+        conclusion = timeline[0].get("attribution")
+    else:
+        conclusion = "request-selector-not-found"
+
+    related_account_ids = {str(row.get("accountId")) for row in timeline if row.get("accountId") is not None}
+    account_evidence = []
+    for account_id in sorted(related_account_ids):
+        account = accounts_by_id.get(account_id) or {}
+        proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
+        load = account_concurrency_map.get(account_id) or {}
+        available = account_availability_map.get(account_id) or {}
+        account_evidence.append({
+            "id": account.get("id"), "name": account.get("name"), "status": account.get("status"), "schedulable": account.get("schedulable"),
+            "currentInUse": load.get("current_in_use"), "waitingInQueue": load.get("waiting_in_queue"), "maxCapacity": load.get("max_capacity"),
+            "available": available.get("is_available"), "rateLimited": available.get("is_rate_limited"), "overloaded": available.get("is_overloaded"),
+            "proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"), "proxyStatus": proxy.get("status"),
+        })
+
+    return {
+        "ok": True,
+        "mode": "feedback",
+        "selector": {"user": USER_SELECTOR, "requestId": REQUEST_SELECTOR, "pageToken": PAGE_TOKEN, "window": WINDOW},
+        "user": {"id": user_id, "email": user.get("email"), "username": user.get("username"), "status": user.get("status"), "maxConcurrency": user.get("concurrency"), "currentInUse": user_current, "waitingInQueue": user_waiting},
+        "apiKeys": compact_keys,
+        "lifecycle": {"notInbound": not requests and user_current == 0 and user_waiting == 0, "queued": user_waiting, "inFlight": user_current, "completed": len(requests)},
+        "timeline": timeline,
+        "timelineSummary": {"returned": len(timeline), "total": len(all_timeline), "moreAvailable": more_available, "nextPageToken": next_page_token, "pageSize": PAGE_SIZE, "systemLogCount": len(logs), "maxGapSeconds": max_gap},
+        "requestDetail": {"logs": detail_logs, "logCount": len(detail_logs)} if REQUEST_SELECTOR else None,
+        "accounts": account_evidence,
+        "infrastructure": infrastructure,
+        "conclusion": {"attribution": conclusion, "boundary": "只读证据归因；无入站记录不等于客户端未调用，运行中请求仅由实时并发佐证。"},
+        "toolCallReduction": {"manualInvestigationCalls": 9, "feedbackCliCalls": 1, "reducedCalls": 8, "reductionPercent": 88.9, "basis": "issue-2000: user, requests, system logs, two request-id traces, account, proxy, process/port, journal"},
+        "sources": ["admin users", "admin user api-keys", "ops requests", "ops system-logs", "ops concurrency", "ops user-concurrency", "ops account-availability", "admin accounts", "admin proxies", "runtime health"],
+        "errors": errors,
+        "admin": {"email": admin_email, "tokenPrinted": False},
+        "valuesPrinted": False,
+    }
+
+try:
+    print(json.dumps(main(), ensure_ascii=False))
+except Exception as exc:
+    print(json.dumps({"ok": False, "mode": "feedback", "error": redact_text(str(exc)), "valuesPrinted": False}, ensure_ascii=False))
+    raise
+PY
+`;
+}
+
+function record(value: unknown): Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
+function records(value: unknown): Record<string, unknown>[] {
+  return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
+}
+
+function renderFeedbackReport(report: Record<string, unknown> | null, options: FeedbackOptions, remote: Record<string, unknown>): string {
+  if (report === null) {
+    return [
+      `SUB2API FEEDBACK user=${options.user} unavailable`,
+      `remote_exit=${remote.exitCode ?? "?"} stdout_bytes=${remote.stdoutBytes ?? "?"} stderr_bytes=${remote.stderrBytes ?? "?"}`,
+      textValue(remote.stderrTail ?? remote.stdoutTail),
+    ].join("\n");
+  }
+  if (report.ok !== true) {
+    return [
+      `SUB2API FEEDBACK user=${options.user} ok=false`,
+      `error=${textValue(report.error)}`,
+      "JSON: add --json for the structured failure envelope.",
+    ].join("\n");
+  }
+  const user = record(report.user);
+  const lifecycle = record(report.lifecycle);
+  const summary = record(report.timelineSummary);
+  const conclusion = record(report.conclusion);
+  const reduction = record(report.toolCallReduction);
+  const timeline = records(report.timeline);
+  const accounts = records(report.accounts);
+  const apiKeys = records(report.apiKeys);
+  const detail = record(report.requestDetail);
+  const detailLogs = records(detail.logs);
+  const infrastructure = record(report.infrastructure);
+  const components = records(infrastructure.components);
+  const errors = records(report.errors);
+  const lines: string[] = [];
+  lines.push(`SUB2API FEEDBACK user=${textValue(user.email)}#${textValue(user.id)} window=${options.window} ok=true`);
+  lines.push(`CONCLUSION attribution=${textValue(conclusion.attribution)} boundary=${textValue(conclusion.boundary)}`);
+  lines.push(`LIFECYCLE not_inbound=${textValue(lifecycle.notInbound)} queued=${textValue(lifecycle.queued)} in_flight=${textValue(lifecycle.inFlight)} completed=${textValue(lifecycle.completed)} max_gap_s=${textValue(summary.maxGapSeconds)}`);
+  lines.push(`TOOLS manual=${textValue(reduction.manualInvestigationCalls)} cli=${textValue(reduction.feedbackCliCalls)} reduced=${textValue(reduction.reducedCalls)} reduction=${textValue(reduction.reductionPercent)}%`);
+  if (apiKeys.length > 0) {
+    lines.push("");
+    lines.push("API KEYS");
+    lines.push(renderTable([
+      ["ID", "NAME", "GROUP", "STATUS", "CURRENT"],
+      ...apiKeys.map((key) => [textValue(key.id), shorten(textValue(key.name), 30), shorten(textValue(key.groupName), 24), textValue(key.status), textValue(key.currentConcurrency)]),
+    ]));
+  }
+  lines.push("");
+  lines.push(`REQUESTS returned=${textValue(summary.returned)} total=${textValue(summary.total)} moreAvailable=${textValue(summary.moreAvailable)} system_logs=${textValue(summary.systemLogCount)}`);
+  if (timeline.length === 0) {
+    lines.push("No matching completed request records in the selected window.");
+  } else {
+    lines.push(renderTable([
+      ["#", "AT", "GAP_S", "STATUS", "DURATION", "MODEL", "ACCOUNT", "ATTRIBUTION"],
+      ...timeline.map((item, index) => [
+        String(index + 1), shortIso(item.at), textValue(item.gapSeconds), textValue(item.statusCode ?? item.kind), textValue(item.durationMs),
+        shorten(textValue(item.model), 18), `${textValue(item.accountName)}#${textValue(item.accountId)}`, textValue(item.attribution),
+      ]),
+    ]));
+    lines.push("");
+    lines.push("REQUEST ID INDEX");
+    lines.push(renderTable([
+      ["#", "REQUEST_ID", "CLIENT_ID"],
+      ...timeline.map((item, index) => [String(index + 1), textValue(item.requestId), textValue(item.clientRequestId)]),
+    ]));
+  }
+  if (accounts.length > 0 || components.length > 0) {
+    lines.push("");
+    lines.push("ACCOUNT / INFRA");
+    lines.push(renderTable([
+      ["ACCOUNT", "STATUS", "SCHED", "IN_USE", "QUEUE", "CAP", "AVAILABLE", "PROXY", "P_STATUS"],
+      ...accounts.map((item) => [
+        `${textValue(item.name)}#${textValue(item.id)}`, textValue(item.status), textValue(item.schedulable),
+        textValue(item.currentInUse), textValue(item.waitingInQueue), textValue(item.maxCapacity), textValue(item.available),
+        shorten(`${textValue(item.proxyName)}#${textValue(item.proxyId)}`, 24), textValue(item.proxyStatus),
+      ]),
+      ...components.map((item) => [shorten(textValue(item.name), 32), textValue(item.status), "-", "-", "-", "-", textValue(item.running), "runtime", textValue(infrastructure.mode)]),
+    ]));
+  }
+  if (options.requestId !== null) {
+    lines.push("");
+    lines.push(`REQUEST DETAIL id=${options.requestId} logs=${textValue(detail.logCount)}`);
+    if (detailLogs.length > 0) {
+      lines.push(renderTable([
+        ["LOG_ID", "AT", "LEVEL", "COMPONENT", "REQUEST_ID", "CLIENT_ID", "MESSAGE"],
+        ...detailLogs.map((item) => [
+          textValue(item.id), shortIso(item.at), textValue(item.level), shorten(textValue(item.component), 22),
+          shorten(textValue(item.requestId), 18), shorten(textValue(item.clientRequestId), 18), shorten(textValue(item.message), 64),
+        ]),
+      ]));
+    }
+  }
+  if (errors.length > 0) {
+    lines.push("");
+    lines.push("PARTIAL EVIDENCE");
+    lines.push(renderTable([["SOURCE", "ERROR"], ...errors.map((item) => [textValue(item.source), shorten(textValue(item.error), 90)])]));
+  }
+  lines.push("");
+  if (summary.moreAvailable === true && typeof summary.nextPageToken === "string") {
+    lines.push(`NEXT_PAGE_TOKEN ${summary.nextPageToken}`);
+    lines.push(`Next: bun scripts/cli.ts platform-infra sub2api codex-pool feedback --target ${options.targetId} --user ${options.user} --window ${options.window} --page-token ${summary.nextPageToken}`);
+  }
+  lines.push("Disclosure: rerun with --request-id <client-or-internal-id> for correlated indexed logs.");
+  lines.push("JSON: add --json for the complete redacted report.");
+  return lines.join("\n");
+}
@@ -13,3 +13,4 @@ export * from "./remote-scripts";
 export * from "./accounts";
 export * from "./remote-python-sync";
 export * from "./remote";
+export * from "./feedback";
@@ -21,10 +21,12 @@ import {
 import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets";
 import { runSshCommandCapture, type SshCaptureResult } from "../ssh";

-import type { ConfirmOptions, DisclosureOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types";
+import type { ConfirmOptions, DisclosureOptions, FeedbackOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types";
 import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions";
+import { codexPoolFeedback } from "./feedback";
 import { renderCodexPoolPlan } from "./render";
 import { codexPoolRuntime } from "./runtime";
+import { codexPoolFaults } from "./faults";
 import { defaultCodexPoolRuntimeTargetId } from "./runtime-target";
 import { codexPoolHelp } from "./types";

@@ -38,7 +40,9 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[])
  if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1)));
  if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1)));
  if (action === "runtime") return await codexPoolRuntime(config, args.slice(1));
+  if (action === "faults") return await codexPoolFaults(config, args.slice(1));
  if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1)));
+  if (action === "feedback") return await codexPoolFeedback(config, parseFeedbackOptions(args.slice(1)));
  if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1)));
  if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1)));
  if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1)));
@@ -181,6 +185,7 @@ export function parseSentinelReportOptions(args: string[]): SentinelReportOption

 export function parseTraceOptions(args: string[]): TraceOptions {
  let requestId: string | null = null;
+  let pageToken: string | null = null;
  let since = "24h";
  let tail = 20_000;
  let contextSeconds = 300;
@@ -258,6 +263,64 @@ export function parseTraceOptions(args: string[]): TraceOptions {
  return { ...disclosure, requestId, since, tail, contextSeconds, showLines };
 }

+export function parseFeedbackOptions(args: string[]): FeedbackOptions {
+  let user: string | null = null;
+  let window: FeedbackOptions["window"] = "30m";
+  let requestId: string | null = null;
+  let pageToken: string | null = null;
+  let json = false;
+  const targetArgs: string[] = [];
+  const windows = new Set<FeedbackOptions["window"]>(["5m", "30m", "1h", "6h", "24h", "7d", "30d"]);
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index]!;
+    if (arg === "--json") {
+      json = true;
+      continue;
+    }
+    if (arg === "--target" || arg.startsWith("--target=")) {
+      targetArgs.push(arg);
+      if (arg === "--target") {
+        const value = args[index + 1];
+        if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
+        targetArgs.push(value);
+        index += 1;
+      }
+      continue;
+    }
+    const readValue = (option: string): string => {
+      const value = args[index + 1];
+      if (value === undefined || value.startsWith("--")) throw new Error(`${option} requires a value`);
+      index += 1;
+      return value.trim();
+    };
+    if (arg === "--user") user = readValue(arg);
+    else if (arg.startsWith("--user=")) user = arg.slice("--user=".length).trim();
+    else if (arg === "--window") {
+      const value = readValue(arg) as FeedbackOptions["window"];
+      if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d");
+      window = value;
+    } else if (arg.startsWith("--window=")) {
+      const value = arg.slice("--window=".length) as FeedbackOptions["window"];
+      if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d");
+      window = value;
+    } else if (arg === "--request-id") requestId = readValue(arg);
+    else if (arg.startsWith("--request-id=")) requestId = arg.slice("--request-id=".length).trim();
+    else if (arg === "--page-token") pageToken = readValue(arg);
+    else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim();
+    else throw new Error(`unsupported option: ${arg}`);
+  }
+  if (user === null || user.length === 0) throw new Error("feedback requires --user <email-or-id>");
+  if (user.length > 320 || /[\r\n]/u.test(user)) throw new Error("--user must be a single email or positive user id");
+  if (requestId !== null && (requestId.length === 0 || requestId.length > 256 || /[\r\n]/u.test(requestId))) {
+    throw new Error("--request-id must be a non-empty stable request id up to 256 characters");
+  }
+  if (pageToken !== null && (pageToken.length === 0 || pageToken.length > 1024 || !/^[A-Za-z0-9_-]+$/u.test(pageToken))) {
+    throw new Error("--page-token must be a URL-safe cursor token");
+  }
+  if (requestId !== null && pageToken !== null) throw new Error("--request-id and --page-token cannot be combined");
+  return { user, window, requestId, pageToken, json, targetId: parseTargetId(targetArgs) };
+}
+
 export function parseKubectlDuration(raw: string): string {
  const value = raw.trim();
  if (!/^[1-9][0-9]*(?:s|m|h)$/u.test(value)) throw new Error("--since must be a kubectl duration such as 24h, 90m, or 300s");
@@ -71,6 +71,15 @@ export interface TraceOptions extends DisclosureOptions {
  showLines: boolean;
 }

+export interface FeedbackOptions {
+  user: string;
+  window: "5m" | "30m" | "1h" | "6h" | "24h" | "7d" | "30d";
+  requestId: string | null;
+  pageToken: string | null;
+  json: boolean;
+  targetId: string;
+}
+
 export interface SentinelImageOptions extends DisclosureOptions {
  action: "status" | "build";
  confirm: boolean;
@@ -341,8 +350,8 @@ export function codexPoolHelp(): unknown {
  const pool = readCodexPoolConfig();
  const runtimeTarget = codexPoolRuntimeTarget();
  return {
-    command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
-    output: "json, except trace and sentinel-report default to low-noise text tables",
+    command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|faults|trace|feedback|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
+    output: "json, except trace, feedback, and sentinel-report default to low-noise text tables",
    usage: [
      "bun scripts/cli.ts platform-infra sub2api codex-pool plan",
      "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601",
@@ -351,11 +360,13 @@ export function codexPoolHelp(): unknown {
      "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--full|--raw]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account <name-or-id> [--target PK01] [--full|--raw]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account <name-or-id>] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]",
+      "bun scripts/cli.ts platform-infra sub2api codex-pool faults [--level P0|P1|P2] [--group <name-or-id>] [--account <name-or-id>] [--model <model>] [--stream sync|stream] [--endpoint <path>] [--request-id <id>] [--page-token <token>] [--target PK01] [--json]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account <name-or-id> --template <id> [--target PK01] [--confirm]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --accounts <name-or-id,...> --template <id> [--target PK01] [--confirm] [--full|--raw]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account <name-or-id> --kind temp-unschedulable [--target PK01] [--confirm]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --accounts <name-or-id,...> --kind temp-unschedulable [--target PK01] [--confirm] [--full|--raw]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id <requestId> [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]",
+      "bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user <email-or-id> [--window 30m] [--page-token <token>|--request-id <client-or-internal-id>] [--target PK01] [--json]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe [--target D601] --account unidesk-codex-hy --confirm",