|
|
|
@@ -0,0 +1,593 @@
|
|
|
|
|
import type { UniDeskConfig } from "../config";
|
|
|
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
|
|
|
|
|
|
import { readCodexPoolConfig } from "./config";
|
|
|
|
|
import { capture, compactCapture, parseJsonOutput } from "./remote";
|
|
|
|
|
import { renderedCliResult, renderTable, shorten, shortIso, textValue } from "./render";
|
|
|
|
|
import { codexPoolRuntimeTarget } from "./runtime-target";
|
|
|
|
|
import type { CodexPoolConfig, CodexPoolRuntimeTarget, FeedbackOptions } from "./types";
|
|
|
|
|
|
|
|
|
|
function pyJson(value: unknown): string {
|
|
|
|
|
return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function codexPoolFeedback(config: UniDeskConfig, options: FeedbackOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
|
|
|
const pool = readCodexPoolConfig();
|
|
|
|
|
const target = codexPoolRuntimeTarget(options.targetId);
|
|
|
|
|
const result = await capture(config, target.route, ["sh"], feedbackScript(pool, options, target));
|
|
|
|
|
const report = parseJsonOutput(result.stdout);
|
|
|
|
|
const ok = result.exitCode === 0 && report?.ok === true;
|
|
|
|
|
const remote = compactCapture(result, { full: result.exitCode !== 0 || report === null });
|
|
|
|
|
if (options.json) {
|
|
|
|
|
return {
|
|
|
|
|
ok,
|
|
|
|
|
action: "platform-infra-sub2api-codex-pool-feedback",
|
|
|
|
|
target: { id: target.id, route: target.route, runtimeMode: target.runtimeMode },
|
|
|
|
|
report,
|
|
|
|
|
remote,
|
|
|
|
|
valuesPrinted: false,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
return renderedCliResult(ok, "platform-infra sub2api codex-pool feedback", renderFeedbackReport(report, options, remote));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function feedbackScript(pool: CodexPoolConfig, options: FeedbackOptions, target: CodexPoolRuntimeTarget): string {
|
|
|
|
|
return `
|
|
|
|
|
set -u
|
|
|
|
|
python3 - <<'PY'
|
|
|
|
|
import base64
|
|
|
|
|
import json
|
|
|
|
|
import re
|
|
|
|
|
import subprocess
|
|
|
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
|
from urllib.parse import quote, urlencode
|
|
|
|
|
|
|
|
|
|
RUNTIME_MODE = ${pyJson(target.runtimeMode)}
|
|
|
|
|
NAMESPACE = ${pyJson(target.namespace)}
|
|
|
|
|
APP_SECRET_NAME = ${pyJson(target.appSecretName)}
|
|
|
|
|
ADMIN_EMAIL_DEFAULT = ${pyJson(pool.adminEmailDefault)}
|
|
|
|
|
HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)}
|
|
|
|
|
HOST_DOCKER_ENV_PATH = ${pyJson(target.hostDockerEnvPath)}
|
|
|
|
|
USER_SELECTOR = ${pyJson(options.user)}
|
|
|
|
|
WINDOW = ${pyJson(options.window)}
|
|
|
|
|
REQUEST_SELECTOR = ${pyJson(options.requestId)}
|
|
|
|
|
PAGE_TOKEN = ${pyJson(options.pageToken)}
|
|
|
|
|
APP_CONTAINER = "sub2api-app"
|
|
|
|
|
PAGE_SIZE = 10
|
|
|
|
|
|
|
|
|
|
def run(command, input_bytes=None):
|
|
|
|
|
return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
|
|
|
|
|
|
|
def text(value, limit=1000):
|
|
|
|
|
if isinstance(value, bytes):
|
|
|
|
|
value = value.decode("utf-8", errors="replace")
|
|
|
|
|
return str(value or "")[-limit:]
|
|
|
|
|
|
|
|
|
|
def docker(args):
|
|
|
|
|
proc = run(["docker", *args])
|
|
|
|
|
if proc.returncode == 0:
|
|
|
|
|
return proc
|
|
|
|
|
fallback = run(["sudo", "-n", "docker", *args])
|
|
|
|
|
return fallback if fallback.returncode == 0 else proc
|
|
|
|
|
|
|
|
|
|
def read_host_env():
|
|
|
|
|
if RUNTIME_MODE != "host-docker":
|
|
|
|
|
return {}
|
|
|
|
|
try:
|
|
|
|
|
with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle:
|
|
|
|
|
lines = handle.read().splitlines()
|
|
|
|
|
except PermissionError:
|
|
|
|
|
proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH])
|
|
|
|
|
if proc.returncode != 0:
|
|
|
|
|
raise RuntimeError("read host-docker env failed: " + text(proc.stderr))
|
|
|
|
|
lines = proc.stdout.decode("utf-8", errors="replace").splitlines()
|
|
|
|
|
values = {}
|
|
|
|
|
for line in lines:
|
|
|
|
|
stripped = line.strip()
|
|
|
|
|
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
|
|
|
|
continue
|
|
|
|
|
key, value = stripped.split("=", 1)
|
|
|
|
|
value = value.strip()
|
|
|
|
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
|
|
|
|
|
value = value[1:-1]
|
|
|
|
|
values[key.strip()] = value
|
|
|
|
|
return values
|
|
|
|
|
|
|
|
|
|
def runtime_secret(key):
|
|
|
|
|
if RUNTIME_MODE == "host-docker":
|
|
|
|
|
return read_host_env().get(key)
|
|
|
|
|
proc = run(["kubectl", "-n", NAMESPACE, "get", "secret", APP_SECRET_NAME, "-o", "json"])
|
|
|
|
|
if proc.returncode != 0:
|
|
|
|
|
raise RuntimeError("read app secret failed: " + text(proc.stderr))
|
|
|
|
|
raw = (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
|
|
|
|
|
return base64.b64decode(raw).decode("utf-8") if raw else None
|
|
|
|
|
|
|
|
|
|
def runtime_config(key):
|
|
|
|
|
if RUNTIME_MODE == "host-docker":
|
|
|
|
|
return read_host_env().get(key)
|
|
|
|
|
proc = run(["kubectl", "-n", NAMESPACE, "get", "configmap", "sub2api-config", "-o", "json"])
|
|
|
|
|
if proc.returncode != 0:
|
|
|
|
|
return None
|
|
|
|
|
return (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
|
|
|
|
|
|
|
|
|
|
def parse_curl(proc):
|
|
|
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
|
|
|
marker = "\\n__HTTP_CODE__:"
|
|
|
|
|
position = stdout.rfind(marker)
|
|
|
|
|
if position < 0:
|
|
|
|
|
return {"ok": False, "httpStatus": 0, "json": None, "message": text(proc.stderr)}
|
|
|
|
|
body = stdout[:position]
|
|
|
|
|
try:
|
|
|
|
|
status = int(stdout[position + len(marker):].strip()[-3:])
|
|
|
|
|
except Exception:
|
|
|
|
|
status = 0
|
|
|
|
|
try:
|
|
|
|
|
parsed = json.loads(body) if body.strip() else None
|
|
|
|
|
except Exception:
|
|
|
|
|
parsed = None
|
|
|
|
|
message = parsed.get("message") if isinstance(parsed, dict) else text(body)
|
|
|
|
|
return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "message": message}
|
|
|
|
|
|
|
|
|
|
def curl_api(method, path, bearer=None, payload=None):
|
|
|
|
|
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
script = r'''set -eu
|
|
|
|
|
method="$1"
|
|
|
|
|
url="$2"
|
|
|
|
|
token="\${3:-}"
|
|
|
|
|
tmp="$(mktemp)"
|
|
|
|
|
trap 'rm -f "$tmp"' EXIT
|
|
|
|
|
cat > "$tmp"
|
|
|
|
|
args=""
|
|
|
|
|
if [ -n "$token" ]; then args="Authorization: Bearer $token"; fi
|
|
|
|
|
if [ "$method" = GET ] && [ ! -s "$tmp" ]; then
|
|
|
|
|
if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -H "$args" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' "$url"; fi
|
|
|
|
|
else
|
|
|
|
|
if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "$args" --data-binary @"$tmp" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"; fi
|
|
|
|
|
fi'''
|
|
|
|
|
if RUNTIME_MODE == "host-docker":
|
|
|
|
|
proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body)
|
|
|
|
|
else:
|
|
|
|
|
proc = run(["kubectl", "-n", NAMESPACE, "exec", "deploy/sub2api", "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""], body)
|
|
|
|
|
return parse_curl(proc)
|
|
|
|
|
|
|
|
|
|
def api_data(method, path, token=None, payload=None, label=None):
|
|
|
|
|
response = curl_api(method, path, token, payload)
|
|
|
|
|
parsed = response.get("json")
|
|
|
|
|
code = parsed.get("code") if isinstance(parsed, dict) else None
|
|
|
|
|
if not response.get("ok") or (code is not None and code != 0):
|
|
|
|
|
raise RuntimeError(f"{label or path} failed: http={response.get('httpStatus')} message={response.get('message')}")
|
|
|
|
|
return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
|
|
|
|
|
|
|
|
|
|
def find_token(value):
|
|
|
|
|
if isinstance(value, dict):
|
|
|
|
|
for key in ("access_token", "token"):
|
|
|
|
|
if isinstance(value.get(key), str) and value.get(key):
|
|
|
|
|
return value[key]
|
|
|
|
|
for item in value.values():
|
|
|
|
|
token = find_token(item)
|
|
|
|
|
if token:
|
|
|
|
|
return token
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
def login():
|
|
|
|
|
email = runtime_config("ADMIN_EMAIL") or ADMIN_EMAIL_DEFAULT
|
|
|
|
|
password = runtime_secret("ADMIN_PASSWORD")
|
|
|
|
|
if not password:
|
|
|
|
|
raise RuntimeError("ADMIN_PASSWORD missing")
|
|
|
|
|
token = find_token(api_data("POST", "/api/v1/auth/login", payload={"email": email, "password": password}, label="admin login"))
|
|
|
|
|
if not token:
|
|
|
|
|
raise RuntimeError("admin login returned no token")
|
|
|
|
|
return email, token
|
|
|
|
|
|
|
|
|
|
def items(data):
|
|
|
|
|
if isinstance(data, list):
|
|
|
|
|
return data
|
|
|
|
|
if isinstance(data, dict):
|
|
|
|
|
if isinstance(data.get("items"), list):
|
|
|
|
|
return data["items"]
|
|
|
|
|
for key in ("users", "api_keys", "keys", "groups", "accounts", "proxies", "logs"):
|
|
|
|
|
if isinstance(data.get(key), list):
|
|
|
|
|
return data[key]
|
|
|
|
|
return []
|
|
|
|
|
|
|
|
|
|
def page_all(token, path, query=None, page_size=100):
|
|
|
|
|
query = dict(query or {})
|
|
|
|
|
collected = []
|
|
|
|
|
page = 1
|
|
|
|
|
while True:
|
|
|
|
|
params = {**query, "page": page, "page_size": page_size}
|
|
|
|
|
separator = "&" if "?" in path else "?"
|
|
|
|
|
data = api_data("GET", path + separator + urlencode(params), token, label=path)
|
|
|
|
|
batch = items(data)
|
|
|
|
|
collected.extend(batch)
|
|
|
|
|
total = data.get("total") if isinstance(data, dict) else None
|
|
|
|
|
if not batch or (isinstance(total, int) and len(collected) >= total) or len(batch) < page_size:
|
|
|
|
|
break
|
|
|
|
|
page += 1
|
|
|
|
|
return collected
|
|
|
|
|
|
|
|
|
|
def exact_user(token):
|
|
|
|
|
if USER_SELECTOR.isdigit() and int(USER_SELECTOR) > 0:
|
|
|
|
|
user = api_data("GET", f"/api/v1/admin/users/{int(USER_SELECTOR)}", token, label="get user")
|
|
|
|
|
return user if isinstance(user, dict) else None
|
|
|
|
|
candidates = page_all(token, "/api/v1/admin/users", {"search": USER_SELECTOR}, 100)
|
|
|
|
|
exact = [user for user in candidates if isinstance(user, dict) and str(user.get("email") or "").lower() == USER_SELECTOR.lower()]
|
|
|
|
|
if len(exact) > 1:
|
|
|
|
|
raise RuntimeError("email matched multiple users")
|
|
|
|
|
return exact[0] if exact else None
|
|
|
|
|
|
|
|
|
|
def safe_call(errors, name, callback, fallback):
|
|
|
|
|
try:
|
|
|
|
|
return callback()
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
errors.append({"source": name, "error": redact_text(str(exc))})
|
|
|
|
|
return fallback
|
|
|
|
|
|
|
|
|
|
def redact_text(value):
|
|
|
|
|
value = str(value or "")
|
|
|
|
|
value = re.sub(r"(?i)(bearer\\s+)[A-Za-z0-9._~+\\-/=]+", r"\\1<redacted>", value)
|
|
|
|
|
value = re.sub(r"(?i)((?:api[_-]?key|token|password)\\s*[=:]\\s*)[^\\s,;]+", r"\\1<redacted>", value)
|
|
|
|
|
value = re.sub(r"\\bsk-[A-Za-z0-9_-]{8,}\\b", "sk-<redacted>", value)
|
|
|
|
|
return value[:500]
|
|
|
|
|
|
|
|
|
|
def iso_epoch(value):
|
|
|
|
|
if not isinstance(value, str) or not value:
|
|
|
|
|
return None
|
|
|
|
|
try:
|
|
|
|
|
match = re.match(r"^(\\d{4}-\\d\\d-\\d\\d[T ]\\d\\d:\\d\\d:\\d\\d)(?:\\.(\\d+))?(Z|[+-]\\d\\d:?\\d\\d)$", value)
|
|
|
|
|
if not match:
|
|
|
|
|
return None
|
|
|
|
|
base = datetime.strptime(match.group(1).replace(" ", "T"), "%Y-%m-%dT%H:%M:%S")
|
|
|
|
|
fraction = int((match.group(2) or "0")[:6].ljust(6, "0"))
|
|
|
|
|
zone = match.group(3)
|
|
|
|
|
if zone == "Z":
|
|
|
|
|
offset = timezone.utc
|
|
|
|
|
else:
|
|
|
|
|
sign = 1 if zone[0] == "+" else -1
|
|
|
|
|
digits = zone[1:].replace(":", "")
|
|
|
|
|
offset = timezone(sign * timedelta(hours=int(digits[:2]), minutes=int(digits[2:])))
|
|
|
|
|
return base.replace(microsecond=fraction, tzinfo=offset).timestamp()
|
|
|
|
|
except Exception:
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
def encode_page_token(row):
|
|
|
|
|
payload = json.dumps({"v": 1, "at": row.get("at"), "requestId": row.get("requestId")}, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=")
|
|
|
|
|
|
|
|
|
|
def decode_page_token(value):
|
|
|
|
|
if not isinstance(value, str) or not value:
|
|
|
|
|
return None
|
|
|
|
|
try:
|
|
|
|
|
padded = value + "=" * (-len(value) % 4)
|
|
|
|
|
parsed = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8"))
|
|
|
|
|
except Exception:
|
|
|
|
|
raise RuntimeError("invalid page token")
|
|
|
|
|
if not isinstance(parsed, dict) or parsed.get("v") != 1 or not parsed.get("at") or not parsed.get("requestId"):
|
|
|
|
|
raise RuntimeError("invalid page token")
|
|
|
|
|
return parsed
|
|
|
|
|
|
|
|
|
|
def dict_by_id(rows):
|
|
|
|
|
return {str(row.get("id")): row for row in rows if isinstance(row, dict) and row.get("id") is not None}
|
|
|
|
|
|
|
|
|
|
def keyed_map(value, key):
|
|
|
|
|
source = value.get(key) if isinstance(value, dict) else None
|
|
|
|
|
return source if isinstance(source, dict) else {}
|
|
|
|
|
|
|
|
|
|
def runtime_health():
|
|
|
|
|
if RUNTIME_MODE == "host-docker":
|
|
|
|
|
proc = docker(["inspect", APP_CONTAINER, "sub2api-redis"])
|
|
|
|
|
if proc.returncode != 0:
|
|
|
|
|
return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
|
|
|
|
|
rows = []
|
|
|
|
|
for item in json.loads(proc.stdout.decode("utf-8")):
|
|
|
|
|
state = item.get("State") or {}
|
|
|
|
|
health = state.get("Health") or {}
|
|
|
|
|
rows.append({"name": str(item.get("Name") or "").lstrip("/"), "running": state.get("Running"), "status": health.get("Status") or state.get("Status")})
|
|
|
|
|
return {"ok": all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
|
|
|
|
|
proc = run(["kubectl", "-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api", "-o", "json"])
|
|
|
|
|
if proc.returncode != 0:
|
|
|
|
|
return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
|
|
|
|
|
rows = []
|
|
|
|
|
for item in json.loads(proc.stdout.decode("utf-8")).get("items") or []:
|
|
|
|
|
status = item.get("status") or {}
|
|
|
|
|
containers = status.get("containerStatuses") or []
|
|
|
|
|
rows.append({"name": (item.get("metadata") or {}).get("name"), "running": status.get("phase") == "Running", "status": "ready" if containers and all(entry.get("ready") is True for entry in containers) else status.get("phase")})
|
|
|
|
|
return {"ok": bool(rows) and all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
|
|
|
|
|
|
|
|
|
|
def compact_log(row):
|
|
|
|
|
extra = row.get("extra") if isinstance(row.get("extra"), dict) else {}
|
|
|
|
|
allow = {}
|
|
|
|
|
for key in ("phase", "error_owner", "status_code", "duration_ms", "latency_ms", "time_to_first_token_ms", "upstream_status", "path", "stream"):
|
|
|
|
|
if key in extra:
|
|
|
|
|
allow[key] = extra.get(key)
|
|
|
|
|
return {
|
|
|
|
|
"id": row.get("id"),
|
|
|
|
|
"at": row.get("created_at"),
|
|
|
|
|
"level": row.get("level"),
|
|
|
|
|
"component": row.get("component"),
|
|
|
|
|
"message": redact_text(row.get("message")),
|
|
|
|
|
"requestId": row.get("request_id"),
|
|
|
|
|
"clientRequestId": row.get("client_request_id"),
|
|
|
|
|
"accountId": row.get("account_id"),
|
|
|
|
|
"extra": allow,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def related_logs(request, logs):
|
|
|
|
|
ids = {str(request.get("request_id") or "")}
|
|
|
|
|
matched = []
|
|
|
|
|
changed = True
|
|
|
|
|
while changed:
|
|
|
|
|
changed = False
|
|
|
|
|
for row in logs:
|
|
|
|
|
request_id = str(row.get("request_id") or "")
|
|
|
|
|
client_id = str(row.get("client_request_id") or "")
|
|
|
|
|
if row in matched or (request_id not in ids and client_id not in ids):
|
|
|
|
|
continue
|
|
|
|
|
matched.append(row)
|
|
|
|
|
before = len(ids)
|
|
|
|
|
ids.update(value for value in (request_id, client_id) if value)
|
|
|
|
|
changed = changed or len(ids) != before
|
|
|
|
|
return matched
|
|
|
|
|
|
|
|
|
|
def classify(request, logs, account, proxy, queued):
|
|
|
|
|
evidence = " ".join([str(request.get("phase") or ""), str(request.get("message") or ""), *[str(log.get("component") or "") + " " + str(log.get("message") or "") for log in logs]]).lower()
|
|
|
|
|
if any(marker in evidence for marker in ("client disconnect", "broken pipe", "context canceled", "client canceled")):
|
|
|
|
|
return "client-connection"
|
|
|
|
|
if any(marker in evidence for marker in ("proxy", "dial tcp", "dns", "connection refused", "network")) or (proxy and proxy.get("status") not in (None, "active")):
|
|
|
|
|
return "proxy-egress"
|
|
|
|
|
if any(marker in evidence for marker in ("upstream", "provider", "first token", "stream")) or str(request.get("phase") or "") == "upstream":
|
|
|
|
|
return "upstream-first-byte-stream"
|
|
|
|
|
if queued > 0:
|
|
|
|
|
return "sub2api-queue"
|
|
|
|
|
if request.get("kind") == "error" or (isinstance(request.get("status_code"), int) and request.get("status_code") >= 400):
|
|
|
|
|
return "sub2api-or-upstream-unknown"
|
|
|
|
|
return "completed"
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
errors = []
|
|
|
|
|
admin_email, token = login()
|
|
|
|
|
user = exact_user(token)
|
|
|
|
|
if not isinstance(user, dict):
|
|
|
|
|
return {"ok": False, "mode": "feedback", "error": "user-not-found", "selector": USER_SELECTOR, "valuesPrinted": False}
|
|
|
|
|
user_id = user.get("id")
|
|
|
|
|
groups = safe_call(errors, "groups", lambda: items(api_data("GET", "/api/v1/admin/groups/all", token, label="groups")), [])
|
|
|
|
|
keys = safe_call(errors, "api-keys", lambda: page_all(token, f"/api/v1/admin/users/{user_id}/api-keys", {}, 100), [])
|
|
|
|
|
requests = safe_call(errors, "ops-requests", lambda: page_all(token, "/api/v1/admin/ops/requests", {"user_id": user_id, "time_range": WINDOW, "kind": "all", "sort": "created_at_desc"}, 100), [])
|
|
|
|
|
logs = safe_call(errors, "ops-system-logs", lambda: page_all(token, "/api/v1/admin/ops/system-logs", {"user_id": user_id, "time_range": WINDOW}, 200), [])
|
|
|
|
|
concurrency = safe_call(errors, "ops-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/concurrency", token, label="concurrency"), {})
|
|
|
|
|
user_concurrency = safe_call(errors, "ops-user-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/user-concurrency", token, label="user concurrency"), {})
|
|
|
|
|
availability = safe_call(errors, "ops-account-availability", lambda: api_data("GET", "/api/v1/admin/ops/account-availability", token, label="account availability"), {})
|
|
|
|
|
accounts = safe_call(errors, "accounts", lambda: page_all(token, "/api/v1/admin/accounts", {}, 100), [])
|
|
|
|
|
proxies = safe_call(errors, "proxies", lambda: page_all(token, "/api/v1/admin/proxies", {}, 100), [])
|
|
|
|
|
infrastructure = safe_call(errors, "infrastructure", runtime_health, {"ok": False, "mode": RUNTIME_MODE})
|
|
|
|
|
|
|
|
|
|
groups_by_id = dict_by_id(groups)
|
|
|
|
|
keys_by_id = dict_by_id(keys)
|
|
|
|
|
accounts_by_id = dict_by_id(accounts)
|
|
|
|
|
proxies_by_id = dict_by_id(proxies)
|
|
|
|
|
user_current_map = keyed_map(user_concurrency, "user")
|
|
|
|
|
account_concurrency_map = keyed_map(concurrency, "account")
|
|
|
|
|
account_availability_map = keyed_map(availability, "account")
|
|
|
|
|
current_user = user_current_map.get(str(user_id)) or {}
|
|
|
|
|
user_current = int(current_user.get("current_in_use") or user.get("current_concurrency") or 0)
|
|
|
|
|
user_waiting = int(current_user.get("waiting_in_queue") or 0)
|
|
|
|
|
|
|
|
|
|
compact_keys = []
|
|
|
|
|
for key in keys:
|
|
|
|
|
group = groups_by_id.get(str(key.get("group_id"))) or {}
|
|
|
|
|
compact_keys.append({
|
|
|
|
|
"id": key.get("id"), "name": key.get("name"), "status": key.get("status"),
|
|
|
|
|
"groupId": key.get("group_id"), "groupName": group.get("name"),
|
|
|
|
|
"currentConcurrency": key.get("current_concurrency"), "keyPrinted": False,
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
timeline = []
|
|
|
|
|
ordered = sorted([row for row in requests if isinstance(row, dict)], key=lambda row: iso_epoch(row.get("created_at")) or 0)
|
|
|
|
|
previous_epoch = None
|
|
|
|
|
for row in ordered:
|
|
|
|
|
matched_logs = related_logs(row, logs)
|
|
|
|
|
client_id = next((log.get("client_request_id") for log in matched_logs if log.get("client_request_id")), None)
|
|
|
|
|
account = accounts_by_id.get(str(row.get("account_id"))) or {}
|
|
|
|
|
proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
|
|
|
|
|
account_load = account_concurrency_map.get(str(row.get("account_id"))) or {}
|
|
|
|
|
queued = int(account_load.get("waiting_in_queue") or 0)
|
|
|
|
|
epoch = iso_epoch(row.get("created_at"))
|
|
|
|
|
gap = round(epoch - previous_epoch, 1) if epoch is not None and previous_epoch is not None else None
|
|
|
|
|
previous_epoch = epoch if epoch is not None else previous_epoch
|
|
|
|
|
timeline.append({
|
|
|
|
|
"at": row.get("created_at"), "gapSeconds": gap, "kind": row.get("kind"),
|
|
|
|
|
"requestId": row.get("request_id"), "clientRequestId": client_id,
|
|
|
|
|
"model": row.get("model"), "statusCode": row.get("status_code"), "durationMs": row.get("duration_ms"),
|
|
|
|
|
"apiKeyId": row.get("api_key_id"), "apiKeyName": (keys_by_id.get(str(row.get("api_key_id"))) or {}).get("name"),
|
|
|
|
|
"accountId": row.get("account_id"), "accountName": account.get("name"),
|
|
|
|
|
"proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"),
|
|
|
|
|
"attribution": classify(row, matched_logs, account, proxy, queued),
|
|
|
|
|
"evidence": {"systemLogCount": len(matched_logs), "accountStatus": account.get("status"), "accountSchedulable": account.get("schedulable"), "proxyStatus": proxy.get("status")},
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
all_timeline = timeline
|
|
|
|
|
max_gap = max([row.get("gapSeconds") for row in all_timeline if isinstance(row.get("gapSeconds"), (int, float))], default=None)
|
|
|
|
|
next_page_token = None
|
|
|
|
|
more_available = False
|
|
|
|
|
if REQUEST_SELECTOR:
|
|
|
|
|
selected_ids = {REQUEST_SELECTOR}
|
|
|
|
|
for row in all_timeline:
|
|
|
|
|
if row.get("requestId") == REQUEST_SELECTOR or row.get("clientRequestId") == REQUEST_SELECTOR:
|
|
|
|
|
selected_ids.update(value for value in (row.get("requestId"), row.get("clientRequestId")) if isinstance(value, str) and value)
|
|
|
|
|
timeline = [row for row in all_timeline if any(isinstance(value, str) and value in selected_ids for value in (row.get("requestId"), row.get("clientRequestId")))]
|
|
|
|
|
detail_logs = [compact_log(row) for row in logs if any(isinstance(value, str) and value in selected_ids for value in (row.get("request_id"), row.get("client_request_id")))]
|
|
|
|
|
else:
|
|
|
|
|
detail_logs = []
|
|
|
|
|
ordered_desc = list(reversed(all_timeline))
|
|
|
|
|
cursor = decode_page_token(PAGE_TOKEN)
|
|
|
|
|
start = 0
|
|
|
|
|
if cursor:
|
|
|
|
|
start = next((index + 1 for index, row in enumerate(ordered_desc) if row.get("at") == cursor.get("at") and row.get("requestId") == cursor.get("requestId")), -1)
|
|
|
|
|
if start < 0:
|
|
|
|
|
raise RuntimeError("page token cursor is no longer present in the selected window")
|
|
|
|
|
timeline = ordered_desc[start:start + PAGE_SIZE]
|
|
|
|
|
more_available = start + len(timeline) < len(ordered_desc)
|
|
|
|
|
if more_available and timeline:
|
|
|
|
|
next_page_token = encode_page_token(timeline[-1])
|
|
|
|
|
if not requests and user_waiting > 0:
|
|
|
|
|
conclusion = "sub2api-queue"
|
|
|
|
|
elif not requests and user_current > 0:
|
|
|
|
|
conclusion = "in-flight"
|
|
|
|
|
elif not requests:
|
|
|
|
|
conclusion = "client-before-submit-or-no-inbound-evidence"
|
|
|
|
|
elif timeline:
|
|
|
|
|
conclusion = timeline[0].get("attribution")
|
|
|
|
|
else:
|
|
|
|
|
conclusion = "request-selector-not-found"
|
|
|
|
|
|
|
|
|
|
related_account_ids = {str(row.get("accountId")) for row in timeline if row.get("accountId") is not None}
|
|
|
|
|
account_evidence = []
|
|
|
|
|
for account_id in sorted(related_account_ids):
|
|
|
|
|
account = accounts_by_id.get(account_id) or {}
|
|
|
|
|
proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
|
|
|
|
|
load = account_concurrency_map.get(account_id) or {}
|
|
|
|
|
available = account_availability_map.get(account_id) or {}
|
|
|
|
|
account_evidence.append({
|
|
|
|
|
"id": account.get("id"), "name": account.get("name"), "status": account.get("status"), "schedulable": account.get("schedulable"),
|
|
|
|
|
"currentInUse": load.get("current_in_use"), "waitingInQueue": load.get("waiting_in_queue"), "maxCapacity": load.get("max_capacity"),
|
|
|
|
|
"available": available.get("is_available"), "rateLimited": available.get("is_rate_limited"), "overloaded": available.get("is_overloaded"),
|
|
|
|
|
"proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"), "proxyStatus": proxy.get("status"),
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
"ok": True,
|
|
|
|
|
"mode": "feedback",
|
|
|
|
|
"selector": {"user": USER_SELECTOR, "requestId": REQUEST_SELECTOR, "pageToken": PAGE_TOKEN, "window": WINDOW},
|
|
|
|
|
"user": {"id": user_id, "email": user.get("email"), "username": user.get("username"), "status": user.get("status"), "maxConcurrency": user.get("concurrency"), "currentInUse": user_current, "waitingInQueue": user_waiting},
|
|
|
|
|
"apiKeys": compact_keys,
|
|
|
|
|
"lifecycle": {"notInbound": not requests and user_current == 0 and user_waiting == 0, "queued": user_waiting, "inFlight": user_current, "completed": len(requests)},
|
|
|
|
|
"timeline": timeline,
|
|
|
|
|
"timelineSummary": {"returned": len(timeline), "total": len(all_timeline), "moreAvailable": more_available, "nextPageToken": next_page_token, "pageSize": PAGE_SIZE, "systemLogCount": len(logs), "maxGapSeconds": max_gap},
|
|
|
|
|
"requestDetail": {"logs": detail_logs, "logCount": len(detail_logs)} if REQUEST_SELECTOR else None,
|
|
|
|
|
"accounts": account_evidence,
|
|
|
|
|
"infrastructure": infrastructure,
|
|
|
|
|
"conclusion": {"attribution": conclusion, "boundary": "只读证据归因；无入站记录不等于客户端未调用，运行中请求仅由实时并发佐证。"},
|
|
|
|
|
"toolCallReduction": {"manualInvestigationCalls": 9, "feedbackCliCalls": 1, "reducedCalls": 8, "reductionPercent": 88.9, "basis": "issue-2000: user, requests, system logs, two request-id traces, account, proxy, process/port, journal"},
|
|
|
|
|
"sources": ["admin users", "admin user api-keys", "ops requests", "ops system-logs", "ops concurrency", "ops user-concurrency", "ops account-availability", "admin accounts", "admin proxies", "runtime health"],
|
|
|
|
|
"errors": errors,
|
|
|
|
|
"admin": {"email": admin_email, "tokenPrinted": False},
|
|
|
|
|
"valuesPrinted": False,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
print(json.dumps(main(), ensure_ascii=False))
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
print(json.dumps({"ok": False, "mode": "feedback", "error": redact_text(str(exc)), "valuesPrinted": False}, ensure_ascii=False))
|
|
|
|
|
raise
|
|
|
|
|
PY
|
|
|
|
|
`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
|
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function records(value: unknown): Record<string, unknown>[] {
|
|
|
|
|
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function renderFeedbackReport(report: Record<string, unknown> | null, options: FeedbackOptions, remote: Record<string, unknown>): string {
|
|
|
|
|
if (report === null) {
|
|
|
|
|
return [
|
|
|
|
|
`SUB2API FEEDBACK user=${options.user} unavailable`,
|
|
|
|
|
`remote_exit=${remote.exitCode ?? "?"} stdout_bytes=${remote.stdoutBytes ?? "?"} stderr_bytes=${remote.stderrBytes ?? "?"}`,
|
|
|
|
|
textValue(remote.stderrTail ?? remote.stdoutTail),
|
|
|
|
|
].join("\n");
|
|
|
|
|
}
|
|
|
|
|
if (report.ok !== true) {
|
|
|
|
|
return [
|
|
|
|
|
`SUB2API FEEDBACK user=${options.user} ok=false`,
|
|
|
|
|
`error=${textValue(report.error)}`,
|
|
|
|
|
"JSON: add --json for the structured failure envelope.",
|
|
|
|
|
].join("\n");
|
|
|
|
|
}
|
|
|
|
|
const user = record(report.user);
|
|
|
|
|
const lifecycle = record(report.lifecycle);
|
|
|
|
|
const summary = record(report.timelineSummary);
|
|
|
|
|
const conclusion = record(report.conclusion);
|
|
|
|
|
const reduction = record(report.toolCallReduction);
|
|
|
|
|
const timeline = records(report.timeline);
|
|
|
|
|
const accounts = records(report.accounts);
|
|
|
|
|
const apiKeys = records(report.apiKeys);
|
|
|
|
|
const detail = record(report.requestDetail);
|
|
|
|
|
const detailLogs = records(detail.logs);
|
|
|
|
|
const infrastructure = record(report.infrastructure);
|
|
|
|
|
const components = records(infrastructure.components);
|
|
|
|
|
const errors = records(report.errors);
|
|
|
|
|
const lines: string[] = [];
|
|
|
|
|
lines.push(`SUB2API FEEDBACK user=${textValue(user.email)}#${textValue(user.id)} window=${options.window} ok=true`);
|
|
|
|
|
lines.push(`CONCLUSION attribution=${textValue(conclusion.attribution)} boundary=${textValue(conclusion.boundary)}`);
|
|
|
|
|
lines.push(`LIFECYCLE not_inbound=${textValue(lifecycle.notInbound)} queued=${textValue(lifecycle.queued)} in_flight=${textValue(lifecycle.inFlight)} completed=${textValue(lifecycle.completed)} max_gap_s=${textValue(summary.maxGapSeconds)}`);
|
|
|
|
|
lines.push(`TOOLS manual=${textValue(reduction.manualInvestigationCalls)} cli=${textValue(reduction.feedbackCliCalls)} reduced=${textValue(reduction.reducedCalls)} reduction=${textValue(reduction.reductionPercent)}%`);
|
|
|
|
|
if (apiKeys.length > 0) {
|
|
|
|
|
lines.push("");
|
|
|
|
|
lines.push("API KEYS");
|
|
|
|
|
lines.push(renderTable([
|
|
|
|
|
["ID", "NAME", "GROUP", "STATUS", "CURRENT"],
|
|
|
|
|
...apiKeys.map((key) => [textValue(key.id), shorten(textValue(key.name), 30), shorten(textValue(key.groupName), 24), textValue(key.status), textValue(key.currentConcurrency)]),
|
|
|
|
|
]));
|
|
|
|
|
}
|
|
|
|
|
lines.push("");
|
|
|
|
|
lines.push(`REQUESTS returned=${textValue(summary.returned)} total=${textValue(summary.total)} moreAvailable=${textValue(summary.moreAvailable)} system_logs=${textValue(summary.systemLogCount)}`);
|
|
|
|
|
if (timeline.length === 0) {
|
|
|
|
|
lines.push("No matching completed request records in the selected window.");
|
|
|
|
|
} else {
|
|
|
|
|
lines.push(renderTable([
|
|
|
|
|
["#", "AT", "GAP_S", "STATUS", "DURATION", "MODEL", "ACCOUNT", "ATTRIBUTION"],
|
|
|
|
|
...timeline.map((item, index) => [
|
|
|
|
|
String(index + 1), shortIso(item.at), textValue(item.gapSeconds), textValue(item.statusCode ?? item.kind), textValue(item.durationMs),
|
|
|
|
|
shorten(textValue(item.model), 18), shorten(`${textValue(item.accountName)}#${textValue(item.accountId)}`, 28), textValue(item.attribution),
|
|
|
|
|
]),
|
|
|
|
|
]));
|
|
|
|
|
lines.push("");
|
|
|
|
|
lines.push("REQUEST ID INDEX");
|
|
|
|
|
lines.push(renderTable([
|
|
|
|
|
["#", "REQUEST_ID", "CLIENT_ID"],
|
|
|
|
|
...timeline.map((item, index) => [String(index + 1), textValue(item.requestId), textValue(item.clientRequestId)]),
|
|
|
|
|
]));
|
|
|
|
|
}
|
|
|
|
|
if (accounts.length > 0 || components.length > 0) {
|
|
|
|
|
lines.push("");
|
|
|
|
|
lines.push("ACCOUNT / INFRA");
|
|
|
|
|
lines.push(renderTable([
|
|
|
|
|
["ACCOUNT", "STATUS", "SCHED", "IN_USE", "QUEUE", "CAP", "AVAILABLE", "PROXY", "P_STATUS"],
|
|
|
|
|
...accounts.map((item) => [
|
|
|
|
|
shorten(`${textValue(item.name)}#${textValue(item.id)}`, 32), textValue(item.status), textValue(item.schedulable),
|
|
|
|
|
textValue(item.currentInUse), textValue(item.waitingInQueue), textValue(item.maxCapacity), textValue(item.available),
|
|
|
|
|
shorten(`${textValue(item.proxyName)}#${textValue(item.proxyId)}`, 24), textValue(item.proxyStatus),
|
|
|
|
|
]),
|
|
|
|
|
...components.map((item) => [shorten(textValue(item.name), 32), textValue(item.status), "-", "-", "-", "-", textValue(item.running), "runtime", textValue(infrastructure.mode)]),
|
|
|
|
|
]));
|
|
|
|
|
}
|
|
|
|
|
if (options.requestId !== null) {
|
|
|
|
|
lines.push("");
|
|
|
|
|
lines.push(`REQUEST DETAIL id=${options.requestId} logs=${textValue(detail.logCount)}`);
|
|
|
|
|
if (detailLogs.length > 0) {
|
|
|
|
|
lines.push(renderTable([
|
|
|
|
|
["LOG_ID", "AT", "LEVEL", "COMPONENT", "REQUEST_ID", "CLIENT_ID", "MESSAGE"],
|
|
|
|
|
...detailLogs.map((item) => [
|
|
|
|
|
textValue(item.id), shortIso(item.at), textValue(item.level), shorten(textValue(item.component), 22),
|
|
|
|
|
shorten(textValue(item.requestId), 18), shorten(textValue(item.clientRequestId), 18), shorten(textValue(item.message), 64),
|
|
|
|
|
]),
|
|
|
|
|
]));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (errors.length > 0) {
|
|
|
|
|
lines.push("");
|
|
|
|
|
lines.push("PARTIAL EVIDENCE");
|
|
|
|
|
lines.push(renderTable([["SOURCE", "ERROR"], ...errors.map((item) => [textValue(item.source), shorten(textValue(item.error), 90)])]));
|
|
|
|
|
}
|
|
|
|
|
lines.push("");
|
|
|
|
|
if (summary.moreAvailable === true && typeof summary.nextPageToken === "string") {
|
|
|
|
|
lines.push(`NEXT_PAGE_TOKEN ${summary.nextPageToken}`);
|
|
|
|
|
lines.push(`Next: bun scripts/cli.ts platform-infra sub2api codex-pool feedback --target ${options.targetId} --user ${options.user} --window ${options.window} --page-token ${summary.nextPageToken}`);
|
|
|
|
|
}
|
|
|
|
|
lines.push("Disclosure: rerun with --request-id <client-or-internal-id> for correlated indexed logs.");
|
|
|
|
|
lines.push("JSON: add --json for the complete redacted report.");
|
|
|
|
|
return lines.join("\n");
|
|
|
|
|
}
|
Codex