Merge remote-tracking branch 'origin/master' into feat/selfmedia-production-release
# Conflicts: # scripts/src/selfmedia-delivery-renderer.ts
This commit is contained in:
Codex
@@ -505,6 +505,9 @@ function digestOf(item) {
|
||||
|
||||
function extractPacArtifactEvidence(recordsInput, logText) {
|
||||
const records = Array.isArray(recordsInput) ? recordsInput.map(record) : [];
|
||||
const envReuseRecord = [...records].reverse().find((item) => item.phase === "env-reuse") || null;
|
||||
const imageBuildRecord = [...records].reverse().find((item) => item.phase === "image-build" || item.phase === "image-build-complete") || null;
|
||||
const stageTimings = Object.assign({}, ...records.map((item) => record(item.stageTimings)));
|
||||
const sourceObservation = extractPacSourceObservation(records);
|
||||
const catalog = [...records].reverse().find((item) => item.status === "published"
|
||||
&& Array.isArray(item.services)
|
||||
@@ -562,10 +565,13 @@ function extractPacArtifactEvidence(recordsInput, logText) {
|
||||
}
|
||||
if (image !== null) {
|
||||
const env = record(image.envReuse);
|
||||
const reuseEnv = record(envReuseRecord?.envReuse);
|
||||
return {
|
||||
imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status) || (stringOrNull(image.digestRef) !== null ? "built" : null),
|
||||
envIdentity: stringOrNull(image.envIdentity),
|
||||
envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.mode) || stringOrNull(image.envReuseStatus),
|
||||
imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status)
|
||||
|| (imageBuildRecord !== null || stringOrNull(image.digestRef) !== null ? "built" : null),
|
||||
envIdentity: stringOrNull(image.envIdentity) || stringOrNull(envReuseRecord?.envIdentity),
|
||||
envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.status) || stringOrNull(image.envReuseStatus)
|
||||
|| stringOrNull(reuseEnv.dependencyReuse) || stringOrNull(reuseEnv.status) || stringOrNull(envReuseRecord?.envReuseStatus),
|
||||
nodeDepsReuse: null,
|
||||
buildCache: null,
|
||||
digest: digestOf(image) || (digests.length === 1 ? digests[0] : null),
|
||||
@@ -576,6 +582,7 @@ function extractPacArtifactEvidence(recordsInput, logText) {
|
||||
baselineSourceCommit: stringOrNull(image.baselineSourceCommit),
|
||||
action: stringOrNull(image.action),
|
||||
reason: stringOrNull(image.reason),
|
||||
stageTimings,
|
||||
sourceObservation,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
|
||||
@@ -703,6 +703,12 @@ function recursiveEnvReuse(value, state) {
|
||||
if (typeof value.reusedServiceCount === 'number') state.serviceReusedCount = value.reusedServiceCount;
|
||||
if (typeof value.envReuse === 'string') state.status = value.envReuse;
|
||||
if (typeof value.envReuseStatus === 'string') state.status = value.envReuseStatus;
|
||||
if (typeof value.envIdentity === 'string') state.identity = value.envIdentity;
|
||||
if (value.stageTimings && typeof value.stageTimings === 'object' && !Array.isArray(value.stageTimings)) {
|
||||
for (const [key, timing] of Object.entries(value.stageTimings)) {
|
||||
if (typeof timing === 'number' && Number.isFinite(timing)) state.stageTimings[key] = timing;
|
||||
}
|
||||
}
|
||||
if (value.envReuse && typeof value.envReuse === 'object') {
|
||||
const env = value.envReuse;
|
||||
if (typeof env.mode === 'string') state.status = env.mode;
|
||||
@@ -742,7 +748,7 @@ function boundedFailureRecord(value) {
|
||||
}
|
||||
|
||||
function envReuseForPipelineRun(namespace, name, taskRuns) {
|
||||
const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null };
|
||||
const state = { status: null, identity: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, stageTimings: {}, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null };
|
||||
const matching = (taskRuns.items || []).filter((item) => item.metadata?.labels?.['tekton.dev/pipelineRun'] === name);
|
||||
const contractTaskRuns = matching.filter((item) => taskTerminalRecord(item) !== null);
|
||||
const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean);
|
||||
@@ -889,9 +895,11 @@ function rowFor(consumer, item, taskRuns) {
|
||||
classification,
|
||||
envReuse: {
|
||||
status: evidence.status,
|
||||
identity: evidence.identity,
|
||||
buildSkippedCount: evidence.buildSkippedCount,
|
||||
serviceReusedCount: evidence.serviceReusedCount,
|
||||
buildCache: evidence.buildCache,
|
||||
stageTimings: evidence.stageTimings,
|
||||
source: evidence.source,
|
||||
},
|
||||
sourceObservation,
|
||||
|
||||
@@ -2307,6 +2307,7 @@ export function renderStatus(result: Record<string, unknown>): RenderedCliResult
|
||||
"",
|
||||
"IMAGE / GITOPS",
|
||||
...table(["IMAGE_STATUS", "ENV_REUSE", "ENV_ID", "DIGEST", "GITOPS"], [[stringValue(artifact.imageStatus), stringValue(artifact.envReuse), stringValue(artifact.envIdentity), short(stringValue(artifact.digest), 18), short(stringValue(artifact.gitopsCommit))]]),
|
||||
` stage-timings: ${stageTimingsText(record(artifact.stageTimings))}`,
|
||||
"",
|
||||
"ARGO",
|
||||
...table(["SYNC", "HEALTH", "REVISION", "RELATION", "TARGET_REVISION"], [[
|
||||
@@ -2912,6 +2913,13 @@ function envReuseText(envReuse: Record<string, unknown>): string {
|
||||
return parts.length === 0 ? "-" : parts.join(",");
|
||||
}
|
||||
|
||||
function stageTimingsText(timings: Record<string, unknown>): string {
|
||||
const parts = Object.entries(timings)
|
||||
.filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1]))
|
||||
.map(([name, value]) => `${name}=${Math.round(value)}${name.endsWith("Seconds") ? "s" : "ms"}`);
|
||||
return parts.length === 0 ? "-" : parts.join(",");
|
||||
}
|
||||
|
||||
function registryText(registry: Record<string, unknown>): string {
|
||||
if (Object.keys(registry).length === 0) return "-";
|
||||
if (registry.status === "not-configured" || registry.applicability === "not-configured") return "N/A";
|
||||
@@ -2973,6 +2981,8 @@ function renderHistoryDetail(row: Record<string, unknown>): string[] {
|
||||
["errorExitCode", stringValue(typedError.exitCode)],
|
||||
["errorSummary", compactLine(stringValue(typedError.stderrSummary ?? typedError.message))],
|
||||
["envReuseSource", stringValue(envReuse.source)],
|
||||
["envIdentity", short(stringValue(envReuse.identity), 24)],
|
||||
["stageTimings", stageTimingsText(record(envReuse.stageTimings))],
|
||||
["deliveryMode", stringValue(sourceObservation.mode)],
|
||||
["deliveryValid", stringValue(sourceObservation.valid)],
|
||||
["deliverySourceMatched", stringValue(sourceObservation.sourceMatched)],
|
||||
|
||||
@@ -0,0 +1,620 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import { CliInputError, type RenderedCliResult } from "../output";
|
||||
|
||||
import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote";
|
||||
import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target";
|
||||
|
||||
type FaultLevel = "P0" | "P1" | "P2";
|
||||
|
||||
interface FaultOptions {
|
||||
level: FaultLevel | null;
|
||||
group: string | null;
|
||||
account: string | null;
|
||||
model: string | null;
|
||||
stream: "sync" | "stream" | null;
|
||||
endpoint: string | null;
|
||||
requestId: string | null;
|
||||
pageToken: string | null;
|
||||
targetId: string;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
export async function codexPoolFaults(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (args.includes("--help")) return renderFaultHelp();
|
||||
const options = parseFaultOptions(args);
|
||||
const target = codexPoolRuntimeTarget(options.targetId);
|
||||
const scope = faultScope(options);
|
||||
const offset = decodePageToken(options.pageToken, scope);
|
||||
const payload = {
|
||||
filters: {
|
||||
level: options.level,
|
||||
group: options.group,
|
||||
account: options.account,
|
||||
model: options.model,
|
||||
stream: options.stream,
|
||||
endpoint: options.endpoint,
|
||||
requestId: options.requestId,
|
||||
},
|
||||
offset,
|
||||
pageSize: PAGE_SIZE,
|
||||
};
|
||||
const result = await runRemoteCodexPoolScript(config, "faults", faultScript(payload, target), target);
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
|
||||
const data = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
||||
const pagination = data && typeof data.pagination === "object" && data.pagination !== null
|
||||
? data.pagination as Record<string, unknown>
|
||||
: null;
|
||||
if (pagination?.hasMore === true) pagination.nextPageToken = encodePageToken(offset + PAGE_SIZE, scope);
|
||||
if (pagination) {
|
||||
pagination.pageSize = PAGE_SIZE;
|
||||
pagination.pageToken = options.pageToken;
|
||||
}
|
||||
const response = {
|
||||
ok,
|
||||
action: "platform-infra-sub2api-codex-pool-faults",
|
||||
target: {
|
||||
id: target.id,
|
||||
route: target.route,
|
||||
runtimeMode: target.runtimeMode,
|
||||
endpoint: target.serviceDns,
|
||||
},
|
||||
source: {
|
||||
kind: "sub2api-native-admin-ops",
|
||||
versionBoundary: "Sub2API native admin Ops facts with explicit, version-neutral UniDesk CLI projection",
|
||||
mutation: false,
|
||||
},
|
||||
filters: payload.filters,
|
||||
faults: data,
|
||||
remote: compactCapture(result, { full: result.exitCode !== 0 || data === null }),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
if (options.json) return response;
|
||||
return renderFaults(response);
|
||||
}
|
||||
|
||||
function renderFaultHelp(): RenderedCliResult {
|
||||
return {
|
||||
ok: true,
|
||||
command: "platform-infra sub2api codex-pool faults --help",
|
||||
renderedText: [
|
||||
"SUB2API CODEX-POOL FAULTS",
|
||||
"Usage:",
|
||||
" bun scripts/cli.ts platform-infra sub2api codex-pool faults [options]",
|
||||
"Options:",
|
||||
" --level P0|P1|P2",
|
||||
" --group <name-or-id>",
|
||||
" --account <name-or-id>",
|
||||
" --model <model>",
|
||||
" --stream sync|stream",
|
||||
" --endpoint <path>",
|
||||
" --request-id <stable-id>",
|
||||
" --page-token <token>",
|
||||
" --target <id>",
|
||||
" --json",
|
||||
"Notes:",
|
||||
` Fixed page size: ${PAGE_SIZE}; --limit is intentionally unsupported.`,
|
||||
" Default output is a bounded Kubernetes-style table; JSON requires --json.",
|
||||
].join("\n"),
|
||||
contentType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
function parseFaultOptions(args: string[]): FaultOptions {
|
||||
let level: FaultLevel | null = null;
|
||||
let group: string | null = null;
|
||||
let account: string | null = null;
|
||||
let model: string | null = null;
|
||||
let stream: "sync" | "stream" | null = null;
|
||||
let endpoint: string | null = null;
|
||||
let requestId: string | null = null;
|
||||
let pageToken: string | null = null;
|
||||
let targetId = defaultCodexPoolRuntimeTargetId();
|
||||
let json = false;
|
||||
const readValue = (index: number, name: string): [string, number] => {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||||
return [validateSelector(value, name), index + 1];
|
||||
};
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index]!;
|
||||
if (arg === "--json") json = true;
|
||||
else if (arg === "--level") {
|
||||
const [value, next] = readValue(index, "--level");
|
||||
level = parseLevel(value);
|
||||
index = next;
|
||||
} else if (arg.startsWith("--level=")) level = parseLevel(arg.slice(8));
|
||||
else if (arg === "--group") [group, index] = readValue(index, "--group");
|
||||
else if (arg.startsWith("--group=")) group = validateSelector(arg.slice(8), "--group");
|
||||
else if (arg === "--account") [account, index] = readValue(index, "--account");
|
||||
else if (arg.startsWith("--account=")) account = validateSelector(arg.slice(10), "--account");
|
||||
else if (arg === "--model") [model, index] = readValue(index, "--model");
|
||||
else if (arg.startsWith("--model=")) model = validateSelector(arg.slice(8), "--model");
|
||||
else if (arg === "--stream") {
|
||||
const [value, next] = readValue(index, "--stream");
|
||||
stream = parseStream(value);
|
||||
index = next;
|
||||
} else if (arg.startsWith("--stream=")) stream = parseStream(arg.slice(9));
|
||||
else if (arg === "--endpoint") [endpoint, index] = readValue(index, "--endpoint");
|
||||
else if (arg.startsWith("--endpoint=")) endpoint = validateSelector(arg.slice(11), "--endpoint");
|
||||
else if (arg === "--request-id") [requestId, index] = readValue(index, "--request-id");
|
||||
else if (arg.startsWith("--request-id=")) requestId = validateSelector(arg.slice(13), "--request-id");
|
||||
else if (arg === "--page-token") [pageToken, index] = readValue(index, "--page-token");
|
||||
else if (arg.startsWith("--page-token=")) pageToken = validateSelector(arg.slice(13), "--page-token");
|
||||
else if (arg === "--target") [targetId, index] = readValue(index, "--target");
|
||||
else if (arg.startsWith("--target=")) targetId = validateSelector(arg.slice(9), "--target");
|
||||
else if (arg === "--limit" || arg.startsWith("--limit=")) throw new Error("faults uses a fixed page size; use --page-token for progressive disclosure");
|
||||
else throw new Error(`unsupported faults option: ${arg}`);
|
||||
}
|
||||
if (level !== null && group === null) {
|
||||
throw new CliInputError("--level requires --group; use the default command for the all-groups overview", {
|
||||
code: "missing-required-option",
|
||||
argument: "--group",
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01 --level P0 --group <name-or-id>",
|
||||
],
|
||||
hint: "Run without --level for the all-groups overview, then drill down with --level and --group.",
|
||||
});
|
||||
}
|
||||
return { level, group, account, model, stream, endpoint, requestId, pageToken, targetId, json };
|
||||
}
|
||||
|
||||
function parseLevel(value: string): FaultLevel {
|
||||
const normalized = value.toUpperCase();
|
||||
if (normalized !== "P0" && normalized !== "P1" && normalized !== "P2") throw new Error("--level must be P0, P1, or P2");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseStream(value: string): "sync" | "stream" {
|
||||
if (value !== "sync" && value !== "stream") throw new Error("--stream must be sync or stream");
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateSelector(value: string, name: string): string {
|
||||
const normalized = value.trim();
|
||||
if (!normalized || normalized.length > 512 || /[\r\n\0]/u.test(normalized)) throw new Error(`${name} has an unsupported format`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function faultScope(options: FaultOptions): string {
|
||||
return createHash("sha256").update(JSON.stringify({
|
||||
level: options.level,
|
||||
group: options.group,
|
||||
account: options.account,
|
||||
model: options.model,
|
||||
stream: options.stream,
|
||||
endpoint: options.endpoint,
|
||||
requestId: options.requestId,
|
||||
targetId: options.targetId,
|
||||
})).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
function encodePageToken(offset: number, scope: string): string {
|
||||
return Buffer.from(JSON.stringify({ v: 1, offset, scope }), "utf8").toString("base64url");
|
||||
}
|
||||
|
||||
function decodePageToken(token: string | null, scope: string): number {
|
||||
if (token === null) return 0;
|
||||
try {
|
||||
const parsed = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as Record<string, unknown>;
|
||||
if (parsed.v !== 1 || parsed.scope !== scope || !Number.isInteger(parsed.offset) || Number(parsed.offset) < 0) throw new Error("invalid");
|
||||
return Number(parsed.offset);
|
||||
} catch {
|
||||
throw new Error("--page-token is invalid or belongs to different filters");
|
||||
}
|
||||
}
|
||||
|
||||
function renderFaults(response: Record<string, unknown>): RenderedCliResult {
|
||||
const faults = response.faults && typeof response.faults === "object" ? response.faults as Record<string, unknown> : {};
|
||||
const lines = ["PLATFORM-INFRA SUB2API FAULTS"];
|
||||
const window = record(faults.window);
|
||||
lines.push(`WINDOW ${text(window.timeRange)} SOURCE native-admin-ops PROJECTION unidesk-cli`);
|
||||
lines.push("");
|
||||
const summary = arrayOfRecords(faults.summary);
|
||||
if (summary.length > 0) {
|
||||
lines.push("GROUPS");
|
||||
lines.push(table(["GROUP", "P0", "CUSTOMER_ERRS", "GROUP_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERRS", "GROUP_UP_ERR%", "ABSORB%"], summary.map((row) => [
|
||||
`${text(row.groupName)} (${text(row.groupId)})`,
|
||||
text(row.p0), text(row.customerErrorCount), percent(row.customerErrorRatePercent),
|
||||
text(row.p1), milliseconds(row.ttftP99Ms),
|
||||
text(row.p2), text(row.upstreamErrorCount), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent),
|
||||
])));
|
||||
}
|
||||
const details = arrayOfRecords(faults.details);
|
||||
if (details.length > 0) {
|
||||
lines.push("", `DETAILS ${text(faults.level ?? "SUMMARY")}`);
|
||||
const columns = Array.isArray(faults.detailColumns) ? faults.detailColumns.map(String) : [];
|
||||
lines.push(table(columns, details.map((row) => columns.map((column) => text(row[column])))));
|
||||
}
|
||||
const boundary = record(faults.boundary);
|
||||
lines.push("", "BOUNDARY");
|
||||
for (const value of Array.isArray(boundary.notes) ? boundary.notes : []) lines.push(`- ${String(value)}`);
|
||||
const pagination = record(faults.pagination);
|
||||
if (pagination.nextPageToken) lines.push("", `NEXT_PAGE_TOKEN ${text(pagination.nextPageToken)}`);
|
||||
if (faults.next) lines.push("", "NEXT", ...String(faults.next).split("\n").map((line) => ` ${line}`));
|
||||
return {
|
||||
ok: response.ok === true,
|
||||
command: "platform-infra sub2api codex-pool faults",
|
||||
renderedText: lines.join("\n"),
|
||||
contentType: "text/plain",
|
||||
projection: response,
|
||||
};
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function arrayOfRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
||||
}
|
||||
|
||||
function text(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
if (typeof value === "boolean") return value ? "yes" : "no";
|
||||
return String(value).replace(/[\r\n\t]+/gu, " ");
|
||||
}
|
||||
|
||||
function percent(value: unknown): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : text(value);
|
||||
}
|
||||
|
||||
function milliseconds(value: unknown): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)}ms` : text(value);
|
||||
}
|
||||
|
||||
function table(headers: string[], rows: string[][]): string {
|
||||
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length)));
|
||||
return [headers, ...rows].map((row) => row.map((cell, index) => cell.padEnd(widths[index]!)).join(" ").trimEnd()).join("\n");
|
||||
}
|
||||
|
||||
function pyJson(value: unknown): string {
|
||||
return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
|
||||
}
|
||||
|
||||
function faultScript(payload: unknown, target: ReturnType<typeof codexPoolRuntimeTarget>): string {
|
||||
return `
|
||||
set -u
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import urlencode
|
||||
|
||||
RUNTIME_MODE = ${pyJson(target.runtimeMode)}
|
||||
NAMESPACE = ${pyJson(target.namespace)}
|
||||
HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)}
|
||||
APP_SECRET_NAME = ${pyJson(target.appSecretName)}
|
||||
PAYLOAD = ${pyJson(payload)}
|
||||
APP_POD = None
|
||||
|
||||
def run(cmd, input_bytes=None):
|
||||
return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def docker(args):
|
||||
proc = run(["docker", *args])
|
||||
if proc.returncode == 0:
|
||||
return proc
|
||||
sudo_proc = run(["sudo", "-n", "docker", *args])
|
||||
return sudo_proc if sudo_proc.returncode == 0 else proc
|
||||
|
||||
def kubectl(args, input_bytes=None):
|
||||
return run(["kubectl", *args], input_bytes)
|
||||
|
||||
def kube_json(args, label):
|
||||
proc = kubectl([*args, "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(label + " failed")
|
||||
return json.loads(proc.stdout.decode("utf-8"))
|
||||
|
||||
if RUNTIME_MODE != "host-docker":
|
||||
pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app=sub2api"], "list pods").get("items") or []
|
||||
ready = [item for item in pods if item.get("status", {}).get("phase") == "Running"]
|
||||
if not ready:
|
||||
raise RuntimeError("sub2api app pod not found")
|
||||
APP_POD = ready[0]["metadata"]["name"]
|
||||
|
||||
def config_value(name, key, default=None):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
proc = docker(["inspect", "sub2api-app", "--format", "{{json .Config.Env}}"])
|
||||
if proc.returncode != 0:
|
||||
return default
|
||||
for item in json.loads(proc.stdout.decode("utf-8")):
|
||||
if item.startswith(key + "="):
|
||||
return item.split("=", 1)[1]
|
||||
return default
|
||||
data = kube_json(["-n", NAMESPACE, "get", "configmap", name], "configmap/" + name).get("data") or {}
|
||||
return data.get(key, default)
|
||||
|
||||
def secret_value(name, key):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
return config_value("", key)
|
||||
data = kube_json(["-n", NAMESPACE, "get", "secret", name], "secret/" + name).get("data") or {}
|
||||
return base64.b64decode(data[key]).decode("utf-8") if key in data else None
|
||||
|
||||
def curl_api(method, path, bearer=None, payload=None):
|
||||
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
script = r'''set -eu
|
||||
method="$1"; url="$2"; token="\${3:-}"; tmp="$(mktemp)"; trap 'rm -f "$tmp"' EXIT; cat > "$tmp"
|
||||
args=""; [ -n "$token" ] && args="Authorization: Bearer $token"
|
||||
if [ -n "$args" ] && [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
|
||||
elif [ -n "$args" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" "$url"
|
||||
elif [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
|
||||
else curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"; fi'''
|
||||
base = f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}" if RUNTIME_MODE == "host-docker" else "http://127.0.0.1:8080"
|
||||
cmd = ["sh", "-c", script, "sh", method, base + path, bearer or ""]
|
||||
proc = run(cmd, body) if RUNTIME_MODE == "host-docker" else run(["kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", *cmd], body)
|
||||
output = proc.stdout.decode("utf-8", errors="replace")
|
||||
marker = "\\n__HTTP_CODE__:"
|
||||
pos = output.rfind(marker)
|
||||
status = int(output[pos + len(marker):].strip()[-3:]) if pos >= 0 else 0
|
||||
raw = output[:pos] if pos >= 0 else output
|
||||
try:
|
||||
parsed = json.loads(raw) if raw.strip() else None
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
return {"ok": proc.returncode == 0 and 200 <= status < 300, "status": status, "json": parsed, "body": raw[:300]}
|
||||
|
||||
def data_of(response, label):
|
||||
parsed = response.get("json")
|
||||
code = parsed.get("code") if isinstance(parsed, dict) else None
|
||||
if response.get("ok") is not True or (code is not None and code != 0):
|
||||
message = parsed.get("message") if isinstance(parsed, dict) else response.get("body")
|
||||
raise RuntimeError(f"{label} failed: http={response.get('status')} message={message}")
|
||||
return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
|
||||
|
||||
def login():
|
||||
email = config_value("sub2api-config", "ADMIN_EMAIL", "admin@example.com")
|
||||
password = secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("ADMIN_PASSWORD missing")
|
||||
data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login")
|
||||
token = data.get("access_token") or data.get("token") if isinstance(data, dict) else None
|
||||
if not token:
|
||||
raise RuntimeError("admin login response has no token")
|
||||
return token
|
||||
|
||||
def get(token, path, params=None, label=None):
|
||||
suffix = "?" + urlencode({key: value for key, value in (params or {}).items() if value is not None}) if params else ""
|
||||
return data_of(curl_api("GET", path + suffix, bearer=token), label or path)
|
||||
|
||||
def items_of(data):
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
for key in ("items", "groups", "accounts"):
|
||||
if isinstance(data.get(key), list):
|
||||
return data[key]
|
||||
return []
|
||||
|
||||
def total_of(data):
|
||||
return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data))
|
||||
|
||||
def all_items(token, path, params, label):
|
||||
page = 1
|
||||
page_size = 500
|
||||
rows = []
|
||||
while True:
|
||||
data = get(token, path, {**params, "page": page, "page_size": page_size}, label)
|
||||
batch = items_of(data)
|
||||
rows.extend(batch)
|
||||
total = total_of(data)
|
||||
if not batch or len(rows) >= total:
|
||||
return rows
|
||||
page += 1
|
||||
|
||||
def percent(value):
|
||||
if not isinstance(value, (int, float)):
|
||||
return None
|
||||
return round(value * 100 if value <= 1 else value, 4)
|
||||
|
||||
def selector_id(value):
|
||||
return int(value) if isinstance(value, str) and value.isdigit() else None
|
||||
|
||||
def scope_filters(group_id=None):
|
||||
result = {"time_range": "24h", "platform": "openai", "group_id": group_id}
|
||||
return result
|
||||
|
||||
def matches_detail_filters(item):
|
||||
filters = PAYLOAD["filters"]
|
||||
account = filters.get("account")
|
||||
if account:
|
||||
account_match = str(item.get("account_id") or "") == str(account) or str(item.get("account_name") or "") == str(account)
|
||||
if not account_match:
|
||||
return False
|
||||
if filters.get("model") and str(item.get("requested_model") or item.get("model") or "") != str(filters["model"]):
|
||||
return False
|
||||
if filters.get("stream"):
|
||||
expected_stream = filters["stream"] == "stream"
|
||||
if item.get("stream") is not expected_stream:
|
||||
return False
|
||||
if filters.get("endpoint") and str(item.get("inbound_endpoint") or item.get("request_path") or item.get("path") or "") != str(filters["endpoint"]):
|
||||
return False
|
||||
if filters.get("requestId"):
|
||||
request_id = filters["requestId"]
|
||||
if item.get("request_id") != request_id and item.get("client_request_id") != request_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
def has_detail_filters():
|
||||
filters = PAYLOAD["filters"]
|
||||
return any(filters.get(key) for key in ("account", "model", "stream", "endpoint", "requestId"))
|
||||
|
||||
DETAIL_ROWS = {}
|
||||
|
||||
def error_rows(token, path, group_id, apply_filters):
|
||||
cache_key = (path, group_id, apply_filters)
|
||||
if cache_key not in DETAIL_ROWS:
|
||||
rows = all_items(token, path, {**scope_filters(group_id), "view": "all", "include_detail": 1}, "fault detail scan")
|
||||
DETAIL_ROWS[cache_key] = [item for item in rows if matches_detail_filters(item)] if apply_filters else rows
|
||||
return DETAIL_ROWS[cache_key]
|
||||
|
||||
def selected_groups(groups):
|
||||
selector = PAYLOAD["filters"].get("group")
|
||||
if not selector:
|
||||
return groups
|
||||
selected = [item for item in groups if str(item.get("id")) == str(selector) or item.get("name") == selector]
|
||||
if len(selected) != 1:
|
||||
raise RuntimeError("group-not-found-or-ambiguous: " + str(selector))
|
||||
return selected
|
||||
|
||||
def threshold_state(value, threshold):
|
||||
if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)):
|
||||
return "unavailable"
|
||||
return "breach" if value >= threshold else "ok"
|
||||
|
||||
def availability_by_account(token, group_id):
|
||||
data = get(token, "/api/v1/admin/ops/account-availability", {"platform": "openai", "group_id": group_id}, "account availability")
|
||||
raw = data.get("account") if isinstance(data, dict) else None
|
||||
return raw if isinstance(raw, dict) else {}
|
||||
|
||||
def detail_row(item, group_metrics, availability, level, absorbed=None):
|
||||
account_id = item.get("account_id")
|
||||
state = availability.get(str(account_id), {}) if isinstance(availability, dict) else {}
|
||||
endpoint = item.get("inbound_endpoint") or item.get("request_path") or item.get("path")
|
||||
base = {
|
||||
"GROUP": f"{item.get('group_name') or group_metrics.get('groupName')} ({item.get('group_id') or group_metrics.get('groupId')})",
|
||||
"ACCOUNT": f"{item.get('account_name') or '-'} ({account_id or '-'})",
|
||||
"MODEL": item.get("requested_model") or item.get("model") or "-",
|
||||
"MODE": "stream" if item.get("stream") is True else "sync" if item.get("stream") is False else "-",
|
||||
"ENDPOINT": endpoint or "-",
|
||||
"STATUS": item.get("status_code") or "-",
|
||||
"REQUEST_ID": item.get("request_id") or item.get("client_request_id") or "-",
|
||||
}
|
||||
if level == "P0":
|
||||
base.update({"CUSTOMER_ERR%": group_metrics.get("customerErrorRatePercent"), "ABSORBED": False})
|
||||
else:
|
||||
unavailable_reason = str(state.get("unavailable_reason") or state.get("reason") or "").lower()
|
||||
if state.get("is_available") is False and ("temporary" in unavailable_reason or "cooldown" in unavailable_reason):
|
||||
temp_unsched = "active"
|
||||
elif state.get("is_available") is True:
|
||||
temp_unsched = "none"
|
||||
else:
|
||||
temp_unsched = "unavailable"
|
||||
base.update({"ROOT": item.get("error_source") or item.get("message") or "upstream", "TEMP_UNSCHED": temp_unsched, "ABSORBED": absorbed})
|
||||
return base
|
||||
|
||||
def execute():
|
||||
token = login()
|
||||
thresholds = get(token, "/api/v1/admin/ops/settings/metric-thresholds", label="metric thresholds")
|
||||
groups = selected_groups(items_of(get(token, "/api/v1/admin/groups/all", {"platform": "openai"}, "list groups")))
|
||||
summaries = []
|
||||
group_data = []
|
||||
for group in groups:
|
||||
group_id = group.get("id")
|
||||
native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview")
|
||||
if has_detail_filters():
|
||||
customer_total = len(error_rows(token, "/api/v1/admin/ops/request-errors", group_id, True))
|
||||
upstream_total = len(error_rows(token, "/api/v1/admin/ops/upstream-errors", group_id, True))
|
||||
else:
|
||||
request_errors = get(token, "/api/v1/admin/ops/request-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors")
|
||||
upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors")
|
||||
customer_total = total_of(request_errors)
|
||||
upstream_total = total_of(upstream_errors)
|
||||
absorbed = max(0, upstream_total - customer_total)
|
||||
absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None
|
||||
customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None)
|
||||
upstream_rate = percent(native.get("upstream_error_rate") if isinstance(native, dict) else None)
|
||||
ttft = native.get("ttft") if isinstance(native, dict) and isinstance(native.get("ttft"), dict) else {}
|
||||
metrics = {
|
||||
"groupId": group_id,
|
||||
"groupName": group.get("name"),
|
||||
"customerErrorCount": customer_total,
|
||||
"customerErrorRatePercent": customer_rate,
|
||||
"upstreamErrorCount": upstream_total,
|
||||
"upstreamErrorRatePercent": upstream_rate,
|
||||
"absorbedCountProjection": absorbed,
|
||||
"absorbedPercent": absorbed_percent,
|
||||
"ttftP99Ms": ttft.get("p99_ms"),
|
||||
}
|
||||
metrics["p0"] = "active" if customer_total > 0 else "clear"
|
||||
metrics["p1"] = threshold_state(metrics["ttftP99Ms"], thresholds.get("ttft_p99_ms_max"))
|
||||
metrics["p2"] = threshold_state(upstream_rate, thresholds.get("upstream_error_rate_percent_max"))
|
||||
summaries.append(metrics)
|
||||
group_data.append((group, metrics))
|
||||
level = PAYLOAD["filters"].get("level")
|
||||
offset = int(PAYLOAD.get("offset") or 0)
|
||||
page = offset // int(PAYLOAD["pageSize"]) + 1
|
||||
details = []
|
||||
detail_columns = []
|
||||
total = 0
|
||||
if level in ("P0", "P2"):
|
||||
for group, metrics in group_data:
|
||||
group_id = group.get("id")
|
||||
endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors"
|
||||
if has_detail_filters():
|
||||
rows = error_rows(token, endpoint, group_id, True)
|
||||
total += len(rows)
|
||||
rows = rows[offset:offset + PAYLOAD["pageSize"]]
|
||||
else:
|
||||
data = get(token, endpoint, {**scope_filters(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details")
|
||||
total += total_of(data)
|
||||
rows = items_of(data)
|
||||
availability = availability_by_account(token, group_id) if level == "P2" else {}
|
||||
customer_request_ids = set()
|
||||
if level == "P2":
|
||||
customer_request_ids = {
|
||||
item.get("request_id") or item.get("client_request_id")
|
||||
for item in error_rows(token, "/api/v1/admin/ops/request-errors", group_id, False)
|
||||
if item.get("request_id") or item.get("client_request_id")
|
||||
}
|
||||
for item in rows:
|
||||
absorbed = None
|
||||
if level == "P2" and item.get("request_id"):
|
||||
absorbed = item.get("request_id") not in customer_request_ids
|
||||
details.append(detail_row(item, metrics, availability, level, absorbed))
|
||||
detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"]
|
||||
elif level == "P1":
|
||||
total = len(group_data) * 2
|
||||
endpoint_names = ["/v1/responses", "/v1/responses/compact"]
|
||||
for group, metrics in group_data:
|
||||
for endpoint in endpoint_names:
|
||||
if PAYLOAD["filters"].get("endpoint") and PAYLOAD["filters"].get("endpoint") != endpoint:
|
||||
continue
|
||||
details.append({
|
||||
"GROUP": f"{metrics.get('groupName')} ({metrics.get('groupId')})",
|
||||
"ENDPOINT": endpoint,
|
||||
"SAMPLES": "unavailable",
|
||||
"P50": "unavailable",
|
||||
"P95": "unavailable",
|
||||
"P99": "unavailable",
|
||||
"MAX": "unavailable",
|
||||
"OUTLIER_REQUEST_ID": "unavailable",
|
||||
"BOUNDARY": "native endpoint/request TTFT unavailable; total duration not substituted",
|
||||
})
|
||||
detail_columns = ["GROUP", "ENDPOINT", "SAMPLES", "P50", "P95", "P99", "MAX", "OUTLIER_REQUEST_ID", "BOUNDARY"]
|
||||
return {
|
||||
"ok": True,
|
||||
"level": level,
|
||||
"window": {"timeRange": "24h"},
|
||||
"thresholds": thresholds,
|
||||
"summary": summaries,
|
||||
"details": details[:PAYLOAD["pageSize"]],
|
||||
"detailColumns": detail_columns,
|
||||
"pagination": {"offset": offset, "total": total, "hasMore": offset + PAYLOAD["pageSize"] < total},
|
||||
"boundary": {"notes": [
|
||||
"P0/P2 facts come from native admin ops request-errors, upstream-errors, overview, and account-availability.",
|
||||
"P0/P1/P2 labels and failover/retry absorption are UniDesk CLI projections; Sub2API native severity is not rewritten.",
|
||||
"P1 endpoint-level samples and request-level TTFT are unavailable in the observed native Ops response; total duration is never used as TTFT.",
|
||||
"Absorption summary is a bounded count projection from native upstream/customer totals; P2 detail absorption is correlated by stable request ID.",
|
||||
]},
|
||||
"next": "Use --level P0|P1|P2 for details; reuse filters with --page-token for the next fixed page; use codex-pool trace --request-id <id> for trace disclosure.",
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
try:
|
||||
output = execute()
|
||||
except Exception as exc:
|
||||
output = {"ok": False, "error": str(exc), "valuesPrinted": False}
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if output.get("ok") else 1)
|
||||
PY
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
import { readCodexPoolConfig } from "./config";
|
||||
import { capture, compactCapture, parseJsonOutput } from "./remote";
|
||||
import { renderedCliResult, renderTable, shorten, shortIso, textValue } from "./render";
|
||||
import { codexPoolRuntimeTarget } from "./runtime-target";
|
||||
import type { CodexPoolConfig, CodexPoolRuntimeTarget, FeedbackOptions } from "./types";
|
||||
|
||||
function pyJson(value: unknown): string {
|
||||
return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
|
||||
}
|
||||
|
||||
export async function codexPoolFeedback(config: UniDeskConfig, options: FeedbackOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const pool = readCodexPoolConfig();
|
||||
const target = codexPoolRuntimeTarget(options.targetId);
|
||||
const result = await capture(config, target.route, ["sh"], feedbackScript(pool, options, target));
|
||||
const report = parseJsonOutput(result.stdout);
|
||||
const ok = result.exitCode === 0 && report?.ok === true;
|
||||
const remote = compactCapture(result, { full: result.exitCode !== 0 || report === null });
|
||||
if (options.json) {
|
||||
return {
|
||||
ok,
|
||||
action: "platform-infra-sub2api-codex-pool-feedback",
|
||||
target: { id: target.id, route: target.route, runtimeMode: target.runtimeMode },
|
||||
report,
|
||||
remote,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
return renderedCliResult(ok, "platform-infra sub2api codex-pool feedback", renderFeedbackReport(report, options, remote));
|
||||
}
|
||||
|
||||
function feedbackScript(pool: CodexPoolConfig, options: FeedbackOptions, target: CodexPoolRuntimeTarget): string {
|
||||
return `
|
||||
set -u
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
RUNTIME_MODE = ${pyJson(target.runtimeMode)}
|
||||
NAMESPACE = ${pyJson(target.namespace)}
|
||||
APP_SECRET_NAME = ${pyJson(target.appSecretName)}
|
||||
ADMIN_EMAIL_DEFAULT = ${pyJson(pool.adminEmailDefault)}
|
||||
HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)}
|
||||
HOST_DOCKER_ENV_PATH = ${pyJson(target.hostDockerEnvPath)}
|
||||
USER_SELECTOR = ${pyJson(options.user)}
|
||||
WINDOW = ${pyJson(options.window)}
|
||||
REQUEST_SELECTOR = ${pyJson(options.requestId)}
|
||||
PAGE_TOKEN = ${pyJson(options.pageToken)}
|
||||
APP_CONTAINER = "sub2api-app"
|
||||
PAGE_SIZE = 10
|
||||
|
||||
def run(command, input_bytes=None):
|
||||
return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def text(value, limit=1000):
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", errors="replace")
|
||||
return str(value or "")[-limit:]
|
||||
|
||||
def docker(args):
|
||||
proc = run(["docker", *args])
|
||||
if proc.returncode == 0:
|
||||
return proc
|
||||
fallback = run(["sudo", "-n", "docker", *args])
|
||||
return fallback if fallback.returncode == 0 else proc
|
||||
|
||||
def read_host_env():
|
||||
if RUNTIME_MODE != "host-docker":
|
||||
return {}
|
||||
try:
|
||||
with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle:
|
||||
lines = handle.read().splitlines()
|
||||
except PermissionError:
|
||||
proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError("read host-docker env failed: " + text(proc.stderr))
|
||||
lines = proc.stdout.decode("utf-8", errors="replace").splitlines()
|
||||
values = {}
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||||
continue
|
||||
key, value = stripped.split("=", 1)
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
|
||||
value = value[1:-1]
|
||||
values[key.strip()] = value
|
||||
return values
|
||||
|
||||
def runtime_secret(key):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
return read_host_env().get(key)
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "get", "secret", APP_SECRET_NAME, "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError("read app secret failed: " + text(proc.stderr))
|
||||
raw = (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
|
||||
return base64.b64decode(raw).decode("utf-8") if raw else None
|
||||
|
||||
def runtime_config(key):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
return read_host_env().get(key)
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "get", "configmap", "sub2api-config", "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
|
||||
|
||||
def parse_curl(proc):
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
marker = "\\n__HTTP_CODE__:"
|
||||
position = stdout.rfind(marker)
|
||||
if position < 0:
|
||||
return {"ok": False, "httpStatus": 0, "json": None, "message": text(proc.stderr)}
|
||||
body = stdout[:position]
|
||||
try:
|
||||
status = int(stdout[position + len(marker):].strip()[-3:])
|
||||
except Exception:
|
||||
status = 0
|
||||
try:
|
||||
parsed = json.loads(body) if body.strip() else None
|
||||
except Exception:
|
||||
parsed = None
|
||||
message = parsed.get("message") if isinstance(parsed, dict) else text(body)
|
||||
return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "message": message}
|
||||
|
||||
def curl_api(method, path, bearer=None, payload=None):
|
||||
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
script = r'''set -eu
|
||||
method="$1"
|
||||
url="$2"
|
||||
token="\${3:-}"
|
||||
tmp="$(mktemp)"
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
cat > "$tmp"
|
||||
args=""
|
||||
if [ -n "$token" ]; then args="Authorization: Bearer $token"; fi
|
||||
if [ "$method" = GET ] && [ ! -s "$tmp" ]; then
|
||||
if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -H "$args" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' "$url"; fi
|
||||
else
|
||||
if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "$args" --data-binary @"$tmp" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"; fi
|
||||
fi'''
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body)
|
||||
else:
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "exec", "deploy/sub2api", "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""], body)
|
||||
return parse_curl(proc)
|
||||
|
||||
def api_data(method, path, token=None, payload=None, label=None):
|
||||
response = curl_api(method, path, token, payload)
|
||||
parsed = response.get("json")
|
||||
code = parsed.get("code") if isinstance(parsed, dict) else None
|
||||
if not response.get("ok") or (code is not None and code != 0):
|
||||
raise RuntimeError(f"{label or path} failed: http={response.get('httpStatus')} message={response.get('message')}")
|
||||
return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
|
||||
|
||||
def find_token(value):
|
||||
if isinstance(value, dict):
|
||||
for key in ("access_token", "token"):
|
||||
if isinstance(value.get(key), str) and value.get(key):
|
||||
return value[key]
|
||||
for item in value.values():
|
||||
token = find_token(item)
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
def login():
|
||||
email = runtime_config("ADMIN_EMAIL") or ADMIN_EMAIL_DEFAULT
|
||||
password = runtime_secret("ADMIN_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("ADMIN_PASSWORD missing")
|
||||
token = find_token(api_data("POST", "/api/v1/auth/login", payload={"email": email, "password": password}, label="admin login"))
|
||||
if not token:
|
||||
raise RuntimeError("admin login returned no token")
|
||||
return email, token
|
||||
|
||||
def items(data):
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
if isinstance(data.get("items"), list):
|
||||
return data["items"]
|
||||
for key in ("users", "api_keys", "keys", "groups", "accounts", "proxies", "logs"):
|
||||
if isinstance(data.get(key), list):
|
||||
return data[key]
|
||||
return []
|
||||
|
||||
def page_all(token, path, query=None, page_size=100):
|
||||
query = dict(query or {})
|
||||
collected = []
|
||||
page = 1
|
||||
while True:
|
||||
params = {**query, "page": page, "page_size": page_size}
|
||||
separator = "&" if "?" in path else "?"
|
||||
data = api_data("GET", path + separator + urlencode(params), token, label=path)
|
||||
batch = items(data)
|
||||
collected.extend(batch)
|
||||
total = data.get("total") if isinstance(data, dict) else None
|
||||
if not batch or (isinstance(total, int) and len(collected) >= total) or len(batch) < page_size:
|
||||
break
|
||||
page += 1
|
||||
return collected
|
||||
|
||||
def exact_user(token):
|
||||
if USER_SELECTOR.isdigit() and int(USER_SELECTOR) > 0:
|
||||
user = api_data("GET", f"/api/v1/admin/users/{int(USER_SELECTOR)}", token, label="get user")
|
||||
return user if isinstance(user, dict) else None
|
||||
candidates = page_all(token, "/api/v1/admin/users", {"search": USER_SELECTOR}, 100)
|
||||
exact = [user for user in candidates if isinstance(user, dict) and str(user.get("email") or "").lower() == USER_SELECTOR.lower()]
|
||||
if len(exact) > 1:
|
||||
raise RuntimeError("email matched multiple users")
|
||||
return exact[0] if exact else None
|
||||
|
||||
def safe_call(errors, name, callback, fallback):
|
||||
try:
|
||||
return callback()
|
||||
except Exception as exc:
|
||||
errors.append({"source": name, "error": redact_text(str(exc))})
|
||||
return fallback
|
||||
|
||||
def redact_text(value):
|
||||
value = str(value or "")
|
||||
value = re.sub(r"(?i)(bearer\\s+)[A-Za-z0-9._~+\\-/=]+", r"\\1<redacted>", value)
|
||||
value = re.sub(r"(?i)((?:api[_-]?key|token|password)\\s*[=:]\\s*)[^\\s,;]+", r"\\1<redacted>", value)
|
||||
value = re.sub(r"\\bsk-[A-Za-z0-9_-]{8,}\\b", "sk-<redacted>", value)
|
||||
return value[:500]
|
||||
|
||||
def iso_epoch(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
match = re.match(r"^(\\d{4}-\\d\\d-\\d\\d[T ]\\d\\d:\\d\\d:\\d\\d)(?:\\.(\\d+))?(Z|[+-]\\d\\d:?\\d\\d)$", value)
|
||||
if not match:
|
||||
return None
|
||||
base = datetime.strptime(match.group(1).replace(" ", "T"), "%Y-%m-%dT%H:%M:%S")
|
||||
fraction = int((match.group(2) or "0")[:6].ljust(6, "0"))
|
||||
zone = match.group(3)
|
||||
if zone == "Z":
|
||||
offset = timezone.utc
|
||||
else:
|
||||
sign = 1 if zone[0] == "+" else -1
|
||||
digits = zone[1:].replace(":", "")
|
||||
offset = timezone(sign * timedelta(hours=int(digits[:2]), minutes=int(digits[2:])))
|
||||
return base.replace(microsecond=fraction, tzinfo=offset).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def encode_page_token(row):
|
||||
payload = json.dumps({"v": 1, "at": row.get("at"), "requestId": row.get("requestId")}, separators=(",", ":")).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=")
|
||||
|
||||
def decode_page_token(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
padded = value + "=" * (-len(value) % 4)
|
||||
parsed = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8"))
|
||||
except Exception:
|
||||
raise RuntimeError("invalid page token")
|
||||
if not isinstance(parsed, dict) or parsed.get("v") != 1 or not parsed.get("at") or not parsed.get("requestId"):
|
||||
raise RuntimeError("invalid page token")
|
||||
return parsed
|
||||
|
||||
def dict_by_id(rows):
|
||||
return {str(row.get("id")): row for row in rows if isinstance(row, dict) and row.get("id") is not None}
|
||||
|
||||
def keyed_map(value, key):
|
||||
source = value.get(key) if isinstance(value, dict) else None
|
||||
return source if isinstance(source, dict) else {}
|
||||
|
||||
def runtime_health():
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
proc = docker(["inspect", APP_CONTAINER, "sub2api-redis"])
|
||||
if proc.returncode != 0:
|
||||
return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
|
||||
rows = []
|
||||
for item in json.loads(proc.stdout.decode("utf-8")):
|
||||
state = item.get("State") or {}
|
||||
health = state.get("Health") or {}
|
||||
rows.append({"name": str(item.get("Name") or "").lstrip("/"), "running": state.get("Running"), "status": health.get("Status") or state.get("Status")})
|
||||
return {"ok": all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api", "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
|
||||
rows = []
|
||||
for item in json.loads(proc.stdout.decode("utf-8")).get("items") or []:
|
||||
status = item.get("status") or {}
|
||||
containers = status.get("containerStatuses") or []
|
||||
rows.append({"name": (item.get("metadata") or {}).get("name"), "running": status.get("phase") == "Running", "status": "ready" if containers and all(entry.get("ready") is True for entry in containers) else status.get("phase")})
|
||||
return {"ok": bool(rows) and all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
|
||||
|
||||
def compact_log(row):
|
||||
extra = row.get("extra") if isinstance(row.get("extra"), dict) else {}
|
||||
allow = {}
|
||||
for key in ("phase", "error_owner", "status_code", "duration_ms", "latency_ms", "time_to_first_token_ms", "upstream_status", "path", "stream"):
|
||||
if key in extra:
|
||||
allow[key] = extra.get(key)
|
||||
return {
|
||||
"id": row.get("id"),
|
||||
"at": row.get("created_at"),
|
||||
"level": row.get("level"),
|
||||
"component": row.get("component"),
|
||||
"message": redact_text(row.get("message")),
|
||||
"requestId": row.get("request_id"),
|
||||
"clientRequestId": row.get("client_request_id"),
|
||||
"accountId": row.get("account_id"),
|
||||
"extra": allow,
|
||||
}
|
||||
|
||||
def related_logs(request, logs):
|
||||
ids = {str(request.get("request_id") or "")}
|
||||
matched = []
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for row in logs:
|
||||
request_id = str(row.get("request_id") or "")
|
||||
client_id = str(row.get("client_request_id") or "")
|
||||
if row in matched or (request_id not in ids and client_id not in ids):
|
||||
continue
|
||||
matched.append(row)
|
||||
before = len(ids)
|
||||
ids.update(value for value in (request_id, client_id) if value)
|
||||
changed = changed or len(ids) != before
|
||||
return matched
|
||||
|
||||
def classify(request, logs, account, proxy, queued):
|
||||
evidence = " ".join([str(request.get("phase") or ""), str(request.get("message") or ""), *[str(log.get("component") or "") + " " + str(log.get("message") or "") for log in logs]]).lower()
|
||||
if any(marker in evidence for marker in ("client disconnect", "broken pipe", "context canceled", "client canceled")):
|
||||
return "client-connection"
|
||||
if any(marker in evidence for marker in ("proxy", "dial tcp", "dns", "connection refused", "network")) or (proxy and proxy.get("status") not in (None, "active")):
|
||||
return "proxy-egress"
|
||||
if any(marker in evidence for marker in ("upstream", "provider", "first token", "stream")) or str(request.get("phase") or "") == "upstream":
|
||||
return "upstream-first-byte-stream"
|
||||
if queued > 0:
|
||||
return "sub2api-queue"
|
||||
if request.get("kind") == "error" or (isinstance(request.get("status_code"), int) and request.get("status_code") >= 400):
|
||||
return "sub2api-or-upstream-unknown"
|
||||
return "completed"
|
||||
|
||||
def main():
|
||||
errors = []
|
||||
admin_email, token = login()
|
||||
user = exact_user(token)
|
||||
if not isinstance(user, dict):
|
||||
return {"ok": False, "mode": "feedback", "error": "user-not-found", "selector": USER_SELECTOR, "valuesPrinted": False}
|
||||
user_id = user.get("id")
|
||||
groups = safe_call(errors, "groups", lambda: items(api_data("GET", "/api/v1/admin/groups/all", token, label="groups")), [])
|
||||
keys = safe_call(errors, "api-keys", lambda: page_all(token, f"/api/v1/admin/users/{user_id}/api-keys", {}, 100), [])
|
||||
requests = safe_call(errors, "ops-requests", lambda: page_all(token, "/api/v1/admin/ops/requests", {"user_id": user_id, "time_range": WINDOW, "kind": "all", "sort": "created_at_desc"}, 100), [])
|
||||
logs = safe_call(errors, "ops-system-logs", lambda: page_all(token, "/api/v1/admin/ops/system-logs", {"user_id": user_id, "time_range": WINDOW}, 200), [])
|
||||
concurrency = safe_call(errors, "ops-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/concurrency", token, label="concurrency"), {})
|
||||
user_concurrency = safe_call(errors, "ops-user-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/user-concurrency", token, label="user concurrency"), {})
|
||||
availability = safe_call(errors, "ops-account-availability", lambda: api_data("GET", "/api/v1/admin/ops/account-availability", token, label="account availability"), {})
|
||||
accounts = safe_call(errors, "accounts", lambda: page_all(token, "/api/v1/admin/accounts", {}, 100), [])
|
||||
proxies = safe_call(errors, "proxies", lambda: page_all(token, "/api/v1/admin/proxies", {}, 100), [])
|
||||
infrastructure = safe_call(errors, "infrastructure", runtime_health, {"ok": False, "mode": RUNTIME_MODE})
|
||||
|
||||
groups_by_id = dict_by_id(groups)
|
||||
keys_by_id = dict_by_id(keys)
|
||||
accounts_by_id = dict_by_id(accounts)
|
||||
proxies_by_id = dict_by_id(proxies)
|
||||
user_current_map = keyed_map(user_concurrency, "user")
|
||||
account_concurrency_map = keyed_map(concurrency, "account")
|
||||
account_availability_map = keyed_map(availability, "account")
|
||||
current_user = user_current_map.get(str(user_id)) or {}
|
||||
user_current = int(current_user.get("current_in_use") or user.get("current_concurrency") or 0)
|
||||
user_waiting = int(current_user.get("waiting_in_queue") or 0)
|
||||
|
||||
compact_keys = []
|
||||
for key in keys:
|
||||
group = groups_by_id.get(str(key.get("group_id"))) or {}
|
||||
compact_keys.append({
|
||||
"id": key.get("id"), "name": key.get("name"), "status": key.get("status"),
|
||||
"groupId": key.get("group_id"), "groupName": group.get("name"),
|
||||
"currentConcurrency": key.get("current_concurrency"), "keyPrinted": False,
|
||||
})
|
||||
|
||||
timeline = []
|
||||
ordered = sorted([row for row in requests if isinstance(row, dict)], key=lambda row: iso_epoch(row.get("created_at")) or 0)
|
||||
previous_epoch = None
|
||||
for row in ordered:
|
||||
matched_logs = related_logs(row, logs)
|
||||
client_id = next((log.get("client_request_id") for log in matched_logs if log.get("client_request_id")), None)
|
||||
account = accounts_by_id.get(str(row.get("account_id"))) or {}
|
||||
proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
|
||||
account_load = account_concurrency_map.get(str(row.get("account_id"))) or {}
|
||||
queued = int(account_load.get("waiting_in_queue") or 0)
|
||||
epoch = iso_epoch(row.get("created_at"))
|
||||
gap = round(epoch - previous_epoch, 1) if epoch is not None and previous_epoch is not None else None
|
||||
previous_epoch = epoch if epoch is not None else previous_epoch
|
||||
timeline.append({
|
||||
"at": row.get("created_at"), "gapSeconds": gap, "kind": row.get("kind"),
|
||||
"requestId": row.get("request_id"), "clientRequestId": client_id,
|
||||
"model": row.get("model"), "statusCode": row.get("status_code"), "durationMs": row.get("duration_ms"),
|
||||
"apiKeyId": row.get("api_key_id"), "apiKeyName": (keys_by_id.get(str(row.get("api_key_id"))) or {}).get("name"),
|
||||
"accountId": row.get("account_id"), "accountName": account.get("name"),
|
||||
"proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"),
|
||||
"attribution": classify(row, matched_logs, account, proxy, queued),
|
||||
"evidence": {"systemLogCount": len(matched_logs), "accountStatus": account.get("status"), "accountSchedulable": account.get("schedulable"), "proxyStatus": proxy.get("status")},
|
||||
})
|
||||
|
||||
all_timeline = timeline
|
||||
max_gap = max([row.get("gapSeconds") for row in all_timeline if isinstance(row.get("gapSeconds"), (int, float))], default=None)
|
||||
next_page_token = None
|
||||
more_available = False
|
||||
if REQUEST_SELECTOR:
|
||||
selected_ids = {REQUEST_SELECTOR}
|
||||
for row in all_timeline:
|
||||
if row.get("requestId") == REQUEST_SELECTOR or row.get("clientRequestId") == REQUEST_SELECTOR:
|
||||
selected_ids.update(value for value in (row.get("requestId"), row.get("clientRequestId")) if isinstance(value, str) and value)
|
||||
timeline = [row for row in all_timeline if any(isinstance(value, str) and value in selected_ids for value in (row.get("requestId"), row.get("clientRequestId")))]
|
||||
detail_logs = [compact_log(row) for row in logs if any(isinstance(value, str) and value in selected_ids for value in (row.get("request_id"), row.get("client_request_id")))]
|
||||
else:
|
||||
detail_logs = []
|
||||
ordered_desc = list(reversed(all_timeline))
|
||||
cursor = decode_page_token(PAGE_TOKEN)
|
||||
start = 0
|
||||
if cursor:
|
||||
start = next((index + 1 for index, row in enumerate(ordered_desc) if row.get("at") == cursor.get("at") and row.get("requestId") == cursor.get("requestId")), -1)
|
||||
if start < 0:
|
||||
raise RuntimeError("page token cursor is no longer present in the selected window")
|
||||
timeline = ordered_desc[start:start + PAGE_SIZE]
|
||||
more_available = start + len(timeline) < len(ordered_desc)
|
||||
if more_available and timeline:
|
||||
next_page_token = encode_page_token(timeline[-1])
|
||||
if not requests and user_waiting > 0:
|
||||
conclusion = "sub2api-queue"
|
||||
elif not requests and user_current > 0:
|
||||
conclusion = "in-flight"
|
||||
elif not requests:
|
||||
conclusion = "client-before-submit-or-no-inbound-evidence"
|
||||
elif timeline:
|
||||
conclusion = timeline[0].get("attribution")
|
||||
else:
|
||||
conclusion = "request-selector-not-found"
|
||||
|
||||
related_account_ids = {str(row.get("accountId")) for row in timeline if row.get("accountId") is not None}
|
||||
account_evidence = []
|
||||
for account_id in sorted(related_account_ids):
|
||||
account = accounts_by_id.get(account_id) or {}
|
||||
proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
|
||||
load = account_concurrency_map.get(account_id) or {}
|
||||
available = account_availability_map.get(account_id) or {}
|
||||
account_evidence.append({
|
||||
"id": account.get("id"), "name": account.get("name"), "status": account.get("status"), "schedulable": account.get("schedulable"),
|
||||
"currentInUse": load.get("current_in_use"), "waitingInQueue": load.get("waiting_in_queue"), "maxCapacity": load.get("max_capacity"),
|
||||
"available": available.get("is_available"), "rateLimited": available.get("is_rate_limited"), "overloaded": available.get("is_overloaded"),
|
||||
"proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"), "proxyStatus": proxy.get("status"),
|
||||
})
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "feedback",
|
||||
"selector": {"user": USER_SELECTOR, "requestId": REQUEST_SELECTOR, "pageToken": PAGE_TOKEN, "window": WINDOW},
|
||||
"user": {"id": user_id, "email": user.get("email"), "username": user.get("username"), "status": user.get("status"), "maxConcurrency": user.get("concurrency"), "currentInUse": user_current, "waitingInQueue": user_waiting},
|
||||
"apiKeys": compact_keys,
|
||||
"lifecycle": {"notInbound": not requests and user_current == 0 and user_waiting == 0, "queued": user_waiting, "inFlight": user_current, "completed": len(requests)},
|
||||
"timeline": timeline,
|
||||
"timelineSummary": {"returned": len(timeline), "total": len(all_timeline), "moreAvailable": more_available, "nextPageToken": next_page_token, "pageSize": PAGE_SIZE, "systemLogCount": len(logs), "maxGapSeconds": max_gap},
|
||||
"requestDetail": {"logs": detail_logs, "logCount": len(detail_logs)} if REQUEST_SELECTOR else None,
|
||||
"accounts": account_evidence,
|
||||
"infrastructure": infrastructure,
|
||||
"conclusion": {"attribution": conclusion, "boundary": "只读证据归因；无入站记录不等于客户端未调用，运行中请求仅由实时并发佐证。"},
|
||||
"toolCallReduction": {"manualInvestigationCalls": 9, "feedbackCliCalls": 1, "reducedCalls": 8, "reductionPercent": 88.9, "basis": "issue-2000: user, requests, system logs, two request-id traces, account, proxy, process/port, journal"},
|
||||
"sources": ["admin users", "admin user api-keys", "ops requests", "ops system-logs", "ops concurrency", "ops user-concurrency", "ops account-availability", "admin accounts", "admin proxies", "runtime health"],
|
||||
"errors": errors,
|
||||
"admin": {"email": admin_email, "tokenPrinted": False},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
try:
|
||||
print(json.dumps(main(), ensure_ascii=False))
|
||||
except Exception as exc:
|
||||
print(json.dumps({"ok": False, "mode": "feedback", "error": redact_text(str(exc)), "valuesPrinted": False}, ensure_ascii=False))
|
||||
raise
|
||||
PY
|
||||
`;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function records(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
||||
}
|
||||
|
||||
function renderFeedbackReport(report: Record<string, unknown> | null, options: FeedbackOptions, remote: Record<string, unknown>): string {
|
||||
if (report === null) {
|
||||
return [
|
||||
`SUB2API FEEDBACK user=${options.user} unavailable`,
|
||||
`remote_exit=${remote.exitCode ?? "?"} stdout_bytes=${remote.stdoutBytes ?? "?"} stderr_bytes=${remote.stderrBytes ?? "?"}`,
|
||||
textValue(remote.stderrTail ?? remote.stdoutTail),
|
||||
].join("\n");
|
||||
}
|
||||
if (report.ok !== true) {
|
||||
return [
|
||||
`SUB2API FEEDBACK user=${options.user} ok=false`,
|
||||
`error=${textValue(report.error)}`,
|
||||
"JSON: add --json for the structured failure envelope.",
|
||||
].join("\n");
|
||||
}
|
||||
const user = record(report.user);
|
||||
const lifecycle = record(report.lifecycle);
|
||||
const summary = record(report.timelineSummary);
|
||||
const conclusion = record(report.conclusion);
|
||||
const reduction = record(report.toolCallReduction);
|
||||
const timeline = records(report.timeline);
|
||||
const accounts = records(report.accounts);
|
||||
const apiKeys = records(report.apiKeys);
|
||||
const detail = record(report.requestDetail);
|
||||
const detailLogs = records(detail.logs);
|
||||
const infrastructure = record(report.infrastructure);
|
||||
const components = records(infrastructure.components);
|
||||
const errors = records(report.errors);
|
||||
const lines: string[] = [];
|
||||
lines.push(`SUB2API FEEDBACK user=${textValue(user.email)}#${textValue(user.id)} window=${options.window} ok=true`);
|
||||
lines.push(`CONCLUSION attribution=${textValue(conclusion.attribution)} boundary=${textValue(conclusion.boundary)}`);
|
||||
lines.push(`LIFECYCLE not_inbound=${textValue(lifecycle.notInbound)} queued=${textValue(lifecycle.queued)} in_flight=${textValue(lifecycle.inFlight)} completed=${textValue(lifecycle.completed)} max_gap_s=${textValue(summary.maxGapSeconds)}`);
|
||||
lines.push(`TOOLS manual=${textValue(reduction.manualInvestigationCalls)} cli=${textValue(reduction.feedbackCliCalls)} reduced=${textValue(reduction.reducedCalls)} reduction=${textValue(reduction.reductionPercent)}%`);
|
||||
if (apiKeys.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("API KEYS");
|
||||
lines.push(renderTable([
|
||||
["ID", "NAME", "GROUP", "STATUS", "CURRENT"],
|
||||
...apiKeys.map((key) => [textValue(key.id), shorten(textValue(key.name), 30), shorten(textValue(key.groupName), 24), textValue(key.status), textValue(key.currentConcurrency)]),
|
||||
]));
|
||||
}
|
||||
lines.push("");
|
||||
lines.push(`REQUESTS returned=${textValue(summary.returned)} total=${textValue(summary.total)} moreAvailable=${textValue(summary.moreAvailable)} system_logs=${textValue(summary.systemLogCount)}`);
|
||||
if (timeline.length === 0) {
|
||||
lines.push("No matching completed request records in the selected window.");
|
||||
} else {
|
||||
lines.push(renderTable([
|
||||
["#", "AT", "GAP_S", "STATUS", "DURATION", "MODEL", "ACCOUNT", "ATTRIBUTION"],
|
||||
...timeline.map((item, index) => [
|
||||
String(index + 1), shortIso(item.at), textValue(item.gapSeconds), textValue(item.statusCode ?? item.kind), textValue(item.durationMs),
|
||||
shorten(textValue(item.model), 18), `${textValue(item.accountName)}#${textValue(item.accountId)}`, textValue(item.attribution),
|
||||
]),
|
||||
]));
|
||||
lines.push("");
|
||||
lines.push("REQUEST ID INDEX");
|
||||
lines.push(renderTable([
|
||||
["#", "REQUEST_ID", "CLIENT_ID"],
|
||||
...timeline.map((item, index) => [String(index + 1), textValue(item.requestId), textValue(item.clientRequestId)]),
|
||||
]));
|
||||
}
|
||||
if (accounts.length > 0 || components.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("ACCOUNT / INFRA");
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "STATUS", "SCHED", "IN_USE", "QUEUE", "CAP", "AVAILABLE", "PROXY", "P_STATUS"],
|
||||
...accounts.map((item) => [
|
||||
`${textValue(item.name)}#${textValue(item.id)}`, textValue(item.status), textValue(item.schedulable),
|
||||
textValue(item.currentInUse), textValue(item.waitingInQueue), textValue(item.maxCapacity), textValue(item.available),
|
||||
shorten(`${textValue(item.proxyName)}#${textValue(item.proxyId)}`, 24), textValue(item.proxyStatus),
|
||||
]),
|
||||
...components.map((item) => [shorten(textValue(item.name), 32), textValue(item.status), "-", "-", "-", "-", textValue(item.running), "runtime", textValue(infrastructure.mode)]),
|
||||
]));
|
||||
}
|
||||
if (options.requestId !== null) {
|
||||
lines.push("");
|
||||
lines.push(`REQUEST DETAIL id=${options.requestId} logs=${textValue(detail.logCount)}`);
|
||||
if (detailLogs.length > 0) {
|
||||
lines.push(renderTable([
|
||||
["LOG_ID", "AT", "LEVEL", "COMPONENT", "REQUEST_ID", "CLIENT_ID", "MESSAGE"],
|
||||
...detailLogs.map((item) => [
|
||||
textValue(item.id), shortIso(item.at), textValue(item.level), shorten(textValue(item.component), 22),
|
||||
shorten(textValue(item.requestId), 18), shorten(textValue(item.clientRequestId), 18), shorten(textValue(item.message), 64),
|
||||
]),
|
||||
]));
|
||||
}
|
||||
}
|
||||
if (errors.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("PARTIAL EVIDENCE");
|
||||
lines.push(renderTable([["SOURCE", "ERROR"], ...errors.map((item) => [textValue(item.source), shorten(textValue(item.error), 90)])]));
|
||||
}
|
||||
lines.push("");
|
||||
if (summary.moreAvailable === true && typeof summary.nextPageToken === "string") {
|
||||
lines.push(`NEXT_PAGE_TOKEN ${summary.nextPageToken}`);
|
||||
lines.push(`Next: bun scripts/cli.ts platform-infra sub2api codex-pool feedback --target ${options.targetId} --user ${options.user} --window ${options.window} --page-token ${summary.nextPageToken}`);
|
||||
}
|
||||
lines.push("Disclosure: rerun with --request-id <client-or-internal-id> for correlated indexed logs.");
|
||||
lines.push("JSON: add --json for the complete redacted report.");
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -13,3 +13,4 @@ export * from "./remote-scripts";
|
||||
export * from "./accounts";
|
||||
export * from "./remote-python-sync";
|
||||
export * from "./remote";
|
||||
export * from "./feedback";
|
||||
|
||||
@@ -21,10 +21,12 @@ import {
|
||||
import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets";
|
||||
import { runSshCommandCapture, type SshCaptureResult } from "../ssh";
|
||||
|
||||
import type { ConfirmOptions, DisclosureOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types";
|
||||
import type { ConfirmOptions, DisclosureOptions, FeedbackOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types";
|
||||
import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions";
|
||||
import { codexPoolFeedback } from "./feedback";
|
||||
import { renderCodexPoolPlan } from "./render";
|
||||
import { codexPoolRuntime } from "./runtime";
|
||||
import { codexPoolFaults } from "./faults";
|
||||
import { defaultCodexPoolRuntimeTargetId } from "./runtime-target";
|
||||
import { codexPoolHelp } from "./types";
|
||||
|
||||
@@ -38,7 +40,9 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[])
|
||||
if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1)));
|
||||
if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1)));
|
||||
if (action === "runtime") return await codexPoolRuntime(config, args.slice(1));
|
||||
if (action === "faults") return await codexPoolFaults(config, args.slice(1));
|
||||
if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1)));
|
||||
if (action === "feedback") return await codexPoolFeedback(config, parseFeedbackOptions(args.slice(1)));
|
||||
if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1)));
|
||||
if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1)));
|
||||
if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1)));
|
||||
@@ -181,6 +185,7 @@ export function parseSentinelReportOptions(args: string[]): SentinelReportOption
|
||||
|
||||
export function parseTraceOptions(args: string[]): TraceOptions {
|
||||
let requestId: string | null = null;
|
||||
let pageToken: string | null = null;
|
||||
let since = "24h";
|
||||
let tail = 20_000;
|
||||
let contextSeconds = 300;
|
||||
@@ -258,6 +263,64 @@ export function parseTraceOptions(args: string[]): TraceOptions {
|
||||
return { ...disclosure, requestId, since, tail, contextSeconds, showLines };
|
||||
}
|
||||
|
||||
export function parseFeedbackOptions(args: string[]): FeedbackOptions {
|
||||
let user: string | null = null;
|
||||
let window: FeedbackOptions["window"] = "30m";
|
||||
let requestId: string | null = null;
|
||||
let pageToken: string | null = null;
|
||||
let json = false;
|
||||
const targetArgs: string[] = [];
|
||||
const windows = new Set<FeedbackOptions["window"]>(["5m", "30m", "1h", "6h", "24h", "7d", "30d"]);
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index]!;
|
||||
if (arg === "--json") {
|
||||
json = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--target" || arg.startsWith("--target=")) {
|
||||
targetArgs.push(arg);
|
||||
if (arg === "--target") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
|
||||
targetArgs.push(value);
|
||||
index += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const readValue = (option: string): string => {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${option} requires a value`);
|
||||
index += 1;
|
||||
return value.trim();
|
||||
};
|
||||
if (arg === "--user") user = readValue(arg);
|
||||
else if (arg.startsWith("--user=")) user = arg.slice("--user=".length).trim();
|
||||
else if (arg === "--window") {
|
||||
const value = readValue(arg) as FeedbackOptions["window"];
|
||||
if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d");
|
||||
window = value;
|
||||
} else if (arg.startsWith("--window=")) {
|
||||
const value = arg.slice("--window=".length) as FeedbackOptions["window"];
|
||||
if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d");
|
||||
window = value;
|
||||
} else if (arg === "--request-id") requestId = readValue(arg);
|
||||
else if (arg.startsWith("--request-id=")) requestId = arg.slice("--request-id=".length).trim();
|
||||
else if (arg === "--page-token") pageToken = readValue(arg);
|
||||
else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim();
|
||||
else throw new Error(`unsupported option: ${arg}`);
|
||||
}
|
||||
if (user === null || user.length === 0) throw new Error("feedback requires --user <email-or-id>");
|
||||
if (user.length > 320 || /[\r\n]/u.test(user)) throw new Error("--user must be a single email or positive user id");
|
||||
if (requestId !== null && (requestId.length === 0 || requestId.length > 256 || /[\r\n]/u.test(requestId))) {
|
||||
throw new Error("--request-id must be a non-empty stable request id up to 256 characters");
|
||||
}
|
||||
if (pageToken !== null && (pageToken.length === 0 || pageToken.length > 1024 || !/^[A-Za-z0-9_-]+$/u.test(pageToken))) {
|
||||
throw new Error("--page-token must be a URL-safe cursor token");
|
||||
}
|
||||
if (requestId !== null && pageToken !== null) throw new Error("--request-id and --page-token cannot be combined");
|
||||
return { user, window, requestId, pageToken, json, targetId: parseTargetId(targetArgs) };
|
||||
}
|
||||
|
||||
export function parseKubectlDuration(raw: string): string {
|
||||
const value = raw.trim();
|
||||
if (!/^[1-9][0-9]*(?:s|m|h)$/u.test(value)) throw new Error("--since must be a kubectl duration such as 24h, 90m, or 300s");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,189 @@
|
||||
export const remotePythonSentinelProbeScript = `def parse_embedded_json(stdout):
|
||||
if not isinstance(stdout, str) or not stdout.strip():
|
||||
return None
|
||||
start = stdout.find("{")
|
||||
end = stdout.rfind("}")
|
||||
if start < 0 or end <= start:
|
||||
return None
|
||||
try:
|
||||
return json.loads(stdout[start:end + 1])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def inject_env(template, name, value):
|
||||
spec = template.setdefault("spec", {})
|
||||
containers = spec.setdefault("containers", [])
|
||||
if not containers:
|
||||
raise RuntimeError("sentinel job template has no containers")
|
||||
container = containers[0]
|
||||
env = container.setdefault("env", [])
|
||||
env[:] = [item for item in env if not (isinstance(item, dict) and item.get("name") == name)]
|
||||
env.append({"name": name, "value": value})
|
||||
|
||||
def sentinel_probe_job_manifest(accounts):
|
||||
cronjob_name = SENTINEL_CONFIG.get("cronJobName")
|
||||
if not isinstance(cronjob_name, str) or not cronjob_name:
|
||||
raise RuntimeError("sentinel cronJobName missing from config")
|
||||
cronjob = kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
|
||||
job_template = ((cronjob.get("spec") or {}).get("jobTemplate") or {}) if isinstance(cronjob, dict) else {}
|
||||
job_spec = copy_json(job_template.get("spec") or {})
|
||||
if not isinstance(job_spec, dict) or not job_spec:
|
||||
raise RuntimeError("sentinel CronJob jobTemplate.spec missing")
|
||||
template = job_spec.get("template")
|
||||
if not isinstance(template, dict):
|
||||
raise RuntimeError("sentinel CronJob jobTemplate.spec.template missing")
|
||||
inject_env(template, "SENTINEL_ACCOUNT_NAMES", ",".join(accounts))
|
||||
job_spec["ttlSecondsAfterFinished"] = int(job_spec.get("ttlSecondsAfterFinished") or 3600)
|
||||
suffix = str(int(time.time()))[-8:] + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(5))
|
||||
job_name = ("sub2api-sentinel-probe-" + suffix)[:63]
|
||||
return job_name, {
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"metadata": {
|
||||
"name": job_name,
|
||||
"namespace": NAMESPACE,
|
||||
"labels": {
|
||||
"app.kubernetes.io/name": cronjob_name,
|
||||
"app.kubernetes.io/part-of": "platform-infra",
|
||||
"app.kubernetes.io/managed-by": "unidesk",
|
||||
"unidesk.ai/job-purpose": "sub2api-account-sentinel-manual-probe",
|
||||
},
|
||||
},
|
||||
"spec": job_spec,
|
||||
}
|
||||
|
||||
def copy_json(value):
|
||||
return json.loads(json.dumps(value))
|
||||
|
||||
def job_condition(job, cond_type):
|
||||
for item in ((job.get("status") or {}).get("conditions") or []):
|
||||
if item.get("type") == cond_type and item.get("status") == "True":
|
||||
return item
|
||||
return None
|
||||
|
||||
def wait_sentinel_probe_job(job_name, timeout_seconds):
|
||||
deadline = time.time() + timeout_seconds
|
||||
latest = None
|
||||
while time.time() < deadline:
|
||||
latest, err = safe_kube_json(["-n", NAMESPACE, "get", "job", job_name], f"job/{job_name}")
|
||||
if isinstance(latest, dict):
|
||||
complete = job_condition(latest, "Complete")
|
||||
failed = job_condition(latest, "Failed")
|
||||
if complete is not None:
|
||||
return "succeeded", latest, complete
|
||||
if failed is not None:
|
||||
return "failed", latest, failed
|
||||
time.sleep(2)
|
||||
return "timeout", latest, None
|
||||
|
||||
def job_logs(job_name):
|
||||
proc = kubectl(["-n", NAMESPACE, "logs", f"job/{job_name}", "--tail=4000"])
|
||||
return {
|
||||
"exitCode": proc.returncode,
|
||||
"stdout": proc.stdout.decode("utf-8", errors="replace"),
|
||||
"stderr": text(proc.stderr, 4000),
|
||||
}
|
||||
|
||||
def run_sentinel_probe():
|
||||
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {}
|
||||
accounts = payload.get("accounts") if isinstance(payload, dict) else None
|
||||
if not isinstance(accounts, list) or not accounts or not all(isinstance(item, str) and item for item in accounts):
|
||||
raise RuntimeError("sentinel-probe payload requires non-empty accounts")
|
||||
job_name, manifest = sentinel_probe_job_manifest(accounts)
|
||||
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"apply sentinel probe job failed: {text(proc.stderr, 2000)}")
|
||||
timeout_seconds = max(300, int(SENTINEL_CONFIG.get("probe", {}).get("timeoutSeconds") or 30) + 240)
|
||||
status, job, condition = wait_sentinel_probe_job(job_name, timeout_seconds)
|
||||
logs = job_logs(job_name)
|
||||
parsed = parse_embedded_json(logs.get("stdout") or "")
|
||||
results = parsed.get("results") if isinstance(parsed, dict) and isinstance(parsed.get("results"), list) else []
|
||||
state_summary = sentinel_state_summary()
|
||||
last_run = state_summary.get("lastRun") if isinstance(state_summary, dict) and isinstance(state_summary.get("lastRun"), dict) else {}
|
||||
if not results and isinstance(last_run.get("results"), list):
|
||||
results = last_run.get("results")
|
||||
requested = set(accounts)
|
||||
measured = {item.get("accountName") for item in results if isinstance(item, dict)}
|
||||
missing = sorted(name for name in requested if name not in measured)
|
||||
marker_ok = len(missing) == 0 and all(isinstance(item, dict) and item.get("accountName") in requested and item.get("markerMatched") is True for item in results if isinstance(item, dict) and item.get("accountName") in requested)
|
||||
if not results and isinstance(last_run, dict):
|
||||
selected = int(last_run.get("selected") or 0)
|
||||
ok_count = int(last_run.get("okCount") or 0)
|
||||
marker_mismatch_count = int(last_run.get("markerMismatchCount") or 0)
|
||||
transport_failure_count = int(last_run.get("transportFailureCount") or 0)
|
||||
if selected >= len(requested) and ok_count >= len(requested) and marker_mismatch_count == 0 and transport_failure_count == 0:
|
||||
missing = []
|
||||
marker_ok = True
|
||||
job_ok = status == "succeeded" and logs.get("exitCode") == 0
|
||||
return {
|
||||
"ok": job_ok and marker_ok,
|
||||
"jobExecutionOk": job_ok,
|
||||
"markerOk": marker_ok,
|
||||
"mode": "sentinel-probe",
|
||||
"namespace": NAMESPACE,
|
||||
"requestedAccounts": accounts,
|
||||
"missingAccounts": missing,
|
||||
"job": {
|
||||
"name": job_name,
|
||||
"status": status,
|
||||
"condition": condition,
|
||||
"timeoutSeconds": timeout_seconds,
|
||||
"applyStdoutTail": text(proc.stdout, 1200),
|
||||
"logsExitCode": logs.get("exitCode"),
|
||||
"logsStderrTail": logs.get("stderr"),
|
||||
},
|
||||
"probe": parsed,
|
||||
"summary": {
|
||||
"at": last_run.get("at"),
|
||||
"selected": last_run.get("selected"),
|
||||
"okCount": last_run.get("okCount"),
|
||||
"markerMismatchCount": last_run.get("markerMismatchCount"),
|
||||
"transportFailureCount": last_run.get("transportFailureCount"),
|
||||
"actionsTaken": last_run.get("actionsTaken"),
|
||||
"selection": last_run.get("selection"),
|
||||
},
|
||||
"results": results,
|
||||
"sentinelState": state_summary,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def run_cleanup_probes():
|
||||
admin_email, token, admin_compliance = login()
|
||||
cleanup = cleanup_probe_resources(token)
|
||||
return {
|
||||
"ok": cleanup.get("ok") is True,
|
||||
"mode": "cleanup-probes",
|
||||
"namespace": NAMESPACE,
|
||||
"serviceDns": SERVICE_DNS,
|
||||
"appPod": APP_POD,
|
||||
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
||||
"cleanup": cleanup,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
try:
|
||||
if MODE == "sync":
|
||||
result = run_sync()
|
||||
elif MODE == "trace":
|
||||
result = run_trace()
|
||||
elif MODE == "cleanup-probes":
|
||||
result = run_cleanup_probes()
|
||||
elif MODE == "sentinel-probe":
|
||||
result = run_sentinel_probe()
|
||||
else:
|
||||
result = run_validate()
|
||||
except Exception as exc:
|
||||
result = {
|
||||
"ok": False,
|
||||
"mode": MODE,
|
||||
"namespace": NAMESPACE,
|
||||
"serviceDns": SERVICE_DNS,
|
||||
"appPod": globals().get("APP_POD"),
|
||||
"error": str(exc),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if result.get("ok") else 1)
|
||||
PY
|
||||
`;
|
||||
@@ -0,0 +1,614 @@
|
||||
export const remotePythonSentinelScript = `def pool_api_key_secret_location():
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
return f"{HOST_DOCKER_ENV_PATH}.{POOL_API_KEY_SECRET_KEY}"
|
||||
return f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}"
|
||||
|
||||
def apply_sentinel_manifest(manifest):
|
||||
if not TARGET_SENTINEL_ENABLED:
|
||||
return {
|
||||
"ok": True,
|
||||
"action": "skipped-target-disabled",
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
if not isinstance(manifest, str) or not manifest.strip():
|
||||
return {
|
||||
"ok": False,
|
||||
"action": "missing-manifest",
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], manifest)
|
||||
if proc.returncode != 0:
|
||||
return {
|
||||
"ok": False,
|
||||
"action": "apply-failed",
|
||||
"stdoutTail": text(proc.stdout, 2000),
|
||||
"stderrTail": text(proc.stderr, 4000),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
status = sentinel_runtime_status()
|
||||
return {
|
||||
"ok": status.get("ok") is True,
|
||||
"action": "applied",
|
||||
"stdoutTail": text(proc.stdout, 2000),
|
||||
"runtime": status,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def safe_kube_json(args, label):
|
||||
proc = kubectl([*args, "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
return None, {"label": label, "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000)}
|
||||
try:
|
||||
return json.loads(proc.stdout.decode("utf-8")), None
|
||||
except Exception as exc:
|
||||
return None, {"label": label, "exitCode": proc.returncode, "error": str(exc), "stdoutTail": text(proc.stdout, 1000)}
|
||||
|
||||
def sentinel_runtime_status():
|
||||
if not TARGET_SENTINEL_ENABLED:
|
||||
return {
|
||||
"ok": True,
|
||||
"action": "skipped-target-disabled",
|
||||
"desired": {
|
||||
"monitorEnabled": SENTINEL_CONFIG.get("monitor", {}).get("enabled"),
|
||||
"actionsEnabled": SENTINEL_CONFIG.get("actions", {}).get("enabled"),
|
||||
},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
cfg = SENTINEL_CONFIG
|
||||
cronjob_name = cfg.get("cronJobName")
|
||||
secret_name = cfg.get("credentialsSecretName")
|
||||
configmap_name = cfg.get("configMapName")
|
||||
state_name = cfg.get("stateConfigMapName")
|
||||
cronjob, cronjob_error = safe_kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
|
||||
secret, secret_error = safe_kube_json(["-n", NAMESPACE, "get", "secret", secret_name], f"secret/{secret_name}")
|
||||
configmap, configmap_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", configmap_name], f"configmap/{configmap_name}")
|
||||
state_cm, state_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
|
||||
state = None
|
||||
if isinstance(state_cm, dict):
|
||||
raw_state = (state_cm.get("data") or {}).get("state.json")
|
||||
if isinstance(raw_state, str) and raw_state:
|
||||
try:
|
||||
state = json.loads(raw_state)
|
||||
except Exception as exc:
|
||||
state = {"parseError": str(exc)}
|
||||
accounts = (state.get("accounts") or {}) if isinstance(state, dict) else {}
|
||||
quarantined = []
|
||||
recent_accounts = []
|
||||
for name, account_state in accounts.items():
|
||||
if not isinstance(account_state, dict):
|
||||
continue
|
||||
quarantine = account_state.get("quarantine")
|
||||
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
||||
quarantined.append({
|
||||
"accountName": name,
|
||||
"until": quarantine.get("until"),
|
||||
"applied": quarantine.get("applied"),
|
||||
"reason": quarantine.get("reason"),
|
||||
"failureKind": quarantine.get("failureKind"),
|
||||
"errorDetails": quarantine.get("errorDetails"),
|
||||
"intervalMinutes": quarantine.get("intervalMinutes"),
|
||||
"sentinelProtect": quarantine.get("sentinelProtect"),
|
||||
})
|
||||
last_probe = account_state.get("lastProbe")
|
||||
if isinstance(last_probe, dict):
|
||||
recent_accounts.append({
|
||||
"accountName": name,
|
||||
"lastProbeAt": account_state.get("lastProbeAt"),
|
||||
"lastStatus": account_state.get("lastStatus"),
|
||||
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
||||
"ok": last_probe.get("ok"),
|
||||
"purpose": last_probe.get("purpose"),
|
||||
"httpStatus": last_probe.get("httpStatus"),
|
||||
"durationMs": last_probe.get("durationMs"),
|
||||
"markerMatched": last_probe.get("markerMatched"),
|
||||
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
|
||||
"outputHash": last_probe.get("outputHash"),
|
||||
"outputPreview": last_probe.get("outputPreview"),
|
||||
"responseBodyHash": last_probe.get("responseBodyHash"),
|
||||
"responseBodyPreview": last_probe.get("responseBodyPreview"),
|
||||
"error": last_probe.get("error"),
|
||||
"errorDetails": last_probe.get("errorDetails"),
|
||||
"usage": last_probe.get("usage"),
|
||||
"failureKind": last_probe.get("failureKind"),
|
||||
"requestShape": last_probe.get("requestShape"),
|
||||
"action": last_probe.get("action"),
|
||||
})
|
||||
recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
|
||||
last_run = state.get("lastRun") if isinstance(state, dict) else None
|
||||
cronjob_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
|
||||
secret_data = secret.get("data") if isinstance(secret, dict) else {}
|
||||
configmap_data = configmap.get("data") if isinstance(configmap, dict) else {}
|
||||
ok = cronjob is not None and secret is not None and configmap is not None
|
||||
return {
|
||||
"ok": ok,
|
||||
"desired": {
|
||||
"monitorEnabled": cfg.get("monitor", {}).get("enabled"),
|
||||
"actionsEnabled": cfg.get("actions", {}).get("enabled"),
|
||||
"schedule": cfg.get("schedule"),
|
||||
"cronJobName": cronjob_name,
|
||||
"configMapName": configmap_name,
|
||||
"credentialsSecretName": secret_name,
|
||||
"stateConfigMapName": state_name,
|
||||
},
|
||||
"cronJob": {
|
||||
"exists": cronjob is not None,
|
||||
"schedule": cronjob_spec.get("schedule") if isinstance(cronjob_spec, dict) else None,
|
||||
"suspend": cronjob_spec.get("suspend") if isinstance(cronjob_spec, dict) else None,
|
||||
"lastScheduleTime": (cronjob.get("status") or {}).get("lastScheduleTime") if isinstance(cronjob, dict) else None,
|
||||
"active": len((cronjob.get("status") or {}).get("active") or []) if isinstance(cronjob, dict) else None,
|
||||
"error": cronjob_error,
|
||||
},
|
||||
"secret": {
|
||||
"exists": secret is not None,
|
||||
"profileSecretPresent": isinstance(secret_data, dict) and "profiles.json" in secret_data,
|
||||
"valuesPrinted": False,
|
||||
"error": secret_error,
|
||||
},
|
||||
"configMap": {
|
||||
"exists": configmap is not None,
|
||||
"configPresent": isinstance(configmap_data, dict) and "config.json" in configmap_data,
|
||||
"runnerPresent": isinstance(configmap_data, dict) and "sentinel.py" in configmap_data,
|
||||
"error": configmap_error,
|
||||
},
|
||||
"state": {
|
||||
"exists": state_cm is not None,
|
||||
"accountCount": len(accounts),
|
||||
"quarantinedCount": len(quarantined),
|
||||
"quarantined": quarantined[-10:],
|
||||
"recentAccounts": recent_accounts[-12:],
|
||||
"lastRun": last_run,
|
||||
"error": state_error,
|
||||
},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def parse_epoch_z(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def sentinel_state_object():
|
||||
if not TARGET_SENTINEL_ENABLED:
|
||||
return None, None
|
||||
state_name = SENTINEL_CONFIG.get("stateConfigMapName")
|
||||
if not state_name:
|
||||
return None, None
|
||||
obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
|
||||
if not isinstance(obj, dict):
|
||||
return None, None
|
||||
raw_state = (obj.get("data") or {}).get("state.json")
|
||||
if not isinstance(raw_state, str) or not raw_state:
|
||||
return obj, None
|
||||
try:
|
||||
return obj, json.loads(raw_state)
|
||||
except Exception:
|
||||
return obj, None
|
||||
|
||||
def active_sentinel_quarantine_names():
|
||||
if not TARGET_SENTINEL_ENABLED:
|
||||
return set()
|
||||
_, state = sentinel_state_object()
|
||||
if not isinstance(state, dict):
|
||||
return set()
|
||||
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
||||
names = set()
|
||||
for name, account_state in accounts_state.items():
|
||||
if not isinstance(name, str) or not isinstance(account_state, dict):
|
||||
continue
|
||||
quarantine = account_state.get("quarantine")
|
||||
if isinstance(quarantine, dict) and quarantine.get("active") is True and quarantine.get("applied") is True:
|
||||
names.add(name)
|
||||
return names
|
||||
|
||||
def default_sentinel_state():
|
||||
return {"version": 1, "accounts": {}, "ledger": {}, "history": []}
|
||||
|
||||
def clamp_sentinel_freezes_for_config(state, now):
|
||||
freeze_config = SENTINEL_CONFIG.get("freeze") if isinstance(SENTINEL_CONFIG.get("freeze"), dict) else {}
|
||||
try:
|
||||
max_interval = int(freeze_config.get("maxTtlMinutes") or 10)
|
||||
except Exception:
|
||||
max_interval = 10
|
||||
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
||||
now_epoch = time.time()
|
||||
items = []
|
||||
for name, account_state in accounts_state.items():
|
||||
if not isinstance(name, str) or not isinstance(account_state, dict):
|
||||
continue
|
||||
quarantine = account_state.get("quarantine")
|
||||
if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
|
||||
continue
|
||||
try:
|
||||
interval = int(quarantine.get("intervalMinutes") or 0)
|
||||
except Exception:
|
||||
interval = 0
|
||||
until_epoch = parse_epoch_z(quarantine.get("until"))
|
||||
old_until = quarantine.get("until")
|
||||
if interval <= max_interval and (until_epoch is None or until_epoch <= now_epoch + max_interval * 60):
|
||||
continue
|
||||
quarantine["previousIntervalMinutes"] = interval
|
||||
quarantine["intervalMinutes"] = max_interval
|
||||
quarantine["until"] = now
|
||||
quarantine["clampedAt"] = now
|
||||
quarantine["clampedBy"] = "sync-freeze-max-ttl"
|
||||
account_state["nextProbeAfter"] = now
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"previousIntervalMinutes": interval,
|
||||
"maxIntervalMinutes": max_interval,
|
||||
"previousUntil": old_until,
|
||||
"nextProbeAfter": now,
|
||||
})
|
||||
return items
|
||||
|
||||
def parse_iso_epoch(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def profile_success_max_interval(profile):
|
||||
cadence = SENTINEL_CONFIG.get("cadence") if isinstance(SENTINEL_CONFIG.get("cadence"), dict) else {}
|
||||
legacy = cadence.get("successMaxIntervalMinutes")
|
||||
if legacy is None:
|
||||
legacy = cadence.get("trustedSuccessMaxIntervalMinutes") or cadence.get("untrustedSuccessMaxIntervalMinutes") or 1
|
||||
if profile.get("trustUpstream") is True:
|
||||
value = cadence.get("trustedSuccessMaxIntervalMinutes") or legacy
|
||||
else:
|
||||
value = cadence.get("untrustedSuccessMaxIntervalMinutes") or legacy
|
||||
try:
|
||||
return int(value)
|
||||
except Exception:
|
||||
return int(legacy)
|
||||
|
||||
def clamp_sentinel_success_cadence_for_config(state, profiles, now):
|
||||
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
||||
profile_map = {item.get("accountName"): item for item in profiles if isinstance(item, dict) and isinstance(item.get("accountName"), str)}
|
||||
now_epoch = time.time()
|
||||
items = []
|
||||
for name, profile in profile_map.items():
|
||||
account_state = accounts_state.get(name)
|
||||
if not isinstance(account_state, dict):
|
||||
continue
|
||||
quarantine = account_state.get("quarantine")
|
||||
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
||||
account_state["trustUpstream"] = profile.get("trustUpstream") is True
|
||||
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
||||
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile)
|
||||
continue
|
||||
try:
|
||||
interval = int(account_state.get("successIntervalMinutes") or 0)
|
||||
except Exception:
|
||||
interval = 0
|
||||
next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter"))
|
||||
max_interval = profile_success_max_interval(profile)
|
||||
account_state["trustUpstream"] = profile.get("trustUpstream") is True
|
||||
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
||||
account_state["successMaxIntervalMinutes"] = max_interval
|
||||
if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60):
|
||||
continue
|
||||
old_next = account_state.get("nextProbeAfter")
|
||||
account_state["previousSuccessIntervalMinutes"] = interval
|
||||
account_state["successIntervalMinutes"] = min(interval, max_interval) if interval > 0 else interval
|
||||
account_state["nextProbeAfter"] = now
|
||||
account_state["cadenceClampedAt"] = now
|
||||
account_state["cadenceClampedBy"] = "sync-success-max-interval"
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"trustUpstream": profile.get("trustUpstream") is True,
|
||||
"previousSuccessIntervalMinutes": interval,
|
||||
"maxIntervalMinutes": max_interval,
|
||||
"previousNextProbeAfter": old_next,
|
||||
"nextProbeAfter": now,
|
||||
})
|
||||
return items
|
||||
|
||||
def update_sentinel_state_configmap(obj, state):
|
||||
state_name = SENTINEL_CONFIG.get("stateConfigMapName")
|
||||
if not state_name:
|
||||
return {"ok": False, "reason": "state-configmap-missing"}
|
||||
state_json = json.dumps(state, ensure_ascii=False, indent=2)
|
||||
manifest = {
|
||||
"apiVersion": "v1",
|
||||
"kind": "ConfigMap",
|
||||
"metadata": {
|
||||
"name": state_name,
|
||||
"namespace": NAMESPACE,
|
||||
"labels": {
|
||||
"app.kubernetes.io/name": SERVICE_NAME,
|
||||
"app.kubernetes.io/component": "account-sentinel",
|
||||
"app.kubernetes.io/managed-by": "unidesk-platform-infra",
|
||||
},
|
||||
},
|
||||
"data": {"state.json": state_json},
|
||||
}
|
||||
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
|
||||
action = "applied" if isinstance(obj, dict) else "created"
|
||||
if proc.returncode != 0:
|
||||
return {"ok": False, "reason": f"{action}-failed", "stderrTail": text(proc.stderr, 2000)}
|
||||
return {"ok": True, "action": action}
|
||||
|
||||
def ensure_sentinel_state_for_sync(account_results, pending_only=False):
|
||||
if not sentinel_quality_gate_enabled():
|
||||
return {"ok": True, "skipped": True, "reason": "sentinel-quality-gate-disabled", "changedCount": 0, "items": [], "valuesPrinted": False}
|
||||
state_obj, state = sentinel_state_object()
|
||||
if not isinstance(state, dict):
|
||||
state = default_sentinel_state()
|
||||
state.setdefault("version", 1)
|
||||
accounts_state = state.setdefault("accounts", {})
|
||||
if not isinstance(accounts_state, dict):
|
||||
accounts_state = {}
|
||||
state["accounts"] = accounts_state
|
||||
now = utc_iso()
|
||||
items = []
|
||||
clamped_items = [] if pending_only else clamp_sentinel_freezes_for_config(state, now)
|
||||
cadence_clamped_items = [] if pending_only else clamp_sentinel_success_cadence_for_config(state, [item.get("profileConfig") for item in account_results if isinstance(item.get("profileConfig"), dict)], now)
|
||||
changed_count = 0
|
||||
fingerprint_only_count = 0
|
||||
for item in account_results:
|
||||
name = item.get("accountName")
|
||||
if not isinstance(name, str) or not name:
|
||||
continue
|
||||
account_state = accounts_state.setdefault(name, {})
|
||||
if not isinstance(account_state, dict):
|
||||
account_state = {}
|
||||
accounts_state[name] = account_state
|
||||
fingerprint_value = item.get("sentinelProbeConfigFingerprint")
|
||||
if isinstance(fingerprint_value, str) and fingerprint_value:
|
||||
account_state["probeConfigFingerprint"] = fingerprint_value
|
||||
if item.get("sentinelProbeRequired") is not True:
|
||||
fingerprint_only_count += 1
|
||||
continue
|
||||
changed_count += 1
|
||||
reasons = item.get("sentinelChangeReasons") if isinstance(item.get("sentinelChangeReasons"), list) else []
|
||||
quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else None
|
||||
if not (isinstance(quarantine, dict) and quarantine.get("active") is True):
|
||||
account_state["quarantine"] = {
|
||||
"active": False,
|
||||
"reason": "yaml-account-change-pending-sentinel-probe",
|
||||
"lastPendingAt": now,
|
||||
"changeReasons": reasons,
|
||||
}
|
||||
account_state["nextProbeAfter"] = now
|
||||
account_state["successStreak"] = 0
|
||||
account_state["successIntervalMinutes"] = 0
|
||||
profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {}
|
||||
account_state["trustUpstream"] = profile_config.get("trustUpstream") is True
|
||||
account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False}
|
||||
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config)
|
||||
account_state["lastStatus"] = "pending-sentinel-quality-gate"
|
||||
account_state["qualityGate"] = {
|
||||
"pending": True,
|
||||
"reason": "yaml-account-change",
|
||||
"changeReasons": reasons,
|
||||
"markedAt": now,
|
||||
"pendingOnly": pending_only,
|
||||
"defaultFrozen": False,
|
||||
}
|
||||
items.append({"accountName": name, "changeReasons": reasons, "nextProbeAfter": now, "defaultFrozen": False, "defaultSchedulable": True, "pendingOnly": pending_only})
|
||||
if changed_count <= 0 and len(clamped_items) <= 0 and len(cadence_clamped_items) <= 0:
|
||||
return {"ok": True, "skipped": False, "reason": "no-new-or-changed-accounts", "changedCount": 0, "fingerprintOnlyCount": fingerprint_only_count, "clampedCount": 0, "cadenceClampedCount": 0, "items": [], "valuesPrinted": False}
|
||||
update = update_sentinel_state_configmap(state_obj, state)
|
||||
if pending_only and changed_count > 0:
|
||||
reason = "new-or-changed-accounts-pending-probe-prepared-default-schedulable"
|
||||
elif changed_count > 0 and (len(clamped_items) > 0 or len(cadence_clamped_items) > 0):
|
||||
reason = "new-or-changed-accounts-default-schedulable-and-sentinel-cadence-clamped"
|
||||
elif changed_count > 0:
|
||||
reason = "new-or-changed-accounts-default-schedulable"
|
||||
elif len(cadence_clamped_items) > 0:
|
||||
reason = "success-cadence-clamped-to-current-config"
|
||||
else:
|
||||
reason = "freeze-backoff-clamped-to-current-config"
|
||||
return {
|
||||
"ok": update.get("ok") is True,
|
||||
"skipped": False,
|
||||
"reason": reason,
|
||||
"changedCount": changed_count,
|
||||
"fingerprintOnlyCount": fingerprint_only_count,
|
||||
"clampedCount": len(clamped_items),
|
||||
"cadenceClampedCount": len(cadence_clamped_items),
|
||||
"pendingOnly": pending_only,
|
||||
"items": items,
|
||||
"clampedItems": clamped_items,
|
||||
"cadenceClampedItems": cadence_clamped_items,
|
||||
"update": update,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def sentinel_state_summary():
|
||||
_, state = sentinel_state_object()
|
||||
if not isinstance(state, dict):
|
||||
return {"exists": False, "valuesPrinted": False}
|
||||
accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
||||
quarantined = []
|
||||
recent_accounts = []
|
||||
for name, account_state in accounts.items():
|
||||
if not isinstance(account_state, dict):
|
||||
continue
|
||||
quarantine = account_state.get("quarantine")
|
||||
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
||||
quarantined.append({
|
||||
"accountName": name,
|
||||
"until": quarantine.get("until"),
|
||||
"applied": quarantine.get("applied"),
|
||||
"reason": quarantine.get("reason"),
|
||||
"failureKind": quarantine.get("failureKind"),
|
||||
"errorDetails": quarantine.get("errorDetails"),
|
||||
"intervalMinutes": quarantine.get("intervalMinutes"),
|
||||
"sentinelProtect": quarantine.get("sentinelProtect"),
|
||||
})
|
||||
last_probe = account_state.get("lastProbe")
|
||||
if isinstance(last_probe, dict):
|
||||
recent_accounts.append({
|
||||
"accountName": name,
|
||||
"lastProbeAt": account_state.get("lastProbeAt"),
|
||||
"lastStatus": account_state.get("lastStatus"),
|
||||
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
||||
"ok": last_probe.get("ok"),
|
||||
"purpose": last_probe.get("purpose"),
|
||||
"httpStatus": last_probe.get("httpStatus"),
|
||||
"durationMs": last_probe.get("durationMs"),
|
||||
"markerMatched": last_probe.get("markerMatched"),
|
||||
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
|
||||
"usage": last_probe.get("usage"),
|
||||
"responseBodyHash": last_probe.get("responseBodyHash"),
|
||||
"responseBodyPreview": last_probe.get("responseBodyPreview"),
|
||||
"error": last_probe.get("error"),
|
||||
"errorDetails": last_probe.get("errorDetails"),
|
||||
"failureKind": last_probe.get("failureKind"),
|
||||
"requestShape": last_probe.get("requestShape"),
|
||||
"action": last_probe.get("action"),
|
||||
})
|
||||
recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
|
||||
return {
|
||||
"exists": True,
|
||||
"accountCount": len(accounts),
|
||||
"quarantinedCount": len(quarantined),
|
||||
"quarantined": quarantined[-10:],
|
||||
"recentAccounts": recent_accounts[-12:],
|
||||
"lastRun": state.get("lastRun"),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def reassert_sentinel_freezes_after_sync(token):
|
||||
if not TARGET_SENTINEL_ENABLED:
|
||||
return {"ok": True, "skipped": True, "reason": "target-disabled", "items": [], "valuesPrinted": False}
|
||||
if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True:
|
||||
return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False}
|
||||
_, state = sentinel_state_object()
|
||||
if not isinstance(state, dict):
|
||||
return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False}
|
||||
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
||||
accounts = list_accounts(token)
|
||||
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
||||
items = []
|
||||
for name, account_state in accounts_state.items():
|
||||
if not isinstance(account_state, dict):
|
||||
continue
|
||||
quarantine = account_state.get("quarantine")
|
||||
if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
|
||||
continue
|
||||
account = by_name.get(name)
|
||||
if not account or account.get("id") is None:
|
||||
items.append({"accountName": name, "ok": False, "reason": "account-not-found"})
|
||||
continue
|
||||
try:
|
||||
ensure_success(
|
||||
curl_api("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", bearer=token, payload={"schedulable": False}),
|
||||
f"reassert sentinel freeze for {name}",
|
||||
)
|
||||
items.append({"accountName": name, "accountId": account.get("id"), "ok": True, "until": quarantine.get("until")})
|
||||
except Exception as exc:
|
||||
items.append({"accountName": name, "accountId": account.get("id"), "ok": False, "error": str(exc)})
|
||||
return {
|
||||
"ok": all(item.get("ok") is True for item in items),
|
||||
"skipped": False,
|
||||
"items": items,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def list_user_keys(token):
|
||||
data = ensure_success(curl_api("GET", "/api/v1/keys?page=1&page_size=200", bearer=token), "list user keys")
|
||||
return extract_items(data)
|
||||
|
||||
def ensure_sub2api_api_key(token, api_key, group_id):
|
||||
keys = list_user_keys(token)
|
||||
existing = next((item for item in keys if item.get("key") == api_key), None)
|
||||
action = "kept-existing"
|
||||
if existing is None:
|
||||
existing = next((item for item in keys if item.get("name") == POOL_API_KEY_NAME), None)
|
||||
if existing is None or existing.get("key") != api_key:
|
||||
payload = {
|
||||
"name": POOL_API_KEY_NAME,
|
||||
"group_id": group_id,
|
||||
"custom_key": api_key,
|
||||
"quota": 0,
|
||||
"rate_limit_5h": 0,
|
||||
"rate_limit_1d": 0,
|
||||
"rate_limit_7d": 0,
|
||||
}
|
||||
created = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload=payload), "create pool API key")
|
||||
existing = created if isinstance(created, dict) else existing
|
||||
action = "created"
|
||||
elif existing.get("id") is not None and (existing.get("group_id") != group_id or existing.get("name") != POOL_API_KEY_NAME):
|
||||
updated = ensure_success(curl_api("PUT", f"/api/v1/keys/{existing['id']}", bearer=token, payload={"name": POOL_API_KEY_NAME, "group_id": group_id}), "update pool API key group")
|
||||
existing = updated if isinstance(updated, dict) else existing
|
||||
action = "updated-name-group"
|
||||
return {
|
||||
"action": action,
|
||||
"id": existing.get("id") if isinstance(existing, dict) else None,
|
||||
"name": existing.get("name") if isinstance(existing, dict) else POOL_API_KEY_NAME,
|
||||
"groupId": existing.get("group_id") if isinstance(existing, dict) else group_id,
|
||||
"userId": existing.get("user_id") if isinstance(existing, dict) else None,
|
||||
}
|
||||
|
||||
def get_admin_user(token, user_id):
|
||||
data = ensure_success(curl_api("GET", f"/api/v1/admin/users/{user_id}", bearer=token), "get API key owner")
|
||||
if not isinstance(data, dict):
|
||||
raise RuntimeError("API key owner response is not an object")
|
||||
return data
|
||||
|
||||
def ensure_pool_owner_balance(token, user_id):
|
||||
user = get_admin_user(token, user_id)
|
||||
current = float(user.get("balance") or 0)
|
||||
if current >= MIN_OWNER_BALANCE_USD:
|
||||
return {
|
||||
"action": "kept-existing",
|
||||
"userId": user_id,
|
||||
"balanceBefore": current,
|
||||
"balanceAfter": current,
|
||||
"minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
|
||||
}
|
||||
updated = ensure_success(curl_api("POST", f"/api/v1/admin/users/{user_id}/balance", bearer=token, payload={
|
||||
"balance": MIN_OWNER_BALANCE_USD,
|
||||
"operation": "set",
|
||||
"notes": "UniDesk Sub2API Codex pool internal API key bootstrap balance.",
|
||||
}), "set API key owner balance")
|
||||
after = float(updated.get("balance") or MIN_OWNER_BALANCE_USD) if isinstance(updated, dict) else MIN_OWNER_BALANCE_USD
|
||||
return {
|
||||
"action": "set",
|
||||
"userId": user_id,
|
||||
"balanceBefore": current,
|
||||
"balanceAfter": after,
|
||||
"minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
|
||||
}
|
||||
|
||||
def ensure_pool_owner_concurrency(token, user_id):
|
||||
user = get_admin_user(token, user_id)
|
||||
try:
|
||||
current = int(user.get("concurrency") or 0)
|
||||
except Exception:
|
||||
current = 0
|
||||
if current >= MIN_OWNER_CONCURRENCY:
|
||||
return {
|
||||
"ok": True,
|
||||
"action": "kept-existing",
|
||||
"userId": user_id,
|
||||
"concurrencyBefore": current,
|
||||
"concurrencyAfter": current,
|
||||
"minimumConcurrency": MIN_OWNER_CONCURRENCY,
|
||||
"minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
||||
}
|
||||
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/users/{user_id}", bearer=token, payload={
|
||||
"concurrency": MIN_OWNER_CONCURRENCY,
|
||||
}), "set API key owner concurrency")
|
||||
try:
|
||||
after = int(updated.get("concurrency") or MIN_OWNER_CONCURRENCY) if isinstance(updated, dict) else MIN_OWNER_CONCURRENCY
|
||||
except Exception:
|
||||
after = MIN_OWNER_CONCURRENCY
|
||||
return {
|
||||
"ok": after >= MIN_OWNER_CONCURRENCY,
|
||||
"action": "set",
|
||||
"userId": user_id,
|
||||
"concurrencyBefore": current,
|
||||
"concurrencyAfter": after,
|
||||
"minimumConcurrency": MIN_OWNER_CONCURRENCY,
|
||||
"minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
||||
}
|
||||
|
||||
`;
|
||||
@@ -0,0 +1,142 @@
|
||||
export const remotePythonSyncValidateScript = `def run_sync():
|
||||
global MANUAL_ACCOUNT_PROTECTIONS
|
||||
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8"))
|
||||
manual_accounts_payload = payload.get("manualAccounts") if isinstance(payload.get("manualAccounts"), dict) else {}
|
||||
resolved_manual_protections = manual_accounts_payload.get("protected")
|
||||
if not isinstance(resolved_manual_protections, list):
|
||||
raise RuntimeError("sync payload has no manualAccounts.protected binding plan")
|
||||
MANUAL_ACCOUNT_PROTECTIONS = resolved_manual_protections
|
||||
profiles = payload.get("profiles") or []
|
||||
prune_removed = bool(payload.get("pruneRemoved"))
|
||||
sentinel_payload = payload.get("sentinel") if isinstance(payload.get("sentinel"), dict) else {}
|
||||
if not profiles and not MANUAL_ACCOUNT_PROTECTIONS:
|
||||
raise RuntimeError("sync payload has no profiles and no manualAccounts.protected binding plan")
|
||||
admin_email, token, admin_compliance = login()
|
||||
group, group_action = ensure_group(token)
|
||||
group_id = group.get("id") if isinstance(group, dict) else None
|
||||
if group_id is None:
|
||||
raise RuntimeError("pool group id missing after ensure")
|
||||
existing_accounts = list_accounts(token)
|
||||
planned_account_results = planned_sentinel_account_results(profiles, existing_accounts)
|
||||
sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True)
|
||||
if sentinel_quality_prepare.get("ok") is not True:
|
||||
raise RuntimeError("prepare sentinel pending probe failed: " + json.dumps(sentinel_quality_prepare, ensure_ascii=False))
|
||||
protected_frozen_names = active_sentinel_quarantine_names()
|
||||
account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed, protected_frozen_names, existing_accounts)
|
||||
manual_account_proxy_bindings = ensure_manual_account_proxy_bindings(token)
|
||||
manual_account_group_bindings = ensure_manual_account_group_bindings(token, group_id)
|
||||
manual_account_protections = manual_account_protection_status(token, group_id)
|
||||
capacity_status = account_capacity_status(token)
|
||||
load_factor_status = account_load_factor_status(token)
|
||||
ws_v2_status = account_ws_v2_status(token)
|
||||
temp_unschedulable_status = account_temp_unschedulable_status(token)
|
||||
api_key, secret_action, secret_apply_stdout = ensure_api_key_secret(group_id, token)
|
||||
api_key_result = ensure_sub2api_api_key(token, api_key, group_id)
|
||||
owner_balance = ensure_pool_owner_balance(token, api_key_result["userId"])
|
||||
owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"])
|
||||
gateway = validate_gateway(api_key)
|
||||
responses_smoke = validate_gateway_responses(api_key)
|
||||
compact_evidence = recent_compact_gateway_evidence()
|
||||
responses_evidence = recent_responses_gateway_evidence()
|
||||
runtime_capabilities = validate_runtime_capabilities(token)
|
||||
sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest"))
|
||||
sentinel_quality = ensure_sentinel_state_for_sync(account_results)
|
||||
sentinel_reassert = reassert_sentinel_freezes_after_sync(token)
|
||||
return {
|
||||
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_proxy_bindings.get("ok") is True and manual_account_group_bindings.get("ok") is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and sentinel_quality_prepare.get("ok") is True and sentinel_quality.get("ok") is True and sentinel_reassert.get("ok") is True and runtime_capabilities.get("ok") is True,
|
||||
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
|
||||
"mode": "sync",
|
||||
"namespace": NAMESPACE,
|
||||
"serviceDns": SERVICE_DNS,
|
||||
"appPod": APP_POD,
|
||||
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
||||
"pool": {"name": POOL_GROUP_NAME, "id": group_id, "action": group_action, "platform": group.get("platform") if isinstance(group, dict) else "openai"},
|
||||
"accounts": {
|
||||
"desired": len(profiles),
|
||||
"created": sum(1 for item in account_results if item["action"] == "created"),
|
||||
"updated": sum(1 for item in account_results if item["action"] == "updated"),
|
||||
"pruned": len(pruned_account_results),
|
||||
"pruneMode": "explicit" if prune_removed else "disabled-by-default",
|
||||
"items": account_results,
|
||||
"prunedItems": pruned_account_results,
|
||||
"processControl": {"schedulableRestore": "sentinel marker probe only; sync does not restore schedulable for existing accounts", "durableConfig": False},
|
||||
"valuesPrinted": False,
|
||||
},
|
||||
"manualAccounts": {**manual_account_protections, "proxySync": manual_account_proxy_bindings, "groupSync": manual_account_group_bindings},
|
||||
"capacity": capacity_status,
|
||||
"loadFactor": load_factor_status,
|
||||
"webSocketsV2": ws_v2_status,
|
||||
"tempUnschedulable": temp_unschedulable_status,
|
||||
"apiKey": {
|
||||
"ok": True,
|
||||
"name": POOL_API_KEY_NAME,
|
||||
"secret": pool_api_key_secret_location(),
|
||||
"secretAction": secret_action,
|
||||
"secretApply": secret_apply_stdout,
|
||||
"sub2apiAction": api_key_result["action"],
|
||||
"sub2apiId": api_key_result["id"],
|
||||
"groupId": api_key_result["groupId"],
|
||||
"userId": api_key_result["userId"],
|
||||
"apiKeyFingerprint": secret_fingerprint(api_key),
|
||||
"valuesPrinted": False,
|
||||
},
|
||||
"ownerBalance": owner_balance,
|
||||
"ownerConcurrency": owner_concurrency,
|
||||
"sentinel": {**sentinel, "qualityGatePrepare": sentinel_quality_prepare, "qualityGate": sentinel_quality, "freezeReassert": sentinel_reassert},
|
||||
"runtimeCapabilities": runtime_capabilities,
|
||||
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
|
||||
}
|
||||
|
||||
def run_validate():
|
||||
admin_email, token, admin_compliance = login()
|
||||
api_key, key_item, api_key_source = resolve_pool_api_key_for_validate(token)
|
||||
api_key_ok = isinstance(key_item, dict) and key_item.get("name") == POOL_API_KEY_NAME
|
||||
owner_balance = None
|
||||
owner_concurrency = None
|
||||
if key_item is not None and key_item.get("user_id") is not None:
|
||||
owner_balance = ensure_pool_owner_balance(token, key_item["user_id"])
|
||||
owner_concurrency = ensure_pool_owner_concurrency(token, key_item["user_id"])
|
||||
capacity_status = account_capacity_status(token)
|
||||
load_factor_status = account_load_factor_status(token)
|
||||
ws_v2_status = account_ws_v2_status(token)
|
||||
temp_unschedulable_status = account_temp_unschedulable_status(token)
|
||||
pool_group_id = key_item.get("group_id") if isinstance(key_item, dict) else None
|
||||
manual_account_protections = manual_account_protection_status(token, pool_group_id)
|
||||
gateway = validate_gateway(api_key)
|
||||
responses_smoke = validate_gateway_responses(api_key)
|
||||
compact_evidence = recent_compact_gateway_evidence()
|
||||
responses_evidence = recent_responses_gateway_evidence()
|
||||
runtime_capabilities = validate_runtime_capabilities(token)
|
||||
sentinel = sentinel_runtime_status()
|
||||
return {
|
||||
"ok": api_key_ok is True and gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and runtime_capabilities.get("ok") is True,
|
||||
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
|
||||
"mode": "validate",
|
||||
"namespace": NAMESPACE,
|
||||
"serviceDns": SERVICE_DNS,
|
||||
"appPod": APP_POD,
|
||||
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
||||
"apiKey": {
|
||||
"ok": api_key_ok,
|
||||
"name": POOL_API_KEY_NAME,
|
||||
"secret": pool_api_key_secret_location(),
|
||||
"source": api_key_source,
|
||||
"sub2apiId": key_item.get("id") if isinstance(key_item, dict) else None,
|
||||
"userId": key_item.get("user_id") if isinstance(key_item, dict) else None,
|
||||
"groupId": key_item.get("group_id") if isinstance(key_item, dict) else None,
|
||||
"apiKeyFingerprint": secret_fingerprint(api_key),
|
||||
"valuesPrinted": False,
|
||||
},
|
||||
"ownerBalance": owner_balance,
|
||||
"ownerConcurrency": owner_concurrency,
|
||||
"capacity": capacity_status,
|
||||
"loadFactor": load_factor_status,
|
||||
"webSocketsV2": ws_v2_status,
|
||||
"tempUnschedulable": temp_unschedulable_status,
|
||||
"manualAccounts": manual_account_protections,
|
||||
"sentinel": sentinel,
|
||||
"runtimeCapabilities": runtime_capabilities,
|
||||
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
|
||||
}
|
||||
|
||||
`;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,345 @@
|
||||
export const remotePythonTraceScript = `def parse_log_line(line):
|
||||
json_start = line.find("{")
|
||||
if json_start < 0:
|
||||
return None
|
||||
prefix = line[:json_start].rstrip()
|
||||
try:
|
||||
item = json.loads(line[json_start:])
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
at = None
|
||||
parts = prefix.split()
|
||||
if parts:
|
||||
at = parts[0]
|
||||
message = ""
|
||||
if len(parts) >= 4:
|
||||
message = " ".join(parts[3:])
|
||||
elif len(parts) >= 3:
|
||||
message = parts[2]
|
||||
elif len(parts) >= 1:
|
||||
message = parts[-1]
|
||||
item["_at"] = at
|
||||
item["_message"] = message
|
||||
item["_line"] = line
|
||||
return item
|
||||
|
||||
def log_time_epoch(item):
|
||||
at = item.get("_at") if isinstance(item, dict) else None
|
||||
if not isinstance(at, str) or not at:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(at, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
|
||||
except Exception:
|
||||
try:
|
||||
return datetime.fromisoformat(at.replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def event_base(item, account_names_by_id):
|
||||
account_id = item.get("account_id")
|
||||
if isinstance(account_id, str) and account_id.isdigit():
|
||||
account_id = int(account_id)
|
||||
account_name = account_names_by_id.get(account_id)
|
||||
return {
|
||||
"at": item.get("_at"),
|
||||
"message": item.get("_message"),
|
||||
"requestId": item.get("request_id"),
|
||||
"clientRequestId": item.get("client_request_id"),
|
||||
"path": item.get("path"),
|
||||
"method": item.get("method"),
|
||||
"model": item.get("model"),
|
||||
"accountId": account_id,
|
||||
"accountName": account_name,
|
||||
"statusCode": item.get("status_code"),
|
||||
"upstreamStatus": item.get("upstream_status"),
|
||||
"latencyMs": item.get("latency_ms"),
|
||||
}
|
||||
|
||||
def classify_trace_event(item, account_names_by_id):
|
||||
message = str(item.get("_message") or "")
|
||||
event = event_base(item, account_names_by_id)
|
||||
if "content_moderation.gateway_check_start" in message:
|
||||
event.update({
|
||||
"type": "request-start",
|
||||
"stream": item.get("stream"),
|
||||
"bodyBytes": item.get("body_bytes"),
|
||||
"groupId": item.get("group_id"),
|
||||
"groupName": item.get("group_name"),
|
||||
"apiKeyName": item.get("api_key_name"),
|
||||
})
|
||||
elif "content_moderation.gateway_check_done" in message:
|
||||
event.update({
|
||||
"type": "gateway-check",
|
||||
"allowed": item.get("allowed"),
|
||||
"blocked": item.get("blocked"),
|
||||
"action": item.get("action"),
|
||||
})
|
||||
elif "openai.upstream_failover_switching" in message:
|
||||
event.update({
|
||||
"type": "failover",
|
||||
"switchCount": item.get("switch_count"),
|
||||
"maxSwitches": item.get("max_switches"),
|
||||
})
|
||||
elif "openai.account_select_failed" in message:
|
||||
event.update({
|
||||
"type": "select-failed",
|
||||
"error": item.get("error"),
|
||||
"excludedAccountCount": item.get("excluded_account_count"),
|
||||
})
|
||||
elif "account_upstream_error" in message:
|
||||
event.update({
|
||||
"type": "upstream-error",
|
||||
"error": item.get("error"),
|
||||
})
|
||||
elif "account_temp_unschedulable" in message:
|
||||
event.update({
|
||||
"type": "temp-unschedulable",
|
||||
"until": item.get("until") or item.get("temp_unschedulable_until"),
|
||||
"ruleIndex": item.get("rule_index"),
|
||||
"matchedKeyword": item.get("matched_keyword"),
|
||||
"reason": item.get("reason") or item.get("error"),
|
||||
})
|
||||
elif "http request completed" in message:
|
||||
event.update({
|
||||
"type": "final",
|
||||
"clientIp": item.get("client_ip"),
|
||||
"protocol": item.get("protocol"),
|
||||
"platform": item.get("platform"),
|
||||
"completedAt": item.get("completed_at"),
|
||||
})
|
||||
elif "admin account schedulable updated" in message or "account schedulable updated" in message or "/schedulable" in str(item.get("path") or ""):
|
||||
event.update({
|
||||
"type": "admin-schedulable",
|
||||
"schedulable": item.get("schedulable"),
|
||||
})
|
||||
else:
|
||||
event.update({"type": "other"})
|
||||
return event
|
||||
|
||||
def with_trace_phase(event, first_epoch, last_epoch):
|
||||
epoch = None
|
||||
at = event.get("at") if isinstance(event, dict) else None
|
||||
if isinstance(at, str) and at:
|
||||
epoch = log_time_epoch({"_at": at})
|
||||
if epoch is None or first_epoch is None:
|
||||
phase = "unknown"
|
||||
elif epoch < first_epoch:
|
||||
phase = "before-request"
|
||||
elif last_epoch is not None and epoch > last_epoch:
|
||||
phase = "after-request"
|
||||
else:
|
||||
phase = "during-request"
|
||||
event["phase"] = phase
|
||||
return event
|
||||
|
||||
def account_snapshot_from_runtime(token):
|
||||
try:
|
||||
accounts = list_accounts(token)
|
||||
except Exception as exc:
|
||||
return [], {"error": str(exc)}
|
||||
rows = []
|
||||
for item in accounts:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
rows.append({
|
||||
"accountId": item.get("id"),
|
||||
"accountName": item.get("name"),
|
||||
"schedulable": item.get("schedulable"),
|
||||
"status": item.get("status"),
|
||||
"concurrency": item.get("concurrency"),
|
||||
"priority": item.get("priority"),
|
||||
"tempUnschedulableUntil": item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil"),
|
||||
"tempUnschedulableSet": (item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil")) is not None or bool(item.get("temp_unschedulable_reason") or item.get("tempUnschedulableReason")),
|
||||
})
|
||||
rows.sort(key=lambda row: (str(row.get("accountName") or ""), int(row.get("accountId") or 0)))
|
||||
return rows, None
|
||||
|
||||
def trace_reason(events, final_event):
|
||||
failovers = [item for item in events if item.get("type") == "failover"]
|
||||
select_failures = [item for item in events if item.get("type") == "select-failed"]
|
||||
upstream_errors = [item for item in events if item.get("type") == "upstream-error"]
|
||||
if failover_budget_exhausted(failovers, final_event):
|
||||
return "failover-budget-exhausted"
|
||||
if failovers and select_failures:
|
||||
return "failover-attempted-no-candidate"
|
||||
if failovers:
|
||||
return "failover-attempted"
|
||||
if select_failures:
|
||||
return "account-select-failed"
|
||||
if upstream_errors:
|
||||
return "upstream-error"
|
||||
if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") >= 400:
|
||||
return "final-http-error"
|
||||
if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int):
|
||||
return "completed"
|
||||
return "unknown"
|
||||
|
||||
def failover_budget_exhausted(failovers, final_event):
|
||||
if not failovers or not isinstance(final_event, dict):
|
||||
return False
|
||||
last = failovers[-1]
|
||||
switch_count = last.get("switchCount")
|
||||
max_switches = last.get("maxSwitches")
|
||||
final_status = final_event.get("statusCode")
|
||||
return (
|
||||
isinstance(switch_count, int)
|
||||
and isinstance(max_switches, int)
|
||||
and max_switches > 0
|
||||
and switch_count >= max_switches
|
||||
and isinstance(final_status, int)
|
||||
and final_status >= 500
|
||||
)
|
||||
|
||||
def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot):
|
||||
if not failover_budget_exhausted(failovers, final_event):
|
||||
return []
|
||||
tried = set()
|
||||
for item in failovers:
|
||||
account_id = item.get("accountId")
|
||||
if isinstance(account_id, int):
|
||||
tried.add(account_id)
|
||||
final_account = final_event.get("accountId") if isinstance(final_event, dict) else None
|
||||
if isinstance(final_account, int):
|
||||
tried.add(final_account)
|
||||
result = []
|
||||
for item in account_snapshot:
|
||||
account_id = item.get("accountId")
|
||||
if not isinstance(account_id, int) or account_id in tried:
|
||||
continue
|
||||
if item.get("schedulable") is True and item.get("status") == "active" and item.get("tempUnschedulableSet") is not True:
|
||||
result.append({
|
||||
"accountId": account_id,
|
||||
"accountName": item.get("accountName"),
|
||||
"priority": item.get("priority"),
|
||||
"concurrency": item.get("concurrency"),
|
||||
})
|
||||
return result
|
||||
|
||||
def run_trace():
|
||||
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {}
|
||||
request_id = payload.get("requestId")
|
||||
since = payload.get("since") or "24h"
|
||||
tail = int(payload.get("tail") or 20000)
|
||||
context_seconds = int(payload.get("contextSeconds") or 300)
|
||||
show_lines = bool(payload.get("showLines"))
|
||||
if not isinstance(request_id, str) or not request_id:
|
||||
raise RuntimeError("trace payload missing requestId")
|
||||
admin_email, token, admin_compliance = login()
|
||||
account_snapshot, account_snapshot_error = account_snapshot_from_runtime(token)
|
||||
account_names_by_id = {}
|
||||
for row in account_snapshot:
|
||||
account_id = row.get("accountId")
|
||||
if isinstance(account_id, str) and account_id.isdigit():
|
||||
account_id = int(account_id)
|
||||
if isinstance(account_id, int) and isinstance(row.get("accountName"), str):
|
||||
account_names_by_id[account_id] = row.get("accountName")
|
||||
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"])
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
parsed_lines = []
|
||||
matched = []
|
||||
for line in stdout.splitlines():
|
||||
parsed = parse_log_line(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
parsed_lines.append(parsed)
|
||||
if request_id in line:
|
||||
matched.append(parsed)
|
||||
first_epoch = None
|
||||
last_epoch = None
|
||||
for item in matched:
|
||||
epoch = log_time_epoch(item)
|
||||
if epoch is None:
|
||||
continue
|
||||
first_epoch = epoch if first_epoch is None else min(first_epoch, epoch)
|
||||
last_epoch = epoch if last_epoch is None else max(last_epoch, epoch)
|
||||
window_lines = []
|
||||
if first_epoch is not None:
|
||||
start_epoch = first_epoch - context_seconds
|
||||
end_epoch = (last_epoch if last_epoch is not None else first_epoch) + context_seconds
|
||||
for item in parsed_lines:
|
||||
epoch = log_time_epoch(item)
|
||||
if epoch is not None and start_epoch <= epoch <= end_epoch:
|
||||
window_lines.append(item)
|
||||
else:
|
||||
window_lines = matched
|
||||
events = [classify_trace_event(item, account_names_by_id) for item in matched]
|
||||
request_start = next((item for item in events if item.get("type") == "request-start"), None)
|
||||
final_event = next((item for item in reversed(events) if item.get("type") == "final"), None)
|
||||
failovers = [item for item in events if item.get("type") == "failover"]
|
||||
select_failures = [item for item in events if item.get("type") == "select-failed"]
|
||||
upstream_errors = [item for item in events if item.get("type") == "upstream-error"]
|
||||
temp_unsched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if "account_temp_unschedulable" in str(item.get("_message") or "")]
|
||||
admin_sched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if ("schedulable" in str(item.get("_message") or "") or "/schedulable" in str(item.get("path") or ""))]
|
||||
window_events = [classify_trace_event(item, account_names_by_id) for item in window_lines]
|
||||
final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400]
|
||||
window_failovers = [item for item in window_events if item.get("type") == "failover"]
|
||||
window_select_failures = [item for item in window_events if item.get("type") == "select-failed"]
|
||||
untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot)
|
||||
reason = trace_reason(events, final_event)
|
||||
if not matched:
|
||||
outcome = "not-found"
|
||||
elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400:
|
||||
outcome = "succeeded"
|
||||
elif isinstance(final_event, dict):
|
||||
outcome = "failed"
|
||||
else:
|
||||
outcome = "incomplete"
|
||||
return {
|
||||
"ok": proc.returncode == 0 and len(matched) > 0,
|
||||
"mode": "trace",
|
||||
"namespace": NAMESPACE,
|
||||
"serviceDns": SERVICE_DNS,
|
||||
"appPod": APP_POD,
|
||||
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
||||
"requestId": request_id,
|
||||
"summary": {
|
||||
"outcome": outcome,
|
||||
"reason": reason,
|
||||
"eventCount": len(events),
|
||||
"matchedLineCount": len(matched),
|
||||
"firstAt": events[0].get("at") if events else None,
|
||||
"lastAt": events[-1].get("at") if events else None,
|
||||
},
|
||||
"window": {
|
||||
"since": since,
|
||||
"tail": tail,
|
||||
"beforeSeconds": context_seconds,
|
||||
"afterSeconds": context_seconds,
|
||||
"lineCount": len(window_lines),
|
||||
},
|
||||
"request": request_start or {},
|
||||
"final": final_event or {},
|
||||
"events": events,
|
||||
"failovers": failovers,
|
||||
"selectFailures": select_failures,
|
||||
"upstreamErrors": upstream_errors,
|
||||
"tempUnschedulable": temp_unsched,
|
||||
"adminSchedulable": admin_sched[-20:],
|
||||
"windowStats": {
|
||||
"matchedLines": len(matched),
|
||||
"eventCount": len(window_events),
|
||||
"finalErrorCount": len(final_errors),
|
||||
"failoverCount": len(window_failovers),
|
||||
"selectFailedCount": len(window_select_failures),
|
||||
"tempUnschedulableCount": len(temp_unsched),
|
||||
"adminSchedulableCount": len(admin_sched),
|
||||
},
|
||||
"diagnostics": {
|
||||
"failoverBudgetExhausted": failover_budget_exhausted(failovers, final_event or {}),
|
||||
"untriedSchedulableAccounts": untried_schedulable_accounts,
|
||||
},
|
||||
"accountSnapshot": account_snapshot,
|
||||
"accountSnapshotError": account_snapshot_error,
|
||||
"rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [],
|
||||
"showLines": show_lines,
|
||||
"logs": {
|
||||
"exitCode": proc.returncode,
|
||||
"stderrTail": text(proc.stderr, 1000),
|
||||
"stdoutLineCount": len(stdout.splitlines()),
|
||||
},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
`;
|
||||
@@ -0,0 +1,809 @@
|
||||
export const remotePythonValidationScript = `def validate_gateway(api_key):
|
||||
resp = curl_api("GET", "/v1/models", bearer=api_key)
|
||||
parsed = resp.get("json")
|
||||
model_count = None
|
||||
if isinstance(parsed, dict) and isinstance(parsed.get("data"), list):
|
||||
model_count = len(parsed["data"])
|
||||
return {
|
||||
"ok": resp.get("ok"),
|
||||
"httpStatus": resp.get("httpStatus"),
|
||||
"transportExitCode": resp.get("transportExitCode"),
|
||||
"modelCount": model_count,
|
||||
"bodyPreview": text(resp.get("body", ""), 500) if not resp.get("ok") else "",
|
||||
"stderr": resp.get("stderr", ""),
|
||||
"method": "GET /v1/models",
|
||||
"serviceDns": SERVICE_DNS,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def response_output_preview(parsed):
|
||||
if not isinstance(parsed, dict):
|
||||
return ""
|
||||
if isinstance(parsed.get("output_text"), str):
|
||||
return parsed["output_text"][:240]
|
||||
output = parsed.get("output")
|
||||
if not isinstance(output, list):
|
||||
return ""
|
||||
parts = []
|
||||
for item in output:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
text_value = block.get("text")
|
||||
if isinstance(text_value, str) and text_value:
|
||||
parts.append(text_value)
|
||||
return "\\n".join(parts)[:240]
|
||||
|
||||
def request_log_evidence(request_id):
|
||||
proc = runtime_logs("5m", 800)
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
lines = [line for line in stdout.splitlines() if request_id in line]
|
||||
failovers = []
|
||||
final = None
|
||||
for line in lines:
|
||||
json_start = line.find("{")
|
||||
if json_start < 0:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line[json_start:])
|
||||
except Exception:
|
||||
continue
|
||||
if "upstream_failover_switching" in line:
|
||||
failovers.append({
|
||||
"accountId": item.get("account_id"),
|
||||
"upstreamStatus": item.get("upstream_status"),
|
||||
"switchCount": item.get("switch_count"),
|
||||
"maxSwitches": item.get("max_switches"),
|
||||
})
|
||||
if "http request completed" in line:
|
||||
final = {
|
||||
"accountId": item.get("account_id"),
|
||||
"statusCode": item.get("status_code"),
|
||||
"latencyMs": item.get("latency_ms"),
|
||||
"path": item.get("path"),
|
||||
}
|
||||
return {
|
||||
"requestId": request_id,
|
||||
"matchedLogLineCount": len(lines),
|
||||
"failovers": failovers,
|
||||
"final": final,
|
||||
"logsExitCode": proc.returncode,
|
||||
"logsStderr": text(proc.stderr, 1000),
|
||||
}
|
||||
|
||||
def recent_compact_gateway_evidence():
|
||||
proc = runtime_logs("6h", 2500)
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
failures = []
|
||||
successes = []
|
||||
failovers = []
|
||||
final_errors = []
|
||||
context_canceled = []
|
||||
for line in stdout.splitlines():
|
||||
if "/responses/compact" not in line and "remote_compact" not in line:
|
||||
continue
|
||||
json_start = line.find("{")
|
||||
if json_start < 0:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line[json_start:])
|
||||
except Exception:
|
||||
continue
|
||||
path = item.get("path")
|
||||
entry = {
|
||||
"requestId": item.get("request_id"),
|
||||
"clientRequestId": item.get("client_request_id"),
|
||||
"accountId": item.get("account_id"),
|
||||
"statusCode": item.get("status_code"),
|
||||
"upstreamStatus": item.get("upstream_status"),
|
||||
"latencyMs": item.get("latency_ms"),
|
||||
"path": path,
|
||||
}
|
||||
if "codex.remote_compact.failed" in line:
|
||||
failures.append(entry)
|
||||
elif "codex.remote_compact.succeeded" in line:
|
||||
successes.append(entry)
|
||||
elif "upstream_failover_switching" in line and path == "/responses/compact":
|
||||
failovers.append({
|
||||
**entry,
|
||||
"switchCount": item.get("switch_count"),
|
||||
"maxSwitches": item.get("max_switches"),
|
||||
})
|
||||
elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
|
||||
final_errors.append(entry)
|
||||
if "context canceled" in line and path == "/responses/compact":
|
||||
context_canceled.append(entry)
|
||||
return {
|
||||
"ok": True,
|
||||
"degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0,
|
||||
"window": "6h",
|
||||
"tailLines": 2500,
|
||||
"failureCount": len(failures),
|
||||
"successCount": len(successes),
|
||||
"failoverCount": len(failovers),
|
||||
"finalErrorCount": len(final_errors),
|
||||
"contextCanceledCount": len(context_canceled),
|
||||
"recentFailures": failures[-5:],
|
||||
"recentSuccesses": successes[-5:],
|
||||
"recentFailovers": failovers[-8:],
|
||||
"recentFinalErrors": final_errors[-5:],
|
||||
"recentContextCanceled": context_canceled[-5:],
|
||||
"logsExitCode": proc.returncode,
|
||||
"logsStderr": text(proc.stderr, 1000),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def is_ignored_probe_noise(entry):
|
||||
for value in (entry.get("requestId"), entry.get("clientRequestId")):
|
||||
if isinstance(value, str) and value.startswith("unidesk-400-model-probe-"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def filter_ignored_probe_noise(items):
|
||||
return [item for item in items if not is_ignored_probe_noise(item)]
|
||||
|
||||
def ignored_probe_noise(items, section):
|
||||
result = []
|
||||
for item in items:
|
||||
if is_ignored_probe_noise(item):
|
||||
probe = dict(item)
|
||||
probe["section"] = section
|
||||
result.append(probe)
|
||||
return result
|
||||
|
||||
def group_by_request_id(items):
|
||||
grouped = {}
|
||||
for item in items:
|
||||
request_id = item.get("requestId")
|
||||
if not isinstance(request_id, str) or not request_id:
|
||||
continue
|
||||
grouped.setdefault(request_id, []).append(item)
|
||||
return grouped
|
||||
|
||||
def failover_budget_exhausted_evidence(failovers, final_errors):
|
||||
final_by_request = {}
|
||||
for item in final_errors:
|
||||
request_id = item.get("requestId")
|
||||
if isinstance(request_id, str) and request_id:
|
||||
final_by_request[request_id] = item
|
||||
exhausted = []
|
||||
for request_id, request_failovers in group_by_request_id(failovers).items():
|
||||
final = final_by_request.get(request_id)
|
||||
if not final:
|
||||
continue
|
||||
last = request_failovers[-1]
|
||||
switch_count = last.get("switchCount")
|
||||
max_switches = last.get("maxSwitches")
|
||||
final_status = final.get("statusCode")
|
||||
if (
|
||||
isinstance(switch_count, int)
|
||||
and isinstance(max_switches, int)
|
||||
and max_switches > 0
|
||||
and switch_count >= max_switches
|
||||
and isinstance(final_status, int)
|
||||
and final_status >= 500
|
||||
):
|
||||
exhausted.append({
|
||||
"requestId": request_id,
|
||||
"clientRequestId": final.get("clientRequestId") or last.get("clientRequestId"),
|
||||
"path": final.get("path") or last.get("path"),
|
||||
"finalAccountId": final.get("accountId"),
|
||||
"finalStatusCode": final_status,
|
||||
"switchCount": switch_count,
|
||||
"maxSwitches": max_switches,
|
||||
"lastFailoverAccountId": last.get("accountId"),
|
||||
"lastUpstreamStatus": last.get("upstreamStatus"),
|
||||
})
|
||||
return exhausted
|
||||
|
||||
def recent_responses_gateway_evidence():
|
||||
proc = runtime_logs("6h", 2500)
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
failovers = []
|
||||
forward_failures = []
|
||||
final_errors = []
|
||||
context_canceled = []
|
||||
slow_final_errors = []
|
||||
for line in stdout.splitlines():
|
||||
if '"/responses"' not in line and '"/v1/responses"' not in line:
|
||||
continue
|
||||
json_start = line.find("{")
|
||||
if json_start < 0:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line[json_start:])
|
||||
except Exception:
|
||||
continue
|
||||
path = item.get("path")
|
||||
if path not in ("/responses", "/v1/responses"):
|
||||
continue
|
||||
entry = {
|
||||
"requestId": item.get("request_id"),
|
||||
"clientRequestId": item.get("client_request_id"),
|
||||
"accountId": item.get("account_id"),
|
||||
"statusCode": item.get("status_code"),
|
||||
"upstreamStatus": item.get("upstream_status"),
|
||||
"latencyMs": item.get("latency_ms"),
|
||||
"path": path,
|
||||
}
|
||||
if "upstream_failover_switching" in line:
|
||||
failovers.append({
|
||||
**entry,
|
||||
"switchCount": item.get("switch_count"),
|
||||
"maxSwitches": item.get("max_switches"),
|
||||
})
|
||||
elif "openai.forward_failed" in line:
|
||||
forward_failures.append({
|
||||
**entry,
|
||||
"errorPreview": text(str(item.get("error") or ""), 500),
|
||||
"fallbackErrorResponseWritten": item.get("fallback_error_response_written"),
|
||||
"upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"),
|
||||
})
|
||||
elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
|
||||
final_errors.append(entry)
|
||||
latency_ms = item.get("latency_ms")
|
||||
if isinstance(latency_ms, int) and latency_ms >= 30000:
|
||||
slow_final_errors.append(entry)
|
||||
if "context canceled" in line:
|
||||
context_canceled.append(entry)
|
||||
visible_failovers = filter_ignored_probe_noise(failovers)
|
||||
visible_forward_failures = filter_ignored_probe_noise(forward_failures)
|
||||
visible_final_errors = filter_ignored_probe_noise(final_errors)
|
||||
visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors)
|
||||
visible_context_canceled = filter_ignored_probe_noise(context_canceled)
|
||||
failover_budget_exhausted = failover_budget_exhausted_evidence(visible_failovers, visible_final_errors)
|
||||
probe_noise = (
|
||||
ignored_probe_noise(failovers, "failovers")
|
||||
+ ignored_probe_noise(forward_failures, "forwardFailures")
|
||||
+ ignored_probe_noise(final_errors, "finalErrors")
|
||||
+ ignored_probe_noise(slow_final_errors, "slowFinalErrors")
|
||||
+ ignored_probe_noise(context_canceled, "contextCanceled")
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"degraded": len(visible_forward_failures) > 0 or len(visible_final_errors) > 0 or len(visible_context_canceled) > 0,
|
||||
"window": "6h",
|
||||
"tailLines": 2500,
|
||||
"failoverCount": len(visible_failovers),
|
||||
"forwardFailureCount": len(visible_forward_failures),
|
||||
"finalErrorCount": len(visible_final_errors),
|
||||
"slowFinalErrorCount": len(visible_slow_final_errors),
|
||||
"contextCanceledCount": len(visible_context_canceled),
|
||||
"ignoredProbeNoiseCount": len(probe_noise),
|
||||
"failoverBudgetExhausted": failover_budget_exhausted[-8:],
|
||||
"rawCounts": {
|
||||
"failoverCount": len(failovers),
|
||||
"forwardFailureCount": len(forward_failures),
|
||||
"finalErrorCount": len(final_errors),
|
||||
"slowFinalErrorCount": len(slow_final_errors),
|
||||
"contextCanceledCount": len(context_canceled),
|
||||
},
|
||||
"recentFailovers": visible_failovers[-8:],
|
||||
"recentForwardFailures": visible_forward_failures[-8:],
|
||||
"recentFinalErrors": visible_final_errors[-8:],
|
||||
"recentSlowFinalErrors": visible_slow_final_errors[-5:],
|
||||
"recentContextCanceled": visible_context_canceled[-5:],
|
||||
"recentProbeNoise": probe_noise[-5:],
|
||||
"logsExitCode": proc.returncode,
|
||||
"logsStderr": text(proc.stderr, 1000),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def validate_gateway_responses(api_key):
|
||||
request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000))
|
||||
payload = {
|
||||
"model": RESPONSES_SMOKE_MODEL,
|
||||
"input": "Reply exactly: unidesk-sub2api-validate-ok",
|
||||
"stream": False,
|
||||
"store": False,
|
||||
"max_output_tokens": 32,
|
||||
}
|
||||
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
script = r'''
|
||||
set -eu
|
||||
token="$1"
|
||||
request_id="$2"
|
||||
url="$3"
|
||||
tmp="$(mktemp)"
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
cat > "$tmp"
|
||||
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "X-Request-ID: $request_id" \
|
||||
-H "OpenAI-Client-Request-ID: $request_id" \
|
||||
--data-binary @"$tmp" \
|
||||
"$url"
|
||||
'''
|
||||
started = time.time()
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
if not isinstance(HOST_DOCKER_APP_PORT, int):
|
||||
raise RuntimeError("host-docker app port missing")
|
||||
proc = run(["sh", "-c", script, "sh", api_key, request_id, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}/v1/responses"], body)
|
||||
else:
|
||||
proc = run([
|
||||
"kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
|
||||
"--", "sh", "-c", script, "sh", api_key, request_id, "http://127.0.0.1:8080/v1/responses",
|
||||
], body)
|
||||
resp = parse_curl_output(proc)
|
||||
evidence = request_log_evidence(request_id)
|
||||
parsed = resp.get("json")
|
||||
failover_count = len(evidence.get("failovers") or [])
|
||||
return {
|
||||
"ok": resp.get("ok"),
|
||||
"degraded": failover_count > 0,
|
||||
"outcome": "succeeded-with-failover" if resp.get("ok") and failover_count > 0 else ("succeeded" if resp.get("ok") else "failed"),
|
||||
"httpStatus": resp.get("httpStatus"),
|
||||
"transportExitCode": resp.get("transportExitCode"),
|
||||
"method": "POST /v1/responses",
|
||||
"model": RESPONSES_SMOKE_MODEL,
|
||||
"requestId": request_id,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"outputTextPreview": response_output_preview(parsed),
|
||||
"bodyPreview": "" if resp.get("ok") else text(resp.get("body", ""), 800),
|
||||
"stderr": resp.get("stderr", ""),
|
||||
"evidence": evidence,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def bool_value(value):
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
if value.lower() == "true":
|
||||
return True
|
||||
if value.lower() == "false":
|
||||
return False
|
||||
return False
|
||||
|
||||
def normalize_temp_unschedulable_credentials(credentials):
|
||||
if not isinstance(credentials, dict):
|
||||
credentials = {}
|
||||
enabled = bool_value(credentials.get("temp_unschedulable_enabled"))
|
||||
raw_rules = credentials.get("temp_unschedulable_rules")
|
||||
if isinstance(raw_rules, str):
|
||||
try:
|
||||
raw_rules = json.loads(raw_rules)
|
||||
except json.JSONDecodeError:
|
||||
raw_rules = []
|
||||
rules = []
|
||||
if isinstance(raw_rules, list):
|
||||
for rule in raw_rules:
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
error_code = rule.get("error_code", rule.get("statusCode"))
|
||||
duration_minutes = rule.get("duration_minutes", rule.get("durationMinutes"))
|
||||
keywords = rule.get("keywords")
|
||||
if not isinstance(error_code, int) or not isinstance(duration_minutes, int) or not isinstance(keywords, list):
|
||||
continue
|
||||
clean_keywords = [item for item in keywords if isinstance(item, str) and item.strip()]
|
||||
if not clean_keywords:
|
||||
continue
|
||||
description = rule.get("description") if isinstance(rule.get("description"), str) else ""
|
||||
rules.append({
|
||||
"error_code": error_code,
|
||||
"keywords": clean_keywords,
|
||||
"duration_minutes": duration_minutes,
|
||||
"description": description,
|
||||
})
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"rules": rules,
|
||||
}
|
||||
|
||||
def summarize_temp_unschedulable_rules(rules):
|
||||
return [{
|
||||
"errorCode": rule.get("error_code"),
|
||||
"durationMinutes": rule.get("duration_minutes"),
|
||||
"keywordCount": len(rule.get("keywords") or []),
|
||||
"keywords": rule.get("keywords") or [],
|
||||
"hasDescription": bool(rule.get("description")),
|
||||
} for rule in rules]
|
||||
|
||||
def success_body_reclassification_requirement():
|
||||
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
||||
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
||||
if expected["enabled"] is not True:
|
||||
continue
|
||||
for rule in expected["rules"]:
|
||||
error_code = rule.get("error_code")
|
||||
keywords = rule.get("keywords") or []
|
||||
if isinstance(error_code, int) and 200 <= error_code < 300 and keywords:
|
||||
return {
|
||||
"required": True,
|
||||
"sourceAccountName": name,
|
||||
"statusCode": error_code,
|
||||
"keywords": keywords,
|
||||
"representativeKeyword": keywords[0],
|
||||
"durationMinutes": rule.get("duration_minutes"),
|
||||
}
|
||||
return {
|
||||
"required": False,
|
||||
"sourceAccountName": None,
|
||||
"statusCode": None,
|
||||
"keywords": [],
|
||||
"representativeKeyword": None,
|
||||
"durationMinutes": None,
|
||||
}
|
||||
|
||||
def model_routing_400_failover_requirement():
|
||||
preferred = ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "暂不支持", "可用模型", "unsupported model", "model not supported", "does not support", "not supported", "model_not_found", "no available channel for model"]
|
||||
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
||||
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
||||
if expected["enabled"] is not True:
|
||||
continue
|
||||
for rule in expected["rules"]:
|
||||
error_code = rule.get("error_code")
|
||||
keywords = rule.get("keywords") or []
|
||||
if error_code != 400 or not keywords:
|
||||
continue
|
||||
representative_keyword = next((item for item in preferred if item in keywords), keywords[0])
|
||||
return {
|
||||
"required": True,
|
||||
"sourceAccountName": name,
|
||||
"statusCode": error_code,
|
||||
"keywords": keywords,
|
||||
"representativeKeyword": representative_keyword,
|
||||
"durationMinutes": rule.get("duration_minutes"),
|
||||
}
|
||||
return {
|
||||
"required": False,
|
||||
"sourceAccountName": None,
|
||||
"statusCode": None,
|
||||
"keywords": [],
|
||||
"representativeKeyword": None,
|
||||
"durationMinutes": None,
|
||||
}
|
||||
|
||||
def delete_probe_resource(token, method, path, label):
|
||||
if not path:
|
||||
return {"label": label, "ok": True, "skipped": True}
|
||||
resp = curl_api(method, path, bearer=token)
|
||||
ok = resp.get("ok") is True or resp.get("httpStatus") in (404, 410)
|
||||
return {
|
||||
"label": label,
|
||||
"ok": ok,
|
||||
"method": method,
|
||||
"path": path,
|
||||
"httpStatus": resp.get("httpStatus"),
|
||||
"transportExitCode": resp.get("transportExitCode"),
|
||||
"bodyPreview": "" if ok else text(resp.get("body", ""), 500),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def validate_runtime_capabilities(token):
|
||||
success_body = success_body_reclassification_requirement()
|
||||
model_routing_400 = model_routing_400_failover_requirement()
|
||||
return {
|
||||
"ok": success_body.get("required") is not True and model_routing_400.get("required") is True,
|
||||
"runtimeImage": app_pod_runtime_image(),
|
||||
"successBodyReclassification": {
|
||||
"ok": success_body.get("required") is not True,
|
||||
"required": success_body.get("required"),
|
||||
"outcome": "not-required-by-yaml" if success_body.get("required") is not True else "requires-source-review-and-real-traffic-evidence",
|
||||
"requirement": success_body,
|
||||
"valuesPrinted": False,
|
||||
},
|
||||
"modelRouting400Failover": {
|
||||
"ok": model_routing_400.get("required") is True,
|
||||
"required": model_routing_400.get("required"),
|
||||
"outcome": "yaml-rules-present-runtime-observed-by-real-traffic",
|
||||
"requirement": model_routing_400,
|
||||
"evidence": "Use Sub2API source review plus validation.gatewayResponsesRecent and Artificer/real request ids for runtime proof; default validate does not create mock upstreams or temporary failover accounts.",
|
||||
"valuesPrinted": False,
|
||||
},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def app_pod_runtime_image():
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
proc = docker(["inspect", HOST_DOCKER_APP_CONTAINER])
|
||||
if proc.returncode != 0:
|
||||
return {
|
||||
"container": HOST_DOCKER_APP_CONTAINER,
|
||||
"error": text(proc.stderr, 1000) or text(proc.stdout, 1000),
|
||||
}
|
||||
try:
|
||||
data = json.loads(proc.stdout.decode("utf-8"))
|
||||
item = data[0] if isinstance(data, list) and data else {}
|
||||
except Exception as exc:
|
||||
return {"container": HOST_DOCKER_APP_CONTAINER, "error": str(exc)}
|
||||
state = item.get("State") if isinstance(item, dict) and isinstance(item.get("State"), dict) else {}
|
||||
health = state.get("Health") if isinstance(state.get("Health"), dict) else {}
|
||||
config = item.get("Config") if isinstance(item, dict) and isinstance(item.get("Config"), dict) else {}
|
||||
return {
|
||||
"container": HOST_DOCKER_APP_CONTAINER,
|
||||
"id": (item.get("Id") or "")[:12] if isinstance(item.get("Id"), str) else None,
|
||||
"image": config.get("Image"),
|
||||
"imageID": item.get("Image"),
|
||||
"ready": state.get("Running") is True and (not health or health.get("Status") in (None, "healthy")),
|
||||
"restartCount": item.get("RestartCount"),
|
||||
"startedAt": state.get("StartedAt"),
|
||||
"health": health.get("Status"),
|
||||
}
|
||||
try:
|
||||
pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], f"pod/{APP_POD}")
|
||||
spec_containers = ((pod.get("spec") or {}).get("containers") or []) if isinstance(pod, dict) else []
|
||||
status_containers = ((pod.get("status") or {}).get("containerStatuses") or []) if isinstance(pod, dict) else []
|
||||
spec = next((item for item in spec_containers if item.get("name") == "sub2api"), spec_containers[0] if spec_containers else {})
|
||||
status = next((item for item in status_containers if item.get("name") == "sub2api"), status_containers[0] if status_containers else {})
|
||||
return {
|
||||
"pod": APP_POD,
|
||||
"image": spec.get("image"),
|
||||
"imageID": status.get("imageID"),
|
||||
"ready": status.get("ready"),
|
||||
"restartCount": status.get("restartCount"),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"pod": APP_POD, "error": str(exc)}
|
||||
|
||||
def get_account_detail(token, account):
|
||||
account_id = account.get("id") if isinstance(account, dict) else None
|
||||
if account_id is None:
|
||||
return account
|
||||
data = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"get account {account.get('name')}")
|
||||
return data if isinstance(data, dict) else account
|
||||
|
||||
def account_temp_unschedulable_status(token):
|
||||
accounts = list_accounts(token)
|
||||
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
||||
items = []
|
||||
missing = []
|
||||
mismatched = []
|
||||
enabled_names = []
|
||||
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
||||
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
||||
account = by_name.get(name)
|
||||
if account is None:
|
||||
missing.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": None,
|
||||
"expectedEnabled": expected["enabled"],
|
||||
"runtimeEnabled": None,
|
||||
"expectedRuleCount": len(expected["rules"]),
|
||||
"runtimeRuleCount": None,
|
||||
"ok": False,
|
||||
})
|
||||
continue
|
||||
detail = get_account_detail(token, account)
|
||||
credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {}
|
||||
runtime = normalize_temp_unschedulable_credentials(credentials)
|
||||
temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil")
|
||||
temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or ""
|
||||
ok = runtime == expected
|
||||
if expected["enabled"]:
|
||||
enabled_names.append(name)
|
||||
if not ok:
|
||||
mismatched.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": account.get("id"),
|
||||
"expectedEnabled": expected["enabled"],
|
||||
"runtimeEnabled": runtime["enabled"],
|
||||
"expectedRuleCount": len(expected["rules"]),
|
||||
"runtimeRuleCount": len(runtime["rules"]),
|
||||
"expectedRules": summarize_temp_unschedulable_rules(expected["rules"]),
|
||||
"runtimeRules": summarize_temp_unschedulable_rules(runtime["rules"]),
|
||||
"status": account.get("status"),
|
||||
"schedulable": account.get("schedulable"),
|
||||
"tempUnschedulableUntil": temp_until,
|
||||
"tempUnschedulableReasonPreview": text(str(temp_reason), 500) if temp_reason else "",
|
||||
"tempUnschedulableSet": temp_until is not None or bool(temp_reason),
|
||||
"ok": ok,
|
||||
})
|
||||
return {
|
||||
"ok": len(missing) == 0 and len(mismatched) == 0,
|
||||
"desired": len(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE),
|
||||
"enabled": enabled_names,
|
||||
"missing": missing,
|
||||
"mismatched": mismatched,
|
||||
"items": items,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def account_capacity_status(token):
|
||||
accounts = list_accounts(token)
|
||||
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
||||
items = []
|
||||
missing = []
|
||||
mismatched = []
|
||||
expected_capacity_total = 0
|
||||
runtime_concurrency_total = 0
|
||||
schedulable_runtime_concurrency_total = 0
|
||||
available_runtime_concurrency_total = 0
|
||||
temp_unschedulable_runtime_concurrency_total = 0
|
||||
for name in sorted(EXPECTED_ACCOUNT_CAPACITIES):
|
||||
expected = int(EXPECTED_ACCOUNT_CAPACITIES[name])
|
||||
expected_capacity_total += expected
|
||||
account = by_name.get(name)
|
||||
if account is None:
|
||||
missing.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": None,
|
||||
"expectedCapacity": expected,
|
||||
"runtimeConcurrency": None,
|
||||
"ok": False,
|
||||
})
|
||||
continue
|
||||
runtime = account.get("concurrency")
|
||||
runtime_int = runtime if isinstance(runtime, int) else 0
|
||||
runtime_concurrency_total += runtime_int
|
||||
if account.get("schedulable") is True:
|
||||
runtime_status = account.get("status")
|
||||
if runtime_status is None or runtime_status == "active":
|
||||
runtime_status_ok = True
|
||||
else:
|
||||
runtime_status_ok = False
|
||||
if runtime_status_ok:
|
||||
schedulable_runtime_concurrency_total += runtime_int
|
||||
temp_until = account.get("temp_unschedulable_until") or account.get("tempUnschedulableUntil")
|
||||
temp_reason = account.get("temp_unschedulable_reason") or account.get("tempUnschedulableReason")
|
||||
if temp_until is None and not bool(temp_reason):
|
||||
available_runtime_concurrency_total += runtime_int
|
||||
else:
|
||||
temp_unschedulable_runtime_concurrency_total += runtime_int
|
||||
ok = runtime == expected
|
||||
if not ok:
|
||||
mismatched.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": account.get("id"),
|
||||
"expectedCapacity": expected,
|
||||
"runtimeConcurrency": runtime,
|
||||
"priority": account.get("priority"),
|
||||
"status": account.get("status"),
|
||||
"schedulable": account.get("schedulable"),
|
||||
"ok": ok,
|
||||
})
|
||||
return {
|
||||
"ok": len(missing) == 0 and len(mismatched) == 0,
|
||||
"defaultAccountCapacity": POOL_DEFAULT_ACCOUNT_CAPACITY,
|
||||
"desired": len(EXPECTED_ACCOUNT_CAPACITIES),
|
||||
"totals": {
|
||||
"expectedCapacityTotal": expected_capacity_total,
|
||||
"runtimeConcurrencyTotal": runtime_concurrency_total,
|
||||
"schedulableRuntimeConcurrencyTotal": schedulable_runtime_concurrency_total,
|
||||
"availableRuntimeConcurrencyTotal": available_runtime_concurrency_total,
|
||||
"tempUnschedulableRuntimeConcurrencyTotal": temp_unschedulable_runtime_concurrency_total,
|
||||
"minOwnerConcurrency": MIN_OWNER_CONCURRENCY,
|
||||
"minOwnerConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
||||
},
|
||||
"missing": missing,
|
||||
"mismatched": mismatched,
|
||||
"items": items,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def account_load_factor_status(token):
|
||||
accounts = list_accounts(token)
|
||||
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
||||
items = []
|
||||
missing = []
|
||||
mismatched = []
|
||||
for name in sorted(EXPECTED_ACCOUNT_LOAD_FACTORS):
|
||||
expected = int(EXPECTED_ACCOUNT_LOAD_FACTORS[name])
|
||||
account = by_name.get(name)
|
||||
if account is None:
|
||||
missing.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": None,
|
||||
"expectedLoadFactor": expected,
|
||||
"runtimeLoadFactor": None,
|
||||
"ok": False,
|
||||
})
|
||||
continue
|
||||
runtime_raw = account.get("load_factor")
|
||||
if runtime_raw is None:
|
||||
detail = get_account_detail(token, account)
|
||||
runtime_raw = detail.get("load_factor") if isinstance(detail, dict) else None
|
||||
try:
|
||||
runtime = int(runtime_raw)
|
||||
except Exception:
|
||||
runtime = runtime_raw
|
||||
ok = runtime == expected
|
||||
if not ok:
|
||||
mismatched.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": account.get("id"),
|
||||
"expectedLoadFactor": expected,
|
||||
"runtimeLoadFactor": runtime,
|
||||
"priority": account.get("priority"),
|
||||
"status": account.get("status"),
|
||||
"schedulable": account.get("schedulable"),
|
||||
"ok": ok,
|
||||
})
|
||||
return {
|
||||
"ok": len(missing) == 0 and len(mismatched) == 0,
|
||||
"defaultAccountLoadFactor": POOL_DEFAULT_ACCOUNT_LOAD_FACTOR,
|
||||
"desired": len(EXPECTED_ACCOUNT_LOAD_FACTORS),
|
||||
"missing": missing,
|
||||
"mismatched": mismatched,
|
||||
"items": items,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def account_ws_v2_status(token):
|
||||
accounts = list_accounts(token)
|
||||
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
||||
items = []
|
||||
missing = []
|
||||
mismatched = []
|
||||
enabled_names = []
|
||||
unschedulable_enabled = []
|
||||
schedulable_enabled = []
|
||||
for name in sorted(EXPECTED_ACCOUNT_WS_MODES):
|
||||
expected_mode = EXPECTED_ACCOUNT_WS_MODES[name]
|
||||
expected_enabled = expected_mode not in (None, "", "off")
|
||||
account = by_name.get(name)
|
||||
if account is None:
|
||||
missing.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": None,
|
||||
"expectedMode": expected_mode,
|
||||
"expectedEnabled": expected_enabled,
|
||||
"runtimeMode": None,
|
||||
"runtimeEnabled": None,
|
||||
"ok": False,
|
||||
})
|
||||
continue
|
||||
extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
|
||||
runtime_mode = extra.get("openai_apikey_responses_websockets_v2_mode")
|
||||
runtime_enabled = extra.get("openai_apikey_responses_websockets_v2_enabled")
|
||||
schedulable = account.get("schedulable")
|
||||
if expected_mode == "off":
|
||||
ok = runtime_mode == "off" and runtime_enabled is False
|
||||
elif expected_mode is None:
|
||||
ok = runtime_mode in (None, "", "off") and runtime_enabled in (None, False)
|
||||
else:
|
||||
ok = runtime_mode == expected_mode and runtime_enabled is True
|
||||
if not ok:
|
||||
mismatched.append(name)
|
||||
if expected_enabled:
|
||||
enabled_names.append(name)
|
||||
if schedulable is True:
|
||||
schedulable_enabled.append(name)
|
||||
else:
|
||||
unschedulable_enabled.append(name)
|
||||
items.append({
|
||||
"accountName": name,
|
||||
"accountId": account.get("id"),
|
||||
"expectedMode": expected_mode,
|
||||
"expectedEnabled": expected_enabled,
|
||||
"runtimeMode": runtime_mode,
|
||||
"runtimeEnabled": runtime_enabled,
|
||||
"status": account.get("status"),
|
||||
"schedulable": schedulable,
|
||||
"ok": ok and ((not expected_enabled) or schedulable is True),
|
||||
})
|
||||
availability_ok = len(enabled_names) == 0 or len(schedulable_enabled) > 0
|
||||
return {
|
||||
"ok": len(missing) == 0 and len(mismatched) == 0 and availability_ok,
|
||||
"desired": len(EXPECTED_ACCOUNT_WS_MODES),
|
||||
"enabled": enabled_names,
|
||||
"schedulableEnabled": schedulable_enabled,
|
||||
"unschedulableEnabled": unschedulable_enabled,
|
||||
"missing": missing,
|
||||
"mismatched": mismatched,
|
||||
"items": items,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def api_key_preview(api_key):
|
||||
if len(api_key) <= 14:
|
||||
return "***"
|
||||
return api_key[:10] + "..." + api_key[-4:]
|
||||
|
||||
def secret_fingerprint(value):
|
||||
return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
`;
|
||||
@@ -1,17 +1,20 @@
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
import { desiredAccountNames } from "./accounts";
|
||||
import { readCodexPoolConfig } from "./config";
|
||||
import { isRecord, stringValue } from "./config-utils";
|
||||
import { protectedManualAccountsForTarget } from "./public-exposure";
|
||||
import { renderSub2ApiTempUnschedulableCredentials } from "./redaction";
|
||||
import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote";
|
||||
import { renderTable } from "./render";
|
||||
import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target";
|
||||
|
||||
type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete";
|
||||
|
||||
interface RuntimeOptions {
|
||||
action: RuntimeAction;
|
||||
account: string | null;
|
||||
accounts: string[];
|
||||
template: string | null;
|
||||
kind: "temp-unschedulable";
|
||||
confirm: boolean;
|
||||
@@ -22,7 +25,7 @@ interface RuntimeOptions {
|
||||
targetId: string;
|
||||
}
|
||||
|
||||
export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
||||
export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const options = parseRuntimeOptions(args);
|
||||
const pool = readCodexPoolConfig();
|
||||
const target = codexPoolRuntimeTarget(options.targetId);
|
||||
@@ -32,7 +35,8 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P
|
||||
}
|
||||
const payload = {
|
||||
action: options.action,
|
||||
account: options.account,
|
||||
account: options.accounts.length === 1 ? options.accounts[0] : null,
|
||||
accounts: options.accounts,
|
||||
kind: options.kind,
|
||||
confirm: options.confirm,
|
||||
full: options.full,
|
||||
@@ -57,6 +61,9 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P
|
||||
const result = await runRemoteCodexPoolScript(config, "runtime", runtimeScript(payload, target), target);
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
|
||||
if (!options.full && !options.raw && (options.action === "apply" || options.action === "delete")) {
|
||||
return renderRuntimeMutation(ok, options, parsed, result.exitCode);
|
||||
}
|
||||
if (options.raw) {
|
||||
return {
|
||||
ok,
|
||||
@@ -78,7 +85,8 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P
|
||||
},
|
||||
request: {
|
||||
operation: options.action,
|
||||
account: options.account,
|
||||
account: options.accounts.length === 1 ? options.accounts[0] : undefined,
|
||||
accounts: options.accounts.length > 1 ? options.accounts : undefined,
|
||||
template: options.template,
|
||||
kind: options.kind,
|
||||
confirm: options.confirm,
|
||||
@@ -100,9 +108,10 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P
|
||||
export function parseRuntimeOptions(args: string[]): RuntimeOptions {
|
||||
const [actionRaw = "list", ...rest] = args;
|
||||
if (actionRaw !== "list" && actionRaw !== "get" && actionRaw !== "errors" && actionRaw !== "apply" && actionRaw !== "delete") {
|
||||
throw new Error("runtime usage: list|get|errors|apply|delete [--account <name-or-id>] [--since 24h] [--tail 50000] [--template <id>] [--kind temp-unschedulable] [--confirm] [--target <id>] [--full|--raw]");
|
||||
throw new Error("runtime usage: list|get|errors|apply|delete [--account <name-or-id>|--accounts <selector,...>] [--since 24h] [--tail 50000] [--template <id>] [--kind temp-unschedulable] [--confirm] [--target <id>] [--full|--raw]");
|
||||
}
|
||||
let account: string | null = null;
|
||||
let accounts: string[] | null = null;
|
||||
let template: string | null = null;
|
||||
let kind = "temp-unschedulable" as const;
|
||||
let confirm = false;
|
||||
@@ -124,6 +133,8 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions {
|
||||
else if (arg === "--raw") raw = true;
|
||||
else if (arg === "--account") account = readValue("--account");
|
||||
else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim();
|
||||
else if (arg === "--accounts") accounts = parseRuntimeAccounts(readValue("--accounts"));
|
||||
else if (arg.startsWith("--accounts=")) accounts = parseRuntimeAccounts(arg.slice("--accounts=".length));
|
||||
else if (arg === "--template") template = readValue("--template");
|
||||
else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim();
|
||||
else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind"));
|
||||
@@ -136,17 +147,27 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions {
|
||||
else if (arg.startsWith("--tail=")) tail = parseRuntimeTail(arg.slice("--tail=".length));
|
||||
else throw new Error(`unsupported runtime option: ${arg}`);
|
||||
}
|
||||
if ((actionRaw === "get" || actionRaw === "apply" || actionRaw === "delete") && !account) {
|
||||
if (account !== null && accounts !== null) throw new Error("use only one of --account or --accounts");
|
||||
const selectors = accounts ?? (account === null ? [] : [account]);
|
||||
if ((actionRaw === "get" || actionRaw === "apply" || actionRaw === "delete") && selectors.length === 0) {
|
||||
throw new Error(`runtime ${actionRaw} requires --account <name-or-id>`);
|
||||
}
|
||||
if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format");
|
||||
if (actionRaw !== "apply" && actionRaw !== "delete" && accounts !== null) throw new Error(`runtime ${actionRaw} does not accept --accounts`);
|
||||
if (selectors.some((selector) => selector.length > 256 || /[\r\n]/u.test(selector))) throw new Error("account selector has an unsupported format");
|
||||
if (actionRaw === "apply" && !template) throw new Error("runtime apply requires --template <id>");
|
||||
if (actionRaw !== "apply" && template !== null) throw new Error(`runtime ${actionRaw} does not accept --template`);
|
||||
if (actionRaw !== "errors" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${actionRaw} does not accept --since or --tail`);
|
||||
if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use <number>m, <number>h, or <number>d");
|
||||
if ((actionRaw === "list" || actionRaw === "get" || actionRaw === "errors") && confirm) throw new Error(`runtime ${actionRaw} does not accept --confirm`);
|
||||
if (full && raw) throw new Error("use only one of --full or --raw");
|
||||
return { action: actionRaw, account, template, kind, confirm, full, raw, since, tail, targetId };
|
||||
return { action: actionRaw, accounts: selectors, template, kind, confirm, full, raw, since, tail, targetId };
|
||||
}
|
||||
|
||||
function parseRuntimeAccounts(value: string): string[] {
|
||||
const accounts = value.split(",").map((item) => item.trim());
|
||||
if (accounts.length === 0 || accounts.some((item) => item.length === 0)) throw new Error("--accounts requires comma-separated selectors");
|
||||
if (new Set(accounts).size !== accounts.length) throw new Error("--accounts contains duplicate selectors");
|
||||
return accounts;
|
||||
}
|
||||
|
||||
function parseRuntimeKind(value: string): "temp-unschedulable" {
|
||||
@@ -160,6 +181,60 @@ function parseRuntimeTail(value: string): number {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function renderRuntimeMutation(ok: boolean, options: RuntimeOptions, parsed: Record<string, unknown> | null, exitCode: number): RenderedCliResult {
|
||||
const summary = parsed !== null && isRecord(parsed.summary) ? parsed.summary : {};
|
||||
const items = parsed !== null && Array.isArray(parsed.items) ? parsed.items.filter(isRecord) : [];
|
||||
const lines = [[
|
||||
"SUB2API RUNTIME",
|
||||
`operation=${options.action}`,
|
||||
`mode=${stringValue(parsed?.mode) ?? (options.confirm ? "confirmed" : "dry-run")}`,
|
||||
`ok=${ok ? "true" : "false"}`,
|
||||
`selected=${summary.selected ?? items.length}`,
|
||||
`changed=${summary.changed ?? "?"}`,
|
||||
`writeSucceeded=${summary.writeSucceeded ?? "?"}`,
|
||||
`writeFailed=${summary.writeFailed ?? "?"}`,
|
||||
`reconciled=${summary.reconciled ?? "?"}`,
|
||||
`mismatched=${summary.mismatched ?? "?"}`,
|
||||
].join(" ")];
|
||||
if (items.length > 0) {
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "ID", "CHANGE", "WRITE", "RECONCILED", "BEFORE", "DESIRED", "ACTUAL", "BEFORE_CODES", "DESIRED_CODES", "ACTUAL_CODES", "FAILURE"],
|
||||
...items.map((item) => {
|
||||
const write = isRecord(item.write) ? item.write : {};
|
||||
const before = isRecord(item.before) ? item.before : {};
|
||||
const desired = isRecord(item.desired) ? item.desired : {};
|
||||
const actual = isRecord(item.actual) ? item.actual : {};
|
||||
const codes = (value: unknown): string => Array.isArray(value) ? value.join(",") || "-" : "-";
|
||||
return [
|
||||
stringValue(item.accountName) ?? "-",
|
||||
String(item.accountId ?? "-"),
|
||||
stringValue(item.change) ?? "-",
|
||||
stringValue(write.status) ?? "-",
|
||||
item.reconciled === true ? "true" : item.reconciled === false ? "false" : "-",
|
||||
String(before.ruleCount ?? "-"),
|
||||
String(desired.ruleCount ?? "-"),
|
||||
String(actual.ruleCount ?? "-"),
|
||||
codes(before.statusCodes),
|
||||
codes(desired.statusCodes),
|
||||
codes(actual.statusCodes),
|
||||
stringValue(write.error) ?? stringValue(item.reconcileError) ?? "-",
|
||||
];
|
||||
}),
|
||||
]));
|
||||
} else if (parsed !== null && typeof parsed.error === "string") {
|
||||
lines.push(`ERROR ${parsed.error}`);
|
||||
} else if (parsed === null) {
|
||||
lines.push(`ERROR runtime output unavailable remote_exit=${exitCode}`);
|
||||
}
|
||||
return {
|
||||
ok,
|
||||
command: "platform-infra sub2api codex-pool runtime",
|
||||
renderedText: lines.join("\n"),
|
||||
contentType: "text/plain",
|
||||
projection: parsed ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function pyJson(value: unknown): string {
|
||||
return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
|
||||
}
|
||||
@@ -487,6 +562,78 @@ def observed_runtime_errors(accounts):
|
||||
"message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence",
|
||||
}
|
||||
|
||||
def selected_account_details(token, accounts):
|
||||
selectors = PAYLOAD.get("accounts")
|
||||
if not isinstance(selectors, list) or not selectors:
|
||||
selector = PAYLOAD.get("account")
|
||||
selectors = [selector] if isinstance(selector, str) and selector else []
|
||||
details = []
|
||||
selected_ids = set()
|
||||
for raw_selector in selectors:
|
||||
selector = str(raw_selector)
|
||||
detail = find_account(token, accounts, selector)
|
||||
account_id = detail.get("id")
|
||||
if account_id in selected_ids:
|
||||
raise RuntimeError("duplicate-account-selector: " + selector)
|
||||
if detail.get("type") != "apikey":
|
||||
raise RuntimeError("runtime temp-unschedulable policy requires an apikey account: " + selector)
|
||||
selected_ids.add(account_id)
|
||||
details.append(detail)
|
||||
return details
|
||||
|
||||
def mutation_plan(detail, action, include_rules):
|
||||
credentials = dict(detail.get("credentials") or {})
|
||||
before = normalize_policy(credentials)
|
||||
if action == "apply":
|
||||
template = PAYLOAD.get("selectedTemplate")
|
||||
if not isinstance(template, dict):
|
||||
raise RuntimeError("selected runtime template missing")
|
||||
desired = dict(credentials)
|
||||
desired.update(template.get("credentials") or {})
|
||||
desired_policy = normalize_policy(desired)
|
||||
exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials
|
||||
change = "noop" if before == desired_policy else ("update" if exists else "create")
|
||||
else:
|
||||
desired = dict(credentials)
|
||||
desired.pop("temp_unschedulable_enabled", None)
|
||||
desired.pop("temp_unschedulable_rules", None)
|
||||
desired_policy = normalize_policy(desired)
|
||||
change = "delete" if before != desired_policy or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop"
|
||||
return {
|
||||
"accountName": detail.get("name"),
|
||||
"accountId": detail.get("id"),
|
||||
"change": change,
|
||||
"before": policy_summary(before, include_rules),
|
||||
"desired": policy_summary(desired_policy, include_rules),
|
||||
"template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None,
|
||||
"_desiredCredentials": desired,
|
||||
"_desiredPolicy": desired_policy,
|
||||
}
|
||||
|
||||
def write_mutation_plan(token, plan):
|
||||
if plan["change"] == "noop":
|
||||
return {"attempted": False, "status": "noop", "error": None}
|
||||
try:
|
||||
data_of(curl_api("PUT", f"/api/v1/admin/accounts/{plan['accountId']}", bearer=token, payload={"credentials": plan["_desiredCredentials"]}), "update account runtime config")
|
||||
return {"attempted": True, "status": "succeeded", "error": None}
|
||||
except Exception as exc:
|
||||
return {"attempted": True, "status": "failed", "error": compact_error(str(exc))}
|
||||
|
||||
def reconcile_mutation_plan(token, plan, write, include_rules):
|
||||
item = {key: value for key, value in plan.items() if not key.startswith("_")}
|
||||
item["write"] = write
|
||||
try:
|
||||
actual_detail = account_detail(token, plan["accountId"])
|
||||
actual_policy = normalize_policy(actual_detail.get("credentials"))
|
||||
item["actual"] = policy_summary(actual_policy, include_rules)
|
||||
item["reconciled"] = actual_policy == plan["_desiredPolicy"]
|
||||
item["reconcileError"] = None
|
||||
except Exception as exc:
|
||||
item["actual"] = None
|
||||
item["reconciled"] = False
|
||||
item["reconcileError"] = compact_error(str(exc))
|
||||
return item
|
||||
|
||||
def runtime_result():
|
||||
token = login()
|
||||
group = next((item for item in list_groups(token) if item.get("name") == PAYLOAD["groupName"]), None)
|
||||
@@ -506,43 +653,33 @@ def runtime_result():
|
||||
if action == "errors":
|
||||
errors = observed_runtime_errors(accounts)
|
||||
return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors}
|
||||
detail = find_account(token, accounts, str(PAYLOAD["account"]))
|
||||
detail = find_account(token, accounts, str(PAYLOAD["account"])) if action == "get" else None
|
||||
if action == "get":
|
||||
return {**base, "ok": True, "operation": "get", "mutation": False, "account": account_summary(detail, True)}
|
||||
if detail.get("type") != "apikey":
|
||||
raise RuntimeError("runtime temp-unschedulable policy requires an apikey account")
|
||||
credentials = dict(detail.get("credentials") or {})
|
||||
before = normalize_policy(credentials)
|
||||
if action == "apply":
|
||||
template = PAYLOAD.get("selectedTemplate")
|
||||
if not isinstance(template, dict):
|
||||
raise RuntimeError("selected runtime template missing")
|
||||
desired_fields = template.get("credentials") or {}
|
||||
desired = dict(credentials)
|
||||
desired.update(desired_fields)
|
||||
after_plan = normalize_policy(desired)
|
||||
exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials
|
||||
change = "noop" if before == after_plan else ("update" if exists else "create")
|
||||
else:
|
||||
desired = dict(credentials)
|
||||
desired.pop("temp_unschedulable_enabled", None)
|
||||
desired.pop("temp_unschedulable_rules", None)
|
||||
after_plan = normalize_policy(desired)
|
||||
change = "delete" if before != after_plan or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop"
|
||||
include_plan_rules = PAYLOAD.get("full") is True
|
||||
plan = {
|
||||
"change": change,
|
||||
"account": account_summary(detail, include_plan_rules),
|
||||
"before": policy_summary(before, include_plan_rules),
|
||||
"after": policy_summary(after_plan, include_plan_rules),
|
||||
"template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None,
|
||||
"credentialsPrinted": False,
|
||||
}
|
||||
if PAYLOAD.get("confirm") is not True or change == "noop":
|
||||
return {**base, "ok": True, "operation": action, "mode": "dry-run" if PAYLOAD.get("confirm") is not True else "confirmed-noop", "mutation": False, "plan": plan}
|
||||
updated = data_of(curl_api("PUT", f"/api/v1/admin/accounts/{detail['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config")
|
||||
after_detail = updated if isinstance(updated, dict) else account_detail(token, detail["id"])
|
||||
return {**base, "ok": True, "operation": action, "mode": "confirmed", "mutation": True, "plan": plan, "account": account_summary(after_detail, include_plan_rules)}
|
||||
details = selected_account_details(token, accounts)
|
||||
plans = [mutation_plan(item, action, include_plan_rules) for item in details]
|
||||
changed = sum(1 for plan in plans if plan["change"] != "noop")
|
||||
if PAYLOAD.get("confirm") is not True:
|
||||
items = []
|
||||
for plan in plans:
|
||||
item = {key: value for key, value in plan.items() if not key.startswith("_")}
|
||||
item["write"] = {"attempted": False, "status": "dry-run", "error": None}
|
||||
item["actual"] = item["before"]
|
||||
item["reconciled"] = None
|
||||
item["reconcileError"] = None
|
||||
items.append(item)
|
||||
summary = {"selected": len(items), "changed": changed, "writeSucceeded": 0, "writeFailed": 0, "reconciled": 0, "mismatched": 0}
|
||||
return {**base, "ok": True, "operation": action, "mode": "dry-run", "mutation": False, "partialWrite": False, "summary": summary, "items": items, "credentialsPrinted": False}
|
||||
writes = [write_mutation_plan(token, plan) for plan in plans]
|
||||
items = [reconcile_mutation_plan(token, plan, write, include_plan_rules) for plan, write in zip(plans, writes)]
|
||||
write_succeeded = sum(1 for item in items if item["write"]["status"] == "succeeded")
|
||||
write_failed = sum(1 for item in items if item["write"]["status"] == "failed")
|
||||
reconciled = sum(1 for item in items if item["reconciled"] is True)
|
||||
mismatched = sum(1 for item in items if item["reconciled"] is False)
|
||||
summary = {"selected": len(items), "changed": changed, "writeSucceeded": write_succeeded, "writeFailed": write_failed, "reconciled": reconciled, "mismatched": mismatched}
|
||||
ok = write_failed == 0 and mismatched == 0
|
||||
return {**base, "ok": ok, "operation": action, "mode": "confirmed", "mutation": write_succeeded > 0, "partialWrite": write_succeeded > 0 and write_failed > 0, "summary": summary, "items": items, "credentialsPrinted": False}
|
||||
|
||||
try:
|
||||
result = runtime_result()
|
||||
|
||||
@@ -71,6 +71,15 @@ export interface TraceOptions extends DisclosureOptions {
|
||||
showLines: boolean;
|
||||
}
|
||||
|
||||
export interface FeedbackOptions {
|
||||
user: string;
|
||||
window: "5m" | "30m" | "1h" | "6h" | "24h" | "7d" | "30d";
|
||||
requestId: string | null;
|
||||
pageToken: string | null;
|
||||
json: boolean;
|
||||
targetId: string;
|
||||
}
|
||||
|
||||
export interface SentinelImageOptions extends DisclosureOptions {
|
||||
action: "status" | "build";
|
||||
confirm: boolean;
|
||||
@@ -341,8 +350,8 @@ export function codexPoolHelp(): unknown {
|
||||
const pool = readCodexPoolConfig();
|
||||
const runtimeTarget = codexPoolRuntimeTarget();
|
||||
return {
|
||||
command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
|
||||
output: "json, except trace and sentinel-report default to low-noise text tables",
|
||||
command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|faults|trace|feedback|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
|
||||
output: "json, except trace, feedback, and sentinel-report default to low-noise text tables",
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601",
|
||||
@@ -351,9 +360,13 @@ export function codexPoolHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account <name-or-id> [--target PK01] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account <name-or-id>] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool faults [--level P0|P1|P2] [--group <name-or-id>] [--account <name-or-id>] [--model <model>] [--stream sync|stream] [--endpoint <path>] [--request-id <id>] [--page-token <token>] [--target PK01] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account <name-or-id> --template <id> [--target PK01] [--confirm]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --accounts <name-or-id,...> --template <id> [--target PK01] [--confirm] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account <name-or-id> --kind temp-unschedulable [--target PK01] [--confirm]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --accounts <name-or-id,...> --kind temp-unschedulable [--target PK01] [--confirm] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id <requestId> [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user <email-or-id> [--window 30m] [--page-token <token>|--request-id <client-or-internal-id>] [--target PK01] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe [--target D601] --account unidesk-codex-hy --confirm",
|
||||
|
||||
@@ -4,6 +4,8 @@ const upstreamVersion = "v0.1.155";
|
||||
const diagnosisRuleSource = "frontend/src/views/admin/ops/components/OpsDashboardHeader.vue";
|
||||
const diagnosisTextSource = "frontend/src/i18n/locales/zh/admin/ops.ts";
|
||||
const channelHistoryPageSize = 20;
|
||||
const monitorResponseHeaderTimeoutMs = 30_000;
|
||||
const monitorTotalRequestTimeoutMs = 45_000;
|
||||
|
||||
export function projectSub2ApiOps(raw: Record<string, unknown>, options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
const target = record(raw.target);
|
||||
@@ -25,6 +27,7 @@ export function projectSub2ApiOps(raw: Record<string, unknown>, options: Sub2Api
|
||||
}
|
||||
|
||||
function projectDiagnosis(target: Record<string, unknown>, data: Record<string, unknown>, options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
const runtimeVersion = record(data.runtimeVersion);
|
||||
const groups = arrayRecords(data.groups).map((entry) => {
|
||||
const group = record(entry.group);
|
||||
const overview = record(entry.overview);
|
||||
@@ -53,7 +56,7 @@ function projectDiagnosis(target: Record<string, unknown>, data: Record<string,
|
||||
action: "platform-infra-sub2api-ops-diagnosis",
|
||||
target,
|
||||
query: querySummary(options),
|
||||
source: sourceSummary("diagnosis", "available"),
|
||||
source: sourceSummary("diagnosis", "available", runtimeVersion),
|
||||
groups,
|
||||
selectedDiagnosis,
|
||||
evidence: options.diagnosisId === null ? null : projectEvidence(record(data.evidence)),
|
||||
@@ -144,6 +147,7 @@ function projectEvidence(evidence: Record<string, unknown>): Record<string, unkn
|
||||
}
|
||||
|
||||
function projectChannels(target: Record<string, unknown>, data: Record<string, unknown>, options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
const runtimeVersion = record(data.runtimeVersion);
|
||||
const channels = arrayRecords(data.channels).map((entry) => projectChannel(entry, options.window));
|
||||
const operational = channels.every((channel) => channel.status === "OPERATIONAL");
|
||||
const history = arrayRecords(data.history)
|
||||
@@ -157,14 +161,15 @@ function projectChannels(target: Record<string, unknown>, data: Record<string, u
|
||||
checkedAt: item.checked_at ?? null,
|
||||
}))
|
||||
.sort((left, right) => String(left.checkedAt).localeCompare(String(right.checkedAt)));
|
||||
const historyProjection = options.channel === null ? null : paginateHistory(history, options);
|
||||
const historyProjection = options.channel === null ? null : paginateHistory(history, options, record(data.correlation), runtimeVersion);
|
||||
const historyError = historyProjection === null ? null : historyProjection.error;
|
||||
return {
|
||||
ok: historyError === null,
|
||||
action: "platform-infra-sub2api-ops-channels",
|
||||
target,
|
||||
query: querySummary(options),
|
||||
source: sourceSummary("channels", "available"),
|
||||
source: sourceSummary("channels", "available", runtimeVersion),
|
||||
timeoutPolicy: monitorTimeoutPolicy(runtimeVersion),
|
||||
overallStatus: operational ? "OPERATIONAL" : "DEGRADED",
|
||||
channels,
|
||||
history: historyProjection,
|
||||
@@ -173,9 +178,10 @@ function projectChannels(target: Record<string, unknown>, data: Record<string, u
|
||||
};
|
||||
}
|
||||
|
||||
function paginateHistory(history: Record<string, unknown>[], options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
function paginateHistory(history: Record<string, unknown>[], options: Sub2ApiOpsOptions, correlation: Record<string, unknown>, runtimeVersion: Record<string, unknown>): Record<string, unknown> {
|
||||
if (options.recordId !== null) {
|
||||
const selectedRecord = history.find((item) => String(item.id) === options.recordId) ?? null;
|
||||
const selectedRecordRaw = history.find((item) => String(item.id) === options.recordId) ?? null;
|
||||
const selectedRecord = selectedRecordRaw === null ? null : projectSelectedRecord(selectedRecordRaw, correlation, runtimeVersion);
|
||||
return {
|
||||
order: "PAST_TO_NOW",
|
||||
pageSize: channelHistoryPageSize,
|
||||
@@ -221,6 +227,53 @@ function paginateHistory(history: Record<string, unknown>[], options: Sub2ApiOps
|
||||
};
|
||||
}
|
||||
|
||||
function projectSelectedRecord(item: Record<string, unknown>, correlation: Record<string, unknown>, runtimeVersion: Record<string, unknown>): Record<string, unknown> {
|
||||
const latencyMs = typeof item.latencyMs === "number" ? item.latencyMs : null;
|
||||
const selected = record(correlation.selected);
|
||||
const final = record(selected.final);
|
||||
const failovers = arrayRecords(selected.failovers);
|
||||
const selectFailures = arrayRecords(selected.selectFailures);
|
||||
const totalLatencyMs = typeof final.latencyMs === "number" ? final.latencyMs : null;
|
||||
const runtimeTag = typeof runtimeVersion.version === "string" && runtimeVersion.version.length > 0 ? runtimeVersion.version : null;
|
||||
const runtimeVersionMismatch = runtimeTag?.replace(/^v/u, "") !== upstreamVersion.replace(/^v/u, "");
|
||||
return {
|
||||
...item,
|
||||
timeout: {
|
||||
observedKind: String(item.message ?? "").toLowerCase().includes("timeout awaiting response headers") ? "response-header" : "unknown",
|
||||
responseHeaderTimeoutMs: monitorResponseHeaderTimeoutMs,
|
||||
totalRequestTimeoutMs: monitorTotalRequestTimeoutMs,
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersion: runtimeTag,
|
||||
runtimeVersionMismatch,
|
||||
runtimePolicyStatus: runtimeVersionMismatch ? "unverified" : "source-version-verified",
|
||||
remainingAfterResponseHeaderMs: monitorTotalRequestTimeoutMs - monitorResponseHeaderTimeoutMs,
|
||||
observedLatencyDeltaMs: latencyMs === null ? null : latencyMs - monitorResponseHeaderTimeoutMs,
|
||||
perUpstreamTimeout: { status: "unsupported", reason: "native monitor history does not expose per-upstream timeout" },
|
||||
overallGatewayDeadline: { status: "unsupported", reason: "native monitor history does not expose the gateway request deadline" },
|
||||
failoverReserve: { status: "unsupported", reason: "native monitor history does not expose a dedicated failover reserve" },
|
||||
},
|
||||
correlation: {
|
||||
status: correlation.status ?? "unsupported",
|
||||
reason: correlation.reason ?? null,
|
||||
requestId: selected.requestId ?? null,
|
||||
match: selected.match ?? null,
|
||||
apiKeyName: selected.apiKeyName ?? null,
|
||||
model: selected.model ?? null,
|
||||
accounts: selected.accounts ?? [],
|
||||
upstreamErrors: selected.upstreamErrors ?? [],
|
||||
failovers,
|
||||
selectFailures,
|
||||
final: Object.keys(final).length === 0 ? null : final,
|
||||
switchingObserved: failovers.length > 0,
|
||||
selectionFailureObserved: selectFailures.length > 0,
|
||||
sufficientForObservedCompletion: runtimeVersionMismatch || totalLatencyMs === null ? null : monitorTotalRequestTimeoutMs >= totalLatencyMs,
|
||||
sufficiencyStatus: runtimeVersionMismatch ? "unverified-runtime-version-mismatch" : totalLatencyMs === null ? "unsupported-no-final-latency" : "evaluated-against-verified-source-version",
|
||||
traceNext: selected.requestId ? `bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id ${String(selected.requestId)}` : null,
|
||||
candidates: correlation.status === "ambiguous" ? correlation.candidates ?? [] : [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function projectChannel(entry: Record<string, unknown>, window: "7d" | "15d" | "30d"): Record<string, unknown> {
|
||||
const summary = record(entry.summary);
|
||||
const detail = record(entry.detail);
|
||||
@@ -261,7 +314,33 @@ function projectChannel(entry: Record<string, unknown>, window: "7d" | "15d" | "
|
||||
collectedAt: timeline.length === 0 ? null : timeline[timeline.length - 1]?.checkedAt ?? null,
|
||||
refreshRemainingSeconds: null,
|
||||
refreshSourceStatus: "unsupported-by-native-response",
|
||||
timeline: { order: "PAST_TO_NOW", records: timeline },
|
||||
timeline: {
|
||||
order: "PAST_TO_NOW",
|
||||
recordCount: timeline.length,
|
||||
firstAt: timeline[0]?.checkedAt ?? null,
|
||||
lastAt: timeline[timeline.length - 1]?.checkedAt ?? null,
|
||||
records: [],
|
||||
detail: "use --channel <id-or-name> for native history pagination",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function monitorTimeoutPolicy(runtimeVersion: Record<string, unknown>): Record<string, unknown> {
|
||||
const runtimeTag = typeof runtimeVersion.version === "string" && runtimeVersion.version.length > 0 ? runtimeVersion.version : null;
|
||||
const runtimeVersionMismatch = runtimeTag?.replace(/^v/u, "") !== upstreamVersion.replace(/^v/u, "");
|
||||
return {
|
||||
runtime: { image: runtimeVersion.image ?? null, version: runtimeTag },
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersionMismatch,
|
||||
runtimePolicyStatus: runtimeVersionMismatch ? "unverified" : "source-version-verified",
|
||||
sourceReference: {
|
||||
responseHeaderTimeoutMs: monitorResponseHeaderTimeoutMs,
|
||||
totalRequestTimeoutMs: monitorTotalRequestTimeoutMs,
|
||||
sourceStatus: "official-source-constant-verified-for-reference-version",
|
||||
},
|
||||
perUpstreamTimeout: { status: "unsupported", reason: "native monitor API does not expose this field" },
|
||||
overallGatewayDeadline: { status: "unsupported", reason: "native monitor API does not expose this field" },
|
||||
failoverReserve: { status: "unsupported", reason: "native monitor API does not expose this field" },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -269,11 +348,12 @@ function availabilityFor(model: Record<string, unknown>, window: "7d" | "15d" |
|
||||
return model[`availability_${window}`] ?? model.availability ?? null;
|
||||
}
|
||||
|
||||
function sourceSummary(action: "diagnosis" | "channels", status: string): Record<string, unknown> {
|
||||
function sourceSummary(action: "diagnosis" | "channels", status: string, runtimeVersion: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
if (action === "diagnosis") {
|
||||
return {
|
||||
status,
|
||||
upstreamVersion,
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersion,
|
||||
metricsEndpoint: "/api/v1/admin/ops/dashboard/overview",
|
||||
nativeDiagnosisEndpoint: { status: "unsupported", reason: "v0.1.155 has no diagnosis/advice response schema" },
|
||||
projectionKind: "native-frontend-projection",
|
||||
@@ -284,10 +364,12 @@ function sourceSummary(action: "diagnosis" | "channels", status: string): Record
|
||||
}
|
||||
return {
|
||||
status,
|
||||
upstreamVersion,
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersion,
|
||||
listEndpoint: "/api/v1/channel-monitors",
|
||||
detailEndpoint: "/api/v1/channel-monitors/:id/status",
|
||||
historyEndpoint: "/api/v1/admin/channel-monitors/:id/history",
|
||||
correlationSource: "heuristic projection by native record checked_at, latency_ms and model; request/failover facts come from request rows and bounded runtime logs when available",
|
||||
windows: ["7d", "15d", "30d"],
|
||||
overallStatusKind: "native-frontend-projection",
|
||||
refreshRemaining: { status: "unsupported", reason: "native response does not include a refresh countdown" },
|
||||
|
||||
@@ -51,6 +51,7 @@ import base64
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
|
||||
TARGET = ${pyJson(target)}
|
||||
@@ -59,6 +60,18 @@ OPTIONS = ${pyJson(options)}
|
||||
def run(command, input_bytes=None):
|
||||
return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def docker(args):
|
||||
proc = run(["docker", *args])
|
||||
if proc.returncode == 0:
|
||||
return proc
|
||||
sudo_proc = run(["sudo", "-n", "docker", *args])
|
||||
return sudo_proc if sudo_proc.returncode == 0 else proc
|
||||
|
||||
def runtime_logs(since="24h", tail=20000):
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
return docker(["logs", f"--since={since}", f"--tail={tail}", "sub2api-app"])
|
||||
return run(["kubectl", "-n", TARGET["namespace"], "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"])
|
||||
|
||||
def tail_text(value, limit=800):
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", errors="replace")
|
||||
@@ -103,6 +116,17 @@ def select_app_pod():
|
||||
|
||||
APP_POD = select_app_pod()
|
||||
|
||||
def runtime_version():
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
proc = docker(["inspect", "--format", "{{.Config.Image}}", "sub2api-app"])
|
||||
image = proc.stdout.decode("utf-8", errors="replace").strip() if proc.returncode == 0 else ""
|
||||
else:
|
||||
pod = kube_json(["-n", TARGET["namespace"], "get", "pod", APP_POD], "sub2api pod")
|
||||
containers = ((pod.get("spec") or {}).get("containers") or [])
|
||||
image = containers[0].get("image") if containers else ""
|
||||
tag = image.rsplit(":", 1)[1] if isinstance(image, str) and ":" in image else None
|
||||
return {"image": image or None, "version": tag}
|
||||
|
||||
def config_value(key):
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
return read_host_env().get(key)
|
||||
@@ -199,6 +223,116 @@ def find_named(items, selector, label):
|
||||
raise RuntimeError(f"unknown {label} {selector}; available={available}")
|
||||
return matches
|
||||
|
||||
def log_item(line):
|
||||
json_start = line.find("{")
|
||||
if json_start < 0:
|
||||
return None
|
||||
try:
|
||||
item = json.loads(line[json_start:])
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
prefix = line[:json_start].strip().split()
|
||||
item["_at"] = prefix[0] if prefix else None
|
||||
item["_message"] = " ".join(prefix[3:]) if len(prefix) >= 4 else " ".join(prefix[2:])
|
||||
return item
|
||||
|
||||
def epoch_of(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def account_names(token, platform):
|
||||
params = {"page": 1, "page_size": 500}
|
||||
if platform:
|
||||
params["platform"] = platform
|
||||
rows = items_of(get(token, "/api/v1/admin/accounts", params, "list accounts for monitor correlation"))
|
||||
return {item.get("id"): item.get("name") for item in rows if isinstance(item, dict) and item.get("id") is not None}
|
||||
|
||||
def correlate_record(token, record, monitor):
|
||||
checked_at = record.get("checked_at")
|
||||
checked_epoch = epoch_of(checked_at)
|
||||
latency_ms = record.get("latency_ms")
|
||||
if checked_epoch is None or not isinstance(latency_ms, (int, float)):
|
||||
return {"status": "unsupported", "reason": "native record has no usable checked_at/latency_ms", "candidates": []}
|
||||
expected_start = checked_epoch - latency_ms / 1000.0
|
||||
record_model = str(record.get("model") or "")
|
||||
request_rows = items_of(get(token, "/api/v1/admin/ops/requests", {"time_range": "24h", "page": 1, "page_size": 500, "kind": "all"}, "list request correlation candidates"))
|
||||
request_candidates = []
|
||||
for item in request_rows:
|
||||
request_id = item.get("request_id")
|
||||
created_epoch = epoch_of(item.get("created_at"))
|
||||
model_match = not record_model or str(item.get("model") or "") == record_model
|
||||
if not isinstance(request_id, str) or not request_id or created_epoch is None or not model_match:
|
||||
continue
|
||||
start_delta_ms = round(abs(created_epoch - expected_start) * 1000)
|
||||
if start_delta_ms <= 15000:
|
||||
request_candidates.append((request_id, item, start_delta_ms))
|
||||
proc = runtime_logs()
|
||||
names = account_names(token, monitor.get("provider"))
|
||||
grouped = {}
|
||||
candidate_ids = {item[0] for item in request_candidates}
|
||||
for line in proc.stdout.decode("utf-8", errors="replace").splitlines() if proc.returncode == 0 else []:
|
||||
item = log_item(line)
|
||||
if item is None:
|
||||
continue
|
||||
request_id = item.get("request_id")
|
||||
at_epoch = epoch_of(item.get("_at"))
|
||||
if not isinstance(request_id, str) or not request_id:
|
||||
continue
|
||||
if request_id not in candidate_ids and (at_epoch is None or at_epoch < expected_start - 15 or at_epoch > checked_epoch + 90):
|
||||
continue
|
||||
grouped.setdefault(request_id, []).append(item)
|
||||
candidates = []
|
||||
rows_by_id = {item[0]: (item[1], item[2]) for item in request_candidates}
|
||||
all_request_ids = set(grouped.keys()) | set(rows_by_id.keys())
|
||||
for request_id in all_request_ids:
|
||||
events = grouped.get(request_id, [])
|
||||
events.sort(key=lambda item: epoch_of(item.get("_at")) or 0)
|
||||
request_row, row_delta_ms = rows_by_id.get(request_id, ({}, None))
|
||||
first = events[0] if events else None
|
||||
models = [str(item.get("model")) for item in events if item.get("model")]
|
||||
first_epoch = epoch_of(first.get("_at")) if first is not None else epoch_of(request_row.get("created_at"))
|
||||
start_delta_ms = row_delta_ms if row_delta_ms is not None else round(abs((first_epoch or expected_start) - expected_start) * 1000)
|
||||
model_match = not record_model or record_model in models or str(request_row.get("model") or "") == record_model
|
||||
if start_delta_ms > 15000 or not model_match:
|
||||
continue
|
||||
failovers = [item for item in events if "upstream_failover_switching" in str(item.get("_message") or "")]
|
||||
select_failures = [item for item in events if "account_select_failed" in str(item.get("_message") or "")]
|
||||
upstream_errors = [item for item in events if "account_upstream_error" in str(item.get("_message") or "")]
|
||||
finals = [item for item in events if "http request completed" in str(item.get("_message") or "")]
|
||||
final = finals[-1] if finals else None
|
||||
account_ids = []
|
||||
for item in events:
|
||||
account_id = item.get("account_id")
|
||||
if isinstance(account_id, str) and account_id.isdigit():
|
||||
account_id = int(account_id)
|
||||
if isinstance(account_id, int) and account_id not in account_ids:
|
||||
account_ids.append(account_id)
|
||||
candidates.append({
|
||||
"requestId": request_id,
|
||||
"match": {"kind": "native-request-start-window", "startDeltaMs": start_delta_ms, "modelMatched": model_match},
|
||||
"firstAt": first.get("_at") if first is not None else request_row.get("created_at"),
|
||||
"lastAt": events[-1].get("_at") if events else request_row.get("created_at"),
|
||||
"model": next((item for item in models if item), request_row.get("model")),
|
||||
"apiKeyName": next((item.get("api_key_name") for item in events if item.get("api_key_name")), None),
|
||||
"accounts": [{"id": account_id, "name": names.get(account_id)} for account_id in account_ids] or ([{"id": request_row.get("account_id"), "name": request_row.get("account_name") or names.get(request_row.get("account_id"))}] if request_row.get("account_id") is not None else []),
|
||||
"upstreamErrors": [{"at": item.get("_at"), "accountId": item.get("account_id"), "error": item.get("error")} for item in upstream_errors],
|
||||
"failovers": [{"at": item.get("_at"), "accountId": item.get("account_id"), "upstreamStatus": item.get("upstream_status"), "switchCount": item.get("switch_count"), "maxSwitches": item.get("max_switches")} for item in failovers],
|
||||
"selectFailures": [{"at": item.get("_at"), "error": item.get("error"), "excludedAccountCount": item.get("excluded_account_count")} for item in select_failures],
|
||||
"final": {"at": request_row.get("created_at"), "statusCode": request_row.get("status_code") or request_row.get("upstream_status_code"), "accountId": request_row.get("account_id"), "latencyMs": request_row.get("duration_ms"), "path": request_row.get("path")} if final is None else {"at": final.get("_at"), "statusCode": final.get("status_code"), "accountId": final.get("account_id"), "latencyMs": final.get("latency_ms"), "path": final.get("path")},
|
||||
})
|
||||
candidates.sort(key=lambda item: item["match"]["startDeltaMs"])
|
||||
if not candidates:
|
||||
return {"status": "unsupported", "reason": "no unique native request_id is available in the record; bounded log correlation found no candidate", "candidates": []}
|
||||
best_delta = candidates[0]["match"]["startDeltaMs"]
|
||||
best = [item for item in candidates if item["match"]["startDeltaMs"] == best_delta]
|
||||
return {"status": "inferred" if len(best) == 1 else "ambiguous", "reason": "heuristic correlation by record start window and model; the native monitor record has no request_id" if len(best) == 1 else "multiple requests have the same nearest start time", "selected": best[0] if len(best) == 1 else None, "candidates": candidates[:5]}
|
||||
|
||||
def diagnosis(token):
|
||||
group_params = {"platform": OPTIONS["platform"]} if OPTIONS.get("platform") else None
|
||||
groups = items_of(get(token, "/api/v1/admin/groups/all", group_params, "list groups"))
|
||||
@@ -247,17 +381,24 @@ def channels(token):
|
||||
detail = get(token, f"/api/v1/channel-monitors/{monitor['id']}/status", None, "channel monitor status")
|
||||
output.append({"summary": monitor, "detail": detail})
|
||||
history = None
|
||||
correlation = None
|
||||
if OPTIONS.get("channel") and output:
|
||||
monitor_id = output[0]["summary"]["id"]
|
||||
params = {"limit": 1000}
|
||||
if OPTIONS.get("model"):
|
||||
params["model"] = OPTIONS["model"]
|
||||
history = items_of(get(token, f"/api/v1/admin/channel-monitors/{monitor_id}/history", params, "channel monitor history"))
|
||||
return {"channels": output, "history": history}
|
||||
record_id = OPTIONS.get("recordId")
|
||||
if record_id:
|
||||
selected = next((item for item in history if str(item.get("id")) == str(record_id)), None)
|
||||
if selected is not None:
|
||||
correlation = correlate_record(token, selected, output[0]["summary"])
|
||||
return {"channels": output, "history": history, "correlation": correlation}
|
||||
|
||||
try:
|
||||
token = login()
|
||||
data = diagnosis(token) if OPTIONS["action"] == "diagnosis" else channels(token)
|
||||
data["runtimeVersion"] = runtime_version()
|
||||
print(json.dumps({"ok": True, "sourceStatus": "available", "data": data}, ensure_ascii=False, separators=(",", ":")))
|
||||
except Exception as error:
|
||||
print(json.dumps({"ok": False, "sourceStatus": "unavailable", "error": str(error), "data": None}, ensure_ascii=False, separators=(",", ":")))
|
||||
|
||||
@@ -125,6 +125,12 @@ function renderChannels(result: Record<string, unknown>): string[] {
|
||||
}
|
||||
const selectedRecord = record(history.selectedRecord);
|
||||
if (Object.keys(selectedRecord).length > 0) {
|
||||
const timeout = record(selectedRecord.timeout);
|
||||
const correlation = record(selectedRecord.correlation);
|
||||
const final = record(correlation.final);
|
||||
const accounts = arrayRecords(correlation.accounts);
|
||||
const failovers = arrayRecords(correlation.failovers);
|
||||
const selectFailures = arrayRecords(correlation.selectFailures);
|
||||
lines.push(
|
||||
"",
|
||||
"RECORD",
|
||||
@@ -136,8 +142,25 @@ function renderChannels(result: Record<string, unknown>): string[] {
|
||||
["pingLatencyMs", stringValue(selectedRecord.pingLatencyMs)],
|
||||
["checkedAt", stringValue(selectedRecord.checkedAt)],
|
||||
["message", stringValue(selectedRecord.message)],
|
||||
["runtimeVersion", stringValue(timeout.runtimeVersion)],
|
||||
["sourceReferenceVersion", stringValue(timeout.sourceReferenceVersion)],
|
||||
["runtimePolicyStatus", stringValue(timeout.runtimePolicyStatus)],
|
||||
["sourceResponseHeaderMs", stringValue(timeout.responseHeaderTimeoutMs)],
|
||||
["sourceTotalRequestMs", stringValue(timeout.totalRequestTimeoutMs)],
|
||||
["remainingAfterHeaderMs", stringValue(timeout.remainingAfterResponseHeaderMs)],
|
||||
["correlationStatus", stringValue(correlation.status)],
|
||||
["requestId", stringValue(correlation.requestId)],
|
||||
["accounts", accounts.map((item) => `${stringValue(item.id)}:${stringValue(item.name)}`).join(",") || "-"],
|
||||
["failoverCount", String(failovers.length)],
|
||||
["selectFailureCount", String(selectFailures.length)],
|
||||
["finalStatus", stringValue(final.statusCode)],
|
||||
["finalLatencyMs", stringValue(final.latencyMs)],
|
||||
["timeoutSufficient", stringValue(correlation.sufficientForObservedCompletion)],
|
||||
["sufficiencyStatus", stringValue(correlation.sufficiencyStatus)],
|
||||
]),
|
||||
);
|
||||
if (correlation.traceNext) lines.push(`Trace: ${stringValue(correlation.traceNext)}`);
|
||||
if (correlation.reason) lines.push(`Correlation: ${stringValue(correlation.reason)}`);
|
||||
}
|
||||
lines.push(
|
||||
"",
|
||||
|
||||
@@ -423,6 +423,7 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <requestId>",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user <email-or-id> [--window 30m] [--page-token <token>|--request-id <client-or-internal-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2rank plan [--target NC01]",
|
||||
@@ -505,6 +506,7 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <requestId>",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user <email-or-id> [--window 30m] [--page-token <token>|--request-id <client-or-internal-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
],
|
||||
module: "scripts/src/platform-infra-sub2api-codex.ts",
|
||||
|
||||
@@ -33,6 +33,12 @@ export interface SelfMediaDeliveryTarget {
|
||||
readonly dockerfile: string;
|
||||
readonly imageRepository: string;
|
||||
readonly networkMode: string;
|
||||
readonly envReuse: {
|
||||
readonly enabled: true;
|
||||
readonly mode: "buildkit-state-host-path";
|
||||
readonly statePath: string;
|
||||
readonly identityFiles: readonly string[];
|
||||
};
|
||||
readonly proxy: Record<string, unknown>;
|
||||
};
|
||||
readonly gitops: Record<string, unknown> & {
|
||||
@@ -151,6 +157,7 @@ function parseDeliveryTarget(id: string, value: Record<string, unknown>): SelfMe
|
||||
const ci = record(value.ci, `${path}.ci`);
|
||||
const source = record(value.source, `${path}.source`);
|
||||
const build = record(value.build, `${path}.build`);
|
||||
const envReuse = record(build.envReuse, `${path}.build.envReuse`);
|
||||
const proxy = record(build.proxy, `${path}.build.proxy`);
|
||||
const gitops = record(value.gitops, `${path}.gitops`);
|
||||
const deployment = record(value.deployment, `${path}.deployment`);
|
||||
@@ -168,6 +175,12 @@ function parseDeliveryTarget(id: string, value: Record<string, unknown>): SelfMe
|
||||
validateAccessSecret(deployment, `${path}.deployment`);
|
||||
const gitopsWriteUrl = text(gitops.writeUrl, `${path}.gitops.writeUrl`);
|
||||
if (!gitopsWriteUrl.startsWith("http://gitea-http.")) throw new Error(`${path}.gitops.writeUrl must use internal Gitea authority`);
|
||||
if (boolean(envReuse.enabled, `${path}.build.envReuse.enabled`) !== true) throw new Error(`${path}.build.envReuse.enabled must be true`);
|
||||
const envReuseMode = text(envReuse.mode, `${path}.build.envReuse.mode`);
|
||||
if (envReuseMode !== "buildkit-state-host-path") throw new Error(`${path}.build.envReuse.mode must be buildkit-state-host-path`);
|
||||
const identityFiles = stringArray(envReuse.identityFiles, `${path}.build.envReuse.identityFiles`)
|
||||
.map((item, index) => relativePath(item, `${path}.build.envReuse.identityFiles[${index}]`));
|
||||
if (identityFiles.length === 0 || new Set(identityFiles).size !== identityFiles.length) throw new Error(`${path}.build.envReuse.identityFiles must be non-empty and unique`);
|
||||
return {
|
||||
id,
|
||||
node,
|
||||
@@ -197,6 +210,12 @@ function parseDeliveryTarget(id: string, value: Record<string, unknown>): SelfMe
|
||||
dockerfile: relativePath(build.dockerfile, `${path}.build.dockerfile`),
|
||||
imageRepository: text(build.imageRepository, `${path}.build.imageRepository`),
|
||||
networkMode: text(build.networkMode, `${path}.build.networkMode`),
|
||||
envReuse: {
|
||||
enabled: true,
|
||||
mode: "buildkit-state-host-path",
|
||||
statePath: absolutePath(envReuse.statePath, `${path}.build.envReuse.statePath`),
|
||||
identityFiles,
|
||||
},
|
||||
proxy: structuredClone(proxy),
|
||||
},
|
||||
gitops: {
|
||||
|
||||
@@ -99,7 +99,10 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin
|
||||
params: taskParams,
|
||||
volumes: [
|
||||
{ name: "workspace", emptyDir: { sizeLimit: target.ci.workspaceSize } },
|
||||
{ name: "buildkit-state", emptyDir: { sizeLimit: "12Gi" } },
|
||||
{
|
||||
name: "buildkit-state",
|
||||
hostPath: { path: target.build.envReuse.statePath, type: "DirectoryOrCreate" },
|
||||
},
|
||||
{ name: "tmp", emptyDir: { sizeLimit: "4Gi" } },
|
||||
{
|
||||
name: "gitops-token",
|
||||
@@ -113,6 +116,7 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin
|
||||
steps: [
|
||||
sourceStep(target, effectiveDeploymentB64),
|
||||
prepareBuildkitStep(target),
|
||||
envReuseStep(target),
|
||||
imageBuildStep(target),
|
||||
gitOpsPublishStep(target),
|
||||
],
|
||||
@@ -137,6 +141,7 @@ function sourceStep(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: str
|
||||
],
|
||||
script: `#!/bin/sh
|
||||
set -eu
|
||||
stage_started_seconds=$(date +%s)
|
||||
source_commit="$(params.revision)"
|
||||
snapshot_prefix="$(params.source-snapshot-prefix)"
|
||||
rm -rf /workspace/source /workspace/release
|
||||
@@ -167,6 +172,8 @@ bun deploy/scripts/prepare-build.ts \\
|
||||
--source-commit "$source_commit" \\
|
||||
--source-snapshot-prefix "$snapshot_prefix" \\
|
||||
--output-dir /workspace/release
|
||||
stage_finished_seconds=$(date +%s)
|
||||
printf '{"ok":true,"phase":"source-prepare","stageTimings":{"sourcePrepareSeconds":%s},"valuesPrinted":false}\n' "$((stage_finished_seconds-stage_started_seconds))"
|
||||
`,
|
||||
securityContext: nonRootSecurity(),
|
||||
volumeMounts: [
|
||||
@@ -182,12 +189,48 @@ function prepareBuildkitStep(target: SelfMediaDeliveryTarget): Record<string, un
|
||||
name: "prepare-buildkit-state",
|
||||
image: target.ci.toolImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
script: "#!/bin/sh\nset -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit\n",
|
||||
script: "#!/bin/sh\nset -eu\nmkdir -p /home/user/.local/share/buildkit/.unidesk\nchown 1000:1000 /home/user/.local/share/buildkit /home/user/.local/share/buildkit/.unidesk\n",
|
||||
securityContext: { runAsUser: 0, runAsGroup: 0 },
|
||||
volumeMounts: [{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }],
|
||||
};
|
||||
}
|
||||
|
||||
function envReuseStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
|
||||
const identityFiles = target.build.envReuse.identityFiles.map((path) => `/workspace/source/${path}`);
|
||||
const identityInputs = [...identityFiles, "/workspace/release/codex-version"]
|
||||
.map(shellSingleQuote)
|
||||
.join(" ");
|
||||
return {
|
||||
name: "env-reuse",
|
||||
image: target.ci.toolImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
script: `#!/bin/sh
|
||||
set -eu
|
||||
stage_started_seconds=$(date +%s)
|
||||
identity_file=/home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity
|
||||
for input in ${identityInputs}; do test -f "$input"; done
|
||||
env_identity="sha256:$(sha256sum ${identityInputs} | sha256sum | awk '{print $1}')"
|
||||
previous_identity=""
|
||||
if [ -f "$identity_file" ]; then previous_identity=$(cat "$identity_file"); fi
|
||||
if [ "$previous_identity" = "$env_identity" ]; then
|
||||
reuse_status=hit
|
||||
previous_present=true
|
||||
else
|
||||
reuse_status=miss
|
||||
if [ -n "$previous_identity" ]; then previous_present=true; else previous_present=false; fi
|
||||
fi
|
||||
printf '%s\n' "$env_identity" > /workspace/env-identity
|
||||
stage_finished_seconds=$(date +%s)
|
||||
printf '{"ok":true,"phase":"env-reuse","envIdentity":"%s","envReuseStatus":"%s","envReuse":{"status":"%s","mode":"${target.build.envReuse.mode}","identitySource":"owning-yaml-source-artifact","previousIdentityPresent":%s},"stageTimings":{"envIdentitySeconds":%s},"valuesPrinted":false}\n' "$env_identity" "$reuse_status" "$reuse_status" "$previous_present" "$((stage_finished_seconds-stage_started_seconds))"
|
||||
`,
|
||||
securityContext: nonRootSecurity(),
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function imageBuildStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
|
||||
return {
|
||||
name: "image-build",
|
||||
@@ -198,7 +241,14 @@ function imageBuildStep(target: SelfMediaDeliveryTarget): Record<string, unknown
|
||||
{ name: "SOURCE_COMMIT", value: "$(params.revision)" },
|
||||
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
||||
],
|
||||
script: "#!/bin/sh\nset -eu\nexec /bin/sh /workspace/source/deploy/scripts/build-image.sh\n",
|
||||
script: `#!/bin/sh
|
||||
set -eu
|
||||
stage_started_seconds=$(date +%s)
|
||||
/bin/sh /workspace/source/deploy/scripts/build-image.sh
|
||||
cp /workspace/env-identity /home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity
|
||||
stage_finished_seconds=$(date +%s)
|
||||
printf '{"ok":true,"phase":"image-build-complete","envIdentity":"%s","stageTimings":{"imageBuildSeconds":%s},"valuesPrinted":false}\n' "$(cat /workspace/env-identity)" "$((stage_finished_seconds-stage_started_seconds))"
|
||||
`,
|
||||
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
@@ -217,6 +267,7 @@ function gitOpsPublishStep(target: SelfMediaDeliveryTarget): Record<string, unkn
|
||||
env: [{ name: "SOURCE_COMMIT", value: "$(params.revision)" }],
|
||||
script: `#!/bin/sh
|
||||
set -eu
|
||||
stage_started_seconds=$(date +%s)
|
||||
askpass=/tmp/selfmedia-git-askpass
|
||||
cat >"$askpass" <<'ASKPASS'
|
||||
#!/bin/sh
|
||||
@@ -237,6 +288,8 @@ bun /workspace/source/deploy/scripts/publish-gitops.ts \\
|
||||
--source-commit "$SOURCE_COMMIT" \\
|
||||
--worktree /workspace/selfmedia-gitops
|
||||
rm -f "$askpass"
|
||||
stage_finished_seconds=$(date +%s)
|
||||
printf '{"ok":true,"phase":"gitops-stage","stageTimings":{"gitopsPublishSeconds":%s},"valuesPrinted":false}\n' "$((stage_finished_seconds-stage_started_seconds))"
|
||||
`,
|
||||
securityContext: nonRootSecurity(),
|
||||
volumeMounts: [
|
||||
@@ -366,3 +419,7 @@ function requiredInteger(value: unknown, path: string): number {
|
||||
if (!Number.isInteger(value)) throw new Error(`${path} must be an integer`);
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function shellSingleQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user