Codex
720 lines
30 KiB
TypeScript
720 lines
30 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/platform-infra-sub2api-codex.ts.
|
|
|
|
// Moved mechanically from scripts/src/platform-infra-sub2api-codex.ts:2701-3096 for #903.
|
|
|
|
import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
import { homedir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { UniDeskConfig } from "../config";
|
|
import { rootPath } from "../config";
|
|
import type { RenderedCliResult } from "../output";
|
|
import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest, type PublicServiceExposure, type PublicServiceTarget } from "../platform-infra-public-service";
|
|
import { shortSha256Fingerprint } from "../platform-infra-ops-library";
|
|
import {
|
|
codexPoolSentinelSummary,
|
|
codexPoolSentinelRuntimeImage,
|
|
readCodexPoolSentinelConfig,
|
|
renderCodexPoolSentinelManifest,
|
|
type CodexPoolSentinelConfig,
|
|
type CodexPoolSentinelProfileSecret,
|
|
} from "../platform-infra-sub2api-codex-sentinel";
|
|
import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets";
|
|
import { runSshCommandCapture, type SshCaptureResult } from "../ssh";
|
|
|
|
import { isRecord, numberValue, stringValue } from "./config-utils";
|
|
import { compactGatewayCompactRecent, compactGatewayModels, compactGatewayResponses, compactGatewayResponsesRecent, compactManualAccounts, compactRuntimeCapability, compactSentinelStatus, compactStatusBlock, compactTempUnschedulableStatus, pickSummaryFields, recordArray } from "./redaction";
|
|
|
|
export function renderedCliResult(ok: boolean, command: string, renderedText: string): RenderedCliResult {
|
|
return { ok, command, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
export function renderCodexPoolPlan(plan: Record<string, unknown>): RenderedCliResult {
|
|
const target = isRecord(plan.target) ? plan.target : {};
|
|
const config = isRecord(plan.config) ? plan.config : {};
|
|
const pool = isRecord(config.pool) ? config.pool : {};
|
|
const manualAccounts = isRecord(pool.manualAccounts) ? pool.manualAccounts : {};
|
|
const sentinel = isRecord(pool.sentinel) ? pool.sentinel : {};
|
|
const profiles = recordArray(plan.profiles);
|
|
const invalidProfiles = profiles.filter((profile) => profile.ok !== true);
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API CODEX POOL PLAN",
|
|
`ok=${plan.ok === true ? "true" : "false"}`,
|
|
`target=${target.id ?? "-"}`,
|
|
`profiles=${profiles.length}`,
|
|
`invalid=${invalidProfiles.length}`,
|
|
].join(" "));
|
|
lines.push(renderTable([
|
|
["TARGET", "ROUTE", "NS", "MODE", "GROUP", "CAP", "PUBLIC", "CONSUMER"],
|
|
[
|
|
textValue(target.id),
|
|
textValue(target.route),
|
|
textValue(target.namespace),
|
|
textValue(target.runtimeMode),
|
|
textValue(pool.groupName ?? target.groupName),
|
|
textValue(pool.accountCapacityTotal ?? target.accountCapacityTotal),
|
|
textValue(target.publicBaseUrl),
|
|
textValue(target.consumerBaseUrl),
|
|
],
|
|
]));
|
|
lines.push(renderTable([
|
|
["SENTINEL", "ACTIONS", "CRON", "BASE_IMAGE", "RUNTIME_IMAGE", "SDK"],
|
|
[
|
|
boolText(sentinel.monitorEnabled),
|
|
boolText(sentinel.actionsEnabled),
|
|
textValue(sentinel.cronJobName),
|
|
textValue(sentinel.image),
|
|
textValue(sentinel.runtimeImage),
|
|
textValue(isRecord(sentinel.sdk) ? sentinel.sdk.openaiPythonVersion : undefined),
|
|
],
|
|
]));
|
|
lines.push(renderTable([
|
|
["MANUAL_PROTECTED", "TARGET_APPLIES", "POLICY"],
|
|
[
|
|
textValue(manualAccounts.protectedCount),
|
|
textValue(manualAccounts.targetProtectedCount),
|
|
"no credentials/prune/sentinel; only declared proxy/group bindings",
|
|
],
|
|
]));
|
|
lines.push("");
|
|
lines.push("PROFILES");
|
|
lines.push(renderTable([
|
|
["PROFILE", "ACCOUNT", "OK", "CAP", "LF", "PRI", "TRUST", "PROT"],
|
|
...profiles.map((profile) => [
|
|
textValue(profile.profile),
|
|
textValue(profile.accountName),
|
|
boolText(profile.ok),
|
|
textValue(profile.capacity),
|
|
textValue(profile.loadFactor),
|
|
textValue(profile.priority),
|
|
boolText(profile.trustUpstream),
|
|
profile.sentinelProtectEnabled === true ? textValue(profile.sentinelProtectConsecutiveFailures) : "-",
|
|
]),
|
|
]));
|
|
if (invalidProfiles.length > 0) {
|
|
lines.push("");
|
|
lines.push("INVALID");
|
|
lines.push(renderTable([
|
|
["PROFILE", "ACCOUNT", "ERROR"],
|
|
...invalidProfiles.map((profile) => [
|
|
textValue(profile.profile),
|
|
textValue(profile.accountName),
|
|
textValue(profile.error),
|
|
]),
|
|
]));
|
|
}
|
|
const next = isRecord(plan.next) ? plan.next : {};
|
|
const nextLines = Object.entries(next)
|
|
.filter(([, value]) => typeof value === "string")
|
|
.map(([key, value]) => ` ${key}: ${value}`);
|
|
if (nextLines.length > 0) {
|
|
lines.push("");
|
|
lines.push("NEXT");
|
|
lines.push(...nextLines);
|
|
}
|
|
lines.push("");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool plan --raw");
|
|
lines.push("Full: bun scripts/cli.ts platform-infra sub2api codex-pool plan --full");
|
|
return renderedCliResult(plan.ok === true, "platform-infra sub2api codex-pool plan", lines.join("\n"));
|
|
}
|
|
|
|
export function renderCodexPoolSyncResult(result: Record<string, unknown>): RenderedCliResult {
|
|
const target = isRecord(result.target) ? result.target : {};
|
|
const local = isRecord(result.local) ? result.local : {};
|
|
const image = isRecord(result.sentinelImage) ? result.sentinelImage : {};
|
|
const imageSummary = isRecord(image.summary) ? image.summary : {};
|
|
const remote = isRecord(result.remote) ? result.remote : {};
|
|
const accounts = isRecord(remote.accounts) ? remote.accounts : {};
|
|
const lines = renderCodexPoolRemoteSummary("SUB2API CODEX POOL SYNC", result, target, remote);
|
|
lines.push(renderTable([
|
|
["LOCAL_PROFILES", "PRUNE", "IMAGE_MODE", "IMAGE", "SDK"],
|
|
[
|
|
textValue(local.profileCount),
|
|
boolText(local.pruneRemoved),
|
|
textValue(imageSummary.mode ?? image.mode),
|
|
textValue(imageSummary.image ?? image.image),
|
|
textValue(imageSummary.openaiPythonVersion),
|
|
],
|
|
]));
|
|
lines.push(renderTable([
|
|
["ACCOUNTS", "CREATED", "UPDATED", "PRUNED", "PRUNE_MODE", "ATTENTION"],
|
|
[
|
|
textValue(accounts.desired ?? accounts.itemCount),
|
|
textValue(accounts.created),
|
|
textValue(accounts.updated),
|
|
textValue(accounts.pruned),
|
|
textValue(accounts.pruneMode),
|
|
textValue(recordArray(accounts.attentionItems).length),
|
|
],
|
|
]));
|
|
appendCodexPoolCheckTable(lines, remote);
|
|
appendNext(lines, result.next);
|
|
lines.push("");
|
|
lines.push("Full: bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --full");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --raw");
|
|
return renderedCliResult(result.ok === true, "platform-infra sub2api codex-pool sync", lines.join("\n"));
|
|
}
|
|
|
|
export function renderCodexPoolValidateResult(result: Record<string, unknown>): RenderedCliResult {
|
|
const target = isRecord(result.target) ? result.target : {};
|
|
const summary = isRecord(result.summary) ? result.summary : {};
|
|
const lines = renderCodexPoolRemoteSummary("SUB2API CODEX POOL VALIDATE", result, target, summary);
|
|
appendCodexPoolCheckTable(lines, summary);
|
|
const validation = isRecord(summary.validation) ? summary.validation : {};
|
|
lines.push(renderTable([
|
|
["GATEWAY_MODELS", "GATEWAY_RESPONSES", "RESPONSES_RECENT", "COMPACT_RECENT"],
|
|
[
|
|
blockOk(validation.gatewayModels),
|
|
blockOk(validation.gatewayResponses),
|
|
blockOk(validation.gatewayResponsesRecent),
|
|
blockOk(validation.gatewayCompactRecent),
|
|
],
|
|
]));
|
|
lines.push("");
|
|
lines.push("Full: bun scripts/cli.ts platform-infra sub2api codex-pool validate --full");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool validate --raw");
|
|
return renderedCliResult(result.ok === true, "platform-infra sub2api codex-pool validate", lines.join("\n"));
|
|
}
|
|
|
|
export function renderCodexPoolSentinelProbeResult(result: Record<string, unknown>): RenderedCliResult {
|
|
const target = isRecord(result.target) ? result.target : {};
|
|
const summary = isRecord(result.summary) ? result.summary : {};
|
|
const job = isRecord(summary.job) ? summary.job : {};
|
|
const runSummary = isRecord(summary.summary) ? summary.summary : {};
|
|
const results = recordArray(summary.results);
|
|
const state = isRecord(summary.sentinelState) ? summary.sentinelState : {};
|
|
const lastRun = isRecord(state.lastRun) ? state.lastRun : {};
|
|
const runtimeSchedulable = isRecord(lastRun.runtimeSchedulable) ? lastRun.runtimeSchedulable : {};
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API CODEX POOL SENTINEL PROBE",
|
|
`ok=${result.ok === true ? "true" : "false"}`,
|
|
`target=${target.id ?? "-"}`,
|
|
`accounts=${Array.isArray(summary.requestedAccounts) ? summary.requestedAccounts.length : results.length}`,
|
|
`job=${job.status ?? "-"}`,
|
|
].join(" "));
|
|
lines.push(renderTable([
|
|
["JOB", "NS", "JOB_OK", "MARKER_OK", "LAST_RUN", "SELECTED", "OK", "TF", "ACTIONS"],
|
|
[
|
|
textValue(job.name),
|
|
textValue(summary.namespace ?? target.namespace),
|
|
boolText(summary.jobExecutionOk),
|
|
boolText(summary.markerOk),
|
|
textValue(runSummary.at ?? lastRun.at),
|
|
textValue(runSummary.selected ?? lastRun.selected),
|
|
textValue(runSummary.okCount ?? lastRun.okCount),
|
|
textValue(runSummary.transportFailureCount ?? lastRun.transportFailureCount),
|
|
textValue(runSummary.actionsTaken ?? lastRun.actionsTaken),
|
|
],
|
|
]));
|
|
lines.push("");
|
|
lines.push("RESULTS");
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "OK", "HTTP", "MARKER", "DUR_MS", "KIND", "ACTION"],
|
|
...results.map((item) => {
|
|
const action = isRecord(item.action) ? item.action : {};
|
|
return [
|
|
textValue(item.accountName),
|
|
boolText(item.ok),
|
|
textValue(item.httpStatus),
|
|
boolText(item.markerMatched),
|
|
textValue(item.durationMs),
|
|
textValue(item.failureKind),
|
|
textValue(action.type),
|
|
];
|
|
}),
|
|
]));
|
|
lines.push(renderTable([
|
|
["QUARANTINED", "SCHEDULABLE", "UNSCHEDULABLE", "MISSING"],
|
|
[
|
|
textValue(state.quarantinedCount),
|
|
textValue(runtimeSchedulable.schedulableCount),
|
|
textValue(runtimeSchedulable.unschedulableCount),
|
|
textValue(runtimeSchedulable.missingCount),
|
|
],
|
|
]));
|
|
lines.push("");
|
|
lines.push("Full: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account <name> --confirm --full");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account <name> --confirm --raw");
|
|
return renderedCliResult(result.ok === true, "platform-infra sub2api codex-pool sentinel-probe", lines.join("\n"));
|
|
}
|
|
|
|
function renderCodexPoolRemoteSummary(title: string, result: Record<string, unknown>, target: Record<string, unknown>, remote: Record<string, unknown>): string[] {
|
|
return [
|
|
[
|
|
title,
|
|
`ok=${result.ok === true ? "true" : "false"}`,
|
|
`target=${target.id ?? "-"}`,
|
|
`mode=${remote.mode ?? "-"}`,
|
|
`degraded=${remote.degraded === true ? "true" : "false"}`,
|
|
].join(" "),
|
|
renderTable([
|
|
["TARGET", "ROUTE", "NS", "SERVICE", "POD", "ADMIN", "API_KEY"],
|
|
[
|
|
textValue(target.id),
|
|
textValue(target.route),
|
|
textValue(remote.namespace ?? target.namespace),
|
|
textValue(remote.serviceDns),
|
|
textValue(remote.appPod),
|
|
blockOk(remote.admin),
|
|
blockOk(remote.apiKey),
|
|
],
|
|
]),
|
|
];
|
|
}
|
|
|
|
function appendCodexPoolCheckTable(lines: string[], remote: Record<string, unknown>): void {
|
|
const runtime = isRecord(remote.runtimeCapabilities) ? remote.runtimeCapabilities : {};
|
|
lines.push(renderTable([
|
|
["BALANCE", "CONCURRENCY", "CAPACITY", "LOAD", "TEMP_UNSCHED", "MANUAL", "SENTINEL", "RUNTIME"],
|
|
[
|
|
blockOk(remote.ownerBalance),
|
|
blockOk(remote.ownerConcurrency),
|
|
blockOk(remote.capacity),
|
|
blockOk(remote.loadFactor),
|
|
blockOk(remote.tempUnschedulable),
|
|
blockOk(remote.manualAccounts),
|
|
blockOk(remote.sentinel),
|
|
blockOk(runtime),
|
|
],
|
|
]));
|
|
}
|
|
|
|
function appendNext(lines: string[], value: unknown): void {
|
|
if (!isRecord(value)) return;
|
|
const entries = Object.entries(value).filter(([, item]) => typeof item === "string");
|
|
if (entries.length === 0) return;
|
|
lines.push("");
|
|
lines.push("NEXT");
|
|
for (const [key, item] of entries) lines.push(` ${key}: ${item}`);
|
|
}
|
|
|
|
function blockOk(value: unknown): string {
|
|
return isRecord(value) ? boolText(value.ok) : "-";
|
|
}
|
|
|
|
export function renderSentinelReport(
|
|
parsed: Record<string, unknown> | null,
|
|
context: { events: number; full: boolean; remote: Record<string, unknown> },
|
|
): string {
|
|
if (parsed === null) {
|
|
return [
|
|
"SUB2API SENTINEL REPORT unavailable",
|
|
`remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
|
|
stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
const metadata = isRecord(parsed.metadata) ? parsed.metadata : {};
|
|
const cronJob = isRecord(parsed.cronJob) ? parsed.cronJob : {};
|
|
const summary = isRecord(parsed.summary) ? parsed.summary : {};
|
|
const accounts = recordArray(parsed.accounts);
|
|
const runs = recordArray(parsed.runs);
|
|
const globalLedger = isRecord(parsed.globalLedger) ? parsed.globalLedger : {};
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API SENTINEL",
|
|
`ok=${parsed.ok === true ? "true" : "false"}`,
|
|
`accounts=${summary.accountCount ?? accounts.length}`,
|
|
`quarantined=${summary.quarantinedCount ?? "?"}`,
|
|
`schedulable=${summary.runtimeSchedulableCount ?? "?"}`,
|
|
`unschedulable=${summary.runtimeUnschedulableCount ?? "?"}`,
|
|
`history=${summary.historyCount ?? runs.length}`,
|
|
`window=${formatWindow(summary.historyFrom, summary.historyTo)}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"CRON",
|
|
`schedule=${cronJob.schedule ?? "-"}`,
|
|
`last=${shortIso(cronJob.lastScheduleTime)}`,
|
|
`active=${cronJob.active ?? "-"}`,
|
|
`state=${metadata.namespace ?? "-"}/${metadata.stateConfigMapName ?? "-"}`,
|
|
`ledger=req:${globalLedger.requestCount ?? 0} tok:${formatNumber(globalLedger.totalTokens)} cost:$${formatCost(globalLedger.estimatedCostUsd)}`,
|
|
].join(" "));
|
|
lines.push("");
|
|
lines.push("ACCOUNTS");
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "STATE", "Q", "SCH", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
|
...accounts.map((account) => [
|
|
stringValue(account.account) ?? "-",
|
|
stringValue(account.status) ?? "-",
|
|
account.quarantineActive === true ? "Y" : "-",
|
|
account.runtimeSchedulable === true ? "Y" : account.runtimeSchedulable === false ? "N" : "-",
|
|
account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-",
|
|
account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-",
|
|
account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-",
|
|
textValue(account.freezeIntervalMin),
|
|
textValue(account.successIntervalMin),
|
|
textValue(account.successMaxIntervalMin),
|
|
textValue(account.probeCount),
|
|
shortIso(account.lastEventAt ?? account.lastProbeAt),
|
|
textValue(account.lastHttp),
|
|
account.lastMarker === true ? "Y" : account.lastMarker === false ? "N" : "-",
|
|
shorten(stringValue(account.lastFailureKind) ?? "-", 20),
|
|
shorten(stringValue(account.lastAction) ?? "-", 16),
|
|
shortIso(account.nextProbeAfter),
|
|
textValue(account.observedLastToNextMin),
|
|
]),
|
|
]));
|
|
if (runs.length > 0 || context.full) {
|
|
lines.push("");
|
|
lines.push(`RUNS last=${Math.min(context.events, runs.length)}`);
|
|
lines.push(renderTable([
|
|
["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "GF", "GACT", "REASSERT"],
|
|
...runs.slice(-context.events).map((run) => [
|
|
shortIso(run.at),
|
|
textValue(run.selected),
|
|
textValue(run.due),
|
|
textValue(run.ok),
|
|
textValue(run.mismatch),
|
|
textValue(run.transportFailures),
|
|
textValue(run.actionsTaken),
|
|
textValue(run.gatewayFailures),
|
|
textValue(run.gatewayActions),
|
|
textValue(run.reasserts),
|
|
]),
|
|
]));
|
|
}
|
|
lines.push("");
|
|
lines.push("LEGEND Q=quarantined SCH=Sub2API runtime schedulable T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
export function renderTraceReport(
|
|
parsed: Record<string, unknown> | null,
|
|
context: { requestId: string; showLines: boolean; remote: Record<string, unknown> },
|
|
): string {
|
|
if (parsed === null) {
|
|
return [
|
|
`SUB2API TRACE ${context.requestId} unavailable`,
|
|
`remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
|
|
stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
const summary = isRecord(parsed.summary) ? parsed.summary : {};
|
|
const request = isRecord(parsed.request) ? parsed.request : {};
|
|
const final = isRecord(parsed.final) ? parsed.final : {};
|
|
const window = isRecord(parsed.window) ? parsed.window : {};
|
|
const events = recordArray(parsed.events);
|
|
const failovers = recordArray(parsed.failovers);
|
|
const selectFailures = recordArray(parsed.selectFailures);
|
|
const upstreamErrors = recordArray(parsed.upstreamErrors);
|
|
const forwardFailures = recordArray(parsed.forwardFailures);
|
|
const tempUnschedulable = recordArray(parsed.tempUnschedulable);
|
|
const windowStats = isRecord(parsed.windowStats) ? parsed.windowStats : {};
|
|
const diagnostics = isRecord(parsed.diagnostics) ? parsed.diagnostics : {};
|
|
const timing = isRecord(diagnostics.timing) ? diagnostics.timing : {};
|
|
const logs = isRecord(parsed.logs) ? parsed.logs : {};
|
|
const accountSnapshot = recordArray(parsed.accountSnapshot);
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API TRACE",
|
|
stringValue(parsed.requestId) ?? context.requestId,
|
|
`ok=${parsed.ok === true ? "true" : "false"}`,
|
|
`outcome=${stringValue(summary.outcome) ?? "-"}`,
|
|
`reason=${stringValue(summary.reason) ?? "-"}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"SOURCE",
|
|
`target=${textValue(parsed.targetId)}`,
|
|
`runtime=${textValue(parsed.runtimeMode)}`,
|
|
`logs=${textValue(logs.source)}`,
|
|
`context=${textValue(logs.contextSource)}`,
|
|
`exit=${textValue(logs.exitCode)}`,
|
|
].join(" "));
|
|
if (parsed.ok !== true && stringValue(logs.stderrTail) !== null) {
|
|
lines.push(`LOG ERROR ${shorten(stringValue(logs.stderrTail) ?? "", 240)}`);
|
|
}
|
|
lines.push([
|
|
"REQUEST",
|
|
`path=${request.path ?? final.path ?? "-"}`,
|
|
`model=${request.model ?? final.model ?? "-"}`,
|
|
`stream=${textValue(request.stream)}`,
|
|
`body=${textValue(request.bodyBytes)}`,
|
|
`first=${shortIso(summary.firstAt)}`,
|
|
`last=${shortIso(summary.lastAt)}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"FINAL",
|
|
`status=${textValue(final.statusCode)}`,
|
|
`account=${formatAccountRef(final)}`,
|
|
`latency_ms=${textValue(final.latencyMs)}`,
|
|
`events=${textValue(summary.eventCount)}`,
|
|
`window=${window.beforeSeconds ?? "?"}s/${window.afterSeconds ?? "?"}s`,
|
|
].join(" "));
|
|
lines.push(renderTable([
|
|
["TIME_REF", "FIRST_FAILOVER_MS", "SELECT_FAILED_MS", "FINAL_MS", "FAILOVER_TO_FINAL_MS", "CLIENT_DEADLINE", "REMAINING_MS"],
|
|
[
|
|
textValue(timing.referenceSource), textValue(timing.firstFailoverElapsedMs), textValue(timing.firstSelectFailedElapsedMs),
|
|
textValue(timing.finalElapsedMs), textValue(timing.failoverToFinalMs), timing.clientDeadlineAvailable === true ? "available" : "unavailable",
|
|
textValue(timing.clientDeadlineRemainingMs),
|
|
],
|
|
]));
|
|
if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`);
|
|
if (failovers.length > 0) {
|
|
lines.push("");
|
|
lines.push("FAILOVER");
|
|
lines.push(renderTable([
|
|
["AT", "ELAPSED_MS", "ACCOUNT", "UPSTREAM", "SWITCH", "MAX"],
|
|
...failovers.map((item) => [
|
|
shortIso(item.at),
|
|
textValue(item.elapsedMs),
|
|
formatAccountRef(item),
|
|
textValue(item.upstreamStatus),
|
|
textValue(item.switchCount),
|
|
textValue(item.maxSwitches),
|
|
]),
|
|
]));
|
|
}
|
|
if (selectFailures.length > 0) {
|
|
lines.push("");
|
|
lines.push("SELECT-FAILED");
|
|
lines.push(renderTable([
|
|
["AT", "ELAPSED_MS", "ERROR", "EXCLUDED"],
|
|
...selectFailures.map((item) => [
|
|
shortIso(item.at),
|
|
textValue(item.elapsedMs),
|
|
shorten(stringValue(item.error) ?? "-", 56),
|
|
textValue(item.excludedAccountCount),
|
|
]),
|
|
]));
|
|
}
|
|
if (forwardFailures.length > 0) {
|
|
lines.push("");
|
|
lines.push("FORWARD-FAILED");
|
|
lines.push(renderTable([
|
|
["AT", "ELAPSED_MS", "ACCOUNT", "STATUS", "ERROR"],
|
|
...forwardFailures.map((item) => [
|
|
shortIso(item.at),
|
|
textValue(item.elapsedMs),
|
|
formatAccountRef(item),
|
|
textValue(item.statusCode),
|
|
shorten(stringValue(item.error) ?? "-", 72),
|
|
]),
|
|
]));
|
|
}
|
|
if (upstreamErrors.length > 0 || tempUnschedulable.length > 0) {
|
|
lines.push("");
|
|
lines.push("ACCOUNT SIGNALS");
|
|
lines.push(renderTable([
|
|
["TYPE", "PHASE", "AT", "ACCOUNT", "STATUS", "RULE", "UNTIL", "DETAIL"],
|
|
...upstreamErrors.map((item) => [
|
|
"upstream-error",
|
|
textValue(item.phase),
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.statusCode),
|
|
"-",
|
|
"-",
|
|
shorten(stringValue(item.error) ?? "-", 36),
|
|
]),
|
|
...tempUnschedulable.map((item) => [
|
|
"temp-unsched",
|
|
textValue(item.phase),
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.statusCode),
|
|
textValue(item.ruleIndex),
|
|
shortIso(item.until),
|
|
shorten(stringValue(item.reason) ?? stringValue(item.matchedKeyword) ?? "-", 36),
|
|
]),
|
|
]));
|
|
}
|
|
lines.push("");
|
|
lines.push("WINDOW STATS");
|
|
lines.push(renderTable([
|
|
["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "FORWARD_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"],
|
|
[
|
|
textValue(windowStats.matchedLines),
|
|
textValue(windowStats.eventCount),
|
|
textValue(windowStats.finalErrorCount),
|
|
textValue(windowStats.failoverCount),
|
|
textValue(windowStats.selectFailedCount),
|
|
textValue(windowStats.forwardFailedCount),
|
|
textValue(windowStats.tempUnschedulableCount),
|
|
textValue(windowStats.adminSchedulableCount),
|
|
],
|
|
]));
|
|
if (accountSnapshot.length > 0) {
|
|
lines.push("");
|
|
lines.push("CURRENT ACCOUNTS");
|
|
lines.push(renderTable([
|
|
["ID", "ACCOUNT", "SCHED", "STATUS", "CONC", "TEMP_UNTIL"],
|
|
...accountSnapshot.slice(0, 20).map((item) => [
|
|
textValue(item.accountId),
|
|
shorten(stringValue(item.accountName) ?? "-", 32),
|
|
textValue(item.schedulable),
|
|
textValue(item.status),
|
|
textValue(item.concurrency),
|
|
shortIso(item.tempUnschedulableUntil),
|
|
]),
|
|
]));
|
|
}
|
|
if (context.showLines || parsed.showLines === true) {
|
|
const rawLines = recordArray(parsed.rawLines);
|
|
if (rawLines.length > 0) {
|
|
lines.push("");
|
|
lines.push("RAW LINES");
|
|
for (const item of rawLines) {
|
|
lines.push(shorten(stringValue(item.line) ?? "", 1000));
|
|
}
|
|
}
|
|
}
|
|
lines.push("");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <id> --raw");
|
|
lines.push("Lines: add --show-lines for bounded matched log lines.");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
export function formatAccountRef(item: Record<string, unknown>): string {
|
|
const name = stringValue(item.accountName);
|
|
const id = textValue(item.accountId);
|
|
if (name !== null && id !== "-") return `${name}#${id}`;
|
|
if (name !== null) return name;
|
|
return id;
|
|
}
|
|
|
|
export function renderTable(rows: string[][]): string {
|
|
if (rows.length === 0) return "";
|
|
const widths: number[] = [];
|
|
for (const row of rows) {
|
|
row.forEach((cell, index) => {
|
|
widths[index] = Math.max(widths[index] ?? 0, displayWidth(cell));
|
|
});
|
|
}
|
|
return rows.map((row) => row.map((cell, index) => padRight(cell, widths[index] ?? 0)).join(" ").trimEnd()).join("\n");
|
|
}
|
|
|
|
export function padRight(value: string, width: number): string {
|
|
const pad = width - displayWidth(value);
|
|
return pad <= 0 ? value : `${value}${" ".repeat(pad)}`;
|
|
}
|
|
|
|
export function displayWidth(value: string): number {
|
|
return [...value].reduce((width, char) => width + (char.charCodeAt(0) > 0x7f ? 2 : 1), 0);
|
|
}
|
|
|
|
export function formatWindow(from: unknown, to: unknown): string {
|
|
const left = shortIso(from);
|
|
const right = shortIso(to);
|
|
return left === "-" && right === "-" ? "-" : `${left}..${right}`;
|
|
}
|
|
|
|
export function shortIso(value: unknown): string {
|
|
const text = stringValue(value);
|
|
if (text === null) return "-";
|
|
return text.replace(/^\d{4}-/u, "").replace(/:00Z$/u, "Z").replace("T", " ");
|
|
}
|
|
|
|
export function textValue(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Math.round(value * 10) / 10);
|
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
return String(value);
|
|
}
|
|
|
|
export function boolText(value: unknown): string {
|
|
if (value === true) return "Y";
|
|
if (value === false) return "N";
|
|
return "-";
|
|
}
|
|
|
|
export function shorten(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : `${value.slice(0, Math.max(0, maxChars - 1))}…`;
|
|
}
|
|
|
|
export function formatNumber(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0";
|
|
if (Math.abs(num) >= 1_000_000) return `${(num / 1_000_000).toFixed(1)}M`;
|
|
if (Math.abs(num) >= 1_000) return `${(num / 1_000).toFixed(1)}K`;
|
|
return String(Math.round(num));
|
|
}
|
|
|
|
export function formatCost(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0.0000";
|
|
return num.toFixed(4);
|
|
}
|
|
|
|
export function codexPoolValidationSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (parsed === null) return null;
|
|
const validation = isRecord(parsed.validation) ? parsed.validation : {};
|
|
const runtimeCapabilities = isRecord(parsed.runtimeCapabilities) ? parsed.runtimeCapabilities : {};
|
|
return {
|
|
ok: parsed.ok,
|
|
degraded: parsed.degraded,
|
|
mode: parsed.mode,
|
|
namespace: parsed.namespace,
|
|
serviceDns: parsed.serviceDns,
|
|
appPod: parsed.appPod,
|
|
admin: parsed.admin,
|
|
apiKey: parsed.apiKey,
|
|
ownerBalance: parsed.ownerBalance,
|
|
ownerConcurrency: parsed.ownerConcurrency,
|
|
capacity: compactStatusBlock(parsed.capacity, ["accountName", "accountId", "expectedCapacity", "runtimeConcurrency", "priority", "status", "schedulable", "ok"]),
|
|
loadFactor: compactStatusBlock(parsed.loadFactor, ["accountName", "accountId", "expectedLoadFactor", "runtimeLoadFactor", "priority", "status", "schedulable", "ok"]),
|
|
webSocketsV2: compactStatusBlock(parsed.webSocketsV2, ["accountName", "accountId", "expectedMode", "runtimeMode", "runtimeEnabled", "status", "schedulable", "ok"]),
|
|
tempUnschedulable: compactTempUnschedulableStatus(parsed.tempUnschedulable),
|
|
manualAccounts: compactManualAccounts(parsed.manualAccounts),
|
|
sentinel: compactSentinelStatus(parsed.sentinel),
|
|
runtimeCapabilities: {
|
|
ok: runtimeCapabilities.ok,
|
|
runtimeImage: runtimeCapabilities.runtimeImage,
|
|
successBodyReclassification: compactRuntimeCapability(runtimeCapabilities.successBodyReclassification),
|
|
modelRouting400Failover: compactRuntimeCapability(runtimeCapabilities.modelRouting400Failover),
|
|
valuesPrinted: false,
|
|
},
|
|
validation: {
|
|
gatewayModels: compactGatewayModels(validation.gatewayModels),
|
|
gatewayResponses: compactGatewayResponses(validation.gatewayResponses),
|
|
gatewayResponsesRecent: compactGatewayResponsesRecent(validation.gatewayResponsesRecent),
|
|
gatewayCompactRecent: compactGatewayCompactRecent(validation.gatewayCompactRecent),
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --raw",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function codexPoolSyncSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
const base = codexPoolValidationSummary(parsed);
|
|
if (parsed === null || base === null) return base;
|
|
const accounts = isRecord(parsed.accounts) ? parsed.accounts : {};
|
|
const compactAccounts = recordArray(accounts.items).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"action",
|
|
"status",
|
|
"runtimeSchedulable",
|
|
"priority",
|
|
"capacity",
|
|
"loadFactor",
|
|
"tempUnschedulableConfigured",
|
|
"tempUnschedulableRuleCount",
|
|
"ok",
|
|
]));
|
|
return {
|
|
...base,
|
|
pool: parsed.pool,
|
|
accounts: {
|
|
desired: accounts.desired,
|
|
created: accounts.created,
|
|
updated: accounts.updated,
|
|
pruned: accounts.pruned,
|
|
pruneMode: accounts.pruneMode,
|
|
itemCount: compactAccounts.length,
|
|
attentionItems: compactAccounts.filter((item) => item.ok === false || item.runtimeSchedulable === false),
|
|
prunedItems: accounts.prunedItems,
|
|
processControl: accounts.processControl,
|
|
valuesPrinted: false,
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --raw",
|
|
pruneRemoved: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --prune-removed",
|
|
},
|
|
};
|
|
}
|