Codex
408 lines
17 KiB
TypeScript
408 lines
17 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/platform-infra-sub2api-codex.ts.
|
|
|
|
// Moved mechanically from scripts/src/platform-infra-sub2api-codex.ts:2701-3096 for #903.
|
|
|
|
import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
import { homedir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { UniDeskConfig } from "../config";
|
|
import { rootPath } from "../config";
|
|
import type { RenderedCliResult } from "../output";
|
|
import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest, type PublicServiceExposure, type PublicServiceTarget } from "../platform-infra-public-service";
|
|
import { shortSha256Fingerprint } from "../platform-infra-ops-library";
|
|
import {
|
|
codexPoolSentinelSummary,
|
|
codexPoolSentinelRuntimeImage,
|
|
readCodexPoolSentinelConfig,
|
|
renderCodexPoolSentinelManifest,
|
|
type CodexPoolSentinelConfig,
|
|
type CodexPoolSentinelProfileSecret,
|
|
} from "../platform-infra-sub2api-codex-sentinel";
|
|
import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets";
|
|
import { runSshCommandCapture, type SshCaptureResult } from "../ssh";
|
|
|
|
import { isRecord, numberValue, stringValue } from "./config-utils";
|
|
import { compactGatewayCompactRecent, compactGatewayModels, compactGatewayResponses, compactGatewayResponsesRecent, compactManualAccounts, compactRuntimeCapability, compactSentinelStatus, compactStatusBlock, compactTempUnschedulableStatus, pickSummaryFields, recordArray } from "./redaction";
|
|
|
|
export function renderedCliResult(ok: boolean, command: string, renderedText: string): RenderedCliResult {
|
|
return { ok, command, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
export function renderSentinelReport(
|
|
parsed: Record<string, unknown> | null,
|
|
context: { events: number; full: boolean; remote: Record<string, unknown> },
|
|
): string {
|
|
if (parsed === null) {
|
|
return [
|
|
"SUB2API SENTINEL REPORT unavailable",
|
|
`remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
|
|
stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
const metadata = isRecord(parsed.metadata) ? parsed.metadata : {};
|
|
const cronJob = isRecord(parsed.cronJob) ? parsed.cronJob : {};
|
|
const summary = isRecord(parsed.summary) ? parsed.summary : {};
|
|
const accounts = recordArray(parsed.accounts);
|
|
const runs = recordArray(parsed.runs);
|
|
const globalLedger = isRecord(parsed.globalLedger) ? parsed.globalLedger : {};
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API SENTINEL",
|
|
`ok=${parsed.ok === true ? "true" : "false"}`,
|
|
`accounts=${summary.accountCount ?? accounts.length}`,
|
|
`quarantined=${summary.quarantinedCount ?? "?"}`,
|
|
`schedulable=${summary.runtimeSchedulableCount ?? "?"}`,
|
|
`unschedulable=${summary.runtimeUnschedulableCount ?? "?"}`,
|
|
`history=${summary.historyCount ?? runs.length}`,
|
|
`window=${formatWindow(summary.historyFrom, summary.historyTo)}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"CRON",
|
|
`schedule=${cronJob.schedule ?? "-"}`,
|
|
`last=${shortIso(cronJob.lastScheduleTime)}`,
|
|
`active=${cronJob.active ?? "-"}`,
|
|
`state=${metadata.namespace ?? "-"}/${metadata.stateConfigMapName ?? "-"}`,
|
|
`ledger=req:${globalLedger.requestCount ?? 0} tok:${formatNumber(globalLedger.totalTokens)} cost:$${formatCost(globalLedger.estimatedCostUsd)}`,
|
|
].join(" "));
|
|
lines.push("");
|
|
lines.push("ACCOUNTS");
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "STATE", "Q", "SCH", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
|
...accounts.map((account) => [
|
|
stringValue(account.account) ?? "-",
|
|
stringValue(account.status) ?? "-",
|
|
account.quarantineActive === true ? "Y" : "-",
|
|
account.runtimeSchedulable === true ? "Y" : account.runtimeSchedulable === false ? "N" : "-",
|
|
account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-",
|
|
account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-",
|
|
account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-",
|
|
textValue(account.freezeIntervalMin),
|
|
textValue(account.successIntervalMin),
|
|
textValue(account.successMaxIntervalMin),
|
|
textValue(account.probeCount),
|
|
shortIso(account.lastEventAt ?? account.lastProbeAt),
|
|
textValue(account.lastHttp),
|
|
account.lastMarker === true ? "Y" : account.lastMarker === false ? "N" : "-",
|
|
shorten(stringValue(account.lastFailureKind) ?? "-", 20),
|
|
shorten(stringValue(account.lastAction) ?? "-", 16),
|
|
shortIso(account.nextProbeAfter),
|
|
textValue(account.observedLastToNextMin),
|
|
]),
|
|
]));
|
|
if (runs.length > 0 || context.full) {
|
|
lines.push("");
|
|
lines.push(`RUNS last=${Math.min(context.events, runs.length)}`);
|
|
lines.push(renderTable([
|
|
["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "GF", "GACT", "REASSERT"],
|
|
...runs.slice(-context.events).map((run) => [
|
|
shortIso(run.at),
|
|
textValue(run.selected),
|
|
textValue(run.due),
|
|
textValue(run.ok),
|
|
textValue(run.mismatch),
|
|
textValue(run.transportFailures),
|
|
textValue(run.actionsTaken),
|
|
textValue(run.gatewayFailures),
|
|
textValue(run.gatewayActions),
|
|
textValue(run.reasserts),
|
|
]),
|
|
]));
|
|
}
|
|
lines.push("");
|
|
lines.push("LEGEND Q=quarantined SCH=Sub2API runtime schedulable T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
export function renderTraceReport(
|
|
parsed: Record<string, unknown> | null,
|
|
context: { requestId: string; showLines: boolean; remote: Record<string, unknown> },
|
|
): string {
|
|
if (parsed === null) {
|
|
return [
|
|
`SUB2API TRACE ${context.requestId} unavailable`,
|
|
`remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
|
|
stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
const summary = isRecord(parsed.summary) ? parsed.summary : {};
|
|
const request = isRecord(parsed.request) ? parsed.request : {};
|
|
const final = isRecord(parsed.final) ? parsed.final : {};
|
|
const window = isRecord(parsed.window) ? parsed.window : {};
|
|
const events = recordArray(parsed.events);
|
|
const failovers = recordArray(parsed.failovers);
|
|
const selectFailures = recordArray(parsed.selectFailures);
|
|
const upstreamErrors = recordArray(parsed.upstreamErrors);
|
|
const tempUnschedulable = recordArray(parsed.tempUnschedulable);
|
|
const windowStats = isRecord(parsed.windowStats) ? parsed.windowStats : {};
|
|
const accountSnapshot = recordArray(parsed.accountSnapshot);
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API TRACE",
|
|
stringValue(parsed.requestId) ?? context.requestId,
|
|
`ok=${parsed.ok === true ? "true" : "false"}`,
|
|
`outcome=${stringValue(summary.outcome) ?? "-"}`,
|
|
`reason=${stringValue(summary.reason) ?? "-"}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"REQUEST",
|
|
`path=${request.path ?? final.path ?? "-"}`,
|
|
`model=${request.model ?? final.model ?? "-"}`,
|
|
`stream=${textValue(request.stream)}`,
|
|
`body=${textValue(request.bodyBytes)}`,
|
|
`first=${shortIso(summary.firstAt)}`,
|
|
`last=${shortIso(summary.lastAt)}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"FINAL",
|
|
`status=${textValue(final.statusCode)}`,
|
|
`account=${formatAccountRef(final)}`,
|
|
`latency_ms=${textValue(final.latencyMs)}`,
|
|
`events=${textValue(summary.eventCount)}`,
|
|
`window=${window.beforeSeconds ?? "?"}s/${window.afterSeconds ?? "?"}s`,
|
|
].join(" "));
|
|
if (failovers.length > 0) {
|
|
lines.push("");
|
|
lines.push("FAILOVER");
|
|
lines.push(renderTable([
|
|
["AT", "ACCOUNT", "UPSTREAM", "SWITCH", "MAX"],
|
|
...failovers.map((item) => [
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.upstreamStatus),
|
|
textValue(item.switchCount),
|
|
textValue(item.maxSwitches),
|
|
]),
|
|
]));
|
|
}
|
|
if (selectFailures.length > 0) {
|
|
lines.push("");
|
|
lines.push("SELECT-FAILED");
|
|
lines.push(renderTable([
|
|
["AT", "ERROR", "EXCLUDED"],
|
|
...selectFailures.map((item) => [
|
|
shortIso(item.at),
|
|
shorten(stringValue(item.error) ?? "-", 56),
|
|
textValue(item.excludedAccountCount),
|
|
]),
|
|
]));
|
|
}
|
|
if (upstreamErrors.length > 0 || tempUnschedulable.length > 0) {
|
|
lines.push("");
|
|
lines.push("ACCOUNT SIGNALS");
|
|
lines.push(renderTable([
|
|
["TYPE", "PHASE", "AT", "ACCOUNT", "STATUS", "RULE", "UNTIL", "DETAIL"],
|
|
...upstreamErrors.map((item) => [
|
|
"upstream-error",
|
|
textValue(item.phase),
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.statusCode),
|
|
"-",
|
|
"-",
|
|
shorten(stringValue(item.error) ?? "-", 36),
|
|
]),
|
|
...tempUnschedulable.map((item) => [
|
|
"temp-unsched",
|
|
textValue(item.phase),
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.statusCode),
|
|
textValue(item.ruleIndex),
|
|
shortIso(item.until),
|
|
shorten(stringValue(item.reason) ?? stringValue(item.matchedKeyword) ?? "-", 36),
|
|
]),
|
|
]));
|
|
}
|
|
lines.push("");
|
|
lines.push("WINDOW STATS");
|
|
lines.push(renderTable([
|
|
["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"],
|
|
[
|
|
textValue(windowStats.matchedLines),
|
|
textValue(windowStats.eventCount),
|
|
textValue(windowStats.finalErrorCount),
|
|
textValue(windowStats.failoverCount),
|
|
textValue(windowStats.selectFailedCount),
|
|
textValue(windowStats.tempUnschedulableCount),
|
|
textValue(windowStats.adminSchedulableCount),
|
|
],
|
|
]));
|
|
if (accountSnapshot.length > 0) {
|
|
lines.push("");
|
|
lines.push("CURRENT ACCOUNTS");
|
|
lines.push(renderTable([
|
|
["ID", "ACCOUNT", "SCHED", "STATUS", "CONC", "TEMP_UNTIL"],
|
|
...accountSnapshot.slice(0, 20).map((item) => [
|
|
textValue(item.accountId),
|
|
shorten(stringValue(item.accountName) ?? "-", 32),
|
|
textValue(item.schedulable),
|
|
textValue(item.status),
|
|
textValue(item.concurrency),
|
|
shortIso(item.tempUnschedulableUntil),
|
|
]),
|
|
]));
|
|
}
|
|
if (context.showLines || parsed.showLines === true) {
|
|
const rawLines = recordArray(parsed.rawLines);
|
|
if (rawLines.length > 0) {
|
|
lines.push("");
|
|
lines.push("RAW LINES");
|
|
for (const item of rawLines) {
|
|
lines.push(shorten(stringValue(item.line) ?? "", 1000));
|
|
}
|
|
}
|
|
}
|
|
lines.push("");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <id> --raw");
|
|
lines.push("Lines: add --show-lines for bounded matched log lines.");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
export function formatAccountRef(item: Record<string, unknown>): string {
|
|
const name = stringValue(item.accountName);
|
|
const id = textValue(item.accountId);
|
|
if (name !== null && id !== "-") return `${name}#${id}`;
|
|
if (name !== null) return name;
|
|
return id;
|
|
}
|
|
|
|
export function renderTable(rows: string[][]): string {
|
|
if (rows.length === 0) return "";
|
|
const widths: number[] = [];
|
|
for (const row of rows) {
|
|
row.forEach((cell, index) => {
|
|
widths[index] = Math.max(widths[index] ?? 0, displayWidth(cell));
|
|
});
|
|
}
|
|
return rows.map((row) => row.map((cell, index) => padRight(cell, widths[index] ?? 0)).join(" ").trimEnd()).join("\n");
|
|
}
|
|
|
|
export function padRight(value: string, width: number): string {
|
|
const pad = width - displayWidth(value);
|
|
return pad <= 0 ? value : `${value}${" ".repeat(pad)}`;
|
|
}
|
|
|
|
export function displayWidth(value: string): number {
|
|
return [...value].reduce((width, char) => width + (char.charCodeAt(0) > 0x7f ? 2 : 1), 0);
|
|
}
|
|
|
|
export function formatWindow(from: unknown, to: unknown): string {
|
|
const left = shortIso(from);
|
|
const right = shortIso(to);
|
|
return left === "-" && right === "-" ? "-" : `${left}..${right}`;
|
|
}
|
|
|
|
export function shortIso(value: unknown): string {
|
|
const text = stringValue(value);
|
|
if (text === null) return "-";
|
|
return text.replace(/^\d{4}-/u, "").replace(/:00Z$/u, "Z").replace("T", " ");
|
|
}
|
|
|
|
export function textValue(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Math.round(value * 10) / 10);
|
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
return String(value);
|
|
}
|
|
|
|
export function shorten(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : `${value.slice(0, Math.max(0, maxChars - 1))}…`;
|
|
}
|
|
|
|
export function formatNumber(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0";
|
|
if (Math.abs(num) >= 1_000_000) return `${(num / 1_000_000).toFixed(1)}M`;
|
|
if (Math.abs(num) >= 1_000) return `${(num / 1_000).toFixed(1)}K`;
|
|
return String(Math.round(num));
|
|
}
|
|
|
|
export function formatCost(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0.0000";
|
|
return num.toFixed(4);
|
|
}
|
|
|
|
export function codexPoolValidationSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (parsed === null) return null;
|
|
const validation = isRecord(parsed.validation) ? parsed.validation : {};
|
|
const runtimeCapabilities = isRecord(parsed.runtimeCapabilities) ? parsed.runtimeCapabilities : {};
|
|
return {
|
|
ok: parsed.ok,
|
|
degraded: parsed.degraded,
|
|
mode: parsed.mode,
|
|
namespace: parsed.namespace,
|
|
serviceDns: parsed.serviceDns,
|
|
appPod: parsed.appPod,
|
|
admin: parsed.admin,
|
|
apiKey: parsed.apiKey,
|
|
ownerBalance: parsed.ownerBalance,
|
|
ownerConcurrency: parsed.ownerConcurrency,
|
|
capacity: compactStatusBlock(parsed.capacity, ["accountName", "accountId", "expectedCapacity", "runtimeConcurrency", "priority", "status", "schedulable", "ok"]),
|
|
loadFactor: compactStatusBlock(parsed.loadFactor, ["accountName", "accountId", "expectedLoadFactor", "runtimeLoadFactor", "priority", "status", "schedulable", "ok"]),
|
|
webSocketsV2: compactStatusBlock(parsed.webSocketsV2, ["accountName", "accountId", "expectedMode", "runtimeMode", "runtimeEnabled", "status", "schedulable", "ok"]),
|
|
tempUnschedulable: compactTempUnschedulableStatus(parsed.tempUnschedulable),
|
|
manualAccounts: compactManualAccounts(parsed.manualAccounts),
|
|
sentinel: compactSentinelStatus(parsed.sentinel),
|
|
runtimeCapabilities: {
|
|
ok: runtimeCapabilities.ok,
|
|
runtimeImage: runtimeCapabilities.runtimeImage,
|
|
successBodyReclassification: compactRuntimeCapability(runtimeCapabilities.successBodyReclassification),
|
|
modelRouting400Failover: compactRuntimeCapability(runtimeCapabilities.modelRouting400Failover),
|
|
valuesPrinted: false,
|
|
},
|
|
validation: {
|
|
gatewayModels: compactGatewayModels(validation.gatewayModels),
|
|
gatewayResponses: compactGatewayResponses(validation.gatewayResponses),
|
|
gatewayResponsesRecent: compactGatewayResponsesRecent(validation.gatewayResponsesRecent),
|
|
gatewayCompactRecent: compactGatewayCompactRecent(validation.gatewayCompactRecent),
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --raw",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function codexPoolSyncSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
const base = codexPoolValidationSummary(parsed);
|
|
if (parsed === null || base === null) return base;
|
|
const accounts = isRecord(parsed.accounts) ? parsed.accounts : {};
|
|
const compactAccounts = recordArray(accounts.items).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"action",
|
|
"status",
|
|
"runtimeSchedulable",
|
|
"priority",
|
|
"capacity",
|
|
"loadFactor",
|
|
"tempUnschedulableConfigured",
|
|
"tempUnschedulableRuleCount",
|
|
"ok",
|
|
]));
|
|
return {
|
|
...base,
|
|
pool: parsed.pool,
|
|
accounts: {
|
|
desired: accounts.desired,
|
|
created: accounts.created,
|
|
updated: accounts.updated,
|
|
pruned: accounts.pruned,
|
|
pruneMode: accounts.pruneMode,
|
|
itemCount: compactAccounts.length,
|
|
attentionItems: compactAccounts.filter((item) => item.ok === false || item.runtimeSchedulable === false),
|
|
prunedItems: accounts.prunedItems,
|
|
processControl: accounts.processControl,
|
|
valuesPrinted: false,
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --raw",
|
|
pruneRemoved: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --prune-removed",
|
|
},
|
|
};
|
|
}
|