feat: 提高 Sub2API 账号评分归因精度
This commit is contained in:
Codex
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
import type { RuntimeRemoteTarget } from "./runtime-remote-script-head";
|
||||
import { runtimeRemoteScriptHead } from "./runtime-remote-script-head";
|
||||
import { runtimeRemoteScriptTail } from "./runtime-remote-script-tail";
|
||||
|
||||
export function runtimeScript(payload: unknown, target: RuntimeRemoteTarget): string {
|
||||
return runtimeRemoteScriptHead(payload, target) + runtimeRemoteScriptTail();
|
||||
}
|
||||
@@ -0,0 +1,659 @@
|
||||
import type { RenderedCliResult } from "../output";
|
||||
import { renderedCliResult, renderTable } from "./render";
|
||||
import type { RuntimeOptions } from "./runtime-options";
|
||||
|
||||
export function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown {
|
||||
if (options.action !== "errors" || options.full || value === null || typeof value !== "object" || Array.isArray(value)) return value;
|
||||
const runtime = value as Record<string, unknown>;
|
||||
const allGroups = runtimeRecord(runtime.allGroups);
|
||||
if (allGroups !== null) {
|
||||
const nextPageToken = typeof allGroups.nextPageToken === "string" ? allGroups.nextPageToken : null;
|
||||
const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`;
|
||||
const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`;
|
||||
return {
|
||||
ok: runtime.ok,
|
||||
operation: runtime.operation,
|
||||
mutation: runtime.mutation,
|
||||
endpoint: runtime.endpoint,
|
||||
allGroups,
|
||||
disclosure: {
|
||||
group: `${baseCommand} --group <group-name-or-id>${platformFlag}`,
|
||||
nextPage: nextPageToken === null ? null : `${baseCommand} --all-groups${platformFlag} --page-token ${nextPageToken}`,
|
||||
note: "The overview uses stable group-id pagination. Use group drilldown for account root causes and request indexes.",
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
const errors = runtimeRecord(runtime.errors);
|
||||
if (errors === null) return value;
|
||||
const effectiveness = runtimeRecord(errors.policyEffectiveness) ?? {};
|
||||
const nativeOps = runtimeRecord(errors.nativeOps) ?? {};
|
||||
const availability = runtimeRecord(nativeOps.accountAvailability) ?? {};
|
||||
const availabilityGroup = runtimeRecord(availability.group) ?? {};
|
||||
const concurrency = runtimeRecord(nativeOps.concurrency) ?? {};
|
||||
const concurrencyGroup = runtimeRecord(concurrency.group) ?? {};
|
||||
const sink = runtimeRecord(nativeOps.systemLogSink) ?? {};
|
||||
const sourceStatus = runtimeRecord(effectiveness.sourceStatus) ?? {};
|
||||
const drilldown = runtimeRecord(effectiveness.requestDrilldown) ?? {};
|
||||
const requestIndexes = Object.fromEntries(Object.entries(drilldown).flatMap(([key, item]) => {
|
||||
if (!Array.isArray(item)) return [];
|
||||
const ids = item.filter((entry): entry is string => typeof entry === "string");
|
||||
if (ids.length === 0) return [];
|
||||
return [[key, {
|
||||
count: ids.length,
|
||||
sampleRequestId: ids[0],
|
||||
moreAvailable: ids.length > 1,
|
||||
}]];
|
||||
}));
|
||||
const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`;
|
||||
const groupFlag = options.group === null ? "" : ` --group ${options.group}`;
|
||||
const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`;
|
||||
if (options.json && options.account === null) {
|
||||
return {
|
||||
ok: runtime.ok,
|
||||
operation: runtime.operation,
|
||||
mutation: runtime.mutation,
|
||||
group: runtime.group,
|
||||
endpoint: runtime.endpoint,
|
||||
errors: {
|
||||
window: errors.window,
|
||||
nativeOps: {
|
||||
source: nativeOps.source,
|
||||
overview: nativeOps.overview,
|
||||
accountQuality: compactAccountQuality(nativeOps.accountQuality, false),
|
||||
accountAvailability: {
|
||||
status: availability.status,
|
||||
totalAccounts: availabilityGroup.total_accounts,
|
||||
availableCount: availabilityGroup.available_count,
|
||||
rateLimitCount: availabilityGroup.rate_limit_count,
|
||||
errorCount: availabilityGroup.error_count,
|
||||
unavailableAccounts: Array.isArray(availability.unavailableAccounts)
|
||||
? availability.unavailableAccounts.map((item) => {
|
||||
const account = runtimeRecord(item) ?? {};
|
||||
return {
|
||||
accountId: account.accountId,
|
||||
accountName: account.accountName,
|
||||
status: account.status,
|
||||
reason: account.error,
|
||||
};
|
||||
})
|
||||
: [],
|
||||
},
|
||||
concurrency: {
|
||||
status: concurrency.status,
|
||||
currentInUse: concurrencyGroup.current_in_use,
|
||||
maxCapacity: concurrencyGroup.max_capacity,
|
||||
loadPercentage: concurrencyGroup.load_percentage,
|
||||
waitingInQueue: concurrencyGroup.waiting_in_queue,
|
||||
},
|
||||
},
|
||||
},
|
||||
disclosure: {
|
||||
account: `${baseCommand}${groupFlag}${platformFlag} --account <name-or-id>`,
|
||||
trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target <target-id> --request-id <request-id>",
|
||||
note: "This group overview keeps every account name and quality summary. Use account drilldown for request indexes and complete failure evidence.",
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: runtime.ok,
|
||||
operation: runtime.operation,
|
||||
mutation: runtime.mutation,
|
||||
group: runtime.group,
|
||||
endpoint: runtime.endpoint,
|
||||
errors: {
|
||||
window: errors.window,
|
||||
nativeOps: {
|
||||
source: nativeOps.source,
|
||||
overview: nativeOps.overview,
|
||||
responseHeaderTimeout: options.account === null || !options.json ? nativeOps.responseHeaderTimeout : undefined,
|
||||
requestTiming: options.account === null || !options.json ? compactRequestTiming(nativeOps.requestTiming) : undefined,
|
||||
customerErrors: compactCustomerErrors(nativeOps.customerErrors, options.account !== null || !options.json),
|
||||
accountQuality: compactAccountQuality(nativeOps.accountQuality, options.account !== null || !options.json),
|
||||
accountAvailability: {
|
||||
status: availability.status,
|
||||
totalAccounts: availabilityGroup.total_accounts,
|
||||
availableCount: availabilityGroup.available_count,
|
||||
rateLimitCount: availabilityGroup.rate_limit_count,
|
||||
errorCount: availabilityGroup.error_count,
|
||||
unavailableAccounts: Array.isArray(availability.unavailableAccounts)
|
||||
? availability.unavailableAccounts.map((item) => {
|
||||
const account = runtimeRecord(item) ?? {};
|
||||
return {
|
||||
accountId: account.accountId,
|
||||
accountName: account.accountName,
|
||||
status: account.status,
|
||||
reason: account.error,
|
||||
};
|
||||
})
|
||||
: [],
|
||||
},
|
||||
concurrency: {
|
||||
status: concurrency.status,
|
||||
currentInUse: concurrencyGroup.current_in_use,
|
||||
maxCapacity: concurrencyGroup.max_capacity,
|
||||
loadPercentage: concurrencyGroup.load_percentage,
|
||||
waitingInQueue: concurrencyGroup.waiting_in_queue,
|
||||
},
|
||||
systemLogSink: {
|
||||
status: sink.status,
|
||||
queueDepth: sink.queueDepth,
|
||||
droppedCount: sink.droppedCount,
|
||||
writeFailedCount: sink.writeFailedCount,
|
||||
},
|
||||
},
|
||||
rootCauseAnalysis: errors.rootCauseAnalysis,
|
||||
policyEffectiveness: {
|
||||
source: effectiveness.source,
|
||||
scope: effectiveness.scope,
|
||||
attribution: effectiveness.attribution,
|
||||
assessment: effectiveness.assessment,
|
||||
ruleMatches: effectiveness.ruleMatches,
|
||||
failover: effectiveness.failover,
|
||||
forwardFailure: effectiveness.forwardFailure,
|
||||
selectFailedEvents: effectiveness.selectFailedEvents,
|
||||
requestIndexes,
|
||||
sourceStatus: {
|
||||
status: sourceStatus.status,
|
||||
fallbackUsed: sourceStatus.fallbackUsed,
|
||||
fallbackReason: sourceStatus.fallbackReason,
|
||||
},
|
||||
},
|
||||
},
|
||||
disclosure: {
|
||||
complete: `${baseCommand}${groupFlag}${platformFlag}${options.account === null ? "" : " --account <same-account-selector>"} --full`,
|
||||
account: `${baseCommand}${groupFlag}${platformFlag} --account <name-or-id>`,
|
||||
trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target <target-id> --request-id <request-id>",
|
||||
note: "Use requestIndexes for trace drilldown and --full for the complete finite index.",
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
function compactRequestTiming(value: unknown): unknown {
|
||||
const timing = runtimeRecord(value);
|
||||
if (timing === null) return value;
|
||||
return {
|
||||
status: timing.status,
|
||||
reason: timing.reason,
|
||||
source: timing.source,
|
||||
ttft: timing.ttft,
|
||||
duration: timing.duration,
|
||||
observedFloorSeconds: timing.observedFloorSeconds,
|
||||
attribution: timing.attribution,
|
||||
};
|
||||
}
|
||||
function compactCustomerErrors(value: unknown, detailed: boolean): unknown {
|
||||
const profile = runtimeRecord(value);
|
||||
if (profile === null) return value;
|
||||
return {
|
||||
status: profile.status,
|
||||
reason: profile.reason,
|
||||
source: profile.source,
|
||||
total: profile.total,
|
||||
modelBuckets: runtimeRecords(profile.modelBuckets).slice(0, 5),
|
||||
accountBuckets: detailed ? runtimeRecords(profile.accountBuckets).slice(0, 5) : undefined,
|
||||
statusBuckets: runtimeRecords(profile.statusBuckets).slice(0, 5),
|
||||
streamBuckets: runtimeRecords(profile.streamBuckets).slice(0, 5),
|
||||
messageBuckets: runtimeRecords(profile.messageBuckets).slice(0, detailed ? 10 : 5),
|
||||
phaseBuckets: detailed ? runtimeRecords(profile.phaseBuckets).slice(0, 5) : undefined,
|
||||
endpointBuckets: detailed ? runtimeRecords(profile.endpointBuckets).slice(0, 5) : undefined,
|
||||
pathBuckets: detailed ? runtimeRecords(profile.pathBuckets).slice(0, 5) : undefined,
|
||||
samples: detailed ? runtimeRecords(profile.samples).slice(0, 15) : undefined,
|
||||
};
|
||||
}
|
||||
function compactAccountQuality(value: unknown, detailed: boolean): unknown {
|
||||
const quality = runtimeRecord(value);
|
||||
if (quality === null) return value;
|
||||
return {
|
||||
source: quality.source,
|
||||
scope: quality.scope,
|
||||
window: quality.window,
|
||||
scoreRule: (() => {
|
||||
const rule = runtimeRecord(quality.scoreRule) ?? {};
|
||||
return detailed ? {
|
||||
reliabilityWeight: rule.reliabilityWeight,
|
||||
latencyWeight: rule.latencyWeight,
|
||||
availabilityWeight: rule.availabilityWeight,
|
||||
failureDefinition: rule.failureDefinition,
|
||||
missingMetric: rule.missingMetric,
|
||||
} : {
|
||||
reliabilityWeight: rule.reliabilityWeight,
|
||||
latencyWeight: rule.latencyWeight,
|
||||
availabilityWeight: rule.availabilityWeight,
|
||||
};
|
||||
})(),
|
||||
accounts: runtimeRecords(quality.accounts).map((account) => detailed ? {
|
||||
accountId: account.accountId,
|
||||
accountName: account.accountName,
|
||||
status: account.status,
|
||||
currentlyAvailable: account.currentlyAvailable,
|
||||
score: account.score,
|
||||
grade: account.grade,
|
||||
assessment: account.assessment,
|
||||
confidence: account.confidence,
|
||||
observedAttempts: account.observedAttempts,
|
||||
successRequests: account.successRequests,
|
||||
failureRequests: account.failureRequests,
|
||||
failureRate: account.failureRate,
|
||||
firstTokenSamples: account.firstTokenSamples,
|
||||
firstTokenCoverage: account.firstTokenCoverage,
|
||||
ttftP95Ms: account.ttftP95Ms,
|
||||
customerErrorRequests: account.customerErrorRequests,
|
||||
scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests,
|
||||
excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests,
|
||||
excludedReasonBuckets: account.excludedReasonBuckets,
|
||||
usage: account.usage,
|
||||
failoverRequests: account.failoverRequests,
|
||||
failoverRecovered: account.failoverRecovered,
|
||||
failoverFailed: account.failoverFailed,
|
||||
sameAccountRetryEvents: account.sameAccountRetryEvents,
|
||||
tempUnschedulableEvents: account.tempUnschedulableEvents,
|
||||
forwardFailedRequests: account.forwardFailedRequests,
|
||||
reasons: account.reasons,
|
||||
} : {
|
||||
accountId: account.accountId,
|
||||
accountName: account.accountName,
|
||||
currentlyAvailable: account.currentlyAvailable,
|
||||
score: account.score,
|
||||
grade: account.grade,
|
||||
confidence: account.confidence,
|
||||
observedAttempts: account.observedAttempts,
|
||||
failureRate: account.failureRate,
|
||||
ttftP95Ms: account.ttftP95Ms,
|
||||
customerErrorRequests: account.customerErrorRequests,
|
||||
scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests,
|
||||
excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests,
|
||||
excludedReasonBuckets: account.excludedReasonBuckets,
|
||||
usage: account.usage,
|
||||
failoverRequests: account.failoverRequests,
|
||||
failoverRecovered: account.failoverRecovered,
|
||||
failoverFailed: account.failoverFailed,
|
||||
}),
|
||||
attribution: detailed ? quality.attribution : undefined,
|
||||
};
|
||||
}
|
||||
function runtimeRecord(value: unknown): Record<string, unknown> | null {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function runtimeRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.flatMap((item) => {
|
||||
const record = runtimeRecord(item);
|
||||
return record === null ? [] : [record];
|
||||
}) : [];
|
||||
}
|
||||
function runtimeText(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
if (typeof value === "boolean") return value ? "true" : "false";
|
||||
if (Array.isArray(value)) return value.map(runtimeText).join(",");
|
||||
return String(value).replace(/\s+/gu, " ").trim();
|
||||
}
|
||||
|
||||
function runtimeShort(value: unknown, max = 42): string {
|
||||
const text = runtimeText(value);
|
||||
return text.length <= max ? text : `${text.slice(0, max - 1)}…`;
|
||||
}
|
||||
|
||||
function runtimeRate(value: unknown): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? `${(value * 100).toFixed(2)}%` : "-";
|
||||
}
|
||||
|
||||
function runtimeMillions(value: unknown): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? `${(value / 1_000_000).toFixed(3)}M` : "-";
|
||||
}
|
||||
|
||||
function runtimeDecimal(value: unknown, digits = 4): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value.toFixed(digits) : "-";
|
||||
}
|
||||
|
||||
export function renderRuntimeResult(response: Record<string, unknown>, options: RuntimeOptions): RenderedCliResult {
|
||||
const runtime = runtimeRecord(response.runtime) ?? {};
|
||||
const ok = response.ok === true;
|
||||
const lines: string[] = [];
|
||||
if (options.action === "errors") renderRuntimeErrors(lines, runtime, options);
|
||||
else if (options.action === "infrastructure") renderRuntimeInfrastructure(lines, runtime);
|
||||
else if (options.action === "list") renderRuntimeAccountList(lines, runtime);
|
||||
else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime);
|
||||
else renderRuntimeMutation(lines, runtime, options);
|
||||
lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} <same-options> --json`);
|
||||
return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n"));
|
||||
}
|
||||
|
||||
function renderRuntimeErrors(lines: string[], runtime: Record<string, unknown>, options: RuntimeOptions): void {
|
||||
const allGroups = runtimeRecord(runtime.allGroups);
|
||||
if (allGroups !== null) {
|
||||
const totals = runtimeRecord(allGroups.pageTotals) ?? {};
|
||||
const timeout = runtimeRecord(allGroups.responseHeaderTimeout) ?? {};
|
||||
const timing = runtimeRecord(allGroups.requestTiming) ?? {};
|
||||
const groups = runtimeRecords(allGroups.groups);
|
||||
lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`);
|
||||
lines.push(renderTable([
|
||||
["GROUPS", "RETURNED", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UPSTREAM_ERR", "UP_RATE", "ATTENTION"],
|
||||
[
|
||||
runtimeText(allGroups.totalGroups), runtimeText(allGroups.returnedGroups), runtimeText(totals.requestCount),
|
||||
runtimeText(totals.errorCount), runtimeRate(totals.errorRate), runtimeText(totals.upstreamErrorCount),
|
||||
runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention),
|
||||
],
|
||||
]));
|
||||
lines.push(renderTable([
|
||||
["OPENAI_HEADER_TIMEOUT", "SOURCE", "GENERIC_TIMEOUT", "GENERIC_APPLIES_TO_OPENAI", "PAGE_MAX_TTFT_P99_MS", "OBSERVED_FLOOR_SECONDS"],
|
||||
[
|
||||
timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source),
|
||||
`${runtimeText(timeout.genericResponseHeaderTimeoutSeconds)}s`, runtimeText(timeout.genericAppliesToOpenAI),
|
||||
runtimeText(timing.maxTtftP99Ms), runtimeText(timing.observedFloorSeconds),
|
||||
],
|
||||
]));
|
||||
if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`);
|
||||
lines.push("", "GROUPS");
|
||||
lines.push(renderTable([
|
||||
["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "TTFT_P99_MS", "DUR_P99_MS", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"],
|
||||
...groups.map((group) => [
|
||||
runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status),
|
||||
runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount),
|
||||
runtimeRate(group.upstreamErrorRate), runtimeText(group.ttftP99Ms), runtimeText(group.durationP99Ms), runtimeText(group.totalAccounts), runtimeText(group.availableCount),
|
||||
`${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no",
|
||||
]),
|
||||
]));
|
||||
const disclosure = runtimeRecord(runtime.disclosure) ?? {};
|
||||
lines.push("", `Detail: ${runtimeText(disclosure.group)}`);
|
||||
if (typeof disclosure.nextPage === "string") lines.push(`Next: ${disclosure.nextPage}`);
|
||||
return;
|
||||
}
|
||||
const group = runtimeRecord(runtime.group) ?? {};
|
||||
const errors = runtimeRecord(runtime.errors) ?? {};
|
||||
const nativeOps = runtimeRecord(errors.nativeOps) ?? {};
|
||||
const overview = runtimeRecord(nativeOps.overview) ?? {};
|
||||
const availability = runtimeRecord(nativeOps.accountAvailability) ?? {};
|
||||
const concurrency = runtimeRecord(nativeOps.concurrency) ?? {};
|
||||
const timeout = runtimeRecord(nativeOps.responseHeaderTimeout) ?? {};
|
||||
const timing = runtimeRecord(nativeOps.requestTiming) ?? {};
|
||||
const ttft = runtimeRecord(timing.ttft) ?? {};
|
||||
const duration = runtimeRecord(timing.duration) ?? {};
|
||||
const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {};
|
||||
const accountQuality = runtimeRecord(nativeOps.accountQuality) ?? {};
|
||||
lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`);
|
||||
lines.push(renderTable([
|
||||
["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"],
|
||||
[
|
||||
runtimeShort(group.name, 30), runtimeText(group.id), runtimeText(group.platform), runtimeText(overview.requestCount),
|
||||
runtimeText(overview.errorCount), runtimeRate(overview.errorRate), runtimeRate(overview.upstreamErrorRate),
|
||||
runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore),
|
||||
],
|
||||
]));
|
||||
lines.push(renderTable([
|
||||
["OPENAI_HEADER_TIMEOUT", "SOURCE", "TTFT_P50_MS", "TTFT_P95_MS", "TTFT_P99_MS", "TTFT_MAX_MS", "DURATION_P99_MS", "DURATION_MAX_MS"],
|
||||
[
|
||||
timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source),
|
||||
runtimeText(ttft.p50_ms), runtimeText(ttft.p95_ms), runtimeText(ttft.p99_ms), runtimeText(ttft.max_ms),
|
||||
runtimeText(duration.p99_ms), runtimeText(duration.max_ms),
|
||||
],
|
||||
]));
|
||||
const candidates = runtimeRecords(timing.candidates);
|
||||
if (candidates.length > 0) lines.push("", "TIMEOUT CANDIDATES", renderTable([
|
||||
["SECONDS", "OBSERVED_FLOOR", "TTFT_P99_HEADROOM_MS", "TTFT_P99_BELOW", "ALL_DUR>=", "ERROR_DUR>=", "ASSESSMENT"],
|
||||
...candidates.map((candidate) => [
|
||||
runtimeText(candidate.seconds), runtimeText(candidate.observedFloor), runtimeText(candidate.ttftP99HeadroomMs), runtimeText(candidate.ttftP99Below),
|
||||
runtimeText(candidate.totalDurationAtLeastCount), runtimeText(candidate.errorDurationAtLeastCount), runtimeText(candidate.assessment),
|
||||
]),
|
||||
]));
|
||||
if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`);
|
||||
const modelTiming = runtimeRecords(timing.modelTiming);
|
||||
if (modelTiming.length > 0) lines.push("", "MODEL TIMING", renderTable([
|
||||
["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"],
|
||||
...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]),
|
||||
]));
|
||||
const qualityAccounts = runtimeRecords(accountQuality.accounts);
|
||||
if (qualityAccounts.length > 0) {
|
||||
lines.push("", "ACCOUNT QUALITY", renderTable([
|
||||
["ACCOUNT", "ID", "SCORE", "GRADE", "QUALITY", "CONF", "ATTEMPTS", "SUCCESS", "FAIL", "UP_CUST", "EXCLUDED", "FAIL_RATE", "TTFT_P95_MS", "TTFT_N", "AVAILABLE", "REASONS"],
|
||||
...qualityAccounts.map((item) => [
|
||||
runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.score), runtimeText(item.grade), runtimeText(item.assessment), runtimeText(item.confidence),
|
||||
runtimeText(item.observedAttempts), runtimeText(item.successRequests), runtimeText(item.failureRequests), runtimeText(item.scoreableUpstreamErrorRequests),
|
||||
runtimeText(item.excludedNonUpstreamErrorRequests), runtimeRate(item.failureRate),
|
||||
runtimeText(item.ttftP95Ms), runtimeText(item.firstTokenSamples), runtimeText(item.currentlyAvailable),
|
||||
runtimeShort(Array.isArray(item.reasons) ? item.reasons.join(",") : "", 58),
|
||||
]),
|
||||
]));
|
||||
lines.push("ACCOUNT USAGE AND COST", renderTable([
|
||||
["ACCOUNT", "ID", "REQUESTS", "TOKENS", "API_USD", "COST_RATE_CNY", "UPSTREAM_COST_CNY", "EXCLUDED_REASONS"],
|
||||
...qualityAccounts.map((item) => {
|
||||
const usage = runtimeRecord(item.usage) ?? {};
|
||||
const excluded = runtimeRecords(item.excludedReasonBuckets).map((entry) => `${runtimeText(entry.reason)}:${runtimeText(entry.count)}`).join(",");
|
||||
return [
|
||||
runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(usage.requestCount), runtimeMillions(usage.tokenCount),
|
||||
runtimeDecimal(usage.apiAmountUsd, 4), runtimeDecimal(usage.costRateCnyPerApiUsd, 3), runtimeDecimal(usage.upstreamCostCny, 4),
|
||||
runtimeShort(excluded, 64),
|
||||
];
|
||||
}),
|
||||
]));
|
||||
const controlRows = qualityAccounts.filter((item) => [
|
||||
item.failoverRequests, item.sameAccountRetryEvents, item.tempUnschedulableEvents, item.forwardFailedRequests, item.customerErrorRequests,
|
||||
].some((value) => typeof value === "number" && value > 0));
|
||||
if (controlRows.length > 0) lines.push("ACCOUNT FAILURE CONTROL", renderTable([
|
||||
["ACCOUNT", "ID", "FAILOVER", "RECOVERED", "FAILED", "MISSING", "RETRY", "TEMP_UNSCHED", "FORWARD_FAIL", "CUSTOMER_ERR"],
|
||||
...controlRows.map((item) => [
|
||||
runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.failoverRequests), runtimeText(item.failoverRecovered),
|
||||
runtimeText(item.failoverFailed), runtimeText(item.failoverOutcomeMissing), runtimeText(item.sameAccountRetryEvents),
|
||||
runtimeText(item.tempUnschedulableEvents), runtimeText(item.forwardFailedRequests), runtimeText(item.customerErrorRequests),
|
||||
]),
|
||||
]));
|
||||
const scoreRule = runtimeRecord(accountQuality.scoreRule) ?? {};
|
||||
lines.push(`SCORE RULE reliability=${runtimeText(scoreRule.reliabilityWeight)} latency=${runtimeText(scoreRule.latencyWeight)} availability=${runtimeText(scoreRule.availabilityWeight)} confidence=separate`);
|
||||
if (typeof accountQuality.attribution === "string") lines.push(`NOTE: ${accountQuality.attribution}`);
|
||||
}
|
||||
const errorModels = runtimeRecords(customerErrors.modelBuckets);
|
||||
const errorAccounts = runtimeRecords(customerErrors.accountBuckets);
|
||||
const errorDimensions = [
|
||||
...runtimeRecords(customerErrors.statusBuckets).map((item) => ["status", runtimeText(item.value), runtimeText(item.count)]),
|
||||
...runtimeRecords(customerErrors.streamBuckets).map((item) => ["mode", runtimeText(item.value), runtimeText(item.count)]),
|
||||
...runtimeRecords(customerErrors.phaseBuckets).map((item) => ["phase", runtimeText(item.value), runtimeText(item.count)]),
|
||||
...runtimeRecords(customerErrors.endpointBuckets).map((item) => ["endpoint", runtimeText(item.value), runtimeText(item.count)]),
|
||||
...runtimeRecords(customerErrors.pathBuckets).map((item) => ["path", runtimeText(item.value), runtimeText(item.count)]),
|
||||
];
|
||||
if (errorModels.length > 0) lines.push("", `CUSTOMER ERRORS source=${runtimeText(customerErrors.source)} total=${runtimeText(customerErrors.total)}`, renderTable([
|
||||
["MODEL", "COUNT"], ...errorModels.map((item) => [runtimeShort(item.value, 40), runtimeText(item.count)]),
|
||||
]));
|
||||
if (errorAccounts.length > 0) lines.push(renderTable([
|
||||
["ACCOUNT_ID", "COUNT"], ...errorAccounts.map((item) => [runtimeText(item.value), runtimeText(item.count)]),
|
||||
]));
|
||||
const errorMessages = runtimeRecords(customerErrors.messageBuckets);
|
||||
if (errorMessages.length > 0) lines.push("CUSTOMER ERROR MESSAGES", renderTable([
|
||||
["MESSAGE", "COUNT"], ...errorMessages.map((item) => [runtimeShort(item.value, 100), runtimeText(item.count)]),
|
||||
]));
|
||||
if (errorDimensions.length > 0) lines.push(renderTable([["DIMENSION", "VALUE", "COUNT"], ...errorDimensions]));
|
||||
const errorSamples = runtimeRecords(customerErrors.samples);
|
||||
if (errorSamples.length > 0) lines.push("CUSTOMER ERROR INDEX", renderTable([
|
||||
["REQUEST_ID", "MODEL", "STATUS", "ACCOUNT", "MODE", "PATH", "MESSAGE"],
|
||||
...errorSamples.map((item) => [runtimeText(item.requestId), runtimeShort(item.model, 26), runtimeText(item.statusCode), runtimeText(item.accountId), runtimeText(item.mode), runtimeShort(item.path, 34), runtimeShort(item.message, 48)]),
|
||||
]));
|
||||
lines.push(renderTable([
|
||||
["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"],
|
||||
[
|
||||
runtimeText(availability.totalAccounts), runtimeText(availability.availableCount), runtimeText(availability.rateLimitCount),
|
||||
runtimeText(availability.errorCount), runtimeText(concurrency.currentInUse), runtimeText(concurrency.maxCapacity), runtimeText(concurrency.waitingInQueue),
|
||||
],
|
||||
]));
|
||||
const unavailable = runtimeRecords(availability.unavailableAccounts);
|
||||
if (unavailable.length > 0) {
|
||||
lines.push("", "UNAVAILABLE ACCOUNTS", renderTable([
|
||||
["ACCOUNT", "ID", "STATUS", "REASON"],
|
||||
...unavailable.map((account) => [runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.status), runtimeShort(account.reason, 70)]),
|
||||
]));
|
||||
}
|
||||
const root = runtimeRecord(errors.rootCauseAnalysis) ?? {};
|
||||
const causes = runtimeRecords(root.causeBuckets);
|
||||
if (causes.length > 0) {
|
||||
lines.push("", `ROOT CAUSES scope=${runtimeText(root.scope)}`, renderTable([
|
||||
["CAUSE", "COUNT"],
|
||||
...causes.map((cause) => [runtimeText(cause.cause), runtimeText(cause.count)]),
|
||||
]));
|
||||
}
|
||||
const policy = runtimeRecord(errors.policyEffectiveness) ?? {};
|
||||
const rules = runtimeRecord(policy.ruleMatches) ?? {};
|
||||
const failover = runtimeRecord(policy.failover) ?? {};
|
||||
const forward = runtimeRecord(policy.forwardFailure) ?? {};
|
||||
lines.push("", `POLICY EFFECTIVENESS scope=${runtimeText(policy.scope)}`);
|
||||
lines.push(renderTable([
|
||||
["TEMP_UNSCHED", "FAILOVER", "SUCCESS", "FAILED", "MISSING", "FORWARD_200", "FORWARD_ERR", "SELECT_FAIL"],
|
||||
[
|
||||
runtimeText(rules.tempUnschedulableEvents), runtimeText(failover.requests), runtimeText(failover.succeeded),
|
||||
runtimeText(failover.failed), runtimeText(failover.outcomeMissing), runtimeText(forward.withHttpSuccess),
|
||||
runtimeText(forward.withHttpError), runtimeText(policy.selectFailedEvents),
|
||||
],
|
||||
]));
|
||||
const indexes = runtimeRecord(policy.requestIndexes) ?? {};
|
||||
const indexRows = Object.entries(indexes).flatMap(([kind, value]) => {
|
||||
const item = runtimeRecord(value);
|
||||
return item === null ? [] : [[kind, runtimeText(item.count), runtimeText(item.sampleRequestId), item.moreAvailable === true ? "yes" : "no"]];
|
||||
});
|
||||
if (indexRows.length > 0) lines.push("", "REQUEST INDEX", renderTable([["TYPE", "COUNT", "SAMPLE_REQUEST_ID", "MORE"], ...indexRows]));
|
||||
if (typeof root.attribution === "string") lines.push("", `NOTE: ${root.attribution}`);
|
||||
if (typeof policy.attribution === "string") lines.push(`NOTE: ${policy.attribution}`);
|
||||
const disclosure = runtimeRecord(runtime.disclosure) ?? {};
|
||||
lines.push("", `Account: ${runtimeText(disclosure.account)}`, `Trace: ${runtimeText(disclosure.trace)}`, `Full: ${runtimeText(disclosure.complete)}`);
|
||||
}
|
||||
|
||||
function renderRuntimeAccountList(lines: string[], runtime: Record<string, unknown>): void {
|
||||
const group = runtimeRecord(runtime.group) ?? {};
|
||||
const accounts = runtimeRecords(runtime.accounts);
|
||||
lines.push(`SUB2API RUNTIME ACCOUNTS ok=${runtime.ok === true ? "true" : "false"} group=${runtimeText(group.name)} count=${accounts.length}`);
|
||||
lines.push(renderTable([
|
||||
["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "CONCURRENCY", "LOAD", "PRIORITY", "TEMP_RULES", "TEMPLATE"],
|
||||
...accounts.map((account) => {
|
||||
const temp = runtimeRecord(account.tempUnschedulable) ?? {};
|
||||
return [
|
||||
runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status),
|
||||
runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`,
|
||||
runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate),
|
||||
];
|
||||
}),
|
||||
]));
|
||||
lines.push("", "Detail: bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account <name-or-id>");
|
||||
}
|
||||
|
||||
function renderRuntimeAccountDetail(lines: string[], runtime: Record<string, unknown>): void {
|
||||
const account = runtimeRecord(runtime.account) ?? {};
|
||||
const temp = runtimeRecord(account.tempUnschedulable) ?? {};
|
||||
lines.push(`SUB2API RUNTIME ACCOUNT ok=${runtime.ok === true ? "true" : "false"}`);
|
||||
lines.push(renderTable([
|
||||
["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "PROXY", "CONCURRENCY", "LOAD", "PRIORITY", "TEMPLATE"],
|
||||
[
|
||||
runtimeShort(account.accountName, 42), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status),
|
||||
runtimeText(account.schedulable), runtimeText(account.proxyId), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`,
|
||||
runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(account.matchingTemplate),
|
||||
],
|
||||
]));
|
||||
lines.push(renderTable([
|
||||
["TEMP_ENABLED", "RULES", "STATUS_CODES", "POOL_MODE"],
|
||||
[runtimeText(temp.enabled), runtimeText(temp.ruleCount), runtimeText(temp.statusCodes), runtimeText(account.poolModeEnabled)],
|
||||
]));
|
||||
}
|
||||
|
||||
function renderRuntimeInfrastructure(lines: string[], runtime: Record<string, unknown>): void {
|
||||
const account = runtimeRecord(runtime.account) ?? {};
|
||||
const infra = runtimeRecord(runtime.infrastructure) ?? {};
|
||||
const proxy = runtimeRecord(infra.accountProxy) ?? {};
|
||||
const app = runtimeRecord(infra.appRuntime) ?? {};
|
||||
const route = runtimeRecord(infra.defaultRoute) ?? {};
|
||||
const dns = runtimeRecord(infra.dns) ?? {};
|
||||
const sockets = runtimeRecord(infra.proxySockets) ?? {};
|
||||
const listener = runtimeRecord(infra.proxyEndpointOwner) ?? {};
|
||||
const serviceLogs = runtimeRecord(infra.proxyServiceEvidence) ?? {};
|
||||
const logs = runtimeRecord(infra.recentNetworkEvidence) ?? {};
|
||||
lines.push(`SUB2API RUNTIME INFRASTRUCTURE ok=${runtime.ok === true ? "true" : "false"} since=${runtimeText(logs.window)}`);
|
||||
if (runtime.ok !== true && typeof runtime.error === "string") lines.push(`ERROR: ${runtime.error}`);
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "ID", "TYPE", "STATUS", "SCHED", "PROXY_ID", "PROXY_NAME", "PROTOCOL", "PROXY_ENDPOINT", "PROXY_STATUS"],
|
||||
[
|
||||
runtimeText(account.accountName), runtimeText(account.accountId), runtimeText(account.type), runtimeText(account.status),
|
||||
runtimeText(account.schedulable), runtimeText(proxy.id), runtimeText(proxy.name), runtimeText(proxy.protocol),
|
||||
`${runtimeText(proxy.host)}:${runtimeText(proxy.port)}`, runtimeText(proxy.status),
|
||||
],
|
||||
]));
|
||||
lines.push("", "APP RUNTIME", renderTable([
|
||||
["MODE", "CONTAINER", "STATE", "RESTARTS", "OOM", "NETWORK_MODE", "STARTED_AT"],
|
||||
[
|
||||
runtimeText(app.runtimeMode), runtimeText(app.name), runtimeText(app.state), runtimeText(app.restartCount),
|
||||
runtimeText(app.oomKilled), runtimeText(app.networkMode), runtimeText(app.startedAt),
|
||||
],
|
||||
]));
|
||||
const envRows = runtimeRecords(infra.proxyEnvironment);
|
||||
if (envRows.length > 0) lines.push("PROXY ENVIRONMENT", renderTable([
|
||||
["VARIABLE", "PRESENT", "SCHEME", "HOST", "PORT", "CREDENTIALS_REDACTED"],
|
||||
...envRows.map((item) => [runtimeText(item.name), runtimeText(item.present), runtimeText(item.scheme), runtimeText(item.host), runtimeText(item.port), runtimeText(item.credentialsRedacted)]),
|
||||
]));
|
||||
lines.push("NETWORK", renderTable([
|
||||
["DEFAULT_IFACE", "DEFAULT_GATEWAY", "DNS_NAMESERVERS", "DNS_SEARCH", "PROXY_RESOLVED_IPS", "SOCKETS", "ESTABLISHED"],
|
||||
[
|
||||
runtimeText(route.interface), runtimeText(route.gateway), runtimeText(dns.nameservers), runtimeText(dns.search),
|
||||
runtimeText(sockets.resolvedIps), runtimeText(sockets.total), runtimeText(sockets.established),
|
||||
],
|
||||
]));
|
||||
const listeners = runtimeRecords(listener.listeners);
|
||||
if (listeners.length > 0) lines.push("PROXY ENDPOINT OWNER", renderTable([
|
||||
["LISTEN", "PROCESS", "PID", "UNIT", "ACTIVE", "RESTARTS", "TCP_PEERS", "UDP_PEERS"],
|
||||
...listeners.map((item) => [runtimeText(item.localEndpoint), runtimeText(item.processName), runtimeText(item.pid), runtimeText(item.systemdUnit), runtimeText(item.activeState), runtimeText(item.serviceRestarts), runtimeText(item.establishedPeers), runtimeText(item.udpPeers)]),
|
||||
]));
|
||||
const serviceBuckets = runtimeRecords(serviceLogs.buckets);
|
||||
if (serviceBuckets.length > 0) lines.push("PROXY SERVICE EVIDENCE", renderTable([
|
||||
["CAUSE", "COUNT"],
|
||||
...serviceBuckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]),
|
||||
]));
|
||||
const serviceSamples = runtimeRecords(serviceLogs.samples);
|
||||
if (serviceSamples.length > 0) lines.push("PROXY SERVICE INDEX", renderTable([
|
||||
["AT", "CAUSE", "MESSAGE"],
|
||||
...serviceSamples.map((item) => [runtimeText(item.at), runtimeText(item.cause), runtimeShort(item.message, 100)]),
|
||||
]));
|
||||
const interfaces = runtimeRecords(infra.interfaces);
|
||||
if (interfaces.length > 0) lines.push("INTERFACES", renderTable([
|
||||
["NAME", "RX_BYTES", "TX_BYTES", "RX_ERRORS", "TX_ERRORS", "RX_DROPPED", "TX_DROPPED"],
|
||||
...interfaces.map((item) => [runtimeText(item.name), runtimeText(item.rxBytes), runtimeText(item.txBytes), runtimeText(item.rxErrors), runtimeText(item.txErrors), runtimeText(item.rxDropped), runtimeText(item.txDropped)]),
|
||||
]));
|
||||
const buckets = runtimeRecords(logs.buckets);
|
||||
lines.push("", `RECENT NETWORK EVIDENCE matched=${runtimeText(logs.matchedEvents)} source=${runtimeText(logs.source)}`);
|
||||
if (buckets.length > 0) lines.push(renderTable([
|
||||
["CAUSE", "COUNT"],
|
||||
...buckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]),
|
||||
]));
|
||||
const samples = runtimeRecords(logs.samples);
|
||||
if (samples.length > 0) lines.push("EVIDENCE INDEX", renderTable([
|
||||
["AT", "REQUEST_ID", "CAUSE", "STATUS", "MESSAGE"],
|
||||
...samples.map((item) => [runtimeText(item.at), runtimeText(item.requestId), runtimeText(item.cause), runtimeText(item.statusCode), runtimeShort(item.message, 90)]),
|
||||
]));
|
||||
if (typeof infra.attribution === "string") lines.push(`NOTE: ${infra.attribution}`);
|
||||
}
|
||||
|
||||
function renderRuntimeMutation(lines: string[], runtime: Record<string, unknown>, options: RuntimeOptions): void {
|
||||
const plans = runtimeRecords(runtime.plans);
|
||||
if (plans.length > 0) {
|
||||
const summary = runtimeRecord(runtime.summary) ?? {};
|
||||
lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`);
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES", "RESULT"],
|
||||
...plans.map((plan) => {
|
||||
const account = runtimeRecord(plan.account) ?? {};
|
||||
const before = runtimeRecord(plan.before) ?? {};
|
||||
const after = runtimeRecord(plan.after) ?? {};
|
||||
return [
|
||||
runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template),
|
||||
runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), runtimeText(plan.result),
|
||||
];
|
||||
}),
|
||||
]));
|
||||
lines.push(`SUMMARY selected=${runtimeText(summary.selected)} changed=${runtimeText(summary.changed)} succeeded=${runtimeText(summary.succeeded)} failed=${runtimeText(summary.failed)}`);
|
||||
return;
|
||||
}
|
||||
const plan = runtimeRecord(runtime.plan) ?? {};
|
||||
const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {};
|
||||
const before = runtimeRecord(plan.before) ?? {};
|
||||
const after = runtimeRecord(plan.after) ?? {};
|
||||
lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`);
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES"],
|
||||
[
|
||||
runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template),
|
||||
runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes),
|
||||
],
|
||||
]));
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user