fix: add sub2api edge retry
This commit is contained in:
Codex
@@ -156,6 +156,17 @@ interface CodexPoolPublicExposureConfig {
|
||||
serviceName: string;
|
||||
upstreamBaseUrl: string;
|
||||
responseHeaderTimeoutSeconds: number;
|
||||
edgeRetry: CodexPoolCaddyEdgeRetryConfig;
|
||||
};
|
||||
}
|
||||
|
||||
interface CodexPoolCaddyEdgeRetryConfig {
|
||||
enabled: boolean;
|
||||
tryDurationSeconds: number;
|
||||
tryIntervalMilliseconds: number;
|
||||
retryMatch: {
|
||||
methods: string[];
|
||||
paths: string[];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -999,6 +1010,15 @@ function defaultCodexPoolConfig(): CodexPoolConfig {
|
||||
serviceName: "caddy",
|
||||
upstreamBaseUrl: "http://127.0.0.1:21880",
|
||||
responseHeaderTimeoutSeconds: 180,
|
||||
edgeRetry: {
|
||||
enabled: false,
|
||||
tryDurationSeconds: 0,
|
||||
tryIntervalMilliseconds: 250,
|
||||
retryMatch: {
|
||||
methods: [],
|
||||
paths: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
localCodex: {
|
||||
@@ -1205,6 +1225,66 @@ function readCaddyTimeoutSeconds(value: unknown, key: string, fallback: number):
|
||||
return seconds;
|
||||
}
|
||||
|
||||
function readCaddyEdgeRetryConfig(value: unknown, fallback: CodexPoolCaddyEdgeRetryConfig, key: string): CodexPoolCaddyEdgeRetryConfig {
|
||||
if (value === undefined || value === null) return cloneCaddyEdgeRetryConfig(fallback);
|
||||
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
||||
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback.enabled);
|
||||
if (!enabled) return { ...cloneCaddyEdgeRetryConfig(fallback), enabled: false };
|
||||
|
||||
const tryDurationSeconds = readPositiveInteger(value.tryDurationSeconds, `${key}.tryDurationSeconds`);
|
||||
const tryIntervalMilliseconds = readPositiveInteger(value.tryIntervalMilliseconds, `${key}.tryIntervalMilliseconds`);
|
||||
const retryMatchValue = isRecord(value.retryMatch) ? value.retryMatch : null;
|
||||
if (retryMatchValue === null) throw new Error(`${codexPoolConfigPath}.${key}.retryMatch must be a YAML object when enabled=true`);
|
||||
const retryMatch = {
|
||||
methods: readCaddyRetryMethods(retryMatchValue.methods, `${key}.retryMatch.methods`),
|
||||
paths: readCaddyRetryPaths(retryMatchValue.paths, `${key}.retryMatch.paths`),
|
||||
};
|
||||
if (retryMatch.methods.length === 0 && retryMatch.paths.length === 0) {
|
||||
throw new Error(`${codexPoolConfigPath}.${key}.retryMatch must include at least one method or path matcher when enabled=true`);
|
||||
}
|
||||
return { enabled, tryDurationSeconds, tryIntervalMilliseconds, retryMatch };
|
||||
}
|
||||
|
||||
function readPositiveInteger(value: unknown, key: string): number {
|
||||
const parsed = numberValue(value);
|
||||
if (parsed === null || !Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`${codexPoolConfigPath}.${key} must be a positive integer`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readCaddyRetryMethods(value: unknown, key: string): string[] {
|
||||
if (value === undefined || value === null) return [];
|
||||
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
||||
const seen = new Set<string>();
|
||||
const methods: string[] = [];
|
||||
for (const item of value) {
|
||||
const method = stringValue(item)?.toUpperCase() ?? null;
|
||||
if (method === null || !/^[A-Z][A-Z0-9_-]*$/u.test(method)) throw new Error(`${codexPoolConfigPath}.${key} entries must be HTTP method tokens`);
|
||||
if (seen.has(method)) continue;
|
||||
seen.add(method);
|
||||
methods.push(method);
|
||||
}
|
||||
return methods;
|
||||
}
|
||||
|
||||
function readCaddyRetryPaths(value: unknown, key: string): string[] {
|
||||
if (value === undefined || value === null) return [];
|
||||
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
||||
const seen = new Set<string>();
|
||||
const paths: string[] = [];
|
||||
for (const item of value) {
|
||||
const path = stringValue(item);
|
||||
if (path === null || !path.startsWith("/") || /[\r\n]/u.test(path)) {
|
||||
throw new Error(`${codexPoolConfigPath}.${key} entries must be Caddy path matchers starting with / and without newlines`);
|
||||
}
|
||||
if (seen.has(path)) continue;
|
||||
seen.add(path);
|
||||
paths.push(path);
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
function readTempUnschedulablePolicy(value: unknown, key: string, fallback: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy {
|
||||
if (value === undefined || value === null) return cloneTempUnschedulablePolicy(fallback);
|
||||
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
||||
@@ -1265,6 +1345,18 @@ function cloneTempUnschedulablePolicy(policy: CodexTempUnschedulablePolicy): Cod
|
||||
};
|
||||
}
|
||||
|
||||
function cloneCaddyEdgeRetryConfig(config: CodexPoolCaddyEdgeRetryConfig): CodexPoolCaddyEdgeRetryConfig {
|
||||
return {
|
||||
enabled: config.enabled,
|
||||
tryDurationSeconds: config.tryDurationSeconds,
|
||||
tryIntervalMilliseconds: config.tryIntervalMilliseconds,
|
||||
retryMatch: {
|
||||
methods: [...config.retryMatch.methods],
|
||||
paths: [...config.retryMatch.paths],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function readBooleanConfig(value: unknown, key: string, fallback: boolean): boolean {
|
||||
if (value === undefined || value === null) return fallback;
|
||||
const parsed = booleanValue(value);
|
||||
@@ -1304,6 +1396,11 @@ function readPublicExposureConfig(value: unknown, defaults: CodexPoolPublicExpos
|
||||
"publicExposure.masterCaddy.responseHeaderTimeoutSeconds",
|
||||
defaults.masterCaddy.responseHeaderTimeoutSeconds,
|
||||
),
|
||||
edgeRetry: readCaddyEdgeRetryConfig(
|
||||
masterCaddyValue.edgeRetry,
|
||||
defaults.masterCaddy.edgeRetry,
|
||||
"publicExposure.masterCaddy.edgeRetry",
|
||||
),
|
||||
},
|
||||
};
|
||||
validateKubernetesName(config.configMapName, "publicExposure.configMapName", true);
|
||||
@@ -2137,6 +2234,7 @@ function publicExposureSummary(pool: CodexPoolConfig): Record<string, unknown> {
|
||||
serviceName: pool.publicExposure.masterCaddy.serviceName,
|
||||
upstreamBaseUrl: pool.publicExposure.masterCaddy.upstreamBaseUrl,
|
||||
responseHeaderTimeoutSeconds: pool.publicExposure.masterCaddy.responseHeaderTimeoutSeconds,
|
||||
edgeRetry: pool.publicExposure.masterCaddy.edgeRetry,
|
||||
},
|
||||
upstream: {
|
||||
localIP: pool.publicExposure.localIP,
|
||||
@@ -2201,7 +2299,7 @@ async function applyMasterCaddySite(pool: CodexPoolConfig): Promise<Record<strin
|
||||
const path = caddy.configPath;
|
||||
if (!existsSync(path)) return { ok: false, error: "master-caddy-config-missing", path, valuesPrinted: false };
|
||||
const before = readFileSync(path, "utf8");
|
||||
const desiredBlock = renderCaddySiteBlock(caddy.domain, caddy.upstreamBaseUrl, caddy.responseHeaderTimeoutSeconds);
|
||||
const desiredBlock = renderCaddySiteBlock(caddy.domain, caddy.upstreamBaseUrl, caddy.responseHeaderTimeoutSeconds, caddy.edgeRetry);
|
||||
const existing = caddySiteBlock(before, caddy.domain);
|
||||
const alreadyConfigured = existing === desiredBlock;
|
||||
let backupPath: string | null = null;
|
||||
@@ -2233,6 +2331,7 @@ async function applyMasterCaddySite(pool: CodexPoolConfig): Promise<Record<strin
|
||||
upstreamBaseUrl: caddy.upstreamBaseUrl,
|
||||
serviceName: caddy.serviceName,
|
||||
responseHeaderTimeoutSeconds: caddy.responseHeaderTimeoutSeconds,
|
||||
edgeRetry: caddy.edgeRetry,
|
||||
validate: {
|
||||
exitCode: validate.exitCode,
|
||||
stdoutTail: Buffer.from(validate.stdout).toString("utf8").slice(-1000),
|
||||
@@ -2252,13 +2351,19 @@ async function applyMasterCaddySite(pool: CodexPoolConfig): Promise<Record<strin
|
||||
};
|
||||
}
|
||||
|
||||
export function renderCaddySiteBlock(domain: string, upstreamBaseUrl: string, responseHeaderTimeoutSeconds = 180): string {
|
||||
export function renderCaddySiteBlock(
|
||||
domain: string,
|
||||
upstreamBaseUrl: string,
|
||||
responseHeaderTimeoutSeconds = 180,
|
||||
edgeRetry: CodexPoolCaddyEdgeRetryConfig | null = null,
|
||||
): string {
|
||||
const upstream = new URL(upstreamBaseUrl);
|
||||
const upstreamHost = `${upstream.hostname}${upstream.port ? `:${upstream.port}` : ""}`;
|
||||
const retryBlock = renderCaddyEdgeRetryBlock(edgeRetry);
|
||||
return `${domain} {
|
||||
encode zstd gzip
|
||||
reverse_proxy ${upstreamHost} {
|
||||
header_up Host {host}
|
||||
${retryBlock} header_up Host {host}
|
||||
header_up X-Real-IP {remote_host}
|
||||
transport http {
|
||||
dial_timeout 5s
|
||||
@@ -2268,6 +2373,20 @@ export function renderCaddySiteBlock(domain: string, upstreamBaseUrl: string, re
|
||||
}`;
|
||||
}
|
||||
|
||||
function renderCaddyEdgeRetryBlock(edgeRetry: CodexPoolCaddyEdgeRetryConfig | null): string {
|
||||
if (edgeRetry === null || !edgeRetry.enabled) return "";
|
||||
const matchLines = [
|
||||
edgeRetry.retryMatch.methods.length > 0 ? ` method ${edgeRetry.retryMatch.methods.join(" ")}` : null,
|
||||
edgeRetry.retryMatch.paths.length > 0 ? ` path ${edgeRetry.retryMatch.paths.join(" ")}` : null,
|
||||
].filter((line): line is string => line !== null);
|
||||
return ` lb_try_duration ${edgeRetry.tryDurationSeconds}s
|
||||
lb_try_interval ${edgeRetry.tryIntervalMilliseconds}ms
|
||||
lb_retry_match {
|
||||
${matchLines.join("\n")}
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
function caddySiteBlock(text: string, domain: string): string | null {
|
||||
const startMatch = new RegExp(`(^|\\n)${escapeRegExp(domain)}\\s*\\{`, "u").exec(text);
|
||||
if (startMatch === null) return null;
|
||||
|
||||
Reference in New Issue
Block a user