merge: codex deploy fallback
# Conflicts: # AGENTS.md # TEST.md # config.json # deploy.json # docs/reference/cli.md # docs/reference/microservices.md # docs/reference/observability.md # scripts/cli.ts # scripts/src/microservices.ts # src/components/backend-core/src/microservice-proxy.ts # src/components/microservices/code-queue/src/index.ts # src/components/microservices/code-queue/src/queue-api.ts
This commit is contained in:
@@ -23,6 +23,9 @@ function isValidEgressPort(port: number): boolean {
|
||||
return Number.isInteger(port) && port > 0 && port <= 65_535;
|
||||
}
|
||||
|
||||
const egressTcpConnectTimeoutMs = 15_000;
|
||||
const egressTcpIdleTimeoutMs = 600_000;
|
||||
|
||||
function sendEgressClose(provider: ProviderSocket, connectionId: string, error?: string): void {
|
||||
const message: CoreEgressTcpCloseMessage = error === undefined
|
||||
? { type: "egress_tcp_close", connectionId }
|
||||
@@ -30,13 +33,31 @@ function sendEgressClose(provider: ProviderSocket, connectionId: string, error?:
|
||||
wsSendJson(provider, message);
|
||||
}
|
||||
|
||||
function clearConnectionTimers(connection: EgressTcpConnection): void {
|
||||
if (connection.connectTimer !== null) clearTimeout(connection.connectTimer);
|
||||
if (connection.idleTimer !== null) clearTimeout(connection.idleTimer);
|
||||
connection.connectTimer = null;
|
||||
connection.idleTimer = null;
|
||||
}
|
||||
|
||||
function closeEgressTcpConnection(providerId: string, connectionId: string, error?: string): void {
|
||||
const key = egressTcpKey(providerId, connectionId);
|
||||
const connection = ctx.activeEgressTcpConnections.get(key);
|
||||
if (connection === undefined) return;
|
||||
ctx.activeEgressTcpConnections.delete(key);
|
||||
clearConnectionTimers(connection);
|
||||
connection.socket.destroy();
|
||||
if (error !== undefined) sendEgressClose(connection.provider, connectionId, error);
|
||||
if (error !== undefined) {
|
||||
logger("warn", "egress_tcp_connection_closed", { providerId, connectionId, error });
|
||||
sendEgressClose(connection.provider, connectionId, error);
|
||||
}
|
||||
}
|
||||
|
||||
function refreshConnectionIdle(connection: EgressTcpConnection): void {
|
||||
if (connection.idleTimer !== null) clearTimeout(connection.idleTimer);
|
||||
connection.idleTimer = setTimeout(() => {
|
||||
closeEgressTcpConnection(connection.providerId, connection.connectionId, "egress tcp idle timeout");
|
||||
}, egressTcpIdleTimeoutMs);
|
||||
}
|
||||
|
||||
export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressTcpOpenMessage): void {
|
||||
@@ -49,13 +70,32 @@ export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressT
|
||||
const key = egressTcpKey(message.providerId, message.connectionId);
|
||||
closeEgressTcpConnection(message.providerId, message.connectionId);
|
||||
const socket = connectTcp({ host, port });
|
||||
const connection: EgressTcpConnection = { providerId: message.providerId, connectionId: message.connectionId, socket, provider: ws };
|
||||
const connection: EgressTcpConnection = {
|
||||
providerId: message.providerId,
|
||||
connectionId: message.connectionId,
|
||||
socket,
|
||||
provider: ws,
|
||||
connectTimer: null,
|
||||
idleTimer: null,
|
||||
};
|
||||
ctx.activeEgressTcpConnections.set(key, connection);
|
||||
connection.connectTimer = setTimeout(() => {
|
||||
closeEgressTcpConnection(message.providerId, message.connectionId, "egress tcp connect timeout");
|
||||
}, egressTcpConnectTimeoutMs);
|
||||
refreshConnectionIdle(connection);
|
||||
socket.on("connect", () => {
|
||||
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
|
||||
if (connection.connectTimer !== null) {
|
||||
clearTimeout(connection.connectTimer);
|
||||
connection.connectTimer = null;
|
||||
}
|
||||
refreshConnectionIdle(connection);
|
||||
const opened: CoreEgressTcpOpenedMessage = { type: "egress_tcp_opened", connectionId: message.connectionId };
|
||||
wsSendJson(ws, opened);
|
||||
});
|
||||
socket.on("data", (chunk) => {
|
||||
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
|
||||
refreshConnectionIdle(connection);
|
||||
const data: CoreEgressTcpDataMessage = {
|
||||
type: "egress_tcp_data",
|
||||
connectionId: message.connectionId,
|
||||
@@ -67,11 +107,13 @@ export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressT
|
||||
socket.on("close", () => {
|
||||
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
|
||||
ctx.activeEgressTcpConnections.delete(key);
|
||||
clearConnectionTimers(connection);
|
||||
sendEgressClose(ws, message.connectionId);
|
||||
});
|
||||
socket.on("error", (error) => {
|
||||
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
|
||||
ctx.activeEgressTcpConnections.delete(key);
|
||||
clearConnectionTimers(connection);
|
||||
sendEgressClose(ws, message.connectionId, error.message);
|
||||
});
|
||||
}
|
||||
@@ -79,6 +121,7 @@ export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressT
|
||||
export function handleEgressTcpData(message: ProviderEgressTcpDataMessage): void {
|
||||
const connection = ctx.activeEgressTcpConnections.get(egressTcpKey(message.providerId, message.connectionId));
|
||||
if (connection === undefined) return;
|
||||
refreshConnectionIdle(connection);
|
||||
connection.socket.write(Buffer.from(message.data, message.encoding === "base64" ? "base64" : "utf8"));
|
||||
}
|
||||
|
||||
@@ -90,6 +133,16 @@ export function closeEgressTcpConnectionsForProvider(providerId: string): void {
|
||||
for (const [key, connection] of ctx.activeEgressTcpConnections) {
|
||||
if (connection.providerId !== providerId) continue;
|
||||
ctx.activeEgressTcpConnections.delete(key);
|
||||
clearConnectionTimers(connection);
|
||||
connection.socket.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
export function closeEgressTcpConnectionsForSocket(provider: ProviderSocket): void {
|
||||
for (const [key, connection] of ctx.activeEgressTcpConnections) {
|
||||
if (connection.provider !== provider) continue;
|
||||
ctx.activeEgressTcpConnections.delete(key);
|
||||
clearConnectionTimers(connection);
|
||||
connection.socket.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,6 +381,7 @@ function isK3sctlManagedMicroservice(service: MicroserviceConfig): boolean {
|
||||
function codeQueueK3sServiceIdForRequest(method: string, targetPath: string): string {
|
||||
const normalizedMethod = method.toUpperCase();
|
||||
if (targetPath === "/" || targetPath === "/health" || targetPath === "/live" || targetPath === "/api/dev-ready") return "code-queue-scheduler";
|
||||
if (targetPath === "/api/queues" || targetPath === "/api/tasks/overview") return "code-queue-scheduler";
|
||||
if (targetPath === "/api/oa/backfill" || targetPath === "/api/notifications/claudeqq/drain" || targetPath === "/api/notifications/claudeqq/backfill") return "code-queue-scheduler";
|
||||
if (targetPath === "/api/judge/probe" || targetPath === "/api/judge/self-test" || targetPath === "/api/queue-order/self-test" || targetPath === "/api/reference-injection/self-test" || targetPath === "/api/trace-port/self-test") return "code-queue-scheduler";
|
||||
if (/^\/api\/tasks\/[^/]+\/(?:steer|interrupt)$/u.test(targetPath)) return "code-queue-scheduler";
|
||||
@@ -778,7 +779,8 @@ async function microserviceTunnelSelfTestResponse(service: MicroserviceConfig):
|
||||
const bodyRecord = isPlainRecord(body) ? body : {};
|
||||
const hasRequestId = typeof bodyRecord.requestId === "string" && bodyRecord.requestId.length > 0;
|
||||
const hasStage = typeof bodyRecord.stage === "string" && bodyRecord.stage.length > 0;
|
||||
const ok = response.status === 502 && hasRequestId && hasStage && headers.requestId === bodyRecord.requestId;
|
||||
const expectedStatus = response.status === 502 || response.status === 504;
|
||||
const ok = expectedStatus && hasRequestId && hasStage && headers.requestId === bodyRecord.requestId;
|
||||
return jsonResponse({
|
||||
ok,
|
||||
serviceId: service.id,
|
||||
@@ -787,7 +789,7 @@ async function microserviceTunnelSelfTestResponse(service: MicroserviceConfig):
|
||||
expectedFailure: true,
|
||||
status: response.status,
|
||||
checks: {
|
||||
expectedStatus: response.status === 502,
|
||||
expectedStatus,
|
||||
bodyHasRequestId: hasRequestId,
|
||||
bodyHasStage: hasStage,
|
||||
headerHasRequestId: typeof headers.requestId === "string" && headers.requestId.length > 0,
|
||||
@@ -956,7 +958,7 @@ async function providerHttpTunnelMicroserviceResponse(
|
||||
const durationMs = Date.now() - startedAt;
|
||||
if (message === null) {
|
||||
attempts.push({ attempt, requestId, ok: false, reason: reason ?? "timeout", durationMs, timeoutMs });
|
||||
if (retryable && tunnelFailureRetryable(reason) && attempt < maxAttempts) {
|
||||
if (retryable && reason !== "timeout" && tunnelFailureRetryable(reason) && attempt < maxAttempts) {
|
||||
logger("warn", "http_tunnel_retry", {
|
||||
providerId: service.providerId,
|
||||
serviceId: service.id,
|
||||
|
||||
@@ -166,6 +166,8 @@ export interface EgressTcpConnection {
|
||||
connectionId: string;
|
||||
socket: Socket;
|
||||
provider: ProviderSocket;
|
||||
connectTimer: ReturnType<typeof setTimeout> | null;
|
||||
idleTimer: ReturnType<typeof setTimeout> | null;
|
||||
}
|
||||
|
||||
export type HttpTunnelFailureReason =
|
||||
|
||||
Reference in New Issue
Block a user