merge: codex deploy fallback

# Conflicts:
#	AGENTS.md
#	TEST.md
#	config.json
#	deploy.json
#	docs/reference/cli.md
#	docs/reference/microservices.md
#	docs/reference/observability.md
#	scripts/cli.ts
#	scripts/src/microservices.ts
#	src/components/backend-core/src/microservice-proxy.ts
#	src/components/microservices/code-queue/src/index.ts
#	src/components/microservices/code-queue/src/queue-api.ts
This commit is contained in:
Codex
2026-05-17 12:33:57 +00:00
20 changed files with 819 additions and 77 deletions
+55 -2
View File
@@ -23,6 +23,9 @@ function isValidEgressPort(port: number): boolean {
return Number.isInteger(port) && port > 0 && port <= 65_535;
}
const egressTcpConnectTimeoutMs = 15_000;
const egressTcpIdleTimeoutMs = 600_000;
function sendEgressClose(provider: ProviderSocket, connectionId: string, error?: string): void {
const message: CoreEgressTcpCloseMessage = error === undefined
? { type: "egress_tcp_close", connectionId }
@@ -30,13 +33,31 @@ function sendEgressClose(provider: ProviderSocket, connectionId: string, error?:
wsSendJson(provider, message);
}
function clearConnectionTimers(connection: EgressTcpConnection): void {
if (connection.connectTimer !== null) clearTimeout(connection.connectTimer);
if (connection.idleTimer !== null) clearTimeout(connection.idleTimer);
connection.connectTimer = null;
connection.idleTimer = null;
}
function closeEgressTcpConnection(providerId: string, connectionId: string, error?: string): void {
const key = egressTcpKey(providerId, connectionId);
const connection = ctx.activeEgressTcpConnections.get(key);
if (connection === undefined) return;
ctx.activeEgressTcpConnections.delete(key);
clearConnectionTimers(connection);
connection.socket.destroy();
if (error !== undefined) sendEgressClose(connection.provider, connectionId, error);
if (error !== undefined) {
logger("warn", "egress_tcp_connection_closed", { providerId, connectionId, error });
sendEgressClose(connection.provider, connectionId, error);
}
}
function refreshConnectionIdle(connection: EgressTcpConnection): void {
if (connection.idleTimer !== null) clearTimeout(connection.idleTimer);
connection.idleTimer = setTimeout(() => {
closeEgressTcpConnection(connection.providerId, connection.connectionId, "egress tcp idle timeout");
}, egressTcpIdleTimeoutMs);
}
export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressTcpOpenMessage): void {
@@ -49,13 +70,32 @@ export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressT
const key = egressTcpKey(message.providerId, message.connectionId);
closeEgressTcpConnection(message.providerId, message.connectionId);
const socket = connectTcp({ host, port });
const connection: EgressTcpConnection = { providerId: message.providerId, connectionId: message.connectionId, socket, provider: ws };
const connection: EgressTcpConnection = {
providerId: message.providerId,
connectionId: message.connectionId,
socket,
provider: ws,
connectTimer: null,
idleTimer: null,
};
ctx.activeEgressTcpConnections.set(key, connection);
connection.connectTimer = setTimeout(() => {
closeEgressTcpConnection(message.providerId, message.connectionId, "egress tcp connect timeout");
}, egressTcpConnectTimeoutMs);
refreshConnectionIdle(connection);
socket.on("connect", () => {
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
if (connection.connectTimer !== null) {
clearTimeout(connection.connectTimer);
connection.connectTimer = null;
}
refreshConnectionIdle(connection);
const opened: CoreEgressTcpOpenedMessage = { type: "egress_tcp_opened", connectionId: message.connectionId };
wsSendJson(ws, opened);
});
socket.on("data", (chunk) => {
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
refreshConnectionIdle(connection);
const data: CoreEgressTcpDataMessage = {
type: "egress_tcp_data",
connectionId: message.connectionId,
@@ -67,11 +107,13 @@ export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressT
socket.on("close", () => {
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
ctx.activeEgressTcpConnections.delete(key);
clearConnectionTimers(connection);
sendEgressClose(ws, message.connectionId);
});
socket.on("error", (error) => {
if (ctx.activeEgressTcpConnections.get(key) !== connection) return;
ctx.activeEgressTcpConnections.delete(key);
clearConnectionTimers(connection);
sendEgressClose(ws, message.connectionId, error.message);
});
}
@@ -79,6 +121,7 @@ export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressT
export function handleEgressTcpData(message: ProviderEgressTcpDataMessage): void {
const connection = ctx.activeEgressTcpConnections.get(egressTcpKey(message.providerId, message.connectionId));
if (connection === undefined) return;
refreshConnectionIdle(connection);
connection.socket.write(Buffer.from(message.data, message.encoding === "base64" ? "base64" : "utf8"));
}
@@ -90,6 +133,16 @@ export function closeEgressTcpConnectionsForProvider(providerId: string): void {
for (const [key, connection] of ctx.activeEgressTcpConnections) {
if (connection.providerId !== providerId) continue;
ctx.activeEgressTcpConnections.delete(key);
clearConnectionTimers(connection);
connection.socket.destroy();
}
}
export function closeEgressTcpConnectionsForSocket(provider: ProviderSocket): void {
for (const [key, connection] of ctx.activeEgressTcpConnections) {
if (connection.provider !== provider) continue;
ctx.activeEgressTcpConnections.delete(key);
clearConnectionTimers(connection);
connection.socket.destroy();
}
}
@@ -381,6 +381,7 @@ function isK3sctlManagedMicroservice(service: MicroserviceConfig): boolean {
function codeQueueK3sServiceIdForRequest(method: string, targetPath: string): string {
const normalizedMethod = method.toUpperCase();
if (targetPath === "/" || targetPath === "/health" || targetPath === "/live" || targetPath === "/api/dev-ready") return "code-queue-scheduler";
if (targetPath === "/api/queues" || targetPath === "/api/tasks/overview") return "code-queue-scheduler";
if (targetPath === "/api/oa/backfill" || targetPath === "/api/notifications/claudeqq/drain" || targetPath === "/api/notifications/claudeqq/backfill") return "code-queue-scheduler";
if (targetPath === "/api/judge/probe" || targetPath === "/api/judge/self-test" || targetPath === "/api/queue-order/self-test" || targetPath === "/api/reference-injection/self-test" || targetPath === "/api/trace-port/self-test") return "code-queue-scheduler";
if (/^\/api\/tasks\/[^/]+\/(?:steer|interrupt)$/u.test(targetPath)) return "code-queue-scheduler";
@@ -778,7 +779,8 @@ async function microserviceTunnelSelfTestResponse(service: MicroserviceConfig):
const bodyRecord = isPlainRecord(body) ? body : {};
const hasRequestId = typeof bodyRecord.requestId === "string" && bodyRecord.requestId.length > 0;
const hasStage = typeof bodyRecord.stage === "string" && bodyRecord.stage.length > 0;
const ok = response.status === 502 && hasRequestId && hasStage && headers.requestId === bodyRecord.requestId;
const expectedStatus = response.status === 502 || response.status === 504;
const ok = expectedStatus && hasRequestId && hasStage && headers.requestId === bodyRecord.requestId;
return jsonResponse({
ok,
serviceId: service.id,
@@ -787,7 +789,7 @@ async function microserviceTunnelSelfTestResponse(service: MicroserviceConfig):
expectedFailure: true,
status: response.status,
checks: {
expectedStatus: response.status === 502,
expectedStatus,
bodyHasRequestId: hasRequestId,
bodyHasStage: hasStage,
headerHasRequestId: typeof headers.requestId === "string" && headers.requestId.length > 0,
@@ -956,7 +958,7 @@ async function providerHttpTunnelMicroserviceResponse(
const durationMs = Date.now() - startedAt;
if (message === null) {
attempts.push({ attempt, requestId, ok: false, reason: reason ?? "timeout", durationMs, timeoutMs });
if (retryable && tunnelFailureRetryable(reason) && attempt < maxAttempts) {
if (retryable && reason !== "timeout" && tunnelFailureRetryable(reason) && attempt < maxAttempts) {
logger("warn", "http_tunnel_retry", {
providerId: service.providerId,
serviceId: service.id,
+2
View File
@@ -166,6 +166,8 @@ export interface EgressTcpConnection {
connectionId: string;
socket: Socket;
provider: ProviderSocket;
connectTimer: ReturnType<typeof setTimeout> | null;
idleTimer: ReturnType<typeof setTimeout> | null;
}
export type HttpTunnelFailureReason =