fix: keep code queue live during db outages
This commit is contained in:
Codex
@@ -689,6 +689,15 @@ data:
|
||||
const rawRoutes = process.env.TCP_EGRESS_ROUTES || "";
|
||||
const startedAt = new Date().toISOString();
|
||||
|
||||
function positiveInt(value, fallback) {
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
const connectAttempts = positiveInt(process.env.TCP_EGRESS_CONNECT_ATTEMPTS, 8);
|
||||
const connectRetryBackoffMs = positiveInt(process.env.TCP_EGRESS_CONNECT_RETRY_BACKOFF_MS, 250);
|
||||
const maxEarlyClientBytes = positiveInt(process.env.TCP_EGRESS_MAX_EARLY_CLIENT_BYTES, 1024 * 1024);
|
||||
|
||||
function parseRoute(text) {
|
||||
const parts = text.trim().split("=");
|
||||
if (parts.length !== 3) throw new Error(`invalid route: ${text}`);
|
||||
@@ -715,55 +724,110 @@ data:
|
||||
}
|
||||
|
||||
function pipeViaHttpConnect(client, route) {
|
||||
const proxy = net.connect({ host: proxyHost, port: proxyPort });
|
||||
const earlyClientChunks = [];
|
||||
const proxyHeaderChunks = [];
|
||||
let earlyClientBytes = 0;
|
||||
let proxy = null;
|
||||
let tunnelReady = false;
|
||||
let closed = false;
|
||||
let attempt = 0;
|
||||
let retryTimer = null;
|
||||
|
||||
const cleanupProxy = () => {
|
||||
if (proxy === null) return;
|
||||
proxy.removeAllListeners();
|
||||
proxy.destroy();
|
||||
proxy = null;
|
||||
};
|
||||
|
||||
const closeBoth = () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
if (retryTimer !== null) clearTimeout(retryTimer);
|
||||
client.destroy();
|
||||
proxy.destroy();
|
||||
cleanupProxy();
|
||||
};
|
||||
|
||||
const retryOrClose = (reason, data = {}) => {
|
||||
cleanupProxy();
|
||||
if (!closed && attempt < connectAttempts) {
|
||||
const delayMs = Math.min(connectRetryBackoffMs * attempt, 3000);
|
||||
log("warn", "connect_retry", { route: route.name, attempt, attempts: connectAttempts, delayMs, reason, ...data });
|
||||
retryTimer = setTimeout(() => {
|
||||
retryTimer = null;
|
||||
startProxy();
|
||||
}, delayMs);
|
||||
retryTimer.unref?.();
|
||||
return;
|
||||
}
|
||||
log("warn", "connect_rejected", { route: route.name, attempt, attempts: connectAttempts, reason, ...data });
|
||||
closeBoth();
|
||||
};
|
||||
|
||||
client.on("data", (chunk) => {
|
||||
if (tunnelReady) proxy.write(chunk);
|
||||
else earlyClientChunks.push(chunk);
|
||||
if (tunnelReady && proxy !== null) {
|
||||
proxy.write(chunk);
|
||||
return;
|
||||
}
|
||||
earlyClientBytes += chunk.length;
|
||||
if (earlyClientBytes > maxEarlyClientBytes) {
|
||||
log("warn", "early_client_buffer_limit_exceeded", { route: route.name, bytes: earlyClientBytes, limit: maxEarlyClientBytes });
|
||||
closeBoth();
|
||||
return;
|
||||
}
|
||||
earlyClientChunks.push(chunk);
|
||||
});
|
||||
client.on("error", closeBoth);
|
||||
client.on("close", closeBoth);
|
||||
|
||||
proxy.on("connect", () => {
|
||||
proxy.write(`CONNECT ${route.targetHost}:${route.targetPort} HTTP/1.1\r\nHost: ${route.targetHost}:${route.targetPort}\r\n\r\n`);
|
||||
});
|
||||
proxy.on("data", (chunk) => {
|
||||
if (tunnelReady) {
|
||||
client.write(chunk);
|
||||
return;
|
||||
}
|
||||
proxyHeaderChunks.push(chunk);
|
||||
const headerBuffer = Buffer.concat(proxyHeaderChunks);
|
||||
const headerEnd = findHeaderEnd(headerBuffer);
|
||||
if (headerEnd < 0) return;
|
||||
const header = headerBuffer.subarray(0, headerEnd).toString("latin1");
|
||||
if (!/^HTTP\/1\.[01] 200\b/u.test(header)) {
|
||||
log("warn", "connect_rejected", { route: route.name, header: header.slice(0, 200) });
|
||||
const startProxy = () => {
|
||||
if (closed) return;
|
||||
attempt += 1;
|
||||
tunnelReady = false;
|
||||
const currentProxy = net.connect({ host: proxyHost, port: proxyPort });
|
||||
const proxyHeaderChunks = [];
|
||||
proxy = currentProxy;
|
||||
|
||||
currentProxy.on("connect", () => {
|
||||
currentProxy.write(`CONNECT ${route.targetHost}:${route.targetPort} HTTP/1.1\r\nHost: ${route.targetHost}:${route.targetPort}\r\n\r\n`);
|
||||
});
|
||||
currentProxy.on("data", (chunk) => {
|
||||
if (currentProxy !== proxy) return;
|
||||
if (tunnelReady) {
|
||||
client.write(chunk);
|
||||
return;
|
||||
}
|
||||
proxyHeaderChunks.push(chunk);
|
||||
const headerBuffer = Buffer.concat(proxyHeaderChunks);
|
||||
const headerEnd = findHeaderEnd(headerBuffer);
|
||||
if (headerEnd < 0) return;
|
||||
const header = headerBuffer.subarray(0, headerEnd).toString("latin1");
|
||||
if (!/^HTTP\/1\.[01] 200\b/u.test(header)) {
|
||||
const body = headerBuffer.subarray(headerEnd).toString("utf8").slice(0, 300);
|
||||
retryOrClose("non_200", { header: header.slice(0, 200), body: body.length > 0 ? body : null });
|
||||
return;
|
||||
}
|
||||
tunnelReady = true;
|
||||
const remaining = headerBuffer.subarray(headerEnd);
|
||||
if (remaining.length > 0) client.write(remaining);
|
||||
for (const early of earlyClientChunks) currentProxy.write(early);
|
||||
earlyClientChunks.length = 0;
|
||||
earlyClientBytes = 0;
|
||||
});
|
||||
currentProxy.on("error", (error) => {
|
||||
if (currentProxy !== proxy) return;
|
||||
retryOrClose("proxy_error", { error: String(error?.message || error) });
|
||||
});
|
||||
currentProxy.on("close", () => {
|
||||
if (currentProxy !== proxy || closed) return;
|
||||
if (!tunnelReady) {
|
||||
retryOrClose("proxy_closed_before_tunnel");
|
||||
return;
|
||||
}
|
||||
closeBoth();
|
||||
return;
|
||||
}
|
||||
tunnelReady = true;
|
||||
const remaining = headerBuffer.subarray(headerEnd);
|
||||
if (remaining.length > 0) client.write(remaining);
|
||||
for (const early of earlyClientChunks) proxy.write(early);
|
||||
earlyClientChunks.length = 0;
|
||||
});
|
||||
proxy.on("error", (error) => {
|
||||
log("warn", "proxy_error", { route: route.name, error: String(error?.message || error) });
|
||||
closeBoth();
|
||||
});
|
||||
proxy.on("close", closeBoth);
|
||||
});
|
||||
};
|
||||
|
||||
startProxy();
|
||||
}
|
||||
|
||||
for (const route of routes) {
|
||||
@@ -776,7 +840,7 @@ data:
|
||||
hostname: "0.0.0.0",
|
||||
port: healthPort,
|
||||
fetch() {
|
||||
return Response.json({ ok: true, service: "tcp-egress-gateway", startedAt, proxy: `${proxyHost}:${proxyPort}`, routes });
|
||||
return Response.json({ ok: true, service: "tcp-egress-gateway", startedAt, proxy: `${proxyHost}:${proxyPort}`, routes, connectAttempts, connectRetryBackoffMs });
|
||||
},
|
||||
});
|
||||
---
|
||||
@@ -826,6 +890,10 @@ spec:
|
||||
value: "postgres=15432=74.48.78.17:15432,oa-event-flow=4255=74.48.78.17:4255"
|
||||
- name: TCP_EGRESS_HEALTH_PORT
|
||||
value: "18080"
|
||||
- name: TCP_EGRESS_CONNECT_ATTEMPTS
|
||||
value: "8"
|
||||
- name: TCP_EGRESS_CONNECT_RETRY_BACKOFF_MS
|
||||
value: "250"
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /etc/unidesk-tcp-egress
|
||||
|
||||
Reference in New Issue
Block a user