fix: cap tran runtime and remove local lock
This commit is contained in:
+2
-1
@@ -185,8 +185,9 @@ export function sshHelp(): unknown {
|
||||
"Do not put operation names in any colon route segment, including nested k3s namespace/workload/container segments.",
|
||||
"Do not use post-provider shorthand such as `ssh G14 k3s ...`; write `ssh G14:k3s ...` so location and operation stay separated.",
|
||||
"If an ssh-like remote command fails with timeout/kex/exit-255 friction, stderr includes one low-noise UNIDESK_SSH_HINT JSON line with the argv retry command.",
|
||||
"Non-interactive ssh/tran operations have a hard top-level runtime timeout capped at 60s. Timeout writes UNIDESK_SSH_RUNTIME_TIMEOUT or UNIDESK_TRAN_TIMEOUT_HINT and disconnects the broker; long CI/CD, trace, logs, build, or hardware work must use submit-and-poll / short query loops instead of keeping tran open.",
|
||||
"Only slow ssh/tran runtime writes UNIDESK_SSH_TIMING JSON to stderr; operations over 10s are marked level=warning even when they succeed, because slow successful calls are a distributed performance monitoring signal. Check provider latency, remote command cost, helper bootstrap, or tran/apply-patch optimization before repeating high-frequency work. Routine short calls do not emit timing noise.",
|
||||
"The local tran wrapper serializes non-interactive calls per provider/plane before opening provider SSH sessions, so parallel Codex file reads do not stampede the provider session allocator; set UNIDESK_TRAN_SESSION_LOCK=0 only for explicit diagnostics.",
|
||||
"The local tran wrapper must not add provider/plane directory locks; rely on k8s/Tekton/Argo/Lease or server-side TTL queues for coordination.",
|
||||
"Use -- before a remote command that intentionally starts with a dash.",
|
||||
],
|
||||
};
|
||||
|
||||
+33
-2
@@ -6,7 +6,18 @@ import { type UniDeskConfig } from "./config";
|
||||
import { type DebugDispatchCommand, isDebugDispatchCommand } from "./debug";
|
||||
import { summarizeMicroserviceHealthResponse, summarizeMicroserviceObservation, summarizeMicroserviceProxyResponse } from "./microservices";
|
||||
import { parseNetworkPerfOptions, runNetworkPerf } from "./network-perf";
|
||||
import { formatSshFailureHint, formatSshRuntimeTimingHint, parseSshInvocation, sshFailureHint, sshRoutePayloadCwd, sshRuntimeTimingHint, wrapSshRemoteCommand } from "./ssh";
|
||||
import {
|
||||
formatSshFailureHint,
|
||||
formatSshRuntimeTimeoutHint,
|
||||
formatSshRuntimeTimingHint,
|
||||
parseSshInvocation,
|
||||
sshFailureHint,
|
||||
sshRoutePayloadCwd,
|
||||
sshRuntimeTimeoutHint,
|
||||
sshRuntimeTimeoutMs,
|
||||
sshRuntimeTimingHint,
|
||||
wrapSshRemoteCommand,
|
||||
} from "./ssh";
|
||||
import { codexJudgeQueryAsync, codexOutputQueryAsync, codexPrPreflightQueryAsync, codexQueuesQueryAsync, codexTaskQueryAsync, codexTasksQueryAsync, codexUnreadTriageAsync } from "./code-queue";
|
||||
import { runDecisionCenterCommandAsync } from "./decision-center";
|
||||
import {
|
||||
@@ -898,6 +909,7 @@ async function runRemoteSshWebSocket(
|
||||
rows: Number(process.stdout.rows) > 0 ? Number(process.stdout.rows) : 30,
|
||||
};
|
||||
const openTimeoutMs = Math.max(15000, Number(process.env.UNIDESK_SSH_OPEN_TIMEOUT_MS || 60000));
|
||||
const runtimeTimeoutMs = sshRuntimeTimeoutMs();
|
||||
const payload = {
|
||||
providerId: invocation.providerId,
|
||||
command: wrapSshRemoteCommand(parsed.remoteCommand, parsed.requiredHelpers),
|
||||
@@ -905,6 +917,7 @@ async function runRemoteSshWebSocket(
|
||||
tty: parsed.remoteCommand === null,
|
||||
stdinEotOnEnd: parsed.remoteCommand !== null,
|
||||
openTimeoutMs,
|
||||
runtimeTimeoutMs,
|
||||
cols: size.cols,
|
||||
rows: size.rows,
|
||||
};
|
||||
@@ -945,6 +958,7 @@ async function runRemoteSshWebSocket(
|
||||
|
||||
return await new Promise<number>((resolve) => {
|
||||
const rawMode = parsed.remoteCommand === null && process.stdin.isTTY && typeof process.stdin.setRawMode === "function";
|
||||
let timedOut = false;
|
||||
const openTimer = setTimeout(() => {
|
||||
if (sessionReady || settled) return;
|
||||
process.stderr.write("unidesk remote frontend ssh bridge timed out waiting for provider session\n");
|
||||
@@ -955,9 +969,26 @@ async function runRemoteSshWebSocket(
|
||||
// Ignore close failures while resolving the timeout path.
|
||||
}
|
||||
}, openTimeoutMs);
|
||||
const runtimeTimer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
timedOut = true;
|
||||
exitCode = 124;
|
||||
process.stderr.write(formatSshRuntimeTimeoutHint(sshRuntimeTimeoutHint({
|
||||
invocation,
|
||||
transport: "frontend-websocket",
|
||||
timeoutMs: runtimeTimeoutMs,
|
||||
})));
|
||||
try {
|
||||
ws.close();
|
||||
} catch {
|
||||
// Ignore close failures while resolving the timeout path.
|
||||
}
|
||||
finish(124);
|
||||
}, runtimeTimeoutMs);
|
||||
|
||||
const restore = (): void => {
|
||||
clearTimeout(openTimer);
|
||||
clearTimeout(runtimeTimer);
|
||||
process.stdin.off("data", onStdinData);
|
||||
process.stdin.off("end", onStdinEnd);
|
||||
if (rawMode) process.stdin.setRawMode(false);
|
||||
@@ -966,7 +997,7 @@ async function runRemoteSshWebSocket(
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
restore();
|
||||
const hint = sshFailureHint(invocation.providerId, parsed, code, "");
|
||||
const hint = timedOut ? null : sshFailureHint(invocation.providerId, parsed, code, "");
|
||||
if (hint !== null) process.stderr.write(formatSshFailureHint(hint));
|
||||
const timingHint = formatSshRuntimeTimingHint(sshRuntimeTimingHint({
|
||||
invocation,
|
||||
|
||||
+96
-1
@@ -57,12 +57,28 @@ export interface SshRuntimeTimingHint {
|
||||
note: string;
|
||||
}
|
||||
|
||||
export interface SshRuntimeTimeoutHint {
|
||||
code: "ssh-runtime-timeout";
|
||||
level: "warning";
|
||||
providerId: string;
|
||||
route: string;
|
||||
transport: "backend-core-broker" | "frontend-websocket";
|
||||
invocationKind: SshInvocationKind;
|
||||
timeoutMs: number;
|
||||
timeoutSeconds: number;
|
||||
message: string;
|
||||
action: string;
|
||||
note: string;
|
||||
}
|
||||
|
||||
const argvQuotedSshSubcommands = new Set(["git", "rg", "grep", "sed", "nl", "stat", "du", "ls", "cat", "head", "tail", "wc", "pwd"]);
|
||||
const nativeK3sKubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
||||
const windowsBridgeCwd = "/mnt/c/Windows";
|
||||
const windowsPowerShellExePath = "/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe";
|
||||
const windowsCmdExeNativePath = "C:\\Windows\\System32\\cmd.exe";
|
||||
const defaultSshSlowWarningMs = 10_000;
|
||||
const defaultSshRuntimeTimeoutMs = 60_000;
|
||||
const maxSshRuntimeTimeoutMs = 60_000;
|
||||
const k3sResourceKindAliases = new Set(["pod", "po", "pods", "deployment", "deploy", "deployments", "statefulset", "sts", "daemonset", "ds", "job", "jobs"]);
|
||||
const legacyK3sOperationRouteSegments = new Set([
|
||||
"guard",
|
||||
@@ -1842,6 +1858,38 @@ export function formatSshRuntimeTimingHint(hint: SshRuntimeTimingHint): string {
|
||||
return `UNIDESK_SSH_TIMING ${JSON.stringify(hint)}\n`;
|
||||
}
|
||||
|
||||
export function sshRuntimeTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
|
||||
const raw = env.UNIDESK_SSH_RUNTIME_TIMEOUT_MS ?? env.UNIDESK_TRAN_RUNTIME_TIMEOUT_MS;
|
||||
const parsed = raw === undefined ? NaN : Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return defaultSshRuntimeTimeoutMs;
|
||||
return Math.min(maxSshRuntimeTimeoutMs, Math.max(1000, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
export function sshRuntimeTimeoutHint(options: {
|
||||
invocation: ParsedSshInvocation;
|
||||
transport: SshRuntimeTimeoutHint["transport"];
|
||||
timeoutMs: number;
|
||||
}): SshRuntimeTimeoutHint {
|
||||
const timeoutSeconds = Number((options.timeoutMs / 1000).toFixed(3));
|
||||
return {
|
||||
code: "ssh-runtime-timeout",
|
||||
level: "warning",
|
||||
providerId: safeProviderId(options.invocation.providerId),
|
||||
route: options.invocation.route.raw,
|
||||
transport: options.transport,
|
||||
invocationKind: options.invocation.parsed.invocationKind,
|
||||
timeoutMs: options.timeoutMs,
|
||||
timeoutSeconds,
|
||||
message: `ssh/tran operation exceeded the ${timeoutSeconds}s top-level runtime limit and was disconnected.`,
|
||||
action: "Use short query plus poll semantics; do not keep tran open waiting for long CI/CD, trace, logs, or build progress.",
|
||||
note: "Timeout hint is written to stderr and intentionally does not echo the original remote command.",
|
||||
};
|
||||
}
|
||||
|
||||
export function formatSshRuntimeTimeoutHint(hint: SshRuntimeTimeoutHint): string {
|
||||
return `UNIDESK_SSH_RUNTIME_TIMEOUT ${JSON.stringify(hint)}\n`;
|
||||
}
|
||||
|
||||
function brokerSource(): string {
|
||||
return String.raw`
|
||||
const open = JSON.parse(process.argv[2] || process.argv[1] || "{}");
|
||||
@@ -1861,6 +1909,13 @@ const openTimer = setTimeout(() => {
|
||||
try { ws.close(); } catch {}
|
||||
process.exit(255);
|
||||
}, Number(open.openTimeoutMs || 15000));
|
||||
const runtimeTimeoutMs = Number(open.runtimeTimeoutMs || 60000);
|
||||
const runtimeTimer = setTimeout(() => {
|
||||
process.stderr.write("unidesk ssh bridge runtime timeout; use short query plus poll semantics instead of keeping tran open\n");
|
||||
exitCode = 124;
|
||||
try { ws.close(); } catch {}
|
||||
setTimeout(() => process.exit(124), 250).unref?.();
|
||||
}, runtimeTimeoutMs);
|
||||
|
||||
function send(value) {
|
||||
const text = JSON.stringify(value);
|
||||
@@ -1933,6 +1988,7 @@ ws.addEventListener("message", (event) => {
|
||||
}
|
||||
if (message.type === "ssh.error") {
|
||||
clearTimeout(openTimer);
|
||||
clearTimeout(runtimeTimer);
|
||||
process.stderr.write(String(message.message || "ssh bridge error") + "\n");
|
||||
exitCode = 255;
|
||||
ws.close();
|
||||
@@ -1940,12 +1996,15 @@ ws.addEventListener("message", (event) => {
|
||||
}
|
||||
if (message.type === "ssh.exit") {
|
||||
clearTimeout(openTimer);
|
||||
clearTimeout(runtimeTimer);
|
||||
exitCode = Number.isInteger(message.exitCode) ? message.exitCode : 255;
|
||||
ws.close();
|
||||
}
|
||||
});
|
||||
|
||||
ws.addEventListener("close", () => {
|
||||
clearTimeout(openTimer);
|
||||
clearTimeout(runtimeTimer);
|
||||
process.exit(exitCode);
|
||||
});
|
||||
|
||||
@@ -1979,6 +2038,7 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
const startedAtMs = Date.now();
|
||||
const size = terminalSize();
|
||||
const openTimeoutMs = Math.max(15000, Number(process.env.UNIDESK_SSH_OPEN_TIMEOUT_MS || 60000));
|
||||
const runtimeTimeoutMs = sshRuntimeTimeoutMs();
|
||||
const payload = {
|
||||
providerId: invocation.providerId,
|
||||
command: wrapSshRemoteCommand(parsed.remoteCommand, parsed.requiredHelpers),
|
||||
@@ -1986,6 +2046,7 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
tty: parsed.remoteCommand === null,
|
||||
stdinEotOnEnd: parsed.remoteCommand !== null,
|
||||
openTimeoutMs,
|
||||
runtimeTimeoutMs,
|
||||
cols: size.cols,
|
||||
rows: size.rows,
|
||||
};
|
||||
@@ -2039,7 +2100,41 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
|
||||
return await new Promise<number>((resolve) => {
|
||||
let settled = false;
|
||||
let timedOut = false;
|
||||
let killTimer: NodeJS.Timeout | null = null;
|
||||
const runtimeTimer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
timedOut = true;
|
||||
const hint = sshRuntimeTimeoutHint({
|
||||
invocation,
|
||||
transport: "backend-core-broker",
|
||||
timeoutMs: runtimeTimeoutMs,
|
||||
});
|
||||
const formatted = formatSshRuntimeTimeoutHint(hint);
|
||||
appendStderrTail(formatted);
|
||||
process.stderr.write(formatted);
|
||||
try {
|
||||
child.stdin.destroy();
|
||||
} catch {
|
||||
// Ignore stdin teardown failures on the timeout path.
|
||||
}
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// Ignore kill failures and fall through to finish.
|
||||
}
|
||||
killTimer = setTimeout(() => {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// Process may have already exited.
|
||||
}
|
||||
}, 2000);
|
||||
finish(124);
|
||||
}, runtimeTimeoutMs);
|
||||
const restore = (): void => {
|
||||
clearTimeout(runtimeTimer);
|
||||
if (killTimer && !timedOut) clearTimeout(killTimer);
|
||||
process.stdin.unpipe(child.stdin);
|
||||
if (rawMode) process.stdin.setRawMode(false);
|
||||
};
|
||||
@@ -2047,7 +2142,7 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
restore();
|
||||
const hint = sshFailureHint(invocation.providerId, parsed, exitCode, stderrTail);
|
||||
const hint = timedOut ? null : sshFailureHint(invocation.providerId, parsed, exitCode, stderrTail);
|
||||
if (hint !== null) process.stderr.write(formatSshFailureHint(hint));
|
||||
const timingHint = formatSshRuntimeTimingHint(sshRuntimeTimingHint({
|
||||
invocation,
|
||||
|
||||
Reference in New Issue
Block a user