fix: route AgentRun bridge from runners
This commit is contained in:
+175
-6
@@ -1,6 +1,8 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
||||
import { runRemoteSshCommandCapture } from "./remote";
|
||||
import { startJob } from "./jobs";
|
||||
|
||||
const g14SourceRoute = "G14:/root/agentrun-v01";
|
||||
@@ -1608,9 +1610,9 @@ type AgentRunOfficialCliBridgeGroup = "queue" | "sessions" | "aipod-specs" | "ai
|
||||
async function runOfficialAgentRunCli(config: UniDeskConfig, group: AgentRunOfficialCliBridgeGroup, args: string[]): Promise<Record<string, unknown>> {
|
||||
const prepared = prepareOfficialAgentRunCliArgs([group, ...args]);
|
||||
const command = `agentrun ${prepared.args.join(" ")}`.trim();
|
||||
const bridge = agentRunQueueBridgeMetadata(prepared.materializedFiles, prepared.stdinPayload);
|
||||
const script = officialAgentRunCliScript(prepared);
|
||||
const result = await capture(config, g14SourceRoute, ["script", "--", script]);
|
||||
const bridge = agentRunQueueBridgeMetadata(prepared.materializedFiles, prepared.stdinPayload, captureBridgeMetadata(result));
|
||||
const payload = captureJsonPayload(result);
|
||||
if (result.exitCode === 0 && Object.keys(payload).length > 0) {
|
||||
return {
|
||||
@@ -1628,10 +1630,14 @@ async function runOfficialAgentRunCli(config: UniDeskConfig, group: AgentRunOffi
|
||||
remote: compactCapture(result, { full: true, stdoutTailChars: 4000, stderrTailChars: 2000 }),
|
||||
};
|
||||
}
|
||||
const bridgeFailureKind = bridgeExecutionFailureKind(result);
|
||||
const agentrunFailureKind = stringOrNull(payload.failureKind) ?? stringOrNull(record(payload.error).failureKind);
|
||||
return {
|
||||
ok: false,
|
||||
command,
|
||||
degradedReason: "agentrun-cli-bridge-failed",
|
||||
degradedReason: bridgeFailureKind === null ? "agentrun-cli-returned-failure" : bridgeFailureKind.degradedReason,
|
||||
...(bridgeFailureKind === null ? {} : { failureKind: bridgeFailureKind.failureKind, recoveryActions: bridgeFailureKind.recoveryActions }),
|
||||
...(bridgeFailureKind === null && agentrunFailureKind !== null ? { failureKind: agentrunFailureKind } : {}),
|
||||
bridge,
|
||||
agentrun: payload,
|
||||
remote: compactCapture(result, { full: true, stdoutTailChars: 8000, stderrTailChars: 4000 }),
|
||||
@@ -1668,7 +1674,7 @@ function rewriteOfficialCommandFieldValue(value: unknown): unknown {
|
||||
}
|
||||
|
||||
function shouldRewriteOfficialCommandField(key: string): boolean {
|
||||
return key === "pollCommands" || key === "drillDownCommands" || key === "recoveryActions" || key === "logPath";
|
||||
return key === "pollCommands" || key === "drillDownCommands" || key === "recoveryActions" || key === "nextActions" || key === "logPath";
|
||||
}
|
||||
|
||||
function rewriteOfficialCommandString(value: string): string {
|
||||
@@ -1804,7 +1810,7 @@ function parentDir(pathValue: string): string {
|
||||
return index > 0 ? pathValue.slice(0, index) : ".";
|
||||
}
|
||||
|
||||
function agentRunQueueBridgeMetadata(materializedFiles: AgentRunCliMaterializedFile[], stdinPayload: AgentRunCliForwardedStdin | null): Record<string, unknown> {
|
||||
function agentRunQueueBridgeMetadata(materializedFiles: AgentRunCliMaterializedFile[], stdinPayload: AgentRunCliForwardedStdin | null, captureBridge: Record<string, unknown> | null): Record<string, unknown> {
|
||||
return {
|
||||
route: g14SourceRoute,
|
||||
sourceWorktree: "/root/agentrun-v01",
|
||||
@@ -1813,6 +1819,7 @@ function agentRunQueueBridgeMetadata(materializedFiles: AgentRunCliMaterializedF
|
||||
officialCli: "./scripts/agentrun",
|
||||
mode: "direct-official-cli",
|
||||
compatibility: "no-code-queue-adapter-no-double-write",
|
||||
capture: captureBridge,
|
||||
stdinForwarded: stdinPayload ? {
|
||||
flag: stdinPayload.flag,
|
||||
source: stdinPayload.source,
|
||||
@@ -1827,8 +1834,162 @@ function agentRunQueueBridgeMetadata(materializedFiles: AgentRunCliMaterializedF
|
||||
};
|
||||
}
|
||||
|
||||
async function capture(config: UniDeskConfig, target: string, args: string[]): Promise<SshCaptureResult> {
|
||||
return await runSshCommandCapture(config, target, args);
|
||||
type AgentRunBridgeCaptureBackend = "local-backend-core-broker" | "remote-frontend-websocket";
|
||||
|
||||
interface LocalBackendCoreStatus {
|
||||
dockerExecutable: boolean;
|
||||
backendCoreContainer: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface AgentRunBridgeCapturePlan {
|
||||
backend: AgentRunBridgeCaptureBackend;
|
||||
route: string;
|
||||
reason: string;
|
||||
remoteHost: string | null;
|
||||
localBackendCore: LocalBackendCoreStatus;
|
||||
}
|
||||
|
||||
type AgentRunBridgeCaptureResult = SshCaptureResult & { bridgeExecution?: AgentRunBridgeCapturePlan };
|
||||
|
||||
let localBackendCoreStatusCache: LocalBackendCoreStatus | null = null;
|
||||
|
||||
async function capture(config: UniDeskConfig, target: string, args: string[]): Promise<AgentRunBridgeCaptureResult> {
|
||||
const plan = agentRunBridgeCapturePlan(config, target);
|
||||
if (plan.backend === "remote-frontend-websocket" && plan.remoteHost !== null) {
|
||||
return await captureRemote(config, plan, target, args);
|
||||
}
|
||||
const local = attachBridgeExecution(await runSshCommandCapture(config, target, args), plan);
|
||||
const fallbackHost = agentRunBridgeRemoteHost(config);
|
||||
if (local.exitCode !== 0 && fallbackHost !== null && bridgeExecutionFailureKind(local)?.degradedReason === "capture-backend-unavailable") {
|
||||
const fallbackPlan: AgentRunBridgeCapturePlan = {
|
||||
...plan,
|
||||
backend: "remote-frontend-websocket",
|
||||
remoteHost: fallbackHost,
|
||||
reason: "local-capture-backend-unavailable",
|
||||
};
|
||||
return await captureRemote(config, fallbackPlan, target, args);
|
||||
}
|
||||
return local;
|
||||
}
|
||||
|
||||
async function captureRemote(config: UniDeskConfig, plan: AgentRunBridgeCapturePlan, target: string, args: string[]): Promise<AgentRunBridgeCaptureResult> {
|
||||
if (plan.remoteHost === null) return attachBridgeExecution(remoteBridgeCaptureFailure(new Error("remote host is not configured")), plan);
|
||||
try {
|
||||
return attachBridgeExecution(await runRemoteSshCommandCapture(config, plan.remoteHost, target, args), plan);
|
||||
} catch (error) {
|
||||
return attachBridgeExecution(remoteBridgeCaptureFailure(error), plan);
|
||||
}
|
||||
}
|
||||
|
||||
function remoteBridgeCaptureFailure(error: unknown): SshCaptureResult {
|
||||
const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
||||
return {
|
||||
exitCode: 255,
|
||||
stdout: "",
|
||||
stderr: `unidesk remote frontend ssh bridge failed: ${message}\n`,
|
||||
};
|
||||
}
|
||||
|
||||
function attachBridgeExecution(result: SshCaptureResult, plan: AgentRunBridgeCapturePlan): AgentRunBridgeCaptureResult {
|
||||
return { ...result, bridgeExecution: plan };
|
||||
}
|
||||
|
||||
function agentRunBridgeCapturePlan(config: UniDeskConfig, target: string, env: NodeJS.ProcessEnv = process.env): AgentRunBridgeCapturePlan {
|
||||
const localBackendCore = detectLocalBackendCoreStatus();
|
||||
const remoteHost = agentRunBridgeRemoteHost(config, env);
|
||||
const runnerEnv = isAgentRunRunnerEnvironment(env);
|
||||
if (runnerEnv && remoteHost !== null) {
|
||||
return { backend: "remote-frontend-websocket", route: target, reason: "runner-environment", remoteHost, localBackendCore };
|
||||
}
|
||||
if (!localBackendCore.backendCoreContainer && remoteHost !== null) {
|
||||
return { backend: "remote-frontend-websocket", route: target, reason: "local-backend-core-unavailable", remoteHost, localBackendCore };
|
||||
}
|
||||
return { backend: "local-backend-core-broker", route: target, reason: "main-server-local-backend-core", remoteHost: null, localBackendCore };
|
||||
}
|
||||
|
||||
function detectLocalBackendCoreStatus(): LocalBackendCoreStatus {
|
||||
if (localBackendCoreStatusCache !== null) return localBackendCoreStatusCache;
|
||||
const result = spawnSync("docker", ["ps", "--format", "{{.Names}}"], { encoding: "utf8", timeout: 2000 });
|
||||
if (result.error !== undefined) {
|
||||
localBackendCoreStatusCache = {
|
||||
dockerExecutable: false,
|
||||
backendCoreContainer: false,
|
||||
error: result.error.message,
|
||||
};
|
||||
return localBackendCoreStatusCache;
|
||||
}
|
||||
const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim();
|
||||
localBackendCoreStatusCache = {
|
||||
dockerExecutable: result.status === 0,
|
||||
backendCoreContainer: result.status === 0 && String(result.stdout ?? "").split(/\r?\n/u).includes("unidesk-backend-core"),
|
||||
error: result.status === 0 ? null : output || `docker ps exited ${result.status ?? "unknown"}`,
|
||||
};
|
||||
return localBackendCoreStatusCache;
|
||||
}
|
||||
|
||||
function isAgentRunRunnerEnvironment(env: NodeJS.ProcessEnv): boolean {
|
||||
return Boolean(
|
||||
env.AGENTRUN_BOOT_MODE
|
||||
|| env.AGENTRUN_RUN_ID
|
||||
|| env.AGENTRUN_K8S_JOB_NAME
|
||||
|| env.CODE_QUEUE_SERVICE_ROLE
|
||||
|| env.CODE_QUEUE_INSTANCE_ID
|
||||
|| env.KUBERNETES_SERVICE_HOST,
|
||||
);
|
||||
}
|
||||
|
||||
function agentRunBridgeRemoteHost(config: UniDeskConfig, env: NodeJS.ProcessEnv = process.env): string | null {
|
||||
return normalizeRemoteHostHint(env.UNIDESK_MAIN_SERVER_IP)
|
||||
?? normalizeRemoteHostHint(env.UNIDESK_MAIN_SERVER_HOST)
|
||||
?? normalizeRemoteHostHint(env.CODE_QUEUE_DEV_CONTAINER_MASTER_HOST)
|
||||
?? normalizeRemoteHostHint(config.network.publicHost);
|
||||
}
|
||||
|
||||
function normalizeRemoteHostHint(raw: string | undefined): string | null {
|
||||
const value = raw?.trim() ?? "";
|
||||
if (value.length === 0 || value === "localhost" || value === "127.0.0.1" || value === "::1") return null;
|
||||
return value.replace(/\/+$/u, "");
|
||||
}
|
||||
|
||||
function captureBridgeMetadata(result: SshCaptureResult): Record<string, unknown> | null {
|
||||
const bridgeExecution = (result as AgentRunBridgeCaptureResult).bridgeExecution;
|
||||
if (bridgeExecution === undefined) return null;
|
||||
return {
|
||||
backend: bridgeExecution.backend,
|
||||
route: bridgeExecution.route,
|
||||
reason: bridgeExecution.reason,
|
||||
remoteHost: bridgeExecution.remoteHost,
|
||||
localBackendCore: bridgeExecution.localBackendCore,
|
||||
};
|
||||
}
|
||||
|
||||
function bridgeExecutionFailureKind(result: SshCaptureResult): { degradedReason: string; failureKind: string; recoveryActions: string[] } | null {
|
||||
if (result.exitCode === 0) return null;
|
||||
const stderr = result.stderr;
|
||||
if (/No such container: unidesk-backend-core|failed to start broker|Executable not found.*"docker"|docker: not found|Cannot connect to the Docker daemon/iu.test(stderr)) {
|
||||
return {
|
||||
degradedReason: "capture-backend-unavailable",
|
||||
failureKind: "bridge-execution-environment",
|
||||
recoveryActions: [
|
||||
"Run the same command from a healthy UniDesk main-server CLI, or provide UNIDESK_MAIN_SERVER_IP/UNIDESK_MAIN_SERVER_HOST so the bridge can use the frontend WebSocket SSH backend.",
|
||||
"For AgentRun runner jobs, request the unidesk-ssh tool credential SecretRef instead of embedding secret values in prompts or payloads.",
|
||||
"After restoring the bridge backend, retry the same bun scripts/cli.ts agentrun ... command; do not use direct kubectl or GitHub API bypasses.",
|
||||
],
|
||||
};
|
||||
}
|
||||
if (/frontend login failed|remote frontend ssh bridge|websocket error|timed out waiting for provider session|route-not-allowed/iu.test(stderr)) {
|
||||
return {
|
||||
degradedReason: "bridge-execution-environment-unavailable",
|
||||
failureKind: "bridge-execution-environment",
|
||||
recoveryActions: [
|
||||
"Verify the UniDesk frontend/backend-core SSH bridge is reachable from this runner and that UNIDESK_MAIN_SERVER_IP or UNIDESK_MAIN_SERVER_HOST points at the main server.",
|
||||
"Verify scoped UNIDESK_SSH_CLIENT_TOKEN route allowlist includes the requested AgentRun route, without printing token values.",
|
||||
"Retry with the same bun scripts/cli.ts agentrun ... command after the controlled bridge path is restored.",
|
||||
],
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function compactCapture(result: SshCaptureResult, options: { full?: boolean; stdoutTailChars?: number; stderrTailChars?: number } = {}): Record<string, unknown> {
|
||||
@@ -1842,6 +2003,14 @@ function compactCapture(result: SshCaptureResult, options: { full?: boolean; std
|
||||
stdoutTailOmitted: !full && result.exitCode === 0,
|
||||
stderrTailOmitted: !full && result.exitCode === 0,
|
||||
};
|
||||
const bridgeExecution = captureBridgeMetadata(result);
|
||||
if (bridgeExecution !== null) payload.bridgeExecution = bridgeExecution;
|
||||
const failureKind = bridgeExecutionFailureKind(result);
|
||||
if (failureKind !== null) {
|
||||
payload.degradedReason = failureKind.degradedReason;
|
||||
payload.failureKind = failureKind.failureKind;
|
||||
payload.recoveryActions = failureKind.recoveryActions;
|
||||
}
|
||||
if (full || result.exitCode !== 0) {
|
||||
payload.stdoutTail = tail(result.stdout, stdoutTailChars);
|
||||
payload.stderrTail = tail(result.stderr, stderrTailChars);
|
||||
|
||||
@@ -535,6 +535,28 @@ function scopedSshFrontendSession(host: string, config: UniDeskConfig, token: st
|
||||
return { baseUrl: frontendBaseUrl(host, config), cookie: "", sshClientToken: token };
|
||||
}
|
||||
|
||||
export async function runRemoteSshCommandCapture(
|
||||
config: UniDeskConfig,
|
||||
host: string,
|
||||
target: string,
|
||||
args: string[],
|
||||
input?: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<SshCaptureResult> {
|
||||
const token = sshClientTokenFromEnv(env);
|
||||
const session = token === null
|
||||
? await loginFrontend(host, config)
|
||||
: scopedSshFrontendSession(host, config, token);
|
||||
const normalizedArgs = normalizeSshOperationArgs(args);
|
||||
const invocation = parseSshInvocation(target, normalizedArgs);
|
||||
const parsed = invocation.parsed;
|
||||
if (parsed.remoteCommand === null) throw new Error(`remote ssh ${target} capture requires a non-interactive operation`);
|
||||
const stdin = parsed.stdinPrefix !== undefined || parsed.stdinSuffix !== undefined
|
||||
? `${parsed.stdinPrefix ?? ""}${input ?? ""}${parsed.stdinSuffix ?? ""}`
|
||||
: input;
|
||||
return await runRemoteSshWebSocketCaptureRemoteCommand(session, invocation, parsed.remoteCommand, stdin);
|
||||
}
|
||||
|
||||
async function frontendJson(session: FrontendSession, path: string, init?: RequestInit, timeoutMs = 8000, maxResponseBytes = 5_000_000): Promise<FetchJsonResult> {
|
||||
const headers = new Headers(init?.headers);
|
||||
headers.set("cookie", session.cookie);
|
||||
|
||||
Reference in New Issue
Block a user