fix: bound cli dump previews from yaml
This commit is contained in:
@@ -2,6 +2,7 @@ version: 1
|
||||
kind: unidesk-cli
|
||||
output:
|
||||
maxStdoutBytes: 10240
|
||||
maxPreviewLines: 240
|
||||
dumpDir: /tmp/unidesk-cli-output
|
||||
includePreview: false
|
||||
warning: "CLI stdout exceeded YAML-configured limit; full output was dumped to /tmp for one-off drill-down only. This is a CLI usability defect: improve the command itself to print concise tables/summaries and id-specific progressive disclosure instead of repeatedly depending on dump extraction."
|
||||
|
||||
@@ -313,7 +313,7 @@ core 只允许声明了 `host.ssh` capability 的 provider 使用 `host.ssh` dis
|
||||
|
||||
ssh-like 远端命令如果出现 `kex_exchange_identification`、`Connection closed by remote host`、provider session timeout 或 exit code 255,CLI 会在原始 stderr 后追加一行 `UNIDESK_SSH_HINT { ... }`。该 JSON 不回显原始远端命令,只包含 `code=ssh-like-command-friction`、`trigger`、`try` 和 `triage`;`try` 固定指向显式 `sh` stdin heredoc 形态,避免把一次 ssh-like 解析/握手摩擦误读成 D601 SSH 整体不可用。`ssh`/`trans`/`tran` 在失败路径识别到 tcp-pool 数据面问题时会追加 `UNIDESK_SSH_TCP_POOL_HINT { ... }`,`failureKind` 固定分为 `provider-data-channel-closed`、`provider-data-channel-missing` 和 `provider-data-pool-exhausted`;这类 hint 表示 transport/data-pool transient,幂等受控操作应先运行 `bun scripts/cli.ts debug ssh-pool <providerId>` 查看 labels,再重试原受控 CLI,不能单独定性为远端 runtime 配置失败。backend-core/provider 控制面返回结构化 `ssh.error` 时 broker 还会输出 `UNIDESK_SSH_ERROR { ... }`,供 `job status` 从旧日志或异步 job 日志中恢复 failureKind。`ssh`/`trans`/`tran` 运行时硬超时会输出 `UNIDESK_SSH_RUNTIME_TIMEOUT { ... }` 或 wrapper 层 `UNIDESK_TRAN_TIMEOUT_HINT { ... }`;这不是远端业务失败,而是调用方需要改成短查询/轮询。`ssh`/`trans`/`tran` 只有在运行耗时超过默认 10000ms 时才会在 stderr 追加一行 `UNIDESK_SSH_TIMING { ... }`,且 `level=warning`;正常短调用不输出 timing 噪声。慢成功命令也必须保留该 warning,因为它是 provider session、远端命令成本、helper bootstrap 和 `trans`/`tran`/远端 patch 性能回归的重要监控信号。warning 包含 `elapsedMs`、`elapsedSeconds`、`transport`、`invocationKind` 和 `exitCode`,提示优先排查 provider/session 延迟、远端命令自身耗时、helper bootstrap 或工具层回归。阈值可用 `UNIDESK_SSH_SLOW_WARNING_MS=<ms>` 临时调节,提示同样不回显原始远端命令。
|
||||
|
||||
非交互 `ssh`/`trans`/`tran` 远端命令的流式 stdout 默认有本地输出上限,避免远端日志、PowerShell JSON 或错误对象一次性输出过大导致上下文被淹没;交互登录 shell 不套该上限。超过上限时,CLI 只继续读取远端流并把完整内容写入 `/tmp/unidesk-cli-output/*.stdout.bin`,本地 stderr 追加 `UNIDESK_SSH_STDOUT_TRUNCATED { ... }`,其中包含 `thresholdBytes`、`observedBytesAtTruncation`、`dumpPath` 和 `dumpError`;stdout 本身只保留上限内的开头内容。默认上限是 256KiB,可用 `UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES=<bytes>` 或 `UNIDESK_TRAN_STDOUT_STREAM_MAX_BYTES=<bytes>` 临时调整,最小 4KiB,最大 16MiB。该机制只做渐进披露和完整 dump,不替代远端命令失败判断;看到该 hint 时应优先改成 `tail`、分页或更窄的结构化查询。
|
||||
非交互 `ssh`/`trans`/`tran` 远端命令的流式 stdout/stderr 服从统一 CLI dump/preview 策略,默认预览预算来自 `config/unidesk-cli.yaml#output.maxStdoutBytes` 和 `#output.maxPreviewLines`,完整流写入同一 YAML 声明的 dump 目录;交互登录 shell 不套该上限。超过预算时,CLI 继续读取远端流并把完整内容写入 `/tmp/unidesk-cli-output/*.stdout.bin` 或 `*.stderr.bin`,本地 stderr 追加 `UNIDESK_SSH_STDOUT_TRUNCATED { ... }` 或 `UNIDESK_SSH_STDERR_TRUNCATED { ... }`,其中包含 `stream`、`trigger`、`thresholdBytes`、`thresholdLines`、`forwardedBytes`、`forwardedLines`、`observedBytesAtTruncation`、`observedLinesAtTruncation`、`dumpPath`、`dumpError`、`disclosurePolicy` 和 `recommendedRerun`;被截断的本地流只保留 YAML 预算内的开头预览。需要一次性扩大字节预算时可显式设置 `UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES` / `UNIDESK_TRAN_STDOUT_STREAM_MAX_BYTES` 或 `UNIDESK_SSH_STDERR_STREAM_MAX_BYTES` / `UNIDESK_TRAN_STDERR_STREAM_MAX_BYTES`,仍受最小 4KiB、最大 16MiB 的工具保护。该机制只做渐进披露和完整 dump,不替代远端命令失败判断;看到 hint 时应优先改成 `rg -m`、`sed -n`、`tail`、`--limit`、`--tail-bytes`、`--raw/--full` 等更窄的结构化 drill-down。
|
||||
|
||||
`trans <providerId>` 透传只在当前 operation 需要 helper 时才注入 `/tmp/unidesk-ssh-tools`,普通 `argv`、`sh`/`bash`、`kubectl`、`logs` 和默认 `apply-patch` 等路径不得传输无关工具源码。`apply-patch-v1` 只注入 `apply_patch`;`glob` 只注入 `glob`;`skills`/`skill discover` 只注入 `skill-discover`。`apply_patch` 接受标准 `*** Begin Patch` / `*** End Patch` patch 格式,便于通过 SSH 透传编辑远端仓库文件;远端存在 `perl` 时必须走快速精确匹配路径,避免大文件 hunk 被 sh 模式匹配拖成几十秒,缺少 `perl` 时才退回 sh-only 实现。`glob` 和 `skill-discover` 需要远端 `python3`。注入工具只写 `/tmp/unidesk-ssh-tools`,不修改目标仓库。
|
||||
|
||||
|
||||
+42
-13
@@ -23,8 +23,9 @@ const CLI_OUTPUT_CONFIG_PATH = rootPath("config", "unidesk-cli.yaml");
|
||||
const EMERGENCY_OUTPUT_DUMP_THRESHOLD_BYTES = 10 * 1024;
|
||||
const EMERGENCY_OUTPUT_DUMP_DIR = join(tmpdir(), "unidesk-cli-output");
|
||||
|
||||
interface CliOutputPolicy {
|
||||
export interface CliOutputPolicy {
|
||||
maxStdoutBytes: number;
|
||||
maxPreviewLines: number;
|
||||
dumpDir: string;
|
||||
includePreview: boolean;
|
||||
warning: string;
|
||||
@@ -71,6 +72,10 @@ export function emitError(command: string, error: unknown): void {
|
||||
safeStdoutWrite(renderEnvelope(command, envelope));
|
||||
}
|
||||
|
||||
export function readCliOutputPolicy(): CliOutputPolicy {
|
||||
return cliOutputPolicy();
|
||||
}
|
||||
|
||||
function normalizeErrorPayload(command: string, error: unknown): Record<string, unknown> {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const parsed = parseStructuredErrorMessage(command, message);
|
||||
@@ -141,16 +146,18 @@ function shouldSuppressStack(prefix: string): boolean {
|
||||
|
||||
function renderEnvelope<T>(command: string, envelope: JsonEnvelope<T>): string {
|
||||
const fullText = `${JSON.stringify(envelope, null, 2)}\n`;
|
||||
if (!shouldDumpLargeOutput(command, fullText, envelope)) return fullText;
|
||||
|
||||
const policy = cliOutputPolicy();
|
||||
const trigger = outputDumpTrigger(fullText, policy, "json");
|
||||
if (trigger === null) return fullText;
|
||||
|
||||
const dump = dumpLargeOutput(command, fullText, "json", policy);
|
||||
const compactPayload = {
|
||||
outputTruncated: true,
|
||||
reason: "stdout-json-exceeded-threshold",
|
||||
reason: trigger.reason,
|
||||
warning: policy.warning,
|
||||
message: "Full JSON output was written to a temporary file; stdout contains only file metadata and a compact summary.",
|
||||
disclosurePolicy: disclosurePolicy(policy),
|
||||
trigger,
|
||||
dump,
|
||||
summary: summarizeEnvelope(envelope),
|
||||
};
|
||||
@@ -160,29 +167,47 @@ function renderEnvelope<T>(command: string, envelope: JsonEnvelope<T>): string {
|
||||
return `${JSON.stringify(compactEnvelope, null, 2)}\n`;
|
||||
}
|
||||
|
||||
function shouldDumpLargeOutput(command: string, text: string, envelope: JsonEnvelope<unknown>): boolean {
|
||||
void command;
|
||||
void envelope;
|
||||
if (process.env.UNIDESK_CLI_OUTPUT_DUMP_DISABLED === "1") return false;
|
||||
const threshold = cliOutputPolicy().maxStdoutBytes;
|
||||
return Buffer.byteLength(text, "utf8") > threshold;
|
||||
function outputDumpTrigger(text: string, policy: CliOutputPolicy, content: "json" | "text"): Record<string, unknown> | null {
|
||||
if (process.env.UNIDESK_CLI_OUTPUT_DUMP_DISABLED === "1") return null;
|
||||
const bytes = Buffer.byteLength(text, "utf8");
|
||||
const lines = countLines(text);
|
||||
if (bytes > policy.maxStdoutBytes) {
|
||||
return {
|
||||
reason: `stdout-${content}-bytes-exceeded-threshold`,
|
||||
thresholdBytes: policy.maxStdoutBytes,
|
||||
observedBytes: bytes,
|
||||
thresholdLines: policy.maxPreviewLines,
|
||||
observedLines: lines,
|
||||
};
|
||||
}
|
||||
if (lines > policy.maxPreviewLines) {
|
||||
return {
|
||||
reason: `stdout-${content}-lines-exceeded-threshold`,
|
||||
thresholdBytes: policy.maxStdoutBytes,
|
||||
observedBytes: bytes,
|
||||
thresholdLines: policy.maxPreviewLines,
|
||||
observedLines: lines,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function renderTextOutput(command: string, text: string): string {
|
||||
const fullText = text.endsWith("\n") ? text : `${text}\n`;
|
||||
if (process.env.UNIDESK_CLI_OUTPUT_DUMP_DISABLED === "1") return fullText;
|
||||
const policy = cliOutputPolicy();
|
||||
if (Buffer.byteLength(fullText, "utf8") <= policy.maxStdoutBytes) return fullText;
|
||||
const trigger = outputDumpTrigger(fullText, policy, "text");
|
||||
if (trigger === null) return fullText;
|
||||
const dump = dumpLargeOutput(command, fullText, "txt", policy);
|
||||
const payload: JsonEnvelope<Record<string, unknown>> = {
|
||||
ok: true,
|
||||
command,
|
||||
data: {
|
||||
outputTruncated: true,
|
||||
reason: "stdout-text-exceeded-threshold",
|
||||
reason: trigger.reason,
|
||||
warning: policy.warning,
|
||||
message: "Full text output was written to a temporary file; stdout contains only file metadata.",
|
||||
disclosurePolicy: disclosurePolicy(policy),
|
||||
trigger,
|
||||
dump,
|
||||
},
|
||||
};
|
||||
@@ -200,6 +225,7 @@ function dumpLargeOutput(command: string, text: string, extension: "json" | "txt
|
||||
path,
|
||||
configPath: policy.configPath,
|
||||
thresholdBytes: policy.maxStdoutBytes,
|
||||
thresholdLines: policy.maxPreviewLines,
|
||||
bytes: Buffer.byteLength(text, "utf8"),
|
||||
chars: text.length,
|
||||
lines: countLines(text),
|
||||
@@ -292,6 +318,7 @@ function disclosurePolicy(policy: CliOutputPolicy): Record<string, unknown> {
|
||||
return {
|
||||
configPath: policy.configPath,
|
||||
maxStdoutBytes: policy.maxStdoutBytes,
|
||||
maxPreviewLines: policy.maxPreviewLines,
|
||||
dumpDir: policy.dumpDir,
|
||||
includePreview: policy.includePreview,
|
||||
recommendation: "Prefer k8s-style concise summaries/tables by default; expose full data through explicit --full/--raw/id-specific drill-down commands instead of large stdout.",
|
||||
@@ -520,6 +547,7 @@ function cliOutputPolicy(): CliOutputPolicy {
|
||||
const output = objectField(root, "output", CLI_OUTPUT_CONFIG_RELATIVE_PATH);
|
||||
cachedOutputPolicy = {
|
||||
maxStdoutBytes: positiveIntegerField(output, "maxStdoutBytes", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
|
||||
maxPreviewLines: positiveIntegerField(output, "maxPreviewLines", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
|
||||
dumpDir: absolutePathField(output, "dumpDir", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
|
||||
includePreview: booleanField(output, "includePreview", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
|
||||
warning: stringField(output, "warning", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
|
||||
@@ -530,6 +558,7 @@ function cliOutputPolicy(): CliOutputPolicy {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
cachedOutputPolicy = {
|
||||
maxStdoutBytes: EMERGENCY_OUTPUT_DUMP_THRESHOLD_BYTES,
|
||||
maxPreviewLines: 240,
|
||||
dumpDir: EMERGENCY_OUTPUT_DUMP_DIR,
|
||||
includePreview: false,
|
||||
warning: "CLI output policy YAML could not be loaded; emergency dump guard is active for one-off drill-down only. Fix config/unidesk-cli.yaml, then improve the noisy command itself to print concise tables/summaries and id-specific progressive disclosure instead of repeatedly depending on dump extraction.",
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
buildWindowsPowerShellInvocation,
|
||||
createPosixApplyPatchFileSystem,
|
||||
createWindowsApplyPatchFileSystem,
|
||||
createSshStderrForwarder,
|
||||
createSshStdoutForwarder,
|
||||
formatSshFailureHint,
|
||||
formatSshRuntimeTimeoutHint,
|
||||
@@ -382,6 +383,10 @@ async function runRemoteSshWebSocket(
|
||||
invocation,
|
||||
transport: "frontend-websocket",
|
||||
});
|
||||
const stderrForwarder = parsed.remoteCommand === null ? null : createSshStderrForwarder({
|
||||
invocation,
|
||||
transport: "frontend-websocket",
|
||||
});
|
||||
let timedOut = false;
|
||||
const openTimer = setTimeout(() => {
|
||||
if (sessionReady || settled) return;
|
||||
@@ -464,8 +469,14 @@ async function runRemoteSshWebSocket(
|
||||
}
|
||||
if (message.type === "ssh.data") {
|
||||
const chunk = Buffer.from(String(message.data ?? ""), message.encoding === "base64" ? "base64" : "utf8");
|
||||
if (message.stream === "stderr") process.stderr.write(chunk);
|
||||
else if (stdoutForwarder === null) {
|
||||
if (message.stream === "stderr") {
|
||||
if (stderrForwarder === null) {
|
||||
process.stderr.write(chunk);
|
||||
} else {
|
||||
const hint = stderrForwarder.write(chunk);
|
||||
if (hint !== null) process.stderr.write(hint);
|
||||
}
|
||||
} else if (stdoutForwarder === null) {
|
||||
process.stdout.write(chunk);
|
||||
} else {
|
||||
const hint = stdoutForwarder.write(chunk);
|
||||
|
||||
+13
-2
@@ -10,6 +10,7 @@ import {
|
||||
buildWindowsPowerShellInvocation,
|
||||
createPosixApplyPatchFileSystem,
|
||||
createWindowsApplyPatchFileSystem,
|
||||
createSshStderrForwarder,
|
||||
createSshStdoutForwarder,
|
||||
formatSshFailureHint,
|
||||
formatSshRuntimeTimeoutHint,
|
||||
@@ -1019,6 +1020,10 @@ async function runRemoteSshWebSocket(
|
||||
invocation,
|
||||
transport: "frontend-websocket",
|
||||
});
|
||||
const stderrForwarder = parsed.remoteCommand === null ? null : createSshStderrForwarder({
|
||||
invocation,
|
||||
transport: "frontend-websocket",
|
||||
});
|
||||
let timedOut = false;
|
||||
const openTimer = setTimeout(() => {
|
||||
if (sessionReady || settled) return;
|
||||
@@ -1101,8 +1106,14 @@ async function runRemoteSshWebSocket(
|
||||
}
|
||||
if (message.type === "ssh.data") {
|
||||
const chunk = Buffer.from(String(message.data ?? ""), message.encoding === "base64" ? "base64" : "utf8");
|
||||
if (message.stream === "stderr") process.stderr.write(chunk);
|
||||
else if (stdoutForwarder === null) {
|
||||
if (message.stream === "stderr") {
|
||||
if (stderrForwarder === null) {
|
||||
process.stderr.write(chunk);
|
||||
} else {
|
||||
const hint = stderrForwarder.write(chunk);
|
||||
if (hint !== null) process.stderr.write(hint);
|
||||
}
|
||||
} else if (stdoutForwarder === null) {
|
||||
process.stdout.write(chunk);
|
||||
} else {
|
||||
const hint = stdoutForwarder.write(chunk);
|
||||
|
||||
+67
-3
@@ -1,20 +1,23 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import {
|
||||
createPosixApplyPatchFileSystem,
|
||||
createSshStderrForwarder,
|
||||
createSshStdoutForwarder,
|
||||
formatSshStdoutTruncationHint,
|
||||
parseSshInvocation,
|
||||
sshCaptureBackendPlan,
|
||||
sshStderrStreamMaxBytes,
|
||||
sshStdoutStreamMaxBytes,
|
||||
sshStdoutTruncationHint,
|
||||
windowsFsReadOnlyScript,
|
||||
windowsPowerShellScriptPrelude,
|
||||
} from "./ssh";
|
||||
import { readCliOutputPolicy } from "./output";
|
||||
|
||||
function sha256Hex(text: string): string {
|
||||
return createHash("sha256").update(text, "utf8").digest("hex");
|
||||
@@ -150,14 +153,18 @@ describe("ssh host apply-patch fs backend", () => {
|
||||
});
|
||||
|
||||
describe("ssh stdout bounded streaming", () => {
|
||||
test("uses bounded defaults and clamps env override", () => {
|
||||
expect(sshStdoutStreamMaxBytes({} as NodeJS.ProcessEnv)).toBe(256 * 1024);
|
||||
test("uses YAML output policy defaults and clamps env override", () => {
|
||||
const policy = readCliOutputPolicy();
|
||||
expect(sshStdoutStreamMaxBytes({} as NodeJS.ProcessEnv)).toBe(policy.maxStdoutBytes);
|
||||
expect(sshStderrStreamMaxBytes({} as NodeJS.ProcessEnv)).toBe(policy.maxStdoutBytes);
|
||||
expect(sshStdoutStreamMaxBytes({ UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES: "8192" } as NodeJS.ProcessEnv)).toBe(8192);
|
||||
expect(sshStderrStreamMaxBytes({ UNIDESK_SSH_STDERR_STREAM_MAX_BYTES: "8192" } as NodeJS.ProcessEnv)).toBe(8192);
|
||||
expect(sshStdoutStreamMaxBytes({ UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES: "1" } as NodeJS.ProcessEnv)).toBe(4096);
|
||||
expect(sshStdoutStreamMaxBytes({ UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES: String(64 * 1024 * 1024) } as NodeJS.ProcessEnv)).toBe(16 * 1024 * 1024);
|
||||
});
|
||||
|
||||
test("formats truncation hint without echoing remote command", () => {
|
||||
const policy = readCliOutputPolicy();
|
||||
const invocation = parseSshInvocation("D601:win", ["ps"]);
|
||||
expect(invocation.parsed.remoteCommand).not.toBeNull();
|
||||
const hint = sshStdoutTruncationHint({
|
||||
@@ -175,6 +182,11 @@ describe("ssh stdout bounded streaming", () => {
|
||||
expect(payload.providerId).toBe("D601");
|
||||
expect(payload.route).toBe("D601:win");
|
||||
expect(payload.thresholdBytes).toBe(4096);
|
||||
expect(payload.disclosurePolicy).toMatchObject({
|
||||
name: "unified-cli-dump-preview",
|
||||
configPath: "config/unidesk-cli.yaml",
|
||||
});
|
||||
expect(payload.thresholdLines).toBe(policy.maxPreviewLines);
|
||||
expect(formatted).not.toContain("Get-Content");
|
||||
});
|
||||
|
||||
@@ -201,6 +213,58 @@ describe("ssh stdout bounded streaming", () => {
|
||||
expect(Buffer.concat(forwarded).toString("utf8")).toBe("abcde");
|
||||
expect(hint).toContain("UNIDESK_SSH_STDOUT_TRUNCATED");
|
||||
expect(hint).toContain("\"transport\":\"frontend-websocket\"");
|
||||
expect(hint).toContain("\"forwardedBytes\":5");
|
||||
const payload = JSON.parse(hint!.slice("UNIDESK_SSH_STDOUT_TRUNCATED ".length)) as { dumpPath: string };
|
||||
expect(readFileSync(payload.dumpPath, "utf8")).toBe("abcdefghijkl");
|
||||
rmSync(payload.dumpPath, { force: true });
|
||||
});
|
||||
|
||||
test("forwarder bounds very short lines by YAML-style line budget", () => {
|
||||
const invocation = parseSshInvocation("D601:win", ["ps"]);
|
||||
const forwarded: Buffer[] = [];
|
||||
const stdout = {
|
||||
write(chunk: string | Buffer): boolean {
|
||||
forwarded.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
return true;
|
||||
},
|
||||
} as NodeJS.WritableStream;
|
||||
const forwarder = createSshStdoutForwarder({
|
||||
invocation,
|
||||
transport: "frontend-websocket",
|
||||
maxBytes: 1000,
|
||||
maxLines: 2,
|
||||
stdout,
|
||||
});
|
||||
|
||||
const hint = forwarder.write(Buffer.from("a\nb\nc\n"));
|
||||
expect(Buffer.concat(forwarded).toString("utf8")).toBe("a\nb\n");
|
||||
expect(hint).toContain("\"trigger\":\"lines\"");
|
||||
const payload = JSON.parse(hint!.slice("UNIDESK_SSH_STDOUT_TRUNCATED ".length)) as { dumpPath: string; forwardedLines: number };
|
||||
expect(payload.forwardedLines).toBe(2);
|
||||
expect(readFileSync(payload.dumpPath, "utf8")).toBe("a\nb\nc\n");
|
||||
rmSync(payload.dumpPath, { force: true });
|
||||
});
|
||||
|
||||
test("stderr forwarder uses the same dump guard and marker", () => {
|
||||
const invocation = parseSshInvocation("D601:win", ["ps"]);
|
||||
const forwarded: Buffer[] = [];
|
||||
const stderr = {
|
||||
write(chunk: string | Buffer): boolean {
|
||||
forwarded.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
return true;
|
||||
},
|
||||
} as NodeJS.WritableStream;
|
||||
const forwarder = createSshStderrForwarder({
|
||||
invocation,
|
||||
transport: "backend-core-broker",
|
||||
maxBytes: 4,
|
||||
stderr,
|
||||
});
|
||||
|
||||
const hint = forwarder.write(Buffer.from("abcdef"));
|
||||
expect(Buffer.concat(forwarded).toString("utf8")).toBe("abcd");
|
||||
expect(hint).toContain("UNIDESK_SSH_STDERR_TRUNCATED");
|
||||
expect(hint).toContain("\"stream\":\"stderr\"");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+190
-24
@@ -1,7 +1,6 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { type UniDeskConfig, repoRoot } from "./config";
|
||||
import {
|
||||
@@ -24,6 +23,7 @@ import {
|
||||
} from "./ssh-file-transfer";
|
||||
import { readTransHostProxyEnvRule, type TransHostProxyEnvRule } from "./trans-host-proxy";
|
||||
import { readTransSshBackendConfig, type TransSshBackendConfig } from "./trans-config";
|
||||
import { readCliOutputPolicy } from "./output";
|
||||
|
||||
export interface ParsedSshArgs {
|
||||
remoteCommand: string | null;
|
||||
@@ -151,16 +151,30 @@ export class SshRemovedShellAliasError extends Error {
|
||||
}
|
||||
|
||||
export interface SshStdoutTruncationHint {
|
||||
code: "ssh-stdout-truncated";
|
||||
code: "ssh-stdout-truncated" | "ssh-stderr-truncated";
|
||||
level: "warning";
|
||||
stream: "stdout" | "stderr";
|
||||
providerId: string;
|
||||
route: string;
|
||||
transport: "backend-core-broker" | "frontend-websocket";
|
||||
invocationKind: SshInvocationKind;
|
||||
thresholdBytes: number;
|
||||
thresholdLines: number;
|
||||
trigger: "bytes" | "lines" | "bytes-and-lines";
|
||||
observedBytesAtTruncation: number;
|
||||
forwardedBytes: number;
|
||||
observedLinesAtTruncation: number;
|
||||
forwardedLines: number;
|
||||
dumpPath: string | null;
|
||||
dumpError: string | null;
|
||||
disclosurePolicy: {
|
||||
name: "unified-cli-dump-preview";
|
||||
configPath: string;
|
||||
maxPreviewBytes: number;
|
||||
maxPreviewLines: number;
|
||||
dumpDir: string;
|
||||
};
|
||||
recommendedRerun: string[];
|
||||
message: string;
|
||||
action: string;
|
||||
note: string;
|
||||
@@ -200,10 +214,8 @@ const defaultSshRuntimeTimeoutMs = 60_000;
|
||||
const maxSshRuntimeTimeoutMs = 60_000;
|
||||
const defaultSshBackendCoreDetectTimeoutMs = 15_000;
|
||||
const maxSshBackendCoreDetectTimeoutMs = 60_000;
|
||||
const defaultSshStdoutStreamMaxBytes = 256 * 1024;
|
||||
const minSshStdoutStreamMaxBytes = 4 * 1024;
|
||||
const maxSshStdoutStreamMaxBytes = 16 * 1024 * 1024;
|
||||
const sshStdoutDumpDir = join(tmpdir(), "unidesk-cli-output");
|
||||
export const sshShellCompatibilityPrelude = 'printf(){ if [ "${1+x}" = x ] && [ "$1" = "-v" ] && [ -n "${BASH_VERSION:-}" ]; then command printf "$@"; return $?; fi; if [ "${1+x}" = x ] && [ "$1" = "--" ]; then shift; fi; command printf -- "$@"; }';
|
||||
export const sshUserToolPathPrelude = [
|
||||
'for unidesk_path_dir in "$HOME/.bun/bin" "$HOME/.local/bin" "$HOME/bin" "/root/.bun/bin"; do',
|
||||
@@ -2653,37 +2665,71 @@ export function formatSshRuntimeTimeoutHint(hint: SshRuntimeTimeoutHint): string
|
||||
export function sshStdoutStreamMaxBytes(env: NodeJS.ProcessEnv = process.env): number {
|
||||
const raw = env.UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES ?? env.UNIDESK_TRAN_STDOUT_STREAM_MAX_BYTES;
|
||||
const parsed = raw === undefined ? NaN : Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return defaultSshStdoutStreamMaxBytes;
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return readCliOutputPolicy().maxStdoutBytes;
|
||||
return Math.min(maxSshStdoutStreamMaxBytes, Math.max(minSshStdoutStreamMaxBytes, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
export function sshStderrStreamMaxBytes(env: NodeJS.ProcessEnv = process.env): number {
|
||||
const raw = env.UNIDESK_SSH_STDERR_STREAM_MAX_BYTES ?? env.UNIDESK_TRAN_STDERR_STREAM_MAX_BYTES;
|
||||
const parsed = raw === undefined ? NaN : Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return readCliOutputPolicy().maxStdoutBytes;
|
||||
return Math.min(maxSshStdoutStreamMaxBytes, Math.max(minSshStdoutStreamMaxBytes, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
export function sshStdoutTruncationHint(options: {
|
||||
invocation: ParsedSshInvocation;
|
||||
transport: SshStdoutTruncationHint["transport"];
|
||||
stream?: SshStdoutTruncationHint["stream"];
|
||||
thresholdBytes: number;
|
||||
thresholdLines?: number;
|
||||
trigger?: SshStdoutTruncationHint["trigger"];
|
||||
observedBytesAtTruncation: number;
|
||||
forwardedBytes?: number;
|
||||
observedLinesAtTruncation?: number;
|
||||
forwardedLines?: number;
|
||||
dumpPath: string | null;
|
||||
dumpError?: string | null;
|
||||
}): SshStdoutTruncationHint {
|
||||
const stream = options.stream ?? "stdout";
|
||||
const policy = readCliOutputPolicy();
|
||||
return {
|
||||
code: "ssh-stdout-truncated",
|
||||
code: stream === "stdout" ? "ssh-stdout-truncated" : "ssh-stderr-truncated",
|
||||
level: "warning",
|
||||
stream,
|
||||
providerId: safeProviderId(options.invocation.providerId),
|
||||
route: options.invocation.route.raw,
|
||||
transport: options.transport,
|
||||
invocationKind: options.invocation.parsed.invocationKind,
|
||||
thresholdBytes: options.thresholdBytes,
|
||||
thresholdLines: options.thresholdLines ?? policy.maxPreviewLines,
|
||||
trigger: options.trigger ?? "bytes",
|
||||
observedBytesAtTruncation: options.observedBytesAtTruncation,
|
||||
forwardedBytes: options.forwardedBytes ?? Math.min(options.thresholdBytes, options.observedBytesAtTruncation),
|
||||
observedLinesAtTruncation: options.observedLinesAtTruncation ?? 0,
|
||||
forwardedLines: options.forwardedLines ?? 0,
|
||||
dumpPath: options.dumpPath,
|
||||
dumpError: options.dumpError ?? null,
|
||||
message: `ssh stdout exceeded ${options.thresholdBytes} bytes; stdout is bounded and the complete stream is written to a local dump when possible.`,
|
||||
action: "Inspect the dump path, or rerun a narrower remote command with tail/paging instead of emitting full logs or huge JSON.",
|
||||
disclosurePolicy: {
|
||||
name: "unified-cli-dump-preview",
|
||||
configPath: policy.configPath,
|
||||
maxPreviewBytes: policy.maxStdoutBytes,
|
||||
maxPreviewLines: policy.maxPreviewLines,
|
||||
dumpDir: policy.dumpDir,
|
||||
},
|
||||
recommendedRerun: [
|
||||
"Use rg -m/--max-count, sed -n, head, tail, --limit, --tail-bytes, or an id-specific query.",
|
||||
"Use --full, --raw, --tail-bytes, --limit, or UNIDESK_*_STREAM_MAX_BYTES only when you intentionally need a wider one-off disclosure.",
|
||||
"Read the dumpPath for the complete captured stream.",
|
||||
],
|
||||
message: `ssh ${stream} exceeded the YAML-configured preview budget; ${stream} is bounded and the complete stream is written to a local dump when possible.`,
|
||||
action: "Inspect dumpPath for the complete stream, or rerun a narrower remote command instead of emitting full logs, huge JSON, or broad search output.",
|
||||
note: "This hint is written to stderr and intentionally does not echo the original remote command.",
|
||||
};
|
||||
}
|
||||
|
||||
export function formatSshStdoutTruncationHint(hint: SshStdoutTruncationHint): string {
|
||||
return `UNIDESK_SSH_STDOUT_TRUNCATED ${JSON.stringify(hint)}\n`;
|
||||
const marker = hint.stream === "stderr" ? "UNIDESK_SSH_STDERR_TRUNCATED" : "UNIDESK_SSH_STDOUT_TRUNCATED";
|
||||
return `${marker} ${JSON.stringify(hint)}\n`;
|
||||
}
|
||||
|
||||
export function classifySshTcpPoolFailure(text: string): SshTcpPoolFailureKind | null {
|
||||
@@ -2739,27 +2785,66 @@ export function formatSshTcpPoolHint(hint: SshTcpPoolHint | null): string {
|
||||
return hint === null ? "" : `UNIDESK_SSH_TCP_POOL_HINT ${JSON.stringify(hint)}\n`;
|
||||
}
|
||||
|
||||
function sshStdoutDumpPath(invocation: ParsedSshInvocation): string {
|
||||
mkdirSync(sshStdoutDumpDir, { recursive: true, mode: 0o700 });
|
||||
function sshStreamDumpPath(invocation: ParsedSshInvocation, stream: SshStdoutTruncationHint["stream"]): string {
|
||||
const policy = readCliOutputPolicy();
|
||||
mkdirSync(policy.dumpDir, { recursive: true, mode: 0o700 });
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/gu, "-");
|
||||
const suffix = randomBytes(4).toString("hex");
|
||||
const slug = `${invocation.providerId}-${invocation.route.raw}-${invocation.route.plane}`
|
||||
.replace(/[^A-Za-z0-9._-]+/gu, "-")
|
||||
.replace(/^-+|-+$/gu, "")
|
||||
.slice(0, 80) || "ssh";
|
||||
return join(sshStdoutDumpDir, `${timestamp}-${process.pid}-${suffix}-${slug}.stdout.bin`);
|
||||
return join(policy.dumpDir, `${timestamp}-${process.pid}-${suffix}-${slug}.${stream}.bin`);
|
||||
}
|
||||
|
||||
export function createSshStdoutForwarder(options: {
|
||||
invocation: ParsedSshInvocation;
|
||||
transport: SshStdoutTruncationHint["transport"];
|
||||
maxBytes?: number;
|
||||
maxLines?: number;
|
||||
stdout?: NodeJS.WritableStream;
|
||||
}): { write: (chunk: Buffer) => string | null } {
|
||||
const stdout = options.stdout ?? process.stdout;
|
||||
const maxBytes = options.maxBytes ?? sshStdoutStreamMaxBytes();
|
||||
return createSshStreamForwarder({
|
||||
invocation: options.invocation,
|
||||
transport: options.transport,
|
||||
stream: "stdout",
|
||||
maxBytes: options.maxBytes ?? sshStdoutStreamMaxBytes(),
|
||||
maxLines: options.maxLines ?? readCliOutputPolicy().maxPreviewLines,
|
||||
target: options.stdout ?? process.stdout,
|
||||
});
|
||||
}
|
||||
|
||||
export function createSshStderrForwarder(options: {
|
||||
invocation: ParsedSshInvocation;
|
||||
transport: SshStdoutTruncationHint["transport"];
|
||||
maxBytes?: number;
|
||||
maxLines?: number;
|
||||
stderr?: NodeJS.WritableStream;
|
||||
}): { write: (chunk: Buffer) => string | null } {
|
||||
return createSshStreamForwarder({
|
||||
invocation: options.invocation,
|
||||
transport: options.transport,
|
||||
stream: "stderr",
|
||||
maxBytes: options.maxBytes ?? sshStderrStreamMaxBytes(),
|
||||
maxLines: options.maxLines ?? readCliOutputPolicy().maxPreviewLines,
|
||||
target: options.stderr ?? process.stderr,
|
||||
});
|
||||
}
|
||||
|
||||
function createSshStreamForwarder(options: {
|
||||
invocation: ParsedSshInvocation;
|
||||
transport: SshStdoutTruncationHint["transport"];
|
||||
stream: SshStdoutTruncationHint["stream"];
|
||||
maxBytes: number;
|
||||
maxLines: number;
|
||||
target: NodeJS.WritableStream;
|
||||
}): { write: (chunk: Buffer) => string | null } {
|
||||
let observedBytes = 0;
|
||||
let forwardedBytes = 0;
|
||||
let observedLineBreaks = 0;
|
||||
let forwardedLineBreaks = 0;
|
||||
let observedEndsWithLineBreak = false;
|
||||
let forwardedEndsWithLineBreak = false;
|
||||
let truncated = false;
|
||||
let dumpPath: string | null = null;
|
||||
let dumpError: string | null = null;
|
||||
@@ -2769,7 +2854,7 @@ export function createSshStdoutForwarder(options: {
|
||||
if (dumpError !== null) return;
|
||||
try {
|
||||
if (dumpPath === null) {
|
||||
dumpPath = sshStdoutDumpPath(options.invocation);
|
||||
dumpPath = sshStreamDumpPath(options.invocation, options.stream);
|
||||
writeFileSync(dumpPath, Buffer.alloc(0), { mode: 0o600 });
|
||||
for (const buffered of bufferedChunks) appendFileSync(dumpPath, buffered);
|
||||
bufferedChunks.length = 0;
|
||||
@@ -2784,26 +2869,48 @@ export function createSshStdoutForwarder(options: {
|
||||
return {
|
||||
write(chunk: Buffer): string | null {
|
||||
observedBytes += chunk.length;
|
||||
if (!truncated && observedBytes <= maxBytes) {
|
||||
observedLineBreaks += countBufferLineBreaks(chunk);
|
||||
if (chunk.length > 0) observedEndsWithLineBreak = chunk[chunk.length - 1] === 10;
|
||||
const observedLines = lineCountFromBreaks(observedBytes, observedLineBreaks, observedEndsWithLineBreak);
|
||||
if (!truncated && observedBytes <= options.maxBytes && observedLines <= options.maxLines) {
|
||||
bufferedChunks.push(Buffer.from(chunk));
|
||||
stdout.write(chunk);
|
||||
options.target.write(chunk);
|
||||
forwardedBytes += chunk.length;
|
||||
forwardedLineBreaks += countBufferLineBreaks(chunk);
|
||||
if (chunk.length > 0) forwardedEndsWithLineBreak = chunk[chunk.length - 1] === 10;
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!truncated) {
|
||||
truncated = true;
|
||||
const remaining = Math.max(0, maxBytes - forwardedBytes);
|
||||
const remainingByBytes = Math.max(0, options.maxBytes - forwardedBytes);
|
||||
const remainingByLines = prefixLengthWithinLineBudget(
|
||||
chunk,
|
||||
forwardedBytes,
|
||||
forwardedLineBreaks,
|
||||
forwardedEndsWithLineBreak,
|
||||
options.maxLines,
|
||||
);
|
||||
const remaining = Math.min(remainingByBytes, remainingByLines);
|
||||
if (remaining > 0) {
|
||||
stdout.write(chunk.subarray(0, remaining));
|
||||
const forwarded = chunk.subarray(0, remaining);
|
||||
options.target.write(forwarded);
|
||||
forwardedBytes += remaining;
|
||||
forwardedLineBreaks += countBufferLineBreaks(forwarded);
|
||||
if (forwarded.length > 0) forwardedEndsWithLineBreak = forwarded[forwarded.length - 1] === 10;
|
||||
}
|
||||
appendDump(chunk);
|
||||
return formatSshStdoutTruncationHint(sshStdoutTruncationHint({
|
||||
invocation: options.invocation,
|
||||
transport: options.transport,
|
||||
thresholdBytes: maxBytes,
|
||||
stream: options.stream,
|
||||
thresholdBytes: options.maxBytes,
|
||||
thresholdLines: options.maxLines,
|
||||
trigger: sshStreamTruncationTrigger(observedBytes, observedLines, options.maxBytes, options.maxLines),
|
||||
observedBytesAtTruncation: observedBytes,
|
||||
forwardedBytes,
|
||||
observedLinesAtTruncation: lineCountFromBreaks(observedBytes, observedLineBreaks, observedEndsWithLineBreak),
|
||||
forwardedLines: lineCountFromBreaks(forwardedBytes, forwardedLineBreaks, forwardedEndsWithLineBreak),
|
||||
dumpPath,
|
||||
dumpError,
|
||||
}));
|
||||
@@ -2815,6 +2922,51 @@ export function createSshStdoutForwarder(options: {
|
||||
};
|
||||
}
|
||||
|
||||
function countBufferLineBreaks(buffer: Buffer): number {
|
||||
let count = 0;
|
||||
for (const byte of buffer) {
|
||||
if (byte === 10) count += 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function lineCountFromBreaks(bytes: number, lineBreaks: number, endsWithLineBreak: boolean): number {
|
||||
return bytes === 0 ? 0 : lineBreaks + (endsWithLineBreak ? 0 : 1);
|
||||
}
|
||||
|
||||
function sshStreamTruncationTrigger(
|
||||
observedBytes: number,
|
||||
observedLines: number,
|
||||
maxBytes: number,
|
||||
maxLines: number,
|
||||
): SshStdoutTruncationHint["trigger"] {
|
||||
const bytes = observedBytes > maxBytes;
|
||||
const lines = observedLines > maxLines;
|
||||
if (bytes && lines) return "bytes-and-lines";
|
||||
return bytes ? "bytes" : "lines";
|
||||
}
|
||||
|
||||
function prefixLengthWithinLineBudget(
|
||||
chunk: Buffer,
|
||||
currentBytes: number,
|
||||
currentLineBreaks: number,
|
||||
currentEndsWithLineBreak: boolean,
|
||||
maxLines: number,
|
||||
): number {
|
||||
if (lineCountFromBreaks(currentBytes, currentLineBreaks, currentEndsWithLineBreak) >= maxLines) return 0;
|
||||
let bytes = currentBytes;
|
||||
let lineBreaks = currentLineBreaks;
|
||||
let endsWithLineBreak = currentEndsWithLineBreak;
|
||||
for (let index = 0; index < chunk.length; index += 1) {
|
||||
const byte = chunk[index] ?? 0;
|
||||
bytes += 1;
|
||||
if (byte === 10) lineBreaks += 1;
|
||||
endsWithLineBreak = byte === 10;
|
||||
if (lineCountFromBreaks(bytes, lineBreaks, endsWithLineBreak) > maxLines) return index;
|
||||
}
|
||||
return chunk.length;
|
||||
}
|
||||
|
||||
function brokerSource(): string {
|
||||
return String.raw`
|
||||
const open = JSON.parse(process.argv[2] || process.argv[1] || "{}");
|
||||
@@ -3769,6 +3921,10 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
invocation,
|
||||
transport: "backend-core-broker",
|
||||
});
|
||||
const stderrForwarder = createSshStderrForwarder({
|
||||
invocation,
|
||||
transport: "backend-core-broker",
|
||||
});
|
||||
child.stdout.on("data", (chunk: Buffer) => {
|
||||
const hint = stdoutForwarder.write(chunk);
|
||||
if (hint !== null) {
|
||||
@@ -3776,11 +3932,21 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
process.stderr.write(hint);
|
||||
}
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer) => {
|
||||
appendStderrTail(chunk);
|
||||
const hint = stderrForwarder.write(chunk);
|
||||
if (hint !== null) {
|
||||
appendStderrTail(hint);
|
||||
process.stderr.write(hint);
|
||||
}
|
||||
});
|
||||
}
|
||||
if (parsed.remoteCommand === null) {
|
||||
child.stderr.on("data", (chunk: Buffer) => {
|
||||
appendStderrTail(chunk);
|
||||
process.stderr.write(chunk);
|
||||
});
|
||||
}
|
||||
child.stderr.on("data", (chunk: Buffer) => {
|
||||
appendStderrTail(chunk);
|
||||
process.stderr.write(chunk);
|
||||
});
|
||||
|
||||
return await new Promise<number>((resolve) => {
|
||||
let settled = false;
|
||||
|
||||
Reference in New Issue
Block a user