Merge pull request #1364 from pikasTech/fix/1354-byte-only-dump

fix: CLI dump 只保留字节预算
2026-07-01 11:36:08 +08:00
parent 8f28cf0f9a 73fbe4d81b
commit 09edf61fcf
5 changed files with 3 additions and 136 deletions
@@ -2,7 +2,6 @@ version: 1
 kind: unidesk-cli
 output:
  maxStdoutBytes: 10240
-  maxPreviewLines: 240
  dumpDir: /tmp/unidesk-cli-output
  includePreview: false
  warning: "CLI stdout exceeded YAML-configured limit; full output was dumped to /tmp for one-off drill-down only. This is a CLI usability defect: improve the command itself to print concise tables/summaries and id-specific progressive disclosure instead of repeatedly depending on dump extraction."
@@ -313,7 +313,7 @@ core 只允许声明了 `host.ssh` capability 的 provider 使用 `host.ssh` dis

 ssh-like 远端命令如果出现 `kex_exchange_identification`、`Connection closed by remote host`、provider session timeout 或 exit code 255，CLI 会在原始 stderr 后追加一行 `UNIDESK_SSH_HINT { ... }`。该 JSON 不回显原始远端命令，只包含 `code=ssh-like-command-friction`、`trigger`、`try` 和 `triage`；`try` 固定指向显式 `sh` stdin heredoc 形态，避免把一次 ssh-like 解析/握手摩擦误读成 D601 SSH 整体不可用。`ssh`/`trans`/`tran` 在失败路径识别到 tcp-pool 数据面问题时会追加 `UNIDESK_SSH_TCP_POOL_HINT { ... }`，`failureKind` 固定分为 `provider-data-channel-closed`、`provider-data-channel-missing` 和 `provider-data-pool-exhausted`；这类 hint 表示 transport/data-pool transient，幂等受控操作应先运行 `bun scripts/cli.ts debug ssh-pool <providerId>` 查看 labels，再重试原受控 CLI，不能单独定性为远端 runtime 配置失败。backend-core/provider 控制面返回结构化 `ssh.error` 时 broker 还会输出 `UNIDESK_SSH_ERROR { ... }`，供 `job status` 从旧日志或异步 job 日志中恢复 failureKind。`ssh`/`trans`/`tran` 运行时硬超时会输出 `UNIDESK_SSH_RUNTIME_TIMEOUT { ... }` 或 wrapper 层 `UNIDESK_TRAN_TIMEOUT_HINT { ... }`；这不是远端业务失败，而是调用方需要改成短查询/轮询。`ssh`/`trans`/`tran` 只有在运行耗时超过默认 10000ms 时才会在 stderr 追加一行 `UNIDESK_SSH_TIMING { ... }`，且 `level=warning`；正常短调用不输出 timing 噪声。慢成功命令也必须保留该 warning，因为它是 provider session、远端命令成本、helper bootstrap 和 `trans`/`tran`/远端 patch 性能回归的重要监控信号。warning 包含 `elapsedMs`、`elapsedSeconds`、`transport`、`invocationKind` 和 `exitCode`，提示优先排查 provider/session 延迟、远端命令自身耗时、helper bootstrap 或工具层回归。阈值可用 `UNIDESK_SSH_SLOW_WARNING_MS=<ms>` 临时调节，提示同样不回显原始远端命令。

-非交互 `ssh`/`trans`/`tran` 远端命令的流式 stdout/stderr 服从统一 CLI dump/preview 策略，默认预览预算来自 `config/unidesk-cli.yaml#output.maxStdoutBytes` 和 `#output.maxPreviewLines`，完整流写入同一 YAML 声明的 dump 目录；交互登录 shell 不套该上限。超过预算时，CLI 继续读取远端流并把完整内容写入 `/tmp/unidesk-cli-output/*.stdout.bin` 或 `*.stderr.bin`，本地 stderr 追加 `UNIDESK_SSH_STDOUT_TRUNCATED { ... }` 或 `UNIDESK_SSH_STDERR_TRUNCATED { ... }`，其中包含 `stream`、`trigger`、`thresholdBytes`、`thresholdLines`、`forwardedBytes`、`forwardedLines`、`observedBytesAtTruncation`、`observedLinesAtTruncation`、`dumpPath`、`dumpError`、`disclosurePolicy` 和 `recommendedRerun`；被截断的本地流只保留 YAML 预算内的开头预览。需要一次性扩大字节预算时可显式设置 `UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES` / `UNIDESK_TRAN_STDOUT_STREAM_MAX_BYTES` 或 `UNIDESK_SSH_STDERR_STREAM_MAX_BYTES` / `UNIDESK_TRAN_STDERR_STREAM_MAX_BYTES`，仍受最小 4KiB、最大 16MiB 的工具保护。该机制只做渐进披露和完整 dump，不替代远端命令失败判断；看到 hint 时应优先改成 `rg -m`、`sed -n`、`tail`、`--limit`、`--tail-bytes`、`--raw/--full` 等更窄的结构化 drill-down。
+非交互 `ssh`/`trans`/`tran` 远端命令的流式 stdout/stderr 服从统一 CLI dump/preview 策略，默认预览预算来自 `config/unidesk-cli.yaml#output.maxStdoutBytes`，完整流写入同一 YAML 声明的 dump 目录；交互登录 shell 不套该上限。超过预算时，CLI 继续读取远端流并把完整内容写入 `/tmp/unidesk-cli-output/*.stdout.bin` 或 `*.stderr.bin`，本地 stderr 追加 `UNIDESK_SSH_STDOUT_TRUNCATED { ... }` 或 `UNIDESK_SSH_STDERR_TRUNCATED { ... }`，其中包含 `stream`、`thresholdBytes`、`forwardedBytes`、`observedBytesAtTruncation`、`dumpPath`、`dumpError`、`disclosurePolicy` 和 `recommendedRerun`；被截断的本地流只保留 YAML 字节预算内的开头预览。需要一次性扩大字节预算时可显式设置 `UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES` / `UNIDESK_TRAN_STDOUT_STREAM_MAX_BYTES` 或 `UNIDESK_SSH_STDERR_STREAM_MAX_BYTES` / `UNIDESK_TRAN_STDERR_STREAM_MAX_BYTES`，仍受最小 4KiB、最大 16MiB 的工具保护。该机制只做渐进披露和完整 dump，不替代远端命令失败判断；看到 hint 时应优先改成 `rg -m`、`sed -n`、`tail`、`--limit`、`--tail-bytes`、`--raw/--full` 等更窄的结构化 drill-down。

 `trans <providerId>` 透传只在当前 operation 需要 helper 时才注入 `/tmp/unidesk-ssh-tools`，普通 `argv`、`sh`/`bash`、`kubectl`、`logs` 和默认 `apply-patch` 等路径不得传输无关工具源码。`apply-patch-v1` 只注入 `apply_patch`；`glob` 只注入 `glob`；`skills`/`skill discover` 只注入 `skill-discover`。`apply_patch` 接受标准 `*** Begin Patch` / `*** End Patch` patch 格式，便于通过 SSH 透传编辑远端仓库文件；远端存在 `perl` 时必须走快速精确匹配路径，避免大文件 hunk 被 sh 模式匹配拖成几十秒，缺少 `perl` 时才退回 sh-only 实现。`glob` 和 `skill-discover` 需要远端 `python3`。注入工具只写 `/tmp/unidesk-ssh-tools`，不修改目标仓库。

@@ -25,7 +25,6 @@ const EMERGENCY_OUTPUT_DUMP_DIR = join(tmpdir(), "unidesk-cli-output");

 export interface CliOutputPolicy {
  maxStdoutBytes: number;
-  maxPreviewLines: number;
  dumpDir: string;
  includePreview: boolean;
  warning: string;
@@ -170,23 +169,11 @@ function renderEnvelope<T>(command: string, envelope: JsonEnvelope<T>): string {
 function outputDumpTrigger(text: string, policy: CliOutputPolicy, content: "json" | "text"): Record<string, unknown> | null {
  if (process.env.UNIDESK_CLI_OUTPUT_DUMP_DISABLED === "1") return null;
  const bytes = Buffer.byteLength(text, "utf8");
-  const lines = countLines(text);
  if (bytes > policy.maxStdoutBytes) {
    return {
      reason: `stdout-${content}-bytes-exceeded-threshold`,
      thresholdBytes: policy.maxStdoutBytes,
      observedBytes: bytes,
-      thresholdLines: policy.maxPreviewLines,
-      observedLines: lines,
-    };
-  }
-  if (lines > policy.maxPreviewLines) {
-    return {
-      reason: `stdout-${content}-lines-exceeded-threshold`,
-      thresholdBytes: policy.maxStdoutBytes,
-      observedBytes: bytes,
-      thresholdLines: policy.maxPreviewLines,
-      observedLines: lines,
    };
  }
  return null;
@@ -225,7 +212,6 @@ function dumpLargeOutput(command: string, text: string, extension: "json" | "txt
    path,
    configPath: policy.configPath,
    thresholdBytes: policy.maxStdoutBytes,
-    thresholdLines: policy.maxPreviewLines,
    bytes: Buffer.byteLength(text, "utf8"),
    chars: text.length,
    lines: countLines(text),
@@ -318,7 +304,6 @@ function disclosurePolicy(policy: CliOutputPolicy): Record<string, unknown> {
  return {
    configPath: policy.configPath,
    maxStdoutBytes: policy.maxStdoutBytes,
-    maxPreviewLines: policy.maxPreviewLines,
    dumpDir: policy.dumpDir,
    includePreview: policy.includePreview,
    recommendation: "Prefer k8s-style concise summaries/tables by default; expose full data through explicit --full/--raw/id-specific drill-down commands instead of large stdout.",
@@ -547,7 +532,6 @@ function cliOutputPolicy(): CliOutputPolicy {
    const output = objectField(root, "output", CLI_OUTPUT_CONFIG_RELATIVE_PATH);
    cachedOutputPolicy = {
      maxStdoutBytes: positiveIntegerField(output, "maxStdoutBytes", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
-      maxPreviewLines: positiveIntegerField(output, "maxPreviewLines", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
      dumpDir: absolutePathField(output, "dumpDir", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
      includePreview: booleanField(output, "includePreview", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
      warning: stringField(output, "warning", `${CLI_OUTPUT_CONFIG_RELATIVE_PATH}.output`),
@@ -558,7 +542,6 @@ function cliOutputPolicy(): CliOutputPolicy {
    const message = error instanceof Error ? error.message : String(error);
    cachedOutputPolicy = {
      maxStdoutBytes: EMERGENCY_OUTPUT_DUMP_THRESHOLD_BYTES,
-      maxPreviewLines: 240,
      dumpDir: EMERGENCY_OUTPUT_DUMP_DIR,
      includePreview: false,
      warning: "CLI output policy YAML could not be loaded; emergency dump guard is active for one-off drill-down only. Fix config/unidesk-cli.yaml, then improve the noisy command itself to print concise tables/summaries and id-specific progressive disclosure instead of repeatedly depending on dump extraction.",
@@ -164,7 +164,6 @@ describe("ssh stdout bounded streaming", () => {
  });

  test("formats truncation hint without echoing remote command", () => {
-    const policy = readCliOutputPolicy();
    const invocation = parseSshInvocation("D601:win", ["ps"]);
    expect(invocation.parsed.remoteCommand).not.toBeNull();
    const hint = sshStdoutTruncationHint({
@@ -186,7 +185,6 @@ describe("ssh stdout bounded streaming", () => {
      name: "unified-cli-dump-preview",
      configPath: "config/unidesk-cli.yaml",
    });
-    expect(payload.thresholdLines).toBe(policy.maxPreviewLines);
    expect(formatted).not.toContain("Get-Content");
  });

@@ -219,32 +217,6 @@ describe("ssh stdout bounded streaming", () => {
    rmSync(payload.dumpPath, { force: true });
  });

-  test("forwarder bounds very short lines by YAML-style line budget", () => {
-    const invocation = parseSshInvocation("D601:win", ["ps"]);
-    const forwarded: Buffer[] = [];
-    const stdout = {
-      write(chunk: string | Buffer): boolean {
-        forwarded.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-        return true;
-      },
-    } as NodeJS.WritableStream;
-    const forwarder = createSshStdoutForwarder({
-      invocation,
-      transport: "frontend-websocket",
-      maxBytes: 1000,
-      maxLines: 2,
-      stdout,
-    });
-
-    const hint = forwarder.write(Buffer.from("a\nb\nc\n"));
-    expect(Buffer.concat(forwarded).toString("utf8")).toBe("a\nb\n");
-    expect(hint).toContain("\"trigger\":\"lines\"");
-    const payload = JSON.parse(hint!.slice("UNIDESK_SSH_STDOUT_TRUNCATED ".length)) as { dumpPath: string; forwardedLines: number };
-    expect(payload.forwardedLines).toBe(2);
-    expect(readFileSync(payload.dumpPath, "utf8")).toBe("a\nb\nc\n");
-    rmSync(payload.dumpPath, { force: true });
-  });
-
  test("stderr forwarder uses the same dump guard and marker", () => {
    const invocation = parseSshInvocation("D601:win", ["ps"]);
    const forwarded: Buffer[] = [];
@@ -159,19 +159,14 @@ export interface SshStdoutTruncationHint {
  transport: "backend-core-broker" | "frontend-websocket";
  invocationKind: SshInvocationKind;
  thresholdBytes: number;
-  thresholdLines: number;
-  trigger: "bytes" | "lines" | "bytes-and-lines";
  observedBytesAtTruncation: number;
  forwardedBytes: number;
-  observedLinesAtTruncation: number;
-  forwardedLines: number;
  dumpPath: string | null;
  dumpError: string | null;
  disclosurePolicy: {
    name: "unified-cli-dump-preview";
    configPath: string;
    maxPreviewBytes: number;
-    maxPreviewLines: number;
    dumpDir: string;
  };
  recommendedRerun: string[];
@@ -2681,12 +2676,8 @@ export function sshStdoutTruncationHint(options: {
  transport: SshStdoutTruncationHint["transport"];
  stream?: SshStdoutTruncationHint["stream"];
  thresholdBytes: number;
-  thresholdLines?: number;
-  trigger?: SshStdoutTruncationHint["trigger"];
  observedBytesAtTruncation: number;
  forwardedBytes?: number;
-  observedLinesAtTruncation?: number;
-  forwardedLines?: number;
  dumpPath: string | null;
  dumpError?: string | null;
 }): SshStdoutTruncationHint {
@@ -2701,19 +2692,14 @@ export function sshStdoutTruncationHint(options: {
    transport: options.transport,
    invocationKind: options.invocation.parsed.invocationKind,
    thresholdBytes: options.thresholdBytes,
-    thresholdLines: options.thresholdLines ?? policy.maxPreviewLines,
-    trigger: options.trigger ?? "bytes",
    observedBytesAtTruncation: options.observedBytesAtTruncation,
    forwardedBytes: options.forwardedBytes ?? Math.min(options.thresholdBytes, options.observedBytesAtTruncation),
-    observedLinesAtTruncation: options.observedLinesAtTruncation ?? 0,
-    forwardedLines: options.forwardedLines ?? 0,
    dumpPath: options.dumpPath,
    dumpError: options.dumpError ?? null,
    disclosurePolicy: {
      name: "unified-cli-dump-preview",
      configPath: policy.configPath,
      maxPreviewBytes: policy.maxStdoutBytes,
-      maxPreviewLines: policy.maxPreviewLines,
      dumpDir: policy.dumpDir,
    },
    recommendedRerun: [
@@ -2801,7 +2787,6 @@ export function createSshStdoutForwarder(options: {
  invocation: ParsedSshInvocation;
  transport: SshStdoutTruncationHint["transport"];
  maxBytes?: number;
-  maxLines?: number;
  stdout?: NodeJS.WritableStream;
 }): { write: (chunk: Buffer) => string | null } {
  return createSshStreamForwarder({
@@ -2809,7 +2794,6 @@ export function createSshStdoutForwarder(options: {
    transport: options.transport,
    stream: "stdout",
    maxBytes: options.maxBytes ?? sshStdoutStreamMaxBytes(),
-    maxLines: options.maxLines ?? readCliOutputPolicy().maxPreviewLines,
    target: options.stdout ?? process.stdout,
  });
 }
@@ -2818,7 +2802,6 @@ export function createSshStderrForwarder(options: {
  invocation: ParsedSshInvocation;
  transport: SshStdoutTruncationHint["transport"];
  maxBytes?: number;
-  maxLines?: number;
  stderr?: NodeJS.WritableStream;
 }): { write: (chunk: Buffer) => string | null } {
  return createSshStreamForwarder({
@@ -2826,7 +2809,6 @@ export function createSshStderrForwarder(options: {
    transport: options.transport,
    stream: "stderr",
    maxBytes: options.maxBytes ?? sshStderrStreamMaxBytes(),
-    maxLines: options.maxLines ?? readCliOutputPolicy().maxPreviewLines,
    target: options.stderr ?? process.stderr,
  });
 }
@@ -2836,15 +2818,10 @@ function createSshStreamForwarder(options: {
  transport: SshStdoutTruncationHint["transport"];
  stream: SshStdoutTruncationHint["stream"];
  maxBytes: number;
-  maxLines: number;
  target: NodeJS.WritableStream;
 }): { write: (chunk: Buffer) => string | null } {
  let observedBytes = 0;
  let forwardedBytes = 0;
-  let observedLineBreaks = 0;
-  let forwardedLineBreaks = 0;
-  let observedEndsWithLineBreak = false;
-  let forwardedEndsWithLineBreak = false;
  let truncated = false;
  let dumpPath: string | null = null;
  let dumpError: string | null = null;
@@ -2869,35 +2846,20 @@ function createSshStreamForwarder(options: {
  return {
    write(chunk: Buffer): string | null {
      observedBytes += chunk.length;
-      observedLineBreaks += countBufferLineBreaks(chunk);
-      if (chunk.length > 0) observedEndsWithLineBreak = chunk[chunk.length - 1] === 10;
-      const observedLines = lineCountFromBreaks(observedBytes, observedLineBreaks, observedEndsWithLineBreak);
-      if (!truncated && observedBytes <= options.maxBytes && observedLines <= options.maxLines) {
+      if (!truncated && observedBytes <= options.maxBytes) {
        bufferedChunks.push(Buffer.from(chunk));
        options.target.write(chunk);
        forwardedBytes += chunk.length;
-        forwardedLineBreaks += countBufferLineBreaks(chunk);
-        if (chunk.length > 0) forwardedEndsWithLineBreak = chunk[chunk.length - 1] === 10;
        return null;
      }

      if (!truncated) {
        truncated = true;
-        const remainingByBytes = Math.max(0, options.maxBytes - forwardedBytes);
-        const remainingByLines = prefixLengthWithinLineBudget(
-          chunk,
-          forwardedBytes,
-          forwardedLineBreaks,
-          forwardedEndsWithLineBreak,
-          options.maxLines,
-        );
-        const remaining = Math.min(remainingByBytes, remainingByLines);
+        const remaining = Math.max(0, options.maxBytes - forwardedBytes);
        if (remaining > 0) {
          const forwarded = chunk.subarray(0, remaining);
          options.target.write(forwarded);
          forwardedBytes += remaining;
-          forwardedLineBreaks += countBufferLineBreaks(forwarded);
-          if (forwarded.length > 0) forwardedEndsWithLineBreak = forwarded[forwarded.length - 1] === 10;
        }
        appendDump(chunk);
        return formatSshStdoutTruncationHint(sshStdoutTruncationHint({
@@ -2905,12 +2867,8 @@ function createSshStreamForwarder(options: {
          transport: options.transport,
          stream: options.stream,
          thresholdBytes: options.maxBytes,
-          thresholdLines: options.maxLines,
-          trigger: sshStreamTruncationTrigger(observedBytes, observedLines, options.maxBytes, options.maxLines),
          observedBytesAtTruncation: observedBytes,
          forwardedBytes,
-          observedLinesAtTruncation: lineCountFromBreaks(observedBytes, observedLineBreaks, observedEndsWithLineBreak),
-          forwardedLines: lineCountFromBreaks(forwardedBytes, forwardedLineBreaks, forwardedEndsWithLineBreak),
          dumpPath,
          dumpError,
        }));
@@ -2922,51 +2880,6 @@ function createSshStreamForwarder(options: {
  };
 }

-function countBufferLineBreaks(buffer: Buffer): number {
-  let count = 0;
-  for (const byte of buffer) {
-    if (byte === 10) count += 1;
-  }
-  return count;
-}
-
-function lineCountFromBreaks(bytes: number, lineBreaks: number, endsWithLineBreak: boolean): number {
-  return bytes === 0 ? 0 : lineBreaks + (endsWithLineBreak ? 0 : 1);
-}
-
-function sshStreamTruncationTrigger(
-  observedBytes: number,
-  observedLines: number,
-  maxBytes: number,
-  maxLines: number,
-): SshStdoutTruncationHint["trigger"] {
-  const bytes = observedBytes > maxBytes;
-  const lines = observedLines > maxLines;
-  if (bytes && lines) return "bytes-and-lines";
-  return bytes ? "bytes" : "lines";
-}
-
-function prefixLengthWithinLineBudget(
-  chunk: Buffer,
-  currentBytes: number,
-  currentLineBreaks: number,
-  currentEndsWithLineBreak: boolean,
-  maxLines: number,
-): number {
-  if (lineCountFromBreaks(currentBytes, currentLineBreaks, currentEndsWithLineBreak) >= maxLines) return 0;
-  let bytes = currentBytes;
-  let lineBreaks = currentLineBreaks;
-  let endsWithLineBreak = currentEndsWithLineBreak;
-  for (let index = 0; index < chunk.length; index += 1) {
-    const byte = chunk[index] ?? 0;
-    bytes += 1;
-    if (byte === 10) lineBreaks += 1;
-    endsWithLineBreak = byte === 10;
-    if (lineCountFromBreaks(bytes, lineBreaks, endsWithLineBreak) > maxLines) return index;
-  }
-  return chunk.length;
-}
-
 function brokerSource(): string {
  return String.raw`
 const open = JSON.parse(process.argv[2] || process.argv[1] || "{}");