mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add provider ssh tcp data pool

Browse Source
This commit is contained in:
Codex
2026-06-07 02:02:38 +00:00
parent d4b7fc95f9
commit fe1b91dcbd
32 changed files with 988 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/artifact-registry.ts
+1 -1
View File
@@ -1678,7 +1678,7 @@ function downloadRemoteFile(options: ArtifactRegistryOptions, remotePath: string
options.providerId,
"download",
"--chunk-bytes",
"45000",
"1048576",
remotePath,
localPath,
];
scripts/src/config.ts
+2
View File
@@ -13,6 +13,7 @@ export interface UniDeskConfig {
devFrontend: { port: number; containerPort: number };
database: { port: number; containerPort: number };
providerIngress: { port: number; containerPort: number };
providerData: { port: number; containerPort: number };
restrictedHostAccess?: { bindHost: string; allowedSourceCidrs: string[] };
};
auth: { username: string; password: string; sessionSecret: string; sessionTtlSeconds: number };
@@ -276,6 +277,7 @@ export function readConfig(): UniDeskConfig {
devFrontend: portPair(network, "devFrontend"),
database: portPair(network, "database"),
providerIngress: portPair(network, "providerIngress"),
providerData: portPair(network, "providerData"),
restrictedHostAccess: optionalRestrictedHostAccess(network),
},
database: {
scripts/src/docker.ts
+7
View File
@@ -154,6 +154,10 @@ export function writeComposeEnv(config: UniDeskConfig, freshLogPrefix: boolean):
UNIDESK_DATABASE_PORT: String(config.network.database.port),
UNIDESK_DATABASE_BIND_HOST: runtimeSecretWithDefault("UNIDESK_DATABASE_BIND_HOST", restrictedHostBind, "127.0.0.1"),
UNIDESK_PROVIDER_INGRESS_PORT: String(config.network.providerIngress.port),
UNIDESK_PROVIDER_DATA_PORT: String(config.network.providerData.port),
UNIDESK_PROVIDER_DATA_POOL_SIZE: runtimeSecret("UNIDESK_PROVIDER_DATA_POOL_SIZE") || "10",
UNIDESK_PROVIDER_DATA_CONNECT_TIMEOUT_MS: runtimeSecret("UNIDESK_PROVIDER_DATA_CONNECT_TIMEOUT_MS") || "5000",
UNIDESK_BACKEND_CORE_CARGO_BUILD_JOBS: runtimeSecret("UNIDESK_BACKEND_CORE_CARGO_BUILD_JOBS") || "1",
UNIDESK_DATABASE_USER: config.database.user,
UNIDESK_DATABASE_PASSWORD: config.database.password,
UNIDESK_DATABASE_NAME: config.database.name,
@@ -397,6 +401,7 @@ function fixedPorts(config: UniDeskConfig): Array<{ name: string; port: number;
{ name: "frontend", port: config.network.frontend.port, listening: isPortListening(config.network.frontend.port) },
{ name: "dev-frontend", port: config.network.devFrontend.port, listening: isPortListening(config.network.devFrontend.port) },
{ name: "provider-ingress", port: config.network.providerIngress.port, listening: isPortListening(config.network.providerIngress.port) },
{ name: "provider-data", port: config.network.providerData.port, listening: isPortListening(config.network.providerData.port) },
];
}
@@ -570,6 +575,7 @@ export async function stackStatus(config: UniDeskConfig): Promise<unknown> {
frontend: await probe(`http://127.0.0.1:${config.network.frontend.port}/health`),
devFrontend: await probe(`http://127.0.0.1:${config.network.devFrontend.port}/health`),
providerIngress: await probe(`http://127.0.0.1:${config.network.providerIngress.port}/health`),
providerDataPortListening: isPortListening(config.network.providerData.port),
database: dockerExec(config, "unidesk-database", ["pg_isready", "-U", config.database.user, "-d", config.database.name]),
overview,
},
@@ -577,6 +583,7 @@ export async function stackStatus(config: UniDeskConfig): Promise<unknown> {
frontend: `http://${config.network.publicHost}:${config.network.frontend.port}`,
devFrontend: `http://${config.network.publicHost}:${config.network.devFrontend.port}`,
providerIngress: `ws://${config.network.publicHost}:${config.network.providerIngress.port}/ws/provider`,
providerData: `tcp://${config.network.publicHost}:${config.network.providerData.port}`,
internalCore: `http://backend-core:${config.network.core.containerPort}`,
internalDatabase: `postgres://${config.database.user}:***@database:${config.network.database.containerPort}/${config.database.name}`,
},
scripts/src/help.ts
+3 -3
View File
@@ -12,7 +12,7 @@ export function rootHelp(): unknown {
{ command: "help", description: "List supported commands." },
{ command: "--main-server-ip <ip> <command>", description: "Run selected commands through the public frontend API; use --main-server-key only for legacy SSH transport." },
{ command: "config show", description: "Validate and print config.json as the single source of truth." },
{ command: "check [--full|--files|--scripts-typecheck|--components|--compose|--logs|--recovery-guardrails|--rust] | check recovery-guardrails", description: "Run the lightweight default syntax/config gate or the low-noise read-only D601 recovery guardrails; Rust is opt-in and only allowed from D601 CI/dev execution." },
{ command: "check [--full|--files|--scripts-typecheck|--components|--compose|--logs|--recovery-guardrails|--rust] | check recovery-guardrails", description: "Run the lightweight default syntax/config gate or the low-noise read-only D601 recovery guardrails; check --rust stays D601-guarded, while backend-core main-server online uses the separate constrained rebuild path." },
{ command: "server start", description: "Fire-and-forget build/start for database, backend-core, frontend, provider gateway, and managed main-server user services." },
{ command: "server stop", description: "Fire-and-forget docker-compose down for the fixed UniDesk stack." },
{ command: "server status", description: "Show fixed ports, containers, service health, and public URLs." },
@@ -22,7 +22,7 @@ export function rootHelp(): unknown {
{ command: "gc plan|run|db-trace|policy|remote [--confirm] [--logs-keep-days N] [--include-browser-cache]", description: "One-time main-server or remote provider disk relief and low-risk anti-bloat policy for logs, journald, allowlisted /tmp artifacts, scoped core dumps and explicit trace telemetry retention; plan is read-only and run requires --confirm." },
{ command: "server rebuild <backend-core|frontend|dev-frontend-proxy|provider-gateway|todo-note|code-queue-mgr|project-manager|baidu-netdisk|oa-event-flow>", description: "Maintenance-only local Compose rebuild for reviewed main-server services; frontend standard release must use CI artifact plus deploy apply dev/prod artifact consumers." },
{ command: "provider attach <providerId> [--master-server URL] [--up] [--force] | provider triage <providerId> [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]", description: "Generate the minimal external provider-gateway env/compose bundle or run the low-noise read-only provider health triage contract." },
{ command: "trans <route> [operation args...] (alias of ssh <route> ...)", description: "Open a Host SSH / WSL SSH maintenance session through the provider-gateway bridge; `trans` is the short alias for this ssh passthrough path." },
{ command: "trans <route> [operation args...] (alias of ssh <route> ...)", description: "Open a Host SSH / WSL SSH maintenance session; provider WebSocket carries control and host.ssh.tcp-pool carries stdin/stdout/stderr data." },
{ command: "trans gh:/owner/repo[/pr|/issue][/number[/1]] ls|cat|rg|patch-apply", description: "Treat GitHub PRs/issues as virtual text directories; `ls --full` shows state/floors/body length, and `patch-apply` updates first-floor `body.md` through UniDesk gh plus apply-patch v2." },
{ command: "trans <route> apply-patch < patch.diff", description: "Default remote text patch entry: apply a standard patch with the local TypeScript v2 engine while the remote route only reads and writes files." },
{ command: "trans <route> upload <local-file> <remote-file> | trans <route> download <remote-file> <local-file>", description: "Transfer whole files through SSH passthrough with remote temp files, automatic endpoint byte/SHA-256 verification, and client-side chunk fallback." },
@@ -133,7 +133,7 @@ export function serverHelp(action: string | undefined = undefined): unknown {
providerIngress: "provider-gateway WebSocket ingress",
},
rustBoundary: {
masterServer: "do not use server rebuild backend-core for Rust iteration; it would build locally",
masterServer: "do not use server rebuild backend-core for routine Rust iteration; only reviewed backend-core main-server online may use the constrained rebuild path",
d601: "use deploy apply --env dev --service backend-core and CI for Rust build/check",
},
maintenanceOnly: {
scripts/src/ssh-file-transfer.ts
+24 -16
View File
@@ -50,12 +50,13 @@ class SshFileTransferError extends Error {
}
}
const fileTransferReadBlockBytes = 45_000;
const fileTransferReadBlockBytes = 1_048_576;
const fileTransferWriteB64ArgvLimit = 48_000;
const fileTransferWriteB64ChunkChars = 12_000;
const fileTransferWriteRawChunkBytes = 1_048_576;
const fileTransferWriteB64ChunkChars = 1_398_104;
const fileTransferReadBlockMaxAttempts = 12;
const fileTransferReadEmptyRetryDelayMs = 250;
const fileTransferProgressEveryChunks = 64;
const fileTransferProgressEveryChunks = 16;
export function isSshFileTransferOperation(args: string[]): boolean {
const subcommand = args[0] ?? "";
@@ -163,7 +164,7 @@ function parseSshFileTransferCliOptions(args: string[]): SshFileTransferCliOptio
function boundedTransferChunkBytes(raw: string, option: string): number {
const value = Number(raw);
if (!Number.isInteger(value) || value <= 0) throw new Error(`${option} must be a positive integer`);
return Math.min(96_000, Math.max(1024, value));
return Math.min(4 * 1024 * 1024, Math.max(1024, value));
}
async function writeRemoteFileVerified(
@@ -173,26 +174,33 @@ async function writeRemoteFileVerified(
remotePath: string,
content: Buffer,
): Promise<SshFileTransferWriteResult> {
const encoded = content.toString("base64");
const expectedBytes = String(content.length);
const expectedSha256 = sha256HexBuffer(content);
const encoded = content.length <= fileTransferWriteB64ArgvLimit
? content.toString("base64")
: "";
if (invocation.route.plane !== "win" && encoded.length <= fileTransferWriteB64ArgvLimit) {
await checkedFileTransfer(invocation, executor, builders, "write-b64-argv", [remotePath, expectedBytes, expectedSha256, ...chunkString(encoded, fileTransferWriteB64ChunkChars)]);
return { strategy: "argv", chunks: encoded.length === 0 ? 0 : Math.ceil(encoded.length / fileTransferWriteB64ChunkChars) };
}
try {
await checkedFileTransfer(invocation, executor, builders, "write-b64-stdin", [remotePath, expectedBytes, expectedSha256], encoded);
return { strategy: "stdin", chunks: 1 };
} catch {
const token = `${process.pid}-${Date.now()}-${randomBytes(4).toString("hex")}-${expectedSha256.slice(0, 12)}`;
const chunks = chunkString(encoded, fileTransferWriteB64ChunkChars);
await checkedFileTransfer(invocation, executor, builders, "write-b64-begin", [remotePath, token]);
for (const chunk of chunks) {
await checkedFileTransfer(invocation, executor, builders, "write-b64-append-stdin", [remotePath, token], chunk);
if (content.length <= fileTransferWriteRawChunkBytes) {
try {
await checkedFileTransfer(invocation, executor, builders, "write-b64-stdin", [remotePath, expectedBytes, expectedSha256], content.toString("base64"));
return { strategy: "stdin", chunks: 1 };
} catch {
// Fall through to chunked upload with per-block base64 encoding.
}
await checkedFileTransfer(invocation, executor, builders, "write-b64-commit", [remotePath, token, expectedBytes, expectedSha256]);
return { strategy: "chunked-stdin", chunks: encoded.length === 0 ? 0 : chunks.length };
}
const token = `${process.pid}-${Date.now()}-${randomBytes(4).toString("hex")}-${expectedSha256.slice(0, 12)}`;
let chunks = 0;
await checkedFileTransfer(invocation, executor, builders, "write-b64-begin", [remotePath, token]);
for (let offset = 0; offset < content.length; offset += fileTransferWriteRawChunkBytes) {
const encodedChunk = content.subarray(offset, Math.min(content.length, offset + fileTransferWriteRawChunkBytes)).toString("base64");
await checkedFileTransfer(invocation, executor, builders, "write-b64-append-stdin", [remotePath, token], encodedChunk);
chunks += 1;
}
await checkedFileTransfer(invocation, executor, builders, "write-b64-commit", [remotePath, token, expectedBytes, expectedSha256]);
return { strategy: "chunked-stdin", chunks };
}
async function readRemoteFileVerified(