fix(sentinel): harden post-deploy report and ssh summaries
This commit is contained in:
+98
-7
@@ -175,6 +175,33 @@ export interface SshStdoutTruncationHint {
|
||||
note: string;
|
||||
}
|
||||
|
||||
export interface SshStreamTruncationSummary {
|
||||
stream: "stdout" | "stderr";
|
||||
thresholdBytes: number;
|
||||
observedBytesAtTruncation: number;
|
||||
forwardedBytes: number;
|
||||
dumpPath: string | null;
|
||||
dumpError: string | null;
|
||||
}
|
||||
|
||||
export interface SshTruncationCompletionSummary {
|
||||
code: "ssh-truncation-summary";
|
||||
level: "info";
|
||||
providerId: string;
|
||||
route: string;
|
||||
transport: "backend-core-broker" | "frontend-websocket";
|
||||
invocationKind: SshInvocationKind;
|
||||
exitCode: number;
|
||||
timedOut: boolean;
|
||||
elapsedMs: number;
|
||||
elapsedSeconds: number;
|
||||
commandOmitted: true;
|
||||
stdout: SshStreamTruncationSummary | null;
|
||||
stderr: SshStreamTruncationSummary | null;
|
||||
message: string;
|
||||
action: string;
|
||||
}
|
||||
|
||||
export type SshTcpPoolFailureKind =
|
||||
| "provider-data-channel-closed"
|
||||
| "provider-data-channel-missing"
|
||||
@@ -2718,6 +2745,40 @@ export function formatSshStdoutTruncationHint(hint: SshStdoutTruncationHint): st
|
||||
return `${marker} ${JSON.stringify(hint)}\n`;
|
||||
}
|
||||
|
||||
export function sshTruncationCompletionSummary(options: {
|
||||
invocation: ParsedSshInvocation;
|
||||
transport: SshTruncationCompletionSummary["transport"];
|
||||
exitCode: number;
|
||||
timedOut: boolean;
|
||||
startedAtMs: number;
|
||||
stdout: SshStreamTruncationSummary | null;
|
||||
stderr: SshStreamTruncationSummary | null;
|
||||
}): SshTruncationCompletionSummary | null {
|
||||
if (options.stdout === null && options.stderr === null) return null;
|
||||
const elapsedMs = Math.max(0, Date.now() - options.startedAtMs);
|
||||
return {
|
||||
code: "ssh-truncation-summary",
|
||||
level: "info",
|
||||
providerId: safeProviderId(options.invocation.providerId),
|
||||
route: options.invocation.route.raw,
|
||||
transport: options.transport,
|
||||
invocationKind: options.invocation.parsed.invocationKind,
|
||||
exitCode: options.exitCode,
|
||||
timedOut: options.timedOut,
|
||||
elapsedMs,
|
||||
elapsedSeconds: Math.round(elapsedMs / 100) / 10,
|
||||
commandOmitted: true,
|
||||
stdout: options.stdout,
|
||||
stderr: options.stderr,
|
||||
message: "SSH output was truncated, but the command has finished; use this completion summary for closeout status and inspect dumpPath only when full output is needed.",
|
||||
action: "Use exitCode/timedOut plus dumpPath from this summary instead of manually inferring success from a long truncated stream.",
|
||||
};
|
||||
}
|
||||
|
||||
export function formatSshTruncationCompletionSummary(summary: SshTruncationCompletionSummary | null): string {
|
||||
return summary === null ? "" : `UNIDESK_SSH_TRUNCATION_SUMMARY ${JSON.stringify(summary)}\n`;
|
||||
}
|
||||
|
||||
export function classifySshTcpPoolFailure(text: string): SshTcpPoolFailureKind | null {
|
||||
const normalized = text.toLowerCase();
|
||||
if (normalized.includes("ssh tcp data channel closed")) return "provider-data-channel-closed";
|
||||
@@ -2783,12 +2844,17 @@ function sshStreamDumpPath(invocation: ParsedSshInvocation, stream: SshStdoutTru
|
||||
return join(policy.dumpDir, `${timestamp}-${process.pid}-${suffix}-${slug}.${stream}.bin`);
|
||||
}
|
||||
|
||||
export interface SshStreamForwarder {
|
||||
write: (chunk: Buffer) => string | null;
|
||||
summary: () => SshStreamTruncationSummary | null;
|
||||
}
|
||||
|
||||
export function createSshStdoutForwarder(options: {
|
||||
invocation: ParsedSshInvocation;
|
||||
transport: SshStdoutTruncationHint["transport"];
|
||||
maxBytes?: number;
|
||||
stdout?: NodeJS.WritableStream;
|
||||
}): { write: (chunk: Buffer) => string | null } {
|
||||
}): SshStreamForwarder {
|
||||
return createSshStreamForwarder({
|
||||
invocation: options.invocation,
|
||||
transport: options.transport,
|
||||
@@ -2803,7 +2869,7 @@ export function createSshStderrForwarder(options: {
|
||||
transport: SshStdoutTruncationHint["transport"];
|
||||
maxBytes?: number;
|
||||
stderr?: NodeJS.WritableStream;
|
||||
}): { write: (chunk: Buffer) => string | null } {
|
||||
}): SshStreamForwarder {
|
||||
return createSshStreamForwarder({
|
||||
invocation: options.invocation,
|
||||
transport: options.transport,
|
||||
@@ -2819,12 +2885,13 @@ function createSshStreamForwarder(options: {
|
||||
stream: SshStdoutTruncationHint["stream"];
|
||||
maxBytes: number;
|
||||
target: NodeJS.WritableStream;
|
||||
}): { write: (chunk: Buffer) => string | null } {
|
||||
}): SshStreamForwarder {
|
||||
let observedBytes = 0;
|
||||
let forwardedBytes = 0;
|
||||
let truncated = false;
|
||||
let dumpPath: string | null = null;
|
||||
let dumpError: string | null = null;
|
||||
let lastHint: SshStdoutTruncationHint | null = null;
|
||||
const bufferedChunks: Buffer[] = [];
|
||||
|
||||
const appendDump = (chunk: Buffer): void => {
|
||||
@@ -2862,7 +2929,7 @@ function createSshStreamForwarder(options: {
|
||||
forwardedBytes += remaining;
|
||||
}
|
||||
appendDump(chunk);
|
||||
return formatSshStdoutTruncationHint(sshStdoutTruncationHint({
|
||||
lastHint = sshStdoutTruncationHint({
|
||||
invocation: options.invocation,
|
||||
transport: options.transport,
|
||||
stream: options.stream,
|
||||
@@ -2871,12 +2938,24 @@ function createSshStreamForwarder(options: {
|
||||
forwardedBytes,
|
||||
dumpPath,
|
||||
dumpError,
|
||||
}));
|
||||
});
|
||||
return formatSshStdoutTruncationHint(lastHint);
|
||||
}
|
||||
|
||||
appendDump(chunk);
|
||||
return null;
|
||||
},
|
||||
summary(): SshStreamTruncationSummary | null {
|
||||
if (lastHint === null) return null;
|
||||
return {
|
||||
stream: lastHint.stream,
|
||||
thresholdBytes: lastHint.thresholdBytes,
|
||||
observedBytesAtTruncation: lastHint.observedBytesAtTruncation,
|
||||
forwardedBytes: lastHint.forwardedBytes,
|
||||
dumpPath,
|
||||
dumpError,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3827,14 +3906,16 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
const text = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : chunk;
|
||||
stderrTail = (stderrTail + text).slice(-16_384);
|
||||
};
|
||||
let stdoutForwarder: SshStreamForwarder | null = null;
|
||||
let stderrForwarder: SshStreamForwarder | null = null;
|
||||
if (parsed.remoteCommand === null) {
|
||||
child.stdout.pipe(process.stdout);
|
||||
} else {
|
||||
const stdoutForwarder = createSshStdoutForwarder({
|
||||
stdoutForwarder = createSshStdoutForwarder({
|
||||
invocation,
|
||||
transport: "backend-core-broker",
|
||||
});
|
||||
const stderrForwarder = createSshStderrForwarder({
|
||||
stderrForwarder = createSshStderrForwarder({
|
||||
invocation,
|
||||
transport: "backend-core-broker",
|
||||
});
|
||||
@@ -3923,6 +4004,16 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
|
||||
startedAtMs,
|
||||
}));
|
||||
if (timingHint) process.stderr.write(timingHint);
|
||||
const truncationSummary = formatSshTruncationCompletionSummary(sshTruncationCompletionSummary({
|
||||
invocation,
|
||||
transport: "backend-core-broker",
|
||||
exitCode,
|
||||
timedOut,
|
||||
startedAtMs,
|
||||
stdout: stdoutForwarder?.summary() ?? null,
|
||||
stderr: stderrForwarder?.summary() ?? null,
|
||||
}));
|
||||
if (truncationSummary) process.stderr.write(truncationSummary);
|
||||
resolve(exitCode);
|
||||
};
|
||||
child.on("error", (error) => {
|
||||
|
||||
Reference in New Issue
Block a user