import { randomBytes } from "node:crypto"; import { basename, join, resolve } from "node:path"; import { repoRoot } from "./config"; import { runCommand, type CommandResult } from "./command"; export interface WebProbeRemoteArtifactJobOptions { route: string; localDir: string; waitTimeoutMs: number; commandTimeoutMs: number; inactivityTimeoutMs?: number; pollIntervalMs?: number; keepRemote?: boolean; runIdPrefix?: string; stdoutTailBytes?: number; } interface RemoteWebProbeArtifactManifest { runId: string; exitCode: number; remoteDir: string; stdoutPath: string; stderrPath: string; stdoutTail: string; stderrTail: string; artifacts: Array<{ remotePath: string; bytes: number | null; sha256: string | null; }>; } interface WebProbeRemoteArtifactJobResult { result: CommandResult; transport: Record; } const manifestBegin = "__UNIDESK_WEB_PROBE_ARTIFACT_MANIFEST_BEGIN__"; const manifestEnd = "__UNIDESK_WEB_PROBE_ARTIFACT_MANIFEST_END__"; export function runWebProbeRemoteArtifactJob(options: WebProbeRemoteArtifactJobOptions, userScript: string): WebProbeRemoteArtifactJobResult { const runIdPrefix = safePathSegment(options.runIdPrefix ?? "web-probe-artifact"); const runId = `${runIdPrefix}-${Date.now()}-${randomBytes(4).toString("hex")}`; const remoteDir = `/tmp/${runId}`; const localDir = resolve(options.localDir); const commandTimeoutMs = Math.max(1_000, options.commandTimeoutMs); const pollIntervalMs = Math.max(250, options.pollIntervalMs ?? 2_000); const stdoutTailBytes = Math.max(4_000, Math.min(256_000, Math.floor(options.stdoutTailBytes ?? 4_000))); const inactivityTimeoutMs = options.inactivityTimeoutMs; const keepRemote = options.keepRemote === true; const submitCommand = [transPath(), options.route, "sh"]; const submit = runCommand(submitCommand, repoRoot, { input: remoteArtifactSubmitScript(remoteDir, runId, userScript, stdoutTailBytes), timeoutMs: Math.min(60_000, commandTimeoutMs), }); const submitRecovery = submit.exitCode !== 0 && isRecoverableSubmitTimeout(submit) ? { recovered: true, reason: "submit-short-connection-timeout", remoteDir, runId, exitCode: submit.exitCode, stdoutTail: submit.stdout.slice(-1000), stderrTail: submit.stderr.slice(-1000), next: "submit may have been cut by the 60s trans runtime limit after the background job was launched; polling remote status by remoteDir/runId", } : null; let manifest: RemoteWebProbeArtifactManifest | null = null; let pollFailure: Record | null = null; let downloadFailure: Record | null = null; const commandResults: CommandResult[] = [submit]; if (submit.exitCode === 0 || submitRecovery !== null) { const polled = pollRemoteManifest(options.route, remoteDir, runId, options.waitTimeoutMs, pollIntervalMs); commandResults.push(...polled.results); manifest = polled.manifest; pollFailure = polled.failure; } else { pollFailure = { phase: "submit", message: "web-probe remote artifact submit failed", exitCode: submit.exitCode, stdoutTail: submit.stdout.slice(-1000), stderrTail: submit.stderr.slice(-1000), }; } const artifacts: Array> = []; if (manifest !== null && pollFailure === null) { for (const artifact of manifest.artifacts) { const localPath = join(localDir, `${runId}-${safePathSegment(basename(artifact.remotePath) || "artifact")}`); const maxAttempts = 3; let downloaded = false; let lastDownload: CommandResult | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const downloadArgs = [transPath(), options.route, "download"]; if (inactivityTimeoutMs !== undefined) downloadArgs.push("--inactivity-timeout-ms", String(inactivityTimeoutMs)); downloadArgs.push(artifact.remotePath, localPath); const download = runCommand(downloadArgs, repoRoot, { timeoutMs: commandTimeoutMs }); commandResults.push(download); lastDownload = download; const parsed = parseJsonObject(download.stdout); if (download.exitCode === 0 && parsed.ok === true && parsed.verified === true) { artifacts.push({ remotePath: artifact.remotePath, localPath, bytes: numberOrNull(parsed.bytes) ?? artifact.bytes, sha256: typeof parsed.sha256 === "string" ? parsed.sha256 : artifact.sha256, verified: true, verification: parsed.verification ?? null, transfer: parsed.transfer ?? null, manifestBytes: artifact.bytes, manifestSha256: artifact.sha256, }); downloaded = true; break; } if (attempt < maxAttempts) sleepSync(750 * attempt); } if (!downloaded) { downloadFailure = { remotePath: artifact.remotePath, attempts: maxAttempts, exitCode: lastDownload?.exitCode ?? null, stdoutTail: lastDownload?.stdout.slice(-1000) ?? "", stderrTail: lastDownload?.stderr.slice(-1000) ?? "", }; break; } } } const stop = pollFailure !== null && manifest === null ? stopRemoteJob(options.route, remoteDir) : { payload: { attempted: false, reason: "not-needed", remoteDir, valuesRedacted: true }, result: null }; if (stop.result !== null) commandResults.push(stop.result); const cleanup = cleanupRemoteDir(options.route, remoteDir, keepRemote); if (cleanup.result !== null) commandResults.push(cleanup.result); const manifestExitCode = manifest?.exitCode ?? null; const ok = pollFailure === null && manifest !== null && manifest.exitCode === 0 && downloadFailure === null; const exitCode = ok ? 0 : manifestExitCode ?? (pollFailure !== null ? 1 : submit.exitCode ?? 1); const transport = { ok, command: "web-probe remote-artifact", route: options.route, runId, localDir, remote: { exitCode: manifestExitCode, remoteDir: manifest?.remoteDir ?? remoteDir, stdoutPath: manifest?.stdoutPath ?? join(remoteDir, "stdout.log"), stderrPath: manifest?.stderrPath ?? join(remoteDir, "stderr.log"), stdoutTail: manifest?.stdoutTail ?? "", stderrTail: manifest?.stderrTail ?? "", defaultScreenshot: null, }, artifacts, artifactCount: artifacts.length, expectedArtifactCount: manifest?.artifacts.length ?? 0, cleanup: cleanup.payload, pollFailure, downloadFailure, remoteCommand: { exitCode: manifestExitCode, submitExitCode: submit.exitCode, submitRecovered: submitRecovery !== null, submitRecovery, submitStdoutBytes: Buffer.byteLength(submit.stdout, "utf8"), submitStderrBytes: Buffer.byteLength(submit.stderr, "utf8"), }, stop: stop.payload, valuesRedacted: true, }; return { result: syntheticCommandResult( [transPath(), options.route, "web-probe-remote-artifact"], exitCode, JSON.stringify(transport, null, 2), commandResults, ), transport, }; } function stopRemoteJob(route: string, remoteDir: string): { payload: Record; result: CommandResult | null } { const result = runCommand([transPath(), route, "sh", "--", remoteArtifactStopScript(remoteDir)], repoRoot, { timeoutMs: 30_000 }); const parsed = parseTabStatus(result.stdout); return { payload: { attempted: true, ok: result.exitCode === 0, remoteDir, status: parsed.status ?? null, pid: parsed.pid ?? null, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1000), stderrTail: result.stderr.slice(-1000), valuesRedacted: true, }, result, }; } function pollRemoteManifest( route: string, remoteDir: string, runId: string, waitTimeoutMs: number, pollIntervalMs: number, ): { manifest: RemoteWebProbeArtifactManifest | null; failure: Record | null; results: CommandResult[] } { const deadline = Date.now() + Math.max(1_000, waitTimeoutMs); const results: CommandResult[] = []; let lastStatus: CommandResult | null = null; while (Date.now() <= deadline) { const status = runCommand([transPath(), route, "sh", "--", remoteArtifactStatusScript(remoteDir)], repoRoot, { timeoutMs: Math.min(60_000, Math.max(5_000, waitTimeoutMs)) }); results.push(status); lastStatus = status; if (status.exitCode === 0 && status.stdout.includes(manifestEnd)) { try { return { manifest: parseRemoteManifest(status, remoteDir, runId), failure: null, results }; } catch (error) { return { manifest: null, failure: { phase: "parse-manifest", message: error instanceof Error ? error.message : String(error), stdoutTail: status.stdout.slice(-1000), stderrTail: status.stderr.slice(-1000), }, results, }; } } if (status.exitCode !== 0 && !/status\t(?:pending|running)/u.test(status.stdout)) { return { manifest: null, failure: { phase: "status", exitCode: status.exitCode, stdoutTail: status.stdout.slice(-1000), stderrTail: status.stderr.slice(-1000), }, results, }; } sleepSync(pollIntervalMs); } return { manifest: null, failure: { phase: "timeout", message: `web-probe remote artifact timed out waiting for remote job after ${waitTimeoutMs}ms`, remoteDir, runId, lastStdoutTail: lastStatus?.stdout.slice(-1000) ?? "", lastStderrTail: lastStatus?.stderr.slice(-1000) ?? "", }, results, }; } function cleanupRemoteDir(route: string, remoteDir: string, keepRemote: boolean): { payload: Record; result: CommandResult | null } { if (keepRemote) return { payload: { attempted: false, kept: true, remoteDir }, result: null }; const result = runCommand([transPath(), route, "sh", "--", `rm -rf -- ${shellQuote(remoteDir)}`], repoRoot, { timeoutMs: 60_000 }); return { payload: { attempted: true, kept: false, remoteDir, exitCode: result.exitCode, ok: result.exitCode === 0, stderrTail: result.stderr.slice(-1000), }, result, }; } function remoteArtifactSubmitScript(remoteDir: string, runId: string, userScript: string, stdoutTailBytes: number): string { return [ "set -eu", `remote_dir=${shellQuote(remoteDir)}`, 'mkdir -p -- "$remote_dir"', 'user_script="$remote_dir/user-script.sh"', 'runner_script="$remote_dir/runner.sh"', 'pid_file="$remote_dir/pid"', 'submit_stdout="$remote_dir/submit.out"', 'submit_stderr="$remote_dir/submit.err"', 'if [ -f "$pid_file" ] && kill -0 "$(cat "$pid_file")" >/dev/null 2>&1; then', ' printf "status\\trunning\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$(cat "$pid_file")"', " exit 0", "fi", writeBase64FileShell("$user_script", userScript, "UNIDESK_WEB_PROBE_ARTIFACT_USER_B64"), "chmod 700 \"$user_script\"", writeBase64FileShell("$runner_script", remoteArtifactRunnerScript(remoteDir, runId, stdoutTailBytes), "UNIDESK_WEB_PROBE_ARTIFACT_RUNNER_B64"), "chmod 700 \"$runner_script\"", 'nohup sh "$runner_script" "$user_script" >"$submit_stdout" 2>"$submit_stderr" < /dev/null &', 'pid=$!', 'printf "%s\\n" "$pid" >"$pid_file"', 'printf "status\\tsubmitted\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$pid"', ].join("\n"); } function remoteArtifactStatusScript(remoteDir: string): string { return [ "set -eu", `remote_dir=${shellQuote(remoteDir)}`, 'manifest="$remote_dir/manifest.tsv"', 'pid_file="$remote_dir/pid"', 'if [ -f "$manifest" ]; then cat "$manifest"; exit 0; fi', 'if [ -f "$pid_file" ] && kill -0 "$(cat "$pid_file")" >/dev/null 2>&1; then', ' printf "status\\trunning\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$(cat "$pid_file")"', " exit 1", "fi", 'if [ -f "$pid_file" ]; then', ' printf "status\\texited-without-manifest\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$(cat "$pid_file")"', ' if [ -f "$remote_dir/submit.err" ]; then tail -c 1000 "$remote_dir/submit.err" >&2; fi', ' if [ -f "$remote_dir/stderr.log" ]; then tail -c 1000 "$remote_dir/stderr.log" >&2; fi', " exit 2", "fi", 'printf "status\\tpending\\nremote_dir\\t%s\\n" "$remote_dir"', 'if [ -f "$remote_dir/submit.err" ]; then tail -c 1000 "$remote_dir/submit.err" >&2; fi', 'if [ -f "$remote_dir/stderr.log" ]; then tail -c 1000 "$remote_dir/stderr.log" >&2; fi', "exit 1", ].join("\n"); } function remoteArtifactStopScript(remoteDir: string): string { return [ "set +e", `remote_dir=${shellQuote(remoteDir)}`, 'pid_file="$remote_dir/pid"', 'if [ ! -f "$pid_file" ]; then printf "status\\tno-pid\\nremote_dir\\t%s\\n" "$remote_dir"; exit 0; fi', 'pid="$(cat "$pid_file" 2>/dev/null | tr -cd "0-9")"', 'if [ -z "$pid" ]; then printf "status\\tinvalid-pid\\nremote_dir\\t%s\\n" "$remote_dir"; exit 0; fi', 'children_of() { command -v pgrep >/dev/null 2>&1 && pgrep -P "$1" 2>/dev/null || true; }', 'term_tree() { _pid="$1"; for _child in $(children_of "$_pid"); do term_tree "$_child"; done; kill -TERM "$_pid" 2>/dev/null || true; }', 'kill_tree() { _pid="$1"; for _child in $(children_of "$_pid"); do kill_tree "$_child"; done; kill -KILL "$_pid" 2>/dev/null || true; }', 'if ! kill -0 "$pid" >/dev/null 2>&1; then printf "status\\tnot-running\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$pid"; exit 0; fi', 'term_tree "$pid"', "sleep 1", 'if kill -0 "$pid" >/dev/null 2>&1; then kill_tree "$pid"; sleep 1; fi', 'if kill -0 "$pid" >/dev/null 2>&1; then printf "status\\tstill-running\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$pid"; exit 1; fi', 'printf "status\\tstopped\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$pid"', "exit 0", ].join("\n"); } function remoteArtifactRunnerScript(remoteDir: string, runId: string, stdoutTailBytes: number): string { return [ "set -eu", `remote_dir=${shellQuote(remoteDir)}`, `run_id=${shellQuote(runId)}`, 'if [ "$#" -lt 1 ]; then printf "missing user script path\\n" >&2; exit 2; fi', 'user_script="$1"', 'stdout_file="$remote_dir/stdout.log"', 'stderr_file="$remote_dir/stderr.log"', 'artifacts_file="$remote_dir/artifacts.tsv"', 'manifest_file="$remote_dir/manifest.tsv"', "sha256_file() {", " if command -v sha256sum >/dev/null 2>&1; then sha256sum -- \"$1\" | awk '{print $1}'; return; fi", " if command -v shasum >/dev/null 2>&1; then shasum -a 256 -- \"$1\" | awk '{print $1}'; return; fi", " if command -v openssl >/dev/null 2>&1; then openssl dgst -sha256 -- \"$1\" | awk '{print $NF}'; return; fi", " printf 'missing sha256 tool\\n' >&2; return 127", "}", 'export UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR="$remote_dir"', "set +e", 'sh "$user_script" >"$stdout_file" 2>"$stderr_file"', "user_rc=$?", ': > "$artifacts_file"', 'find "$remote_dir" -type f | while IFS= read -r file; do', ' case "$file" in "$user_script"|"$stdout_file"|"$stderr_file"|"$artifacts_file"|"$manifest_file"|"$remote_dir/runner.sh"|"$remote_dir/submit.out"|"$remote_dir/submit.err"|"$remote_dir/pid") continue ;; esac', ' case "$file" in *.png|*.jpg|*.jpeg|*.webp)', ' bytes=$(wc -c < "$file" | tr -d "[:space:]")', ' digest=$(sha256_file "$file")', ' printf "artifact\\t%s\\t%s\\t%s\\n" "$bytes" "$digest" "$file" >> "$artifacts_file"', " ;;", " esac", "done", `b64_tail() { if [ -f "$1" ]; then tail -c ${stdoutTailBytes} "$1" | base64 | tr -d '\\n'; fi; }`, "{", `printf '%s\\n' ${shellQuote(manifestBegin)}`, "printf 'run_id\\t%s\\n' \"$run_id\"", "printf 'exit_code\\t%s\\n' \"$user_rc\"", "printf 'remote_dir\\t%s\\n' \"$remote_dir\"", "printf 'stdout_path\\t%s\\n' \"$stdout_file\"", "printf 'stderr_path\\t%s\\n' \"$stderr_file\"", "printf 'stdout_tail_b64\\t%s\\n' \"$(b64_tail \"$stdout_file\")\"", "printf 'stderr_tail_b64\\t%s\\n' \"$(b64_tail \"$stderr_file\")\"", 'cat "$artifacts_file"', `printf '%s\\n' ${shellQuote(manifestEnd)}`, "} > \"$manifest_file\"", 'exit "$user_rc"', ].join("\n"); } function parseRemoteManifest(result: CommandResult, fallbackRemoteDir: string, fallbackRunId: string): RemoteWebProbeArtifactManifest { const start = result.stdout.indexOf(manifestBegin); const end = result.stdout.indexOf(manifestEnd, start < 0 ? 0 : start); if (start < 0 || end < 0) { throw new Error(`web-probe remote artifact did not emit a manifest; exitCode=${result.exitCode}; stdoutTail=${JSON.stringify(result.stdout.slice(-1000))}; stderrTail=${JSON.stringify(result.stderr.slice(-1000))}`); } const body = result.stdout.slice(start + manifestBegin.length, end).trim(); const manifest: RemoteWebProbeArtifactManifest = { runId: fallbackRunId, exitCode: result.exitCode ?? 1, remoteDir: fallbackRemoteDir, stdoutPath: join(fallbackRemoteDir, "stdout.log"), stderrPath: join(fallbackRemoteDir, "stderr.log"), stdoutTail: "", stderrTail: "", artifacts: [], }; for (const rawLine of body.split(/\r?\n/u)) { const line = rawLine.trimEnd(); if (line.length === 0) continue; const parts = line.split("\t"); const key = parts[0] ?? ""; if (key === "artifact") { const bytes = Number(parts[1] ?? ""); const sha256 = parts[2] ?? ""; const remotePath = parts.slice(3).join("\t"); if (remotePath.length > 0) { manifest.artifacts.push({ remotePath, bytes: Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : null, sha256: /^[0-9a-f]{64}$/u.test(sha256) ? sha256 : null, }); } continue; } const value = parts.slice(1).join("\t"); if (key === "run_id" && value.length > 0) manifest.runId = value; else if (key === "exit_code") { const exitCode = Number(value); manifest.exitCode = Number.isInteger(exitCode) ? exitCode : result.exitCode ?? 1; } else if (key === "remote_dir" && value.length > 0) manifest.remoteDir = value; else if (key === "stdout_path" && value.length > 0) manifest.stdoutPath = value; else if (key === "stderr_path" && value.length > 0) manifest.stderrPath = value; else if (key === "stdout_tail_b64") manifest.stdoutTail = decodeBase64(value); else if (key === "stderr_tail_b64") manifest.stderrTail = decodeBase64(value); } return manifest; } function syntheticCommandResult(command: string[], exitCode: number | null, stdout: string, results: readonly CommandResult[]): CommandResult { return { command, cwd: repoRoot, exitCode, stdout, stderr: results.map((item) => item.stderr).filter((item) => item.length > 0).join("\n").slice(-8000), signal: null, timedOut: results.some((item) => item.timedOut), }; } function writeBase64FileShell(pathExpression: string, content: string, marker: string): string { return [ `cat > ${pathExpression}.b64 <<'${marker}'`, wrapBase64(Buffer.from(content, "utf8").toString("base64")), marker, `base64 -d < ${pathExpression}.b64 > ${pathExpression}`, `rm -f -- ${pathExpression}.b64`, ].join("\n"); } function wrapBase64(value: string): string { return value.replace(/.{1,76}/gu, "$&\n").trimEnd(); } function parseJsonObject(text: string): Record { const trimmed = text.trim(); if (trimmed.length === 0) return {}; try { const parsed = JSON.parse(trimmed) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; } catch { return {}; } } function parseTabStatus(text: string): Record { const out: Record = {}; for (const rawLine of text.split(/\r?\n/u)) { const line = rawLine.trimEnd(); if (line.length === 0) continue; const [key, ...rest] = line.split("\t"); if (key) out[key] = rest.join("\t"); } return out; } function numberOrNull(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } function decodeBase64(value: string): string { if (value.length === 0) return ""; try { return Buffer.from(value, "base64").toString("utf8"); } catch { return ""; } } function isRecoverableSubmitTimeout(submit: CommandResult): boolean { if (submit.exitCode === 124 || submit.timedOut) return true; const combined = `${submit.stdout}\n${submit.stderr}`; return /(?:timed?\s*out|timeout|60s|exitCode=124|signal\s+TERM|signal\s+KILL)/iu.test(combined); } function sleepSync(ms: number): void { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, ms)); } function safePathSegment(value: string): string { const cleaned = value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, ""); return cleaned.length > 0 ? cleaned.slice(0, 120) : "artifact"; } function shellQuote(value: string): string { return `'${value.replace(/'/g, "'\\''")}'`; } function transPath(): string { return join(repoRoot, "scripts", "trans"); }