Codex
469 lines
18 KiB
TypeScript
469 lines
18 KiB
TypeScript
import { randomBytes } from "node:crypto";
|
|
import { basename, join, resolve } from "node:path";
|
|
import { repoRoot } from "./config";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
|
|
export interface WebProbeRemoteArtifactJobOptions {
|
|
route: string;
|
|
localDir: string;
|
|
waitTimeoutMs: number;
|
|
commandTimeoutMs: number;
|
|
inactivityTimeoutMs?: number;
|
|
pollIntervalMs?: number;
|
|
keepRemote?: boolean;
|
|
runIdPrefix?: string;
|
|
}
|
|
|
|
interface RemoteWebProbeArtifactManifest {
|
|
runId: string;
|
|
exitCode: number;
|
|
remoteDir: string;
|
|
stdoutPath: string;
|
|
stderrPath: string;
|
|
stdoutTail: string;
|
|
stderrTail: string;
|
|
artifacts: Array<{
|
|
remotePath: string;
|
|
bytes: number | null;
|
|
sha256: string | null;
|
|
}>;
|
|
}
|
|
|
|
interface WebProbeRemoteArtifactJobResult {
|
|
result: CommandResult;
|
|
transport: Record<string, unknown>;
|
|
}
|
|
|
|
const manifestBegin = "__UNIDESK_WEB_PROBE_ARTIFACT_MANIFEST_BEGIN__";
|
|
const manifestEnd = "__UNIDESK_WEB_PROBE_ARTIFACT_MANIFEST_END__";
|
|
|
|
export function runWebProbeRemoteArtifactJob(options: WebProbeRemoteArtifactJobOptions, userScript: string): WebProbeRemoteArtifactJobResult {
|
|
const runIdPrefix = safePathSegment(options.runIdPrefix ?? "web-probe-artifact");
|
|
const runId = `${runIdPrefix}-${Date.now()}-${randomBytes(4).toString("hex")}`;
|
|
const remoteDir = `/tmp/${runId}`;
|
|
const localDir = resolve(options.localDir);
|
|
const commandTimeoutMs = Math.max(1_000, options.commandTimeoutMs);
|
|
const pollIntervalMs = Math.max(250, options.pollIntervalMs ?? 2_000);
|
|
const inactivityTimeoutMs = options.inactivityTimeoutMs;
|
|
const keepRemote = options.keepRemote === true;
|
|
const submitCommand = [transPath(), options.route, "sh"];
|
|
const submit = runCommand(submitCommand, repoRoot, {
|
|
input: remoteArtifactSubmitScript(remoteDir, runId, userScript),
|
|
timeoutMs: Math.min(60_000, commandTimeoutMs),
|
|
});
|
|
const submitRecovery = submit.exitCode !== 0 && isRecoverableSubmitTimeout(submit)
|
|
? {
|
|
recovered: true,
|
|
reason: "submit-short-connection-timeout",
|
|
remoteDir,
|
|
runId,
|
|
exitCode: submit.exitCode,
|
|
stdoutTail: submit.stdout.slice(-1000),
|
|
stderrTail: submit.stderr.slice(-1000),
|
|
next: "submit may have been cut by the 60s trans runtime limit after the background job was launched; polling remote status by remoteDir/runId",
|
|
}
|
|
: null;
|
|
|
|
let manifest: RemoteWebProbeArtifactManifest | null = null;
|
|
let pollFailure: Record<string, unknown> | null = null;
|
|
let downloadFailure: Record<string, unknown> | null = null;
|
|
const commandResults: CommandResult[] = [submit];
|
|
if (submit.exitCode === 0 || submitRecovery !== null) {
|
|
const polled = pollRemoteManifest(options.route, remoteDir, runId, options.waitTimeoutMs, pollIntervalMs);
|
|
commandResults.push(...polled.results);
|
|
manifest = polled.manifest;
|
|
pollFailure = polled.failure;
|
|
} else {
|
|
pollFailure = {
|
|
phase: "submit",
|
|
message: "web-probe remote artifact submit failed",
|
|
exitCode: submit.exitCode,
|
|
stdoutTail: submit.stdout.slice(-1000),
|
|
stderrTail: submit.stderr.slice(-1000),
|
|
};
|
|
}
|
|
|
|
const artifacts: Array<Record<string, unknown>> = [];
|
|
if (manifest !== null && pollFailure === null) {
|
|
for (const artifact of manifest.artifacts) {
|
|
const localPath = join(localDir, `${runId}-${safePathSegment(basename(artifact.remotePath) || "artifact")}`);
|
|
const maxAttempts = 3;
|
|
let downloaded = false;
|
|
let lastDownload: CommandResult | null = null;
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
const downloadArgs = [transPath(), options.route, "download"];
|
|
if (inactivityTimeoutMs !== undefined) downloadArgs.push("--inactivity-timeout-ms", String(inactivityTimeoutMs));
|
|
downloadArgs.push(artifact.remotePath, localPath);
|
|
const download = runCommand(downloadArgs, repoRoot, { timeoutMs: commandTimeoutMs });
|
|
commandResults.push(download);
|
|
lastDownload = download;
|
|
const parsed = parseJsonObject(download.stdout);
|
|
if (download.exitCode === 0 && parsed.ok === true && parsed.verified === true) {
|
|
artifacts.push({
|
|
remotePath: artifact.remotePath,
|
|
localPath,
|
|
bytes: numberOrNull(parsed.bytes) ?? artifact.bytes,
|
|
sha256: typeof parsed.sha256 === "string" ? parsed.sha256 : artifact.sha256,
|
|
verified: true,
|
|
verification: parsed.verification ?? null,
|
|
transfer: parsed.transfer ?? null,
|
|
manifestBytes: artifact.bytes,
|
|
manifestSha256: artifact.sha256,
|
|
});
|
|
downloaded = true;
|
|
break;
|
|
}
|
|
if (attempt < maxAttempts) sleepSync(750 * attempt);
|
|
}
|
|
if (!downloaded) {
|
|
downloadFailure = {
|
|
remotePath: artifact.remotePath,
|
|
attempts: maxAttempts,
|
|
exitCode: lastDownload?.exitCode ?? null,
|
|
stdoutTail: lastDownload?.stdout.slice(-1000) ?? "",
|
|
stderrTail: lastDownload?.stderr.slice(-1000) ?? "",
|
|
};
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
const cleanup = cleanupRemoteDir(options.route, remoteDir, keepRemote);
|
|
if (cleanup.result !== null) commandResults.push(cleanup.result);
|
|
const manifestExitCode = manifest?.exitCode ?? null;
|
|
const ok = pollFailure === null && manifest !== null && manifest.exitCode === 0 && downloadFailure === null;
|
|
const transport = {
|
|
ok,
|
|
command: "web-probe remote-artifact",
|
|
route: options.route,
|
|
runId,
|
|
localDir,
|
|
remote: {
|
|
exitCode: manifestExitCode,
|
|
remoteDir: manifest?.remoteDir ?? remoteDir,
|
|
stdoutPath: manifest?.stdoutPath ?? join(remoteDir, "stdout.log"),
|
|
stderrPath: manifest?.stderrPath ?? join(remoteDir, "stderr.log"),
|
|
stdoutTail: manifest?.stdoutTail ?? "",
|
|
stderrTail: manifest?.stderrTail ?? "",
|
|
defaultScreenshot: null,
|
|
},
|
|
artifacts,
|
|
artifactCount: artifacts.length,
|
|
expectedArtifactCount: manifest?.artifacts.length ?? 0,
|
|
cleanup: cleanup.payload,
|
|
pollFailure,
|
|
downloadFailure,
|
|
remoteCommand: {
|
|
exitCode: manifestExitCode,
|
|
submitExitCode: submit.exitCode,
|
|
submitRecovered: submitRecovery !== null,
|
|
submitRecovery,
|
|
submitStdoutBytes: Buffer.byteLength(submit.stdout, "utf8"),
|
|
submitStderrBytes: Buffer.byteLength(submit.stderr, "utf8"),
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
return {
|
|
result: syntheticCommandResult(
|
|
[transPath(), options.route, "web-probe-remote-artifact"],
|
|
ok ? 0 : manifestExitCode ?? submit.exitCode ?? 1,
|
|
JSON.stringify(transport, null, 2),
|
|
commandResults,
|
|
),
|
|
transport,
|
|
};
|
|
}
|
|
|
|
function pollRemoteManifest(
|
|
route: string,
|
|
remoteDir: string,
|
|
runId: string,
|
|
waitTimeoutMs: number,
|
|
pollIntervalMs: number,
|
|
): { manifest: RemoteWebProbeArtifactManifest | null; failure: Record<string, unknown> | null; results: CommandResult[] } {
|
|
const deadline = Date.now() + Math.max(1_000, waitTimeoutMs);
|
|
const results: CommandResult[] = [];
|
|
let lastStatus: CommandResult | null = null;
|
|
while (Date.now() <= deadline) {
|
|
const status = runCommand([transPath(), route, "sh", "--", remoteArtifactStatusScript(remoteDir)], repoRoot, { timeoutMs: Math.min(60_000, Math.max(5_000, waitTimeoutMs)) });
|
|
results.push(status);
|
|
lastStatus = status;
|
|
if (status.exitCode === 0 && status.stdout.includes(manifestEnd)) {
|
|
try {
|
|
return { manifest: parseRemoteManifest(status, remoteDir, runId), failure: null, results };
|
|
} catch (error) {
|
|
return {
|
|
manifest: null,
|
|
failure: {
|
|
phase: "parse-manifest",
|
|
message: error instanceof Error ? error.message : String(error),
|
|
stdoutTail: status.stdout.slice(-1000),
|
|
stderrTail: status.stderr.slice(-1000),
|
|
},
|
|
results,
|
|
};
|
|
}
|
|
}
|
|
if (status.exitCode !== 0 && !/status\t(?:pending|running)/u.test(status.stdout)) {
|
|
return {
|
|
manifest: null,
|
|
failure: {
|
|
phase: "status",
|
|
exitCode: status.exitCode,
|
|
stdoutTail: status.stdout.slice(-1000),
|
|
stderrTail: status.stderr.slice(-1000),
|
|
},
|
|
results,
|
|
};
|
|
}
|
|
sleepSync(pollIntervalMs);
|
|
}
|
|
return {
|
|
manifest: null,
|
|
failure: {
|
|
phase: "timeout",
|
|
message: `web-probe remote artifact timed out waiting for remote job after ${waitTimeoutMs}ms`,
|
|
remoteDir,
|
|
runId,
|
|
lastStdoutTail: lastStatus?.stdout.slice(-1000) ?? "",
|
|
lastStderrTail: lastStatus?.stderr.slice(-1000) ?? "",
|
|
},
|
|
results,
|
|
};
|
|
}
|
|
|
|
function cleanupRemoteDir(route: string, remoteDir: string, keepRemote: boolean): { payload: Record<string, unknown>; result: CommandResult | null } {
|
|
if (keepRemote) return { payload: { attempted: false, kept: true, remoteDir }, result: null };
|
|
const result = runCommand([transPath(), route, "sh", "--", `rm -rf -- ${shellQuote(remoteDir)}`], repoRoot, { timeoutMs: 60_000 });
|
|
return {
|
|
payload: {
|
|
attempted: true,
|
|
kept: false,
|
|
remoteDir,
|
|
exitCode: result.exitCode,
|
|
ok: result.exitCode === 0,
|
|
stderrTail: result.stderr.slice(-1000),
|
|
},
|
|
result,
|
|
};
|
|
}
|
|
|
|
function remoteArtifactSubmitScript(remoteDir: string, runId: string, userScript: string): string {
|
|
return [
|
|
"set -eu",
|
|
`remote_dir=${shellQuote(remoteDir)}`,
|
|
'mkdir -p -- "$remote_dir"',
|
|
'user_script="$remote_dir/user-script.sh"',
|
|
'runner_script="$remote_dir/runner.sh"',
|
|
'pid_file="$remote_dir/pid"',
|
|
'submit_stdout="$remote_dir/submit.out"',
|
|
'submit_stderr="$remote_dir/submit.err"',
|
|
'if [ -f "$pid_file" ] && kill -0 "$(cat "$pid_file")" >/dev/null 2>&1; then',
|
|
' printf "status\\trunning\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$(cat "$pid_file")"',
|
|
" exit 0",
|
|
"fi",
|
|
writeBase64FileShell("$user_script", userScript, "UNIDESK_WEB_PROBE_ARTIFACT_USER_B64"),
|
|
"chmod 700 \"$user_script\"",
|
|
writeBase64FileShell("$runner_script", remoteArtifactRunnerScript(remoteDir, runId), "UNIDESK_WEB_PROBE_ARTIFACT_RUNNER_B64"),
|
|
"chmod 700 \"$runner_script\"",
|
|
'nohup sh "$runner_script" "$user_script" >"$submit_stdout" 2>"$submit_stderr" < /dev/null &',
|
|
'pid=$!',
|
|
'printf "%s\\n" "$pid" >"$pid_file"',
|
|
'printf "status\\tsubmitted\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$pid"',
|
|
].join("\n");
|
|
}
|
|
|
|
function remoteArtifactStatusScript(remoteDir: string): string {
|
|
return [
|
|
"set -eu",
|
|
`remote_dir=${shellQuote(remoteDir)}`,
|
|
'manifest="$remote_dir/manifest.tsv"',
|
|
'pid_file="$remote_dir/pid"',
|
|
'if [ -f "$manifest" ]; then cat "$manifest"; exit 0; fi',
|
|
'if [ -f "$pid_file" ] && kill -0 "$(cat "$pid_file")" >/dev/null 2>&1; then',
|
|
' printf "status\\trunning\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$(cat "$pid_file")"',
|
|
" exit 1",
|
|
"fi",
|
|
'if [ -f "$pid_file" ]; then',
|
|
' printf "status\\texited-without-manifest\\nremote_dir\\t%s\\npid\\t%s\\n" "$remote_dir" "$(cat "$pid_file")"',
|
|
' if [ -f "$remote_dir/submit.err" ]; then tail -c 1000 "$remote_dir/submit.err" >&2; fi',
|
|
' if [ -f "$remote_dir/stderr.log" ]; then tail -c 1000 "$remote_dir/stderr.log" >&2; fi',
|
|
" exit 2",
|
|
"fi",
|
|
'printf "status\\tpending\\nremote_dir\\t%s\\n" "$remote_dir"',
|
|
'if [ -f "$remote_dir/submit.err" ]; then tail -c 1000 "$remote_dir/submit.err" >&2; fi',
|
|
'if [ -f "$remote_dir/stderr.log" ]; then tail -c 1000 "$remote_dir/stderr.log" >&2; fi',
|
|
"exit 1",
|
|
].join("\n");
|
|
}
|
|
|
|
function remoteArtifactRunnerScript(remoteDir: string, runId: string): string {
|
|
return [
|
|
"set -eu",
|
|
`remote_dir=${shellQuote(remoteDir)}`,
|
|
`run_id=${shellQuote(runId)}`,
|
|
'if [ "$#" -lt 1 ]; then printf "missing user script path\\n" >&2; exit 2; fi',
|
|
'user_script="$1"',
|
|
'stdout_file="$remote_dir/stdout.log"',
|
|
'stderr_file="$remote_dir/stderr.log"',
|
|
'artifacts_file="$remote_dir/artifacts.tsv"',
|
|
'manifest_file="$remote_dir/manifest.tsv"',
|
|
"sha256_file() {",
|
|
" if command -v sha256sum >/dev/null 2>&1; then sha256sum -- \"$1\" | awk '{print $1}'; return; fi",
|
|
" if command -v shasum >/dev/null 2>&1; then shasum -a 256 -- \"$1\" | awk '{print $1}'; return; fi",
|
|
" if command -v openssl >/dev/null 2>&1; then openssl dgst -sha256 -- \"$1\" | awk '{print $NF}'; return; fi",
|
|
" printf 'missing sha256 tool\\n' >&2; return 127",
|
|
"}",
|
|
'export UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR="$remote_dir"',
|
|
"set +e",
|
|
'sh "$user_script" >"$stdout_file" 2>"$stderr_file"',
|
|
"user_rc=$?",
|
|
': > "$artifacts_file"',
|
|
'find "$remote_dir" -type f | while IFS= read -r file; do',
|
|
' case "$file" in "$user_script"|"$stdout_file"|"$stderr_file"|"$artifacts_file"|"$manifest_file"|"$remote_dir/runner.sh"|"$remote_dir/submit.out"|"$remote_dir/submit.err"|"$remote_dir/pid") continue ;; esac',
|
|
' case "$file" in *.png|*.jpg|*.jpeg|*.webp)',
|
|
' bytes=$(wc -c < "$file" | tr -d "[:space:]")',
|
|
' digest=$(sha256_file "$file")',
|
|
' printf "artifact\\t%s\\t%s\\t%s\\n" "$bytes" "$digest" "$file" >> "$artifacts_file"',
|
|
" ;;",
|
|
" esac",
|
|
"done",
|
|
"b64_tail() { if [ -f \"$1\" ]; then tail -c 4000 \"$1\" | base64 | tr -d '\\n'; fi; }",
|
|
"{",
|
|
`printf '%s\\n' ${shellQuote(manifestBegin)}`,
|
|
"printf 'run_id\\t%s\\n' \"$run_id\"",
|
|
"printf 'exit_code\\t%s\\n' \"$user_rc\"",
|
|
"printf 'remote_dir\\t%s\\n' \"$remote_dir\"",
|
|
"printf 'stdout_path\\t%s\\n' \"$stdout_file\"",
|
|
"printf 'stderr_path\\t%s\\n' \"$stderr_file\"",
|
|
"printf 'stdout_tail_b64\\t%s\\n' \"$(b64_tail \"$stdout_file\")\"",
|
|
"printf 'stderr_tail_b64\\t%s\\n' \"$(b64_tail \"$stderr_file\")\"",
|
|
'cat "$artifacts_file"',
|
|
`printf '%s\\n' ${shellQuote(manifestEnd)}`,
|
|
"} > \"$manifest_file\"",
|
|
'exit "$user_rc"',
|
|
].join("\n");
|
|
}
|
|
|
|
function parseRemoteManifest(result: CommandResult, fallbackRemoteDir: string, fallbackRunId: string): RemoteWebProbeArtifactManifest {
|
|
const start = result.stdout.indexOf(manifestBegin);
|
|
const end = result.stdout.indexOf(manifestEnd, start < 0 ? 0 : start);
|
|
if (start < 0 || end < 0) {
|
|
throw new Error(`web-probe remote artifact did not emit a manifest; exitCode=${result.exitCode}; stdoutTail=${JSON.stringify(result.stdout.slice(-1000))}; stderrTail=${JSON.stringify(result.stderr.slice(-1000))}`);
|
|
}
|
|
const body = result.stdout.slice(start + manifestBegin.length, end).trim();
|
|
const manifest: RemoteWebProbeArtifactManifest = {
|
|
runId: fallbackRunId,
|
|
exitCode: result.exitCode ?? 1,
|
|
remoteDir: fallbackRemoteDir,
|
|
stdoutPath: join(fallbackRemoteDir, "stdout.log"),
|
|
stderrPath: join(fallbackRemoteDir, "stderr.log"),
|
|
stdoutTail: "",
|
|
stderrTail: "",
|
|
artifacts: [],
|
|
};
|
|
for (const rawLine of body.split(/\r?\n/u)) {
|
|
const line = rawLine.trimEnd();
|
|
if (line.length === 0) continue;
|
|
const parts = line.split("\t");
|
|
const key = parts[0] ?? "";
|
|
if (key === "artifact") {
|
|
const bytes = Number(parts[1] ?? "");
|
|
const sha256 = parts[2] ?? "";
|
|
const remotePath = parts.slice(3).join("\t");
|
|
if (remotePath.length > 0) {
|
|
manifest.artifacts.push({
|
|
remotePath,
|
|
bytes: Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : null,
|
|
sha256: /^[0-9a-f]{64}$/u.test(sha256) ? sha256 : null,
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
const value = parts.slice(1).join("\t");
|
|
if (key === "run_id" && value.length > 0) manifest.runId = value;
|
|
else if (key === "exit_code") {
|
|
const exitCode = Number(value);
|
|
manifest.exitCode = Number.isInteger(exitCode) ? exitCode : result.exitCode ?? 1;
|
|
} else if (key === "remote_dir" && value.length > 0) manifest.remoteDir = value;
|
|
else if (key === "stdout_path" && value.length > 0) manifest.stdoutPath = value;
|
|
else if (key === "stderr_path" && value.length > 0) manifest.stderrPath = value;
|
|
else if (key === "stdout_tail_b64") manifest.stdoutTail = decodeBase64(value);
|
|
else if (key === "stderr_tail_b64") manifest.stderrTail = decodeBase64(value);
|
|
}
|
|
return manifest;
|
|
}
|
|
|
|
function syntheticCommandResult(command: string[], exitCode: number | null, stdout: string, results: readonly CommandResult[]): CommandResult {
|
|
return {
|
|
command,
|
|
cwd: repoRoot,
|
|
exitCode,
|
|
stdout,
|
|
stderr: results.map((item) => item.stderr).filter((item) => item.length > 0).join("\n").slice(-8000),
|
|
signal: null,
|
|
timedOut: results.some((item) => item.timedOut),
|
|
};
|
|
}
|
|
|
|
function writeBase64FileShell(pathExpression: string, content: string, marker: string): string {
|
|
return [
|
|
`cat > ${pathExpression}.b64 <<'${marker}'`,
|
|
wrapBase64(Buffer.from(content, "utf8").toString("base64")),
|
|
marker,
|
|
`base64 -d < ${pathExpression}.b64 > ${pathExpression}`,
|
|
`rm -f -- ${pathExpression}.b64`,
|
|
].join("\n");
|
|
}
|
|
|
|
function wrapBase64(value: string): string {
|
|
return value.replace(/.{1,76}/gu, "$&\n").trimEnd();
|
|
}
|
|
|
|
function parseJsonObject(text: string): Record<string, unknown> {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return {};
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
function numberOrNull(value: unknown): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function decodeBase64(value: string): string {
|
|
if (value.length === 0) return "";
|
|
try {
|
|
return Buffer.from(value, "base64").toString("utf8");
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function isRecoverableSubmitTimeout(submit: CommandResult): boolean {
|
|
if (submit.exitCode === 124 || submit.timedOut) return true;
|
|
const combined = `${submit.stdout}\n${submit.stderr}`;
|
|
return /(?:timed?\s*out|timeout|60s|exitCode=124|signal\s+TERM|signal\s+KILL)/iu.test(combined);
|
|
}
|
|
|
|
function sleepSync(ms: number): void {
|
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, ms));
|
|
}
|
|
|
|
function safePathSegment(value: string): string {
|
|
const cleaned = value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
return cleaned.length > 0 ? cleaned.slice(0, 120) : "artifact";
|
|
}
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/g, "'\\''")}'`;
|
|
}
|
|
|
|
function transPath(): string {
|
|
return join(repoRoot, "scripts", "trans");
|
|
}
|