diff --git a/scripts/src/platform-infra-nginx.ts b/scripts/src/platform-infra-nginx.ts index 146a522f..1a642345 100644 --- a/scripts/src/platform-infra-nginx.ts +++ b/scripts/src/platform-infra-nginx.ts @@ -114,6 +114,13 @@ interface SecretMaterial { fingerprint: string; } +interface RemoteJobOutcome { + exitCode: number; + stdout: string; + stderr: string; + polls: number; +} + export function nginxHelp(): Record { return { command: "platform-infra nginx plan|apply|status", @@ -327,35 +334,65 @@ async function apply(config: UniDeskConfig, options: OpsApplyOptions): Promise>; + let outcome: RemoteJobOutcome; try { - remote = await capture( + const remoteScript = applyRemoteScript(nginx, target, compose, { + image: `${distribution.publicBaseUrl}/${fileName(distributedImage)}`, + composePlugin: `${distribution.publicBaseUrl}/${fileName(distributedPlugin)}`, + }); + const start = await capture( config, target.route, ["sh"], - applyRemoteScript(nginx, target, compose, { - image: `${distribution.publicBaseUrl}/${fileName(distributedImage)}`, - composePlugin: `${distribution.publicBaseUrl}/${fileName(distributedPlugin)}`, - }), + startRemoteJobScript(target, remoteScript), ); + if (start.exitCode !== 0) return failedApply(id, target, "remote-job-start", start, secret); + outcome = await waitRemoteJob(config, target); } finally { await capture(config, distribution.route, ["sh"], stopArtifactSourceScript(distribution)); } - const parsed = parseJsonOutput(remote.stdout); + const parsed = parseJsonOutput(outcome.stdout); return { - ok: remote.exitCode === 0 && parsed?.ok === true, + ok: outcome.exitCode === 0 && parsed?.ok === true, action: "platform-infra-nginx-apply", mode: "confirmed", mutation: true, target: targetSummary(id, target), uploads, apiKey: secretSummary(target, secret), - validation: parsed ?? compactCapture(remote, { full: true }), - remote: compactCapture(remote, { full: options.full || remote.exitCode !== 0 }), + validation: parsed ?? remoteJobSummary(outcome, true), + remote: remoteJobSummary(outcome, options.full || outcome.exitCode !== 0), next: { status: `bun scripts/cli.ts platform-infra nginx status --target ${id}` }, }; } +async function waitRemoteJob(config: UniDeskConfig, target: NginxTarget): Promise { + const maximumPolls = 4000; + for (let poll = 1; poll <= maximumPolls; poll += 1) { + const remote = await capture(config, target.route, ["sh"], pollRemoteJobScript(target)); + if (remote.exitCode !== 0) { + return { exitCode: remote.exitCode, stdout: remote.stdout, stderr: remote.stderr, polls: poll }; + } + const result = parseJsonOutput(remote.stdout); + if (result?.done === true) { + return { + exitCode: Number(result.exitCode), + stdout: decodeBase64(String(result.stdoutBase64 ?? "")), + stderr: decodeBase64(String(result.stderrBase64 ?? "")), + polls: poll, + }; + } + await Bun.sleep(2000); + } + const stopped = await capture(config, target.route, ["sh"], stopRemoteJobScript(target)); + return { + exitCode: 124, + stdout: stopped.stdout, + stderr: `remote nginx apply job exceeded ${maximumPolls * 2} seconds`, + polls: maximumPolls, + }; +} + async function status(config: UniDeskConfig, options: OpsCommonOptions): Promise> { const nginx = readConfig(); const { id, target } = resolveTarget(nginx, options.targetId); @@ -415,6 +452,60 @@ rm -rf ${shQuote(distribution.workDir)} `; } +function remoteJobDir(target: NginxTarget): string { + return `${target.runtime.workDir}/apply-job`; +} + +function startRemoteJobScript(target: NginxTarget, script: string): string { + const jobDir = remoteJobDir(target); + const runner = `#!/bin/sh +set +e +sh ${jobDir}/apply.sh >${jobDir}/stdout.log 2>${jobDir}/stderr.log +code=$? +printf '%s\\n' "$code" >${jobDir}/exit +`; + return `set -eu +job_dir=${shQuote(jobDir)} +if [ -f "$job_dir/pid" ] && kill -0 "$(cat "$job_dir/pid")" 2>/dev/null; then + printf '%s\n' 'nginx apply job is already running' >&2 + exit 1 +fi +rm -rf "$job_dir" +install -d -m 0700 "$job_dir" +printf '%s' ${shQuote(Buffer.from(script).toString("base64"))} | base64 -d > "$job_dir/apply.sh" +printf '%s' ${shQuote(Buffer.from(runner).toString("base64"))} | base64 -d > "$job_dir/runner.sh" +chmod 0700 "$job_dir/apply.sh" "$job_dir/runner.sh" +nohup sh "$job_dir/runner.sh" >/dev/null 2>&1 & +printf '%s\n' "$!" > "$job_dir/pid" +printf '{"ok":true,"started":true}\n' +`; +} + +function pollRemoteJobScript(target: NginxTarget): string { + const jobDir = shQuote(remoteJobDir(target)); + return `set -eu +job_dir=${jobDir} +if [ ! -f "$job_dir/exit" ]; then + printf '{"ok":true,"done":false}\n' + exit 0 +fi +code=$(cat "$job_dir/exit") +stdout=$(tail -c 20000 "$job_dir/stdout.log" 2>/dev/null | base64 -w0) +stderr=$(tail -c 20000 "$job_dir/stderr.log" 2>/dev/null | base64 -w0) +printf '{"ok":true,"done":true,"exitCode":%s,"stdoutBase64":"%s","stderrBase64":"%s"}\n' \ + "$code" "$stdout" "$stderr" +`; +} + +function stopRemoteJobScript(target: NginxTarget): string { + const jobDir = shQuote(remoteJobDir(target)); + return `set -eu +job_dir=${jobDir} +[ ! -f "$job_dir/pid" ] || kill "$(cat "$job_dir/pid")" 2>/dev/null || true +printf '{"ok":true,"stopped":true}\n' +`; +} + function applyRemoteScript( config: NginxConfig, target: NginxTarget, @@ -471,9 +562,10 @@ trap cleanup_backends EXIT gzip -dc "$env_file.gz" > "$env_file" chmod 0600 "$env_file" rm -f "$env_file.gz" -curl -fsSL --retry 2 --connect-timeout 5 --max-time 120 \ +rm -f "$image_artifact" "$plugin" +curl -fsSL --retry 2 --connect-timeout 5 --max-time 3600 \ ${shQuote(artifactUrls.image)} -o "$image_artifact" -curl -fsSL --retry 2 --connect-timeout 5 --max-time 120 \ +curl -fsSL --retry 2 --connect-timeout 5 --max-time 3600 \ ${shQuote(artifactUrls.composePlugin)} -o "$plugin" [ "$(sha256sum "$image_artifact" | awk '{print $1}')" = ${shQuote(config.artifacts.image.sha256)} ] [ "$(sha256sum "$plugin" | awk '{print $1}')" = ${shQuote(config.artifacts.composePlugin.sha256)} ] @@ -701,6 +793,21 @@ function uploadSecret(route: string, sourcePath: string, remotePath: string): Re } } +function decodeBase64(value: string): string { + return Buffer.from(value, "base64").toString("utf8"); +} + +function remoteJobSummary(outcome: RemoteJobOutcome, full: boolean): Record { + return { + exitCode: outcome.exitCode, + polls: outcome.polls, + stdoutBytes: Buffer.byteLength(outcome.stdout), + stderrBytes: Buffer.byteLength(outcome.stderr), + stdoutTail: full ? outcome.stdout.slice(-8000) : "", + stderrTail: full ? outcome.stderr.slice(-4000) : "", + }; +} + function targetSummary(id: string, target: NginxTarget): Record { return { id,