fix(platform-infra): run PK01 nginx apply as remote job
This commit is contained in:
@@ -114,6 +114,13 @@ interface SecretMaterial {
|
||||
fingerprint: string;
|
||||
}
|
||||
|
||||
interface RemoteJobOutcome {
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
polls: number;
|
||||
}
|
||||
|
||||
export function nginxHelp(): Record<string, unknown> {
|
||||
return {
|
||||
command: "platform-infra nginx plan|apply|status",
|
||||
@@ -327,35 +334,65 @@ async function apply(config: UniDeskConfig, options: OpsApplyOptions): Promise<R
|
||||
await capture(config, distribution.route, ["sh"], stopArtifactSourceScript(distribution));
|
||||
return failedApply(id, target, "artifact-source", source, secret);
|
||||
}
|
||||
let remote: Awaited<ReturnType<typeof capture>>;
|
||||
let outcome: RemoteJobOutcome;
|
||||
try {
|
||||
remote = await capture(
|
||||
const remoteScript = applyRemoteScript(nginx, target, compose, {
|
||||
image: `${distribution.publicBaseUrl}/${fileName(distributedImage)}`,
|
||||
composePlugin: `${distribution.publicBaseUrl}/${fileName(distributedPlugin)}`,
|
||||
});
|
||||
const start = await capture(
|
||||
config,
|
||||
target.route,
|
||||
["sh"],
|
||||
applyRemoteScript(nginx, target, compose, {
|
||||
image: `${distribution.publicBaseUrl}/${fileName(distributedImage)}`,
|
||||
composePlugin: `${distribution.publicBaseUrl}/${fileName(distributedPlugin)}`,
|
||||
}),
|
||||
startRemoteJobScript(target, remoteScript),
|
||||
);
|
||||
if (start.exitCode !== 0) return failedApply(id, target, "remote-job-start", start, secret);
|
||||
outcome = await waitRemoteJob(config, target);
|
||||
} finally {
|
||||
await capture(config, distribution.route, ["sh"], stopArtifactSourceScript(distribution));
|
||||
}
|
||||
const parsed = parseJsonOutput(remote.stdout);
|
||||
const parsed = parseJsonOutput(outcome.stdout);
|
||||
return {
|
||||
ok: remote.exitCode === 0 && parsed?.ok === true,
|
||||
ok: outcome.exitCode === 0 && parsed?.ok === true,
|
||||
action: "platform-infra-nginx-apply",
|
||||
mode: "confirmed",
|
||||
mutation: true,
|
||||
target: targetSummary(id, target),
|
||||
uploads,
|
||||
apiKey: secretSummary(target, secret),
|
||||
validation: parsed ?? compactCapture(remote, { full: true }),
|
||||
remote: compactCapture(remote, { full: options.full || remote.exitCode !== 0 }),
|
||||
validation: parsed ?? remoteJobSummary(outcome, true),
|
||||
remote: remoteJobSummary(outcome, options.full || outcome.exitCode !== 0),
|
||||
next: { status: `bun scripts/cli.ts platform-infra nginx status --target ${id}` },
|
||||
};
|
||||
}
|
||||
|
||||
async function waitRemoteJob(config: UniDeskConfig, target: NginxTarget): Promise<RemoteJobOutcome> {
|
||||
const maximumPolls = 4000;
|
||||
for (let poll = 1; poll <= maximumPolls; poll += 1) {
|
||||
const remote = await capture(config, target.route, ["sh"], pollRemoteJobScript(target));
|
||||
if (remote.exitCode !== 0) {
|
||||
return { exitCode: remote.exitCode, stdout: remote.stdout, stderr: remote.stderr, polls: poll };
|
||||
}
|
||||
const result = parseJsonOutput(remote.stdout);
|
||||
if (result?.done === true) {
|
||||
return {
|
||||
exitCode: Number(result.exitCode),
|
||||
stdout: decodeBase64(String(result.stdoutBase64 ?? "")),
|
||||
stderr: decodeBase64(String(result.stderrBase64 ?? "")),
|
||||
polls: poll,
|
||||
};
|
||||
}
|
||||
await Bun.sleep(2000);
|
||||
}
|
||||
const stopped = await capture(config, target.route, ["sh"], stopRemoteJobScript(target));
|
||||
return {
|
||||
exitCode: 124,
|
||||
stdout: stopped.stdout,
|
||||
stderr: `remote nginx apply job exceeded ${maximumPolls * 2} seconds`,
|
||||
polls: maximumPolls,
|
||||
};
|
||||
}
|
||||
|
||||
async function status(config: UniDeskConfig, options: OpsCommonOptions): Promise<Record<string, unknown>> {
|
||||
const nginx = readConfig();
|
||||
const { id, target } = resolveTarget(nginx, options.targetId);
|
||||
@@ -415,6 +452,60 @@ rm -rf ${shQuote(distribution.workDir)}
|
||||
`;
|
||||
}
|
||||
|
||||
function remoteJobDir(target: NginxTarget): string {
|
||||
return `${target.runtime.workDir}/apply-job`;
|
||||
}
|
||||
|
||||
function startRemoteJobScript(target: NginxTarget, script: string): string {
|
||||
const jobDir = remoteJobDir(target);
|
||||
const runner = `#!/bin/sh
|
||||
set +e
|
||||
sh ${jobDir}/apply.sh >${jobDir}/stdout.log 2>${jobDir}/stderr.log
|
||||
code=$?
|
||||
printf '%s\\n' "$code" >${jobDir}/exit
|
||||
`;
|
||||
return `set -eu
|
||||
job_dir=${shQuote(jobDir)}
|
||||
if [ -f "$job_dir/pid" ] && kill -0 "$(cat "$job_dir/pid")" 2>/dev/null; then
|
||||
printf '%s\n' 'nginx apply job is already running' >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -rf "$job_dir"
|
||||
install -d -m 0700 "$job_dir"
|
||||
printf '%s' ${shQuote(Buffer.from(script).toString("base64"))} | base64 -d > "$job_dir/apply.sh"
|
||||
printf '%s' ${shQuote(Buffer.from(runner).toString("base64"))} | base64 -d > "$job_dir/runner.sh"
|
||||
chmod 0700 "$job_dir/apply.sh" "$job_dir/runner.sh"
|
||||
nohup sh "$job_dir/runner.sh" >/dev/null 2>&1 &
|
||||
printf '%s\n' "$!" > "$job_dir/pid"
|
||||
printf '{"ok":true,"started":true}\n'
|
||||
`;
|
||||
}
|
||||
|
||||
function pollRemoteJobScript(target: NginxTarget): string {
|
||||
const jobDir = shQuote(remoteJobDir(target));
|
||||
return `set -eu
|
||||
job_dir=${jobDir}
|
||||
if [ ! -f "$job_dir/exit" ]; then
|
||||
printf '{"ok":true,"done":false}\n'
|
||||
exit 0
|
||||
fi
|
||||
code=$(cat "$job_dir/exit")
|
||||
stdout=$(tail -c 20000 "$job_dir/stdout.log" 2>/dev/null | base64 -w0)
|
||||
stderr=$(tail -c 20000 "$job_dir/stderr.log" 2>/dev/null | base64 -w0)
|
||||
printf '{"ok":true,"done":true,"exitCode":%s,"stdoutBase64":"%s","stderrBase64":"%s"}\n' \
|
||||
"$code" "$stdout" "$stderr"
|
||||
`;
|
||||
}
|
||||
|
||||
function stopRemoteJobScript(target: NginxTarget): string {
|
||||
const jobDir = shQuote(remoteJobDir(target));
|
||||
return `set -eu
|
||||
job_dir=${jobDir}
|
||||
[ ! -f "$job_dir/pid" ] || kill "$(cat "$job_dir/pid")" 2>/dev/null || true
|
||||
printf '{"ok":true,"stopped":true}\n'
|
||||
`;
|
||||
}
|
||||
|
||||
function applyRemoteScript(
|
||||
config: NginxConfig,
|
||||
target: NginxTarget,
|
||||
@@ -471,9 +562,10 @@ trap cleanup_backends EXIT
|
||||
gzip -dc "$env_file.gz" > "$env_file"
|
||||
chmod 0600 "$env_file"
|
||||
rm -f "$env_file.gz"
|
||||
curl -fsSL --retry 2 --connect-timeout 5 --max-time 120 \
|
||||
rm -f "$image_artifact" "$plugin"
|
||||
curl -fsSL --retry 2 --connect-timeout 5 --max-time 3600 \
|
||||
${shQuote(artifactUrls.image)} -o "$image_artifact"
|
||||
curl -fsSL --retry 2 --connect-timeout 5 --max-time 120 \
|
||||
curl -fsSL --retry 2 --connect-timeout 5 --max-time 3600 \
|
||||
${shQuote(artifactUrls.composePlugin)} -o "$plugin"
|
||||
[ "$(sha256sum "$image_artifact" | awk '{print $1}')" = ${shQuote(config.artifacts.image.sha256)} ]
|
||||
[ "$(sha256sum "$plugin" | awk '{print $1}')" = ${shQuote(config.artifacts.composePlugin.sha256)} ]
|
||||
@@ -701,6 +793,21 @@ function uploadSecret(route: string, sourcePath: string, remotePath: string): Re
|
||||
}
|
||||
}
|
||||
|
||||
function decodeBase64(value: string): string {
|
||||
return Buffer.from(value, "base64").toString("utf8");
|
||||
}
|
||||
|
||||
function remoteJobSummary(outcome: RemoteJobOutcome, full: boolean): Record<string, unknown> {
|
||||
return {
|
||||
exitCode: outcome.exitCode,
|
||||
polls: outcome.polls,
|
||||
stdoutBytes: Buffer.byteLength(outcome.stdout),
|
||||
stderrBytes: Buffer.byteLength(outcome.stderr),
|
||||
stdoutTail: full ? outcome.stdout.slice(-8000) : "",
|
||||
stderrTail: full ? outcome.stderr.slice(-4000) : "",
|
||||
};
|
||||
}
|
||||
|
||||
function targetSummary(id: string, target: NginxTarget): Record<string, unknown> {
|
||||
return {
|
||||
id,
|
||||
|
||||
Reference in New Issue
Block a user