Codex
930 lines
34 KiB
TypeScript
930 lines
34 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { spawn } from "node:child_process";
|
|
import {
|
|
chmodSync,
|
|
existsSync,
|
|
mkdtempSync,
|
|
mkdirSync,
|
|
readFileSync,
|
|
rmSync,
|
|
statSync,
|
|
symlinkSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { isAbsolute, join } from "node:path";
|
|
import { gzipSync } from "node:zlib";
|
|
import type { UniDeskConfig } from "./config";
|
|
import { repoRoot, rootPath } from "./config";
|
|
import { startJob } from "./jobs";
|
|
import {
|
|
capture,
|
|
compactCapture,
|
|
fingerprintValues,
|
|
parseEnvFile,
|
|
parseJsonOutput,
|
|
parseOpsApplyOptions,
|
|
parseOpsCommonOptions,
|
|
readYamlRecord,
|
|
shQuote,
|
|
type OpsApplyOptions,
|
|
type OpsCommonOptions,
|
|
} from "./platform-infra-ops-library";
|
|
|
|
const configPath = rootPath("config", "platform-infra", "nginx.yaml");
|
|
const configLabel = "config/platform-infra/nginx.yaml";
|
|
|
|
interface Artifact {
|
|
localPath: string;
|
|
remotePath: string;
|
|
sha256: string;
|
|
}
|
|
|
|
interface ImageArtifact extends Artifact {
|
|
buildTarget: string;
|
|
image: string;
|
|
imageId: string;
|
|
bytes: number;
|
|
}
|
|
|
|
interface ArtifactDistribution {
|
|
mode: "local-http";
|
|
workDir: string;
|
|
bindHost: string;
|
|
port: number;
|
|
publicBaseUrl: string;
|
|
}
|
|
|
|
interface Route {
|
|
listenPort: number;
|
|
upstreams: Array<{ host: string; port: number }>;
|
|
}
|
|
|
|
interface Backend {
|
|
id: string;
|
|
bindHost: string;
|
|
port: number;
|
|
unit: string;
|
|
}
|
|
|
|
interface NginxTarget {
|
|
route: string;
|
|
runtime: {
|
|
workDir: string;
|
|
composePath: string;
|
|
envPath: string;
|
|
projectName: string;
|
|
containerName: string;
|
|
networkMode: "host";
|
|
restart: string;
|
|
resources: { cpus: string; memory: string; pidsLimit: number };
|
|
logDir: string;
|
|
routes: Route[];
|
|
environment: Record<string, string>;
|
|
};
|
|
apiKey: {
|
|
sourceRef: string;
|
|
sourceKey: string;
|
|
targetKey: string;
|
|
requestHeader: string;
|
|
};
|
|
validation: {
|
|
requestCount: number;
|
|
cleanupAfterSuccess: boolean;
|
|
backendScriptPath: string;
|
|
backends: Backend[];
|
|
};
|
|
}
|
|
|
|
interface NginxConfig {
|
|
version: number;
|
|
kind: "platform-infra-nginx";
|
|
metadata: { name: string };
|
|
defaults: { target: string };
|
|
artifacts: {
|
|
image: ImageArtifact;
|
|
composePlugin: Artifact;
|
|
distribution: ArtifactDistribution;
|
|
};
|
|
targets: Record<string, NginxTarget>;
|
|
}
|
|
|
|
interface SecretMaterial {
|
|
sourcePath: string;
|
|
value: string;
|
|
fingerprint: string;
|
|
}
|
|
|
|
interface RemoteJobOutcome {
|
|
exitCode: number;
|
|
stdout: string;
|
|
stderr: string;
|
|
polls: number;
|
|
}
|
|
|
|
export function nginxHelp(): Record<string, unknown> {
|
|
return {
|
|
command: "platform-infra nginx plan|apply|status",
|
|
output: "json",
|
|
usage: [
|
|
"bun scripts/cli.ts platform-infra nginx plan [--target PK01]",
|
|
"bun scripts/cli.ts platform-infra nginx apply [--target PK01] --dry-run",
|
|
"bun scripts/cli.ts platform-infra nginx apply [--target PK01] --confirm",
|
|
"bun scripts/cli.ts platform-infra nginx status [--target PK01] [--full|--raw]",
|
|
],
|
|
configTruth: configLabel,
|
|
secretPolicy: "API key values are never printed; output contains presence and fingerprint only.",
|
|
};
|
|
}
|
|
|
|
export async function runPlatformInfraNginxCommand(
|
|
config: UniDeskConfig,
|
|
args: string[],
|
|
): Promise<Record<string, unknown>> {
|
|
const [action = "plan"] = args;
|
|
if (action === "plan") return plan(parseOpsCommonOptions(args.slice(1)));
|
|
if (action === "apply") return await apply(config, parseOpsApplyOptions(args.slice(1)));
|
|
if (action === "status") return await status(config, parseOpsCommonOptions(args.slice(1)));
|
|
return { ok: false, error: "unsupported-platform-infra-nginx-command", args, help: nginxHelp() };
|
|
}
|
|
|
|
function readConfig(): NginxConfig {
|
|
const config = readYamlRecord<NginxConfig>(configPath, "platform-infra-nginx");
|
|
assert(config.version === 1, `${configLabel}.version must be 1`);
|
|
assert(config.metadata.name === "nginx", `${configLabel}.metadata.name must be nginx`);
|
|
assert(Object.keys(config.targets).length > 0, `${configLabel}.targets must not be empty`);
|
|
validateArtifact(config.artifacts.image, "artifacts.image");
|
|
validateArtifact(config.artifacts.composePlugin, "artifacts.composePlugin");
|
|
validateDistribution(config.artifacts.distribution);
|
|
assert(Number.isInteger(config.artifacts.image.bytes), `${configLabel}.artifacts.image.bytes must be an integer`);
|
|
assert(config.artifacts.image.bytes > 0, `${configLabel}.artifacts.image.bytes must be positive`);
|
|
assert(/^sha256:[a-f0-9]{64}$/u.test(config.artifacts.image.imageId), "artifacts.image.imageId is invalid");
|
|
for (const [targetId, target] of Object.entries(config.targets)) validateTarget(targetId, target);
|
|
return config;
|
|
}
|
|
|
|
function validateDistribution(distribution: ArtifactDistribution): void {
|
|
assert(distribution.mode === "local-http", "artifacts.distribution.mode must be local-http");
|
|
absolutePath(distribution.workDir, "artifacts.distribution.workDir");
|
|
assert(distribution.bindHost === "0.0.0.0", "artifacts.distribution.bindHost must be 0.0.0.0");
|
|
port(distribution.port, "artifacts.distribution.port");
|
|
const expectedSuffix = `:${distribution.port}`;
|
|
assert(
|
|
distribution.publicBaseUrl.startsWith("http://") && distribution.publicBaseUrl.endsWith(expectedSuffix),
|
|
"artifacts.distribution.publicBaseUrl must be HTTP and end with the declared port",
|
|
);
|
|
}
|
|
|
|
function validateArtifact(artifact: Artifact, path: string): void {
|
|
absolutePath(artifact.localPath, `${path}.localPath`);
|
|
absolutePath(artifact.remotePath, `${path}.remotePath`);
|
|
assert(/^[a-f0-9]{64}$/u.test(artifact.sha256), `${configLabel}.${path}.sha256 is invalid`);
|
|
}
|
|
|
|
function validateTarget(id: string, target: NginxTarget): void {
|
|
simpleName(id, `targets.${id}`);
|
|
simpleName(target.route, `targets.${id}.route`);
|
|
const runtime = target.runtime;
|
|
for (const [key, value] of [
|
|
["workDir", runtime.workDir],
|
|
["composePath", runtime.composePath],
|
|
["envPath", runtime.envPath],
|
|
["logDir", runtime.logDir],
|
|
["backendScriptPath", target.validation.backendScriptPath],
|
|
]) absolutePath(value, `targets.${id}.${key}`);
|
|
assert(runtime.networkMode === "host", `targets.${id}.runtime.networkMode must be host`);
|
|
simpleName(runtime.projectName, `targets.${id}.runtime.projectName`);
|
|
simpleName(runtime.containerName, `targets.${id}.runtime.containerName`);
|
|
assert(runtime.routes.length > 0, `targets.${id}.runtime.routes must not be empty`);
|
|
const listenPorts = new Set<number>();
|
|
for (const route of runtime.routes) {
|
|
port(route.listenPort, `targets.${id}.runtime.routes.listenPort`);
|
|
assert(!listenPorts.has(route.listenPort), `targets.${id}.runtime.routes contains a duplicate listenPort`);
|
|
listenPorts.add(route.listenPort);
|
|
assert(route.upstreams.length > 0, `targets.${id}.runtime.routes.upstreams must not be empty`);
|
|
for (const upstream of route.upstreams) {
|
|
assert(/^[A-Za-z0-9_.-]+$/u.test(upstream.host), `targets.${id} upstream host is invalid`);
|
|
port(upstream.port, `targets.${id} upstream port`);
|
|
}
|
|
}
|
|
assert(target.validation.requestCount === 4, `targets.${id}.validation.requestCount must be 4`);
|
|
assert(target.validation.cleanupAfterSuccess, `targets.${id}.validation.cleanupAfterSuccess must be true`);
|
|
assert(target.validation.backends.length === 2, `targets.${id}.validation.backends must contain two backends`);
|
|
for (const backend of target.validation.backends) {
|
|
simpleName(backend.id, `targets.${id}.validation.backends.id`);
|
|
assert(backend.bindHost === "127.0.0.1", `targets.${id} validation backend must bind to 127.0.0.1`);
|
|
port(backend.port, `targets.${id} validation backend port`);
|
|
assert(/^[A-Za-z0-9_.@-]+\.service$/u.test(backend.unit), `targets.${id} backend unit is invalid`);
|
|
}
|
|
for (const key of Object.keys(runtime.environment)) {
|
|
assert(/^(NGINX|LOGGER)_[A-Z0-9_]+$/u.test(key), `targets.${id}.runtime.environment key ${key} is invalid`);
|
|
}
|
|
assert(target.apiKey.sourceKey === "LOGGER_API_KEY", `targets.${id}.apiKey.sourceKey must be LOGGER_API_KEY`);
|
|
assert(target.apiKey.targetKey === "LOGGER_API_KEY", `targets.${id}.apiKey.targetKey must be LOGGER_API_KEY`);
|
|
assert(target.apiKey.requestHeader === "X-API-Key", `targets.${id}.apiKey.requestHeader must be X-API-Key`);
|
|
}
|
|
|
|
function resolveTarget(config: NginxConfig, requested: string | null): { id: string; target: NginxTarget } {
|
|
const id = requested ?? config.defaults.target;
|
|
const target = config.targets[id];
|
|
if (target === undefined) throw new Error(`unknown nginx target: ${id}`);
|
|
return { id, target };
|
|
}
|
|
|
|
function plan(options: OpsCommonOptions): Record<string, unknown> {
|
|
const config = readConfig();
|
|
const { id, target } = resolveTarget(config, options.targetId);
|
|
const secret = readSecret(target);
|
|
const image = localArtifactSummary(config.artifacts.image);
|
|
const composePlugin = localArtifactSummary(config.artifacts.composePlugin);
|
|
const compose = renderCompose(config, target);
|
|
return {
|
|
ok: image.valid && composePlugin.valid,
|
|
action: "platform-infra-nginx-plan",
|
|
mutation: false,
|
|
target: targetSummary(id, target),
|
|
artifacts: { image, composePlugin },
|
|
compose: {
|
|
sha256: sha256(compose),
|
|
bytes: Buffer.byteLength(compose),
|
|
routes: renderRoutes(target.runtime.routes),
|
|
},
|
|
apiKey: secretSummary(target, secret),
|
|
validation: validationSummary(target),
|
|
next: {
|
|
dryRun: `bun scripts/cli.ts platform-infra nginx apply --target ${id} --dry-run`,
|
|
apply: `bun scripts/cli.ts platform-infra nginx apply --target ${id} --confirm`,
|
|
status: `bun scripts/cli.ts platform-infra nginx status --target ${id}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function apply(config: UniDeskConfig, options: OpsApplyOptions): Promise<Record<string, unknown>> {
|
|
const nginx = readConfig();
|
|
const { id, target } = resolveTarget(nginx, options.targetId);
|
|
const secret = readSecret(target);
|
|
const image = localArtifactSummary(nginx.artifacts.image);
|
|
const composePlugin = localArtifactSummary(nginx.artifacts.composePlugin);
|
|
if (!image.valid || !composePlugin.valid) {
|
|
return { ok: false, action: "platform-infra-nginx-apply", mode: "artifact-invalid", image, composePlugin };
|
|
}
|
|
if (options.confirm && !options.wait) {
|
|
const job = startJob(
|
|
`platform_infra_nginx_apply_${id.toLowerCase()}`,
|
|
["bun", "scripts/cli.ts", "platform-infra", "nginx", "apply", "--target", id, "--confirm", "--wait"],
|
|
`Deploy YAML-controlled Nginx to ${id}, run four serial requests, and remove host test backends`,
|
|
);
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-nginx-apply",
|
|
mode: "async-job",
|
|
mutation: true,
|
|
target: targetSummary(id, target),
|
|
job,
|
|
statusCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
|
|
};
|
|
}
|
|
const compose = renderCompose(nginx, target);
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-nginx-apply",
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
target: targetSummary(id, target),
|
|
compose: {
|
|
sha256: sha256(compose),
|
|
bytes: Buffer.byteLength(compose),
|
|
routes: renderRoutes(target.runtime.routes),
|
|
},
|
|
apiKey: secretSummary(target, secret),
|
|
artifacts: { image, composePlugin },
|
|
validation: validationSummary(target),
|
|
};
|
|
}
|
|
|
|
const prepare = await capture(config, target.route, ["sh"], prepareRemoteScript(target));
|
|
if (prepare.exitCode !== 0) return failedApply(id, target, "prepare", prepare, secret);
|
|
const distribution = nginx.artifacts.distribution;
|
|
const distributedImage = `${distribution.workDir}/${fileName(nginx.artifacts.image.remotePath)}`;
|
|
const distributedPlugin = `${distribution.workDir}/${fileName(nginx.artifacts.composePlugin.remotePath)}`;
|
|
const uploads = [
|
|
uploadSecret(target.route, secret.sourcePath, target.runtime.envPath),
|
|
];
|
|
const failedUpload = uploads.find((upload) => !upload.ok);
|
|
if (failedUpload !== undefined) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-nginx-apply",
|
|
mode: "confirmed",
|
|
stage: "upload",
|
|
target: targetSummary(id, target),
|
|
uploads,
|
|
apiKey: secretSummary(target, secret),
|
|
};
|
|
}
|
|
const source = await startLocalArtifactSource(nginx);
|
|
let outcome: RemoteJobOutcome;
|
|
try {
|
|
const remoteScript = applyRemoteScript(nginx, target, compose, {
|
|
image: `${distribution.publicBaseUrl}/${fileName(distributedImage)}`,
|
|
composePlugin: `${distribution.publicBaseUrl}/${fileName(distributedPlugin)}.gz`,
|
|
});
|
|
const start = await capture(
|
|
config,
|
|
target.route,
|
|
["sh"],
|
|
startRemoteJobScript(target, remoteScript),
|
|
);
|
|
if (start.exitCode !== 0) return failedApply(id, target, "remote-job-start", start, secret);
|
|
outcome = await waitRemoteJob(config, target);
|
|
} finally {
|
|
stopLocalArtifactSource(source);
|
|
}
|
|
const parsed = parseJsonOutput(outcome.stdout);
|
|
return {
|
|
ok: outcome.exitCode === 0 && parsed?.ok === true,
|
|
action: "platform-infra-nginx-apply",
|
|
mode: "confirmed",
|
|
mutation: true,
|
|
target: targetSummary(id, target),
|
|
uploads,
|
|
apiKey: secretSummary(target, secret),
|
|
validation: parsed ?? remoteJobSummary(outcome, true),
|
|
remote: remoteJobSummary(outcome, options.full || outcome.exitCode !== 0),
|
|
next: { status: `bun scripts/cli.ts platform-infra nginx status --target ${id}` },
|
|
};
|
|
}
|
|
|
|
async function waitRemoteJob(config: UniDeskConfig, target: NginxTarget): Promise<RemoteJobOutcome> {
|
|
const maximumPolls = 4000;
|
|
for (let poll = 1; poll <= maximumPolls; poll += 1) {
|
|
const remote = await capture(config, target.route, ["sh"], pollRemoteJobScript(target));
|
|
if (remote.exitCode !== 0) {
|
|
return { exitCode: remote.exitCode, stdout: remote.stdout, stderr: remote.stderr, polls: poll };
|
|
}
|
|
const result = parseJsonOutput(remote.stdout);
|
|
if (result?.done === true) {
|
|
return {
|
|
exitCode: Number(result.exitCode),
|
|
stdout: decodeBase64(String(result.stdoutBase64 ?? "")),
|
|
stderr: decodeBase64(String(result.stderrBase64 ?? "")),
|
|
polls: poll,
|
|
};
|
|
}
|
|
await Bun.sleep(2000);
|
|
}
|
|
const stopped = await capture(config, target.route, ["sh"], stopRemoteJobScript(target));
|
|
return {
|
|
exitCode: 124,
|
|
stdout: stopped.stdout,
|
|
stderr: `remote nginx apply job exceeded ${maximumPolls * 2} seconds`,
|
|
polls: maximumPolls,
|
|
};
|
|
}
|
|
|
|
async function status(config: UniDeskConfig, options: OpsCommonOptions): Promise<Record<string, unknown>> {
|
|
const nginx = readConfig();
|
|
const { id, target } = resolveTarget(nginx, options.targetId);
|
|
const secret = readSecret(target);
|
|
const remote = await capture(config, target.route, ["sh"], statusRemoteScript(nginx, target));
|
|
const parsed = parseJsonOutput(remote.stdout);
|
|
return {
|
|
ok: remote.exitCode === 0 && parsed?.ok === true,
|
|
action: "platform-infra-nginx-status",
|
|
target: targetSummary(id, target),
|
|
apiKey: secretSummary(target, secret),
|
|
summary: parsed,
|
|
remote: compactCapture(remote, { full: options.full || remote.exitCode !== 0 }),
|
|
...(options.raw ? { raw: remote } : {}),
|
|
};
|
|
}
|
|
|
|
function prepareRemoteScript(target: NginxTarget): string {
|
|
return `set -eu
|
|
install -d -m 0755 ${shQuote(target.runtime.workDir)} ${shQuote(target.runtime.logDir)}
|
|
`;
|
|
}
|
|
|
|
async function startLocalArtifactSource(
|
|
config: NginxConfig,
|
|
): Promise<{ pid: number; workDir: string }> {
|
|
const distribution = config.artifacts.distribution;
|
|
const imageLink = `${distribution.workDir}/${fileName(config.artifacts.image.remotePath)}`;
|
|
const pluginLink = `${distribution.workDir}/${fileName(config.artifacts.composePlugin.remotePath)}`;
|
|
stopOwnedLocalServer(distribution.workDir);
|
|
rmSync(distribution.workDir, { recursive: true, force: true });
|
|
mkdirSync(distribution.workDir, { recursive: true, mode: 0o755 });
|
|
symlinkSync(config.artifacts.image.localPath, imageLink);
|
|
writeFileSync(
|
|
`${pluginLink}.gz`,
|
|
gzipSync(readFileSync(config.artifacts.composePlugin.localPath), { level: 1 }),
|
|
{ mode: 0o644 },
|
|
);
|
|
const child = spawn(
|
|
"python3",
|
|
[
|
|
"-m",
|
|
"http.server",
|
|
String(distribution.port),
|
|
"--bind",
|
|
distribution.bindHost,
|
|
"--directory",
|
|
distribution.workDir,
|
|
],
|
|
{ detached: true, stdio: "ignore" },
|
|
);
|
|
child.unref();
|
|
if (child.pid === undefined) throw new Error("failed to start local artifact HTTP server");
|
|
writeFileSync(join(distribution.workDir, "server.pid"), `${child.pid}\n`, { mode: 0o600 });
|
|
const expectedBytes = String(config.artifacts.image.bytes);
|
|
for (let attempt = 1; attempt <= 20; attempt += 1) {
|
|
try {
|
|
const response = await fetch(
|
|
`http://127.0.0.1:${distribution.port}/${fileName(imageLink)}`,
|
|
{ method: "HEAD" },
|
|
);
|
|
if (response.ok && response.headers.get("content-length") === expectedBytes) {
|
|
return { pid: child.pid, workDir: distribution.workDir };
|
|
}
|
|
} catch {
|
|
// The server may still be binding its socket.
|
|
}
|
|
await Bun.sleep(100);
|
|
}
|
|
stopLocalArtifactSource({ pid: child.pid, workDir: distribution.workDir });
|
|
throw new Error("local artifact HTTP server did not become ready");
|
|
}
|
|
|
|
function stopOwnedLocalServer(workDir: string): void {
|
|
const pidPath = join(workDir, "server.pid");
|
|
if (!existsSync(pidPath)) return;
|
|
const pid = Number(readFileSync(pidPath, "utf8").trim());
|
|
if (Number.isInteger(pid) && pid > 1) {
|
|
try {
|
|
process.kill(pid, "SIGTERM");
|
|
} catch {
|
|
// A previous server may already have exited.
|
|
}
|
|
}
|
|
}
|
|
|
|
function stopLocalArtifactSource(source: { pid: number; workDir: string }): void {
|
|
try {
|
|
process.kill(source.pid, "SIGTERM");
|
|
} catch {
|
|
// The server may already have exited.
|
|
}
|
|
rmSync(source.workDir, { recursive: true, force: true });
|
|
}
|
|
|
|
function remoteJobDir(target: NginxTarget): string {
|
|
return `${target.runtime.workDir}/apply-job`;
|
|
}
|
|
|
|
function startRemoteJobScript(target: NginxTarget, script: string): string {
|
|
const jobDir = remoteJobDir(target);
|
|
const runner = `#!/bin/sh
|
|
set +e
|
|
sh ${jobDir}/apply.sh >${jobDir}/stdout.log 2>${jobDir}/stderr.log
|
|
code=$?
|
|
printf '%s\\n' "$code" >${jobDir}/exit
|
|
`;
|
|
return `set -eu
|
|
job_dir=${shQuote(jobDir)}
|
|
if [ -f "$job_dir/pid" ] && kill -0 "$(cat "$job_dir/pid")" 2>/dev/null; then
|
|
printf '%s\n' 'nginx apply job is already running' >&2
|
|
exit 1
|
|
fi
|
|
rm -rf "$job_dir"
|
|
install -d -m 0700 "$job_dir"
|
|
printf '%s' ${shQuote(Buffer.from(script).toString("base64"))} | base64 -d > "$job_dir/apply.sh"
|
|
printf '%s' ${shQuote(Buffer.from(runner).toString("base64"))} | base64 -d > "$job_dir/runner.sh"
|
|
chmod 0700 "$job_dir/apply.sh" "$job_dir/runner.sh"
|
|
nohup sh "$job_dir/runner.sh" >/dev/null 2>&1 &
|
|
printf '%s\n' "$!" > "$job_dir/pid"
|
|
printf '{"ok":true,"started":true}\n'
|
|
`;
|
|
}
|
|
|
|
function pollRemoteJobScript(target: NginxTarget): string {
|
|
const jobDir = shQuote(remoteJobDir(target));
|
|
return `set -eu
|
|
job_dir=${jobDir}
|
|
if [ ! -f "$job_dir/exit" ]; then
|
|
printf '{"ok":true,"done":false}\n'
|
|
exit 0
|
|
fi
|
|
code=$(cat "$job_dir/exit")
|
|
stdout=$(tail -c 20000 "$job_dir/stdout.log" 2>/dev/null | base64 -w0)
|
|
stderr=$(tail -c 20000 "$job_dir/stderr.log" 2>/dev/null | base64 -w0)
|
|
printf '{"ok":true,"done":true,"exitCode":%s,"stdoutBase64":"%s","stderrBase64":"%s"}\n' \
|
|
"$code" "$stdout" "$stderr"
|
|
`;
|
|
}
|
|
|
|
function stopRemoteJobScript(target: NginxTarget): string {
|
|
const jobDir = shQuote(remoteJobDir(target));
|
|
return `set -eu
|
|
job_dir=${jobDir}
|
|
[ ! -f "$job_dir/pid" ] || kill "$(cat "$job_dir/pid")" 2>/dev/null || true
|
|
printf '{"ok":true,"stopped":true}\n'
|
|
`;
|
|
}
|
|
|
|
function applyRemoteScript(
|
|
config: NginxConfig,
|
|
target: NginxTarget,
|
|
compose: string,
|
|
artifactUrls: { image: string; composePlugin: string },
|
|
): string {
|
|
const backendScript = renderBackendScript();
|
|
const unitFiles = target.validation.backends.map((backend) => ({
|
|
path: `/etc/systemd/system/${backend.unit}`,
|
|
content: renderBackendUnit(target.validation.backendScriptPath, backend),
|
|
}));
|
|
const firstRoute = target.runtime.routes[0]!;
|
|
const expected = target.validation.backends.map((backend) => backend.id).join(",");
|
|
const actualExpected = Array.from(
|
|
{ length: target.validation.requestCount },
|
|
(_, index) => target.validation.backends[index % target.validation.backends.length]!.id,
|
|
).join(",");
|
|
const unitNames = target.validation.backends.map((backend) => shQuote(backend.unit)).join(" ");
|
|
const unitPaths = target.validation.backends
|
|
.map((backend) => shQuote(`/etc/systemd/system/${backend.unit}`))
|
|
.join(" ");
|
|
const unitWrites = unitFiles
|
|
.map((unit) => {
|
|
const encoded = shQuote(Buffer.from(unit.content).toString("base64"));
|
|
return `printf '%s' ${encoded} | base64 -d > ${shQuote(unit.path)}`;
|
|
})
|
|
.join("\n");
|
|
const adminBase = [
|
|
"http://127.0.0.1:",
|
|
target.runtime.environment.NGINX_ADMIN_PORT,
|
|
target.runtime.environment.LOGGER_ADMIN_PREFIX,
|
|
].join("");
|
|
const validationFormat = [
|
|
'{"ok":true,"requests":',
|
|
String(target.validation.requestCount),
|
|
',"backendSequence":"%s","loggedSequence":"%s",',
|
|
'"unauthorizedStatus":%s,"activeTestBackends":%s,"logCleared":true}\\n',
|
|
].join("");
|
|
assert(expected.length > 0, "validation backends are missing");
|
|
return `set -eu
|
|
image_artifact=${shQuote(config.artifacts.image.remotePath)}
|
|
plugin=${shQuote(config.artifacts.composePlugin.remotePath)}
|
|
plugin_archive="$plugin.gz"
|
|
env_file=${shQuote(target.runtime.envPath)}
|
|
compose_file=${shQuote(target.runtime.composePath)}
|
|
log_file=${shQuote(`${target.runtime.logDir}/requests.jsonl`)}
|
|
backend_script=${shQuote(target.validation.backendScriptPath)}
|
|
cleanup_backends() {
|
|
systemctl stop ${unitNames} >/dev/null 2>&1 || true
|
|
systemctl disable ${unitNames} >/dev/null 2>&1 || true
|
|
rm -f ${unitPaths} "$backend_script"
|
|
systemctl daemon-reload >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup_backends EXIT
|
|
gzip -dc "$env_file.gz" > "$env_file"
|
|
chmod 0600 "$env_file"
|
|
rm -f "$env_file.gz"
|
|
image_sha=$(sha256sum "$image_artifact" 2>/dev/null | awk '{print $1}' || true)
|
|
if [ "$image_sha" != ${shQuote(config.artifacts.image.sha256)} ]; then
|
|
rm -f "$image_artifact"
|
|
curl -fsSL --retry 2 --connect-timeout 5 --max-time 3600 \
|
|
${shQuote(artifactUrls.image)} -o "$image_artifact"
|
|
fi
|
|
rm -f "$plugin" "$plugin_archive"
|
|
curl -fsSL --retry 2 --connect-timeout 5 --max-time 3600 \
|
|
${shQuote(artifactUrls.composePlugin)} -o "$plugin_archive"
|
|
gzip -dc "$plugin_archive" > "$plugin"
|
|
rm -f "$plugin_archive"
|
|
[ "$(sha256sum "$image_artifact" | awk '{print $1}')" = ${shQuote(config.artifacts.image.sha256)} ]
|
|
[ "$(sha256sum "$plugin" | awk '{print $1}')" = ${shQuote(config.artifacts.composePlugin.sha256)} ]
|
|
chmod 0755 "$plugin"
|
|
docker compose version >/dev/null
|
|
chmod 0600 "$env_file"
|
|
printf '%s' ${shQuote(Buffer.from(compose).toString("base64"))} | base64 -d > "$compose_file"
|
|
printf '%s' ${shQuote(Buffer.from(backendScript).toString("base64"))} | base64 -d > "$backend_script"
|
|
chmod 0755 "$backend_script"
|
|
${unitWrites}
|
|
systemctl daemon-reload
|
|
systemctl enable --now ${unitNames} >/dev/null
|
|
for backend_port in ${target.validation.backends.map((backend) => backend.port).join(" ")}; do
|
|
for attempt in $(seq 1 10); do
|
|
if curl -fsS http://127.0.0.1:"$backend_port"/ >/dev/null; then break; fi
|
|
[ "$attempt" -lt 10 ] || exit 1
|
|
sleep 1
|
|
done
|
|
done
|
|
docker load -i "$image_artifact" >/dev/null
|
|
loaded_image_id=$(docker image inspect ${shQuote(config.artifacts.image.image)} --format '{{.Id}}')
|
|
[ "$loaded_image_id" = ${shQuote(config.artifacts.image.imageId)} ]
|
|
docker compose -f "$compose_file" -p ${shQuote(target.runtime.projectName)} up -d --force-recreate
|
|
for attempt in $(seq 1 20); do
|
|
if wget -qO- ${adminBase}/health >/dev/null; then break; fi
|
|
[ "$attempt" -lt 20 ] || exit 1
|
|
sleep 1
|
|
done
|
|
actual=
|
|
for sequence in $(seq 1 ${target.validation.requestCount}); do
|
|
backend=$(curl -fsS -D - -o /dev/null http://127.0.0.1:${firstRoute.listenPort}/validation?sequence="$sequence" \
|
|
| awk 'BEGIN { IGNORECASE=1 } /^X-Backend:/ { gsub("\\r", "", $2); print $2; exit }')
|
|
actual="\${actual}\${actual:+,}\${backend}"
|
|
done
|
|
[ "$actual" = ${shQuote(actualExpected)} ]
|
|
set -a
|
|
. "$env_file"
|
|
set +a
|
|
logger_curl() {
|
|
printf 'header = "${target.apiKey.requestHeader}: %s"\n' "$LOGGER_API_KEY" | curl --config - "$@"
|
|
}
|
|
unauthorized=$(curl -sS -o /dev/null -w '%{http_code}' \
|
|
${adminBase}/logs/info)
|
|
[ "$unauthorized" = 401 ]
|
|
logger_curl -fsS -o /tmp/nginx-logger-info.json \
|
|
${adminBase}/logs/info
|
|
logger_curl -fsS -o /tmp/nginx-logger-tail.jsonl \
|
|
'${adminBase}/logs/tail?lines=${target.validation.requestCount}'
|
|
logger_curl -fsS -o /tmp/nginx-logger-download.jsonl \
|
|
${adminBase}/logs/download
|
|
[ -s /tmp/nginx-logger-download.jsonl ]
|
|
logged=$(python3 -c 'import json,sys; print(",".join(json.loads(x)["backend"] for x in open(sys.argv[1])))' \
|
|
/tmp/nginx-logger-tail.jsonl)
|
|
[ "$logged" = ${shQuote(actualExpected)} ]
|
|
logger_curl -fsS -X POST -o /tmp/nginx-logger-clear.json \
|
|
${adminBase}/logs/clear
|
|
[ ! -s "$log_file" ]
|
|
cleanup_backends
|
|
trap - EXIT
|
|
active_backends=0
|
|
for unit in ${unitNames}; do
|
|
systemctl is-active --quiet "$unit" && active_backends=$((active_backends + 1)) || true
|
|
done
|
|
printf ${shQuote(validationFormat)} \
|
|
"$actual" "$logged" "$unauthorized" "$active_backends"
|
|
`;
|
|
}
|
|
|
|
function statusRemoteScript(config: NginxConfig, target: NginxTarget): string {
|
|
const adminPort = target.runtime.environment.NGINX_ADMIN_PORT;
|
|
const prefix = target.runtime.environment.LOGGER_ADMIN_PREFIX;
|
|
const statusFormat = [
|
|
'{"ok":%s,"running":%s,"imageId":"%s","expectedImageId":"%s",',
|
|
'"healthStatus":%s,"logBytes":%s,"activeTestBackends":%s}\\n',
|
|
].join("");
|
|
return `set -eu
|
|
container=${shQuote(target.runtime.containerName)}
|
|
running=false
|
|
if [ "$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || true)" = true ]; then running=true; fi
|
|
image_id=$(docker inspect -f '{{.Image}}' "$container" 2>/dev/null || true)
|
|
health_code=$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:${adminPort}${prefix}/health 2>/dev/null || true)
|
|
log_bytes=$(wc -c < ${shQuote(`${target.runtime.logDir}/requests.jsonl`)} 2>/dev/null || printf 0)
|
|
active_backends=0
|
|
for unit in ${target.validation.backends.map((backend) => shQuote(backend.unit)).join(" ")}; do
|
|
systemctl is-active --quiet "$unit" && active_backends=$((active_backends + 1)) || true
|
|
done
|
|
ok=false
|
|
[ "$running" = true ] && [ "$image_id" = ${shQuote(config.artifacts.image.imageId)} ] && \
|
|
[ "$health_code" = 200 ] && [ "$active_backends" -eq 0 ] && ok=true
|
|
printf ${shQuote(statusFormat)} \
|
|
"$ok" "$running" "$image_id" ${shQuote(config.artifacts.image.imageId)} \
|
|
"\${health_code:-0}" "\${log_bytes:-0}" "$active_backends"
|
|
`;
|
|
}
|
|
|
|
function renderCompose(config: NginxConfig, target: NginxTarget): string {
|
|
const environment = {
|
|
...target.runtime.environment,
|
|
NGINX_ROUTES: renderRoutes(target.runtime.routes),
|
|
};
|
|
const envLines = Object.entries(environment)
|
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
.map(([key, value]) => ` ${key}: ${yamlString(value.replaceAll("$", () => "$$"))}`)
|
|
.join("\n");
|
|
return `name: ${target.runtime.projectName}
|
|
|
|
services:
|
|
nginx:
|
|
image: ${config.artifacts.image.image}
|
|
container_name: ${target.runtime.containerName}
|
|
restart: ${target.runtime.restart}
|
|
network_mode: ${target.runtime.networkMode}
|
|
mem_limit: ${target.runtime.resources.memory}
|
|
cpus: ${yamlString(target.runtime.resources.cpus)}
|
|
pids_limit: ${target.runtime.resources.pidsLimit}
|
|
env_file:
|
|
- ${target.runtime.envPath}
|
|
environment:
|
|
${envLines}
|
|
volumes:
|
|
- ${target.runtime.logDir}:/var/log/logger
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: 5m
|
|
max-file: "2"
|
|
`;
|
|
}
|
|
|
|
function renderBackendScript(): string {
|
|
return `#!/usr/bin/env python3
|
|
import sys
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
|
|
host, port, backend = sys.argv[1], int(sys.argv[2]), sys.argv[3]
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
def do_GET(self):
|
|
body = ('{"backend":"%s"}\\n' % backend).encode("ascii")
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("X-Backend", backend)
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
def log_message(self, format, *args):
|
|
return
|
|
|
|
HTTPServer((host, port), Handler).serve_forever()
|
|
`;
|
|
}
|
|
|
|
function renderBackendUnit(scriptPath: string, backend: Backend): string {
|
|
return `[Unit]
|
|
Description=Nginx validation backend ${backend.id}
|
|
After=network.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
ExecStart=/usr/bin/python3 ${scriptPath} ${backend.bindHost} ${backend.port} ${backend.id}
|
|
Restart=on-failure
|
|
MemoryMax=32M
|
|
CPUQuota=10%
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
`;
|
|
}
|
|
|
|
function readSecret(target: NginxTarget): SecretMaterial {
|
|
const sourcePath = isAbsolute(target.apiKey.sourceRef)
|
|
? target.apiKey.sourceRef
|
|
: join(repoRoot, target.apiKey.sourceRef);
|
|
assert(existsSync(sourcePath), `${target.apiKey.sourceRef} is missing`);
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const value = values[target.apiKey.sourceKey];
|
|
assert(
|
|
typeof value === "string" && value.length >= 16,
|
|
`${target.apiKey.sourceRef} is missing ${target.apiKey.sourceKey}`,
|
|
);
|
|
return {
|
|
sourcePath,
|
|
value,
|
|
fingerprint: fingerprintValues({ [target.apiKey.sourceKey]: value }, [target.apiKey.sourceKey]),
|
|
};
|
|
}
|
|
|
|
function localArtifactSummary(artifact: Artifact & { bytes?: number }): Record<string, unknown> & { valid: boolean } {
|
|
const present = existsSync(artifact.localPath);
|
|
const bytes = present ? statSync(artifact.localPath).size : 0;
|
|
const digest = present ? sha256(readFileSync(artifact.localPath)) : null;
|
|
const valid = present && digest === artifact.sha256 && (artifact.bytes === undefined || bytes === artifact.bytes);
|
|
return { present, bytes, sha256: digest, expectedSha256: artifact.sha256, valid };
|
|
}
|
|
|
|
function uploadArtifact(route: string, localPath: string, remotePath: string): Record<string, unknown> {
|
|
const result = Bun.spawnSync(["trans", route, "upload", localPath, remotePath], {
|
|
stdout: "pipe",
|
|
stderr: "pipe",
|
|
});
|
|
const stdout = new TextDecoder().decode(result.stdout);
|
|
const parsed = parseJsonOutput(stdout);
|
|
return {
|
|
ok: result.exitCode === 0 && parsed?.ok === true,
|
|
localPath,
|
|
remotePath,
|
|
bytes: parsed?.bytes ?? 0,
|
|
sha256: parsed?.sha256 ?? null,
|
|
verified: parsed?.verified === true,
|
|
stderrBytes: result.stderr.length,
|
|
};
|
|
}
|
|
|
|
function uploadSecret(route: string, sourcePath: string, remotePath: string): Record<string, unknown> {
|
|
const directory = mkdtempSync(join(tmpdir(), "unidesk-nginx-secret-"));
|
|
const transferPath = join(directory, "runtime.env.gz");
|
|
try {
|
|
writeFileSync(transferPath, gzipSync(readFileSync(sourcePath)));
|
|
chmodSync(transferPath, 0o600);
|
|
const result = uploadArtifact(route, transferPath, `${remotePath}.gz`);
|
|
return { ...result, localPath: "gzip-secret-transfer", valuesPrinted: false };
|
|
} finally {
|
|
rmSync(directory, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function decodeBase64(value: string): string {
|
|
return Buffer.from(value, "base64").toString("utf8");
|
|
}
|
|
|
|
function remoteJobSummary(outcome: RemoteJobOutcome, full: boolean): Record<string, unknown> {
|
|
return {
|
|
exitCode: outcome.exitCode,
|
|
polls: outcome.polls,
|
|
stdoutBytes: Buffer.byteLength(outcome.stdout),
|
|
stderrBytes: Buffer.byteLength(outcome.stderr),
|
|
stdoutTail: full ? outcome.stdout.slice(-8000) : "",
|
|
stderrTail: full ? outcome.stderr.slice(-4000) : "",
|
|
};
|
|
}
|
|
|
|
function targetSummary(id: string, target: NginxTarget): Record<string, unknown> {
|
|
return {
|
|
id,
|
|
route: target.route,
|
|
runtime: "host-docker-compose",
|
|
container: target.runtime.containerName,
|
|
resources: target.runtime.resources,
|
|
routes: target.runtime.routes,
|
|
adminPort: target.runtime.environment.NGINX_ADMIN_PORT,
|
|
};
|
|
}
|
|
|
|
function validationSummary(target: NginxTarget): Record<string, unknown> {
|
|
return {
|
|
requestCount: target.validation.requestCount,
|
|
concurrency: 1,
|
|
backends: "host-python-systemd-temporary",
|
|
cleanupAfterSuccess: target.validation.cleanupAfterSuccess,
|
|
};
|
|
}
|
|
|
|
function secretSummary(target: NginxTarget, secret: SecretMaterial): Record<string, unknown> {
|
|
return {
|
|
sourceRef: target.apiKey.sourceRef,
|
|
sourceKey: target.apiKey.sourceKey,
|
|
targetKey: target.apiKey.targetKey,
|
|
present: true,
|
|
fingerprint: secret.fingerprint,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function failedApply(
|
|
id: string,
|
|
target: NginxTarget,
|
|
stage: string,
|
|
remote: Awaited<ReturnType<typeof capture>>,
|
|
secret: SecretMaterial,
|
|
): Record<string, unknown> {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-nginx-apply",
|
|
mode: "confirmed",
|
|
stage,
|
|
target: targetSummary(id, target),
|
|
apiKey: secretSummary(target, secret),
|
|
remote: compactCapture(remote, { full: true }),
|
|
};
|
|
}
|
|
|
|
function renderRoutes(routes: Route[]): string {
|
|
return routes
|
|
.map((route) => {
|
|
const upstreams = route.upstreams.map((upstream) => `${upstream.host}:${upstream.port}`).join(",");
|
|
return `${route.listenPort}=${upstreams}`;
|
|
})
|
|
.join(";");
|
|
}
|
|
|
|
function sha256(value: string | Buffer): string {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
|
|
function yamlString(value: string): string {
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
function fileName(path: string): string {
|
|
return path.slice(path.lastIndexOf("/") + 1);
|
|
}
|
|
|
|
function absolutePath(value: string, path: string): void {
|
|
assert(typeof value === "string" && isAbsolute(value), `${configLabel}.${path} must be an absolute path`);
|
|
}
|
|
|
|
function simpleName(value: string, path: string): void {
|
|
assert(typeof value === "string" && /^[A-Za-z0-9._-]+$/u.test(value), `${configLabel}.${path} is invalid`);
|
|
}
|
|
|
|
function port(value: number, path: string): void {
|
|
assert(Number.isInteger(value) && value >= 1 && value <= 65535, `${configLabel}.${path} must be a valid port`);
|
|
}
|
|
|
|
function assert(condition: unknown, message: string): asserts condition {
|
|
if (!condition) throw new Error(message);
|
|
}
|