fix: harden hwlab cd cache

This commit is contained in:
Codex
2026-05-24 05:59:29 +00:00
parent b517a08b95
commit bfdfd10923
3 changed files with 932 additions and 45 deletions
+314 -32
View File
@@ -3,6 +3,7 @@ import { randomBytes } from "node:crypto";
import { createWriteStream, mkdirSync, readFileSync } from "node:fs";
import { mkdir } from "node:fs/promises";
import { join } from "node:path";
import { gzipSync } from "node:zlib";
import { readConfig, repoRoot, rootPath, type UniDeskConfig } from "./config";
import { d601NativeKubeconfig } from "./d601-k3s-guard";
@@ -46,6 +47,24 @@ interface FrontendSession {
cookie: string;
}
interface FrontendHostSshResult {
ok: boolean;
taskId: string | null;
taskStatus: unknown;
stdout: string;
stderr: string;
exitCode: number | null;
error: string;
raw: unknown;
}
interface RemoteRunnerJob {
pid: number | null;
remoteRunDir: string | null;
remoteCompactResultPath: string | null;
remoteFullResultPath: string | null;
}
interface FetchJsonResult {
ok: boolean;
status?: number;
@@ -54,7 +73,8 @@ interface FetchJsonResult {
}
const defaultProviderId = "D601";
const defaultHwlabCdRepoPath = "/home/ubuntu/hwlab_cd";
const defaultHwlabCdRepoPath = "~/.state/unidesk-hwlab-cd/<run-id>/ephemeral-repo/HWLAB";
const defaultHwlabCdCachePath = "~/.cache/unidesk/hwlab-cd/git-cache/HWLAB.git";
const rejectedRunnerHistoryRepoPath = "/home/ubuntu/hwlab";
const namespace = "hwlab-dev";
const lockName = "hwlab-dev-cd-lock";
@@ -160,8 +180,8 @@ function b64Json(value: unknown): string {
return Buffer.from(JSON.stringify(value), "utf8").toString("base64");
}
function buildRemoteCommand(options: HwlabCdOptions, runId: string): string {
const remoteOptions = {
function buildRemoteOptions(options: HwlabCdOptions, runId: string): Record<string, unknown> {
return {
action: options.action,
environment: options.environment,
dryRun: options.dryRun,
@@ -169,14 +189,113 @@ function buildRemoteCommand(options: HwlabCdOptions, runId: string): string {
kubeconfig: options.kubeconfig,
timeoutMs: options.timeoutMs,
runId,
compactStdout: options.transport !== "local",
};
}
function buildRemoteInlineCommand(options: HwlabCdOptions, runId: string): string {
return [
`UNIDESK_HWLAB_CD_OPTIONS_B64=${shellQuote(b64Json(remoteOptions))} node <<'UNIDESK_HWLAB_CD_JS'`,
`UNIDESK_HWLAB_CD_OPTIONS_B64=${shellQuote(b64Json(buildRemoteOptions(options, runId)))} node <<'UNIDESK_HWLAB_CD_JS'`,
remoteScriptSource,
"UNIDESK_HWLAB_CD_JS",
].join("\n");
}
function buildRemoteUploadedCommand(options: HwlabCdOptions, runId: string, remoteScriptPath: string): string {
return `UNIDESK_HWLAB_CD_OPTIONS_B64=${shellQuote(b64Json(buildRemoteOptions(options, runId)))} node ${shellQuote(remoteScriptPath)}`;
}
function remoteRunnerDir(runId: string): string {
return `$HOME/.state/unidesk-hwlab-cd/${safeRemoteId(runId)}`;
}
function buildRemoteStartCommand(options: HwlabCdOptions, runId: string, remoteScriptPath: string): string {
const runDir = remoteRunnerDir(runId);
const encodedOptions = shellQuote(b64Json(buildRemoteOptions(options, runId)));
return [
"umask 077",
`run_dir="${runDir}"`,
`mkdir -p "$run_dir"`,
`rm -f "$run_dir/result.full.json" "$run_dir/result.compact.json" "$run_dir/runner.stdout.txt" "$run_dir/runner.stderr.txt" "$run_dir/runner.exit"`,
`nohup sh -c 'UNIDESK_HWLAB_CD_OPTIONS_B64="$1" node "$2" > "$3" 2> "$4"; printf "%s\\n" "$?" > "$5"' sh ${encodedOptions} ${shellQuote(remoteScriptPath)} "$run_dir/runner.stdout.txt" "$run_dir/runner.stderr.txt" "$run_dir/runner.exit" >/dev/null 2>&1 & pid=$!`,
`printf '{"pid":%s,"remoteRunDir":"%s","remoteCompactResultPath":"%s/result.compact.json","remoteFullResultPath":"%s/result.full.json"}\\n' "$pid" "$run_dir" "$run_dir" "$run_dir"`,
].join("; ");
}
function buildRemotePollCommand(runId: string): string {
const runDir = remoteRunnerDir(runId);
const pollJs = [
"const fs=require(\"fs\")",
"const dir=process.argv[1]",
"const compact=`${dir}/result.compact.json`",
"const exitPath=`${dir}/runner.exit`",
"const tail=(p)=>{try{const b=fs.readFileSync(p);return b.slice(Math.max(0,b.length-500)).toString(\"utf8\")}catch{return \"\"}}",
"if(fs.existsSync(compact)&&fs.statSync(compact).size>0){process.stdout.write(fs.readFileSync(compact,\"utf8\"));process.exit(0)}",
"if(fs.existsSync(exitPath)){const raw=fs.readFileSync(exitPath,\"utf8\").trim();console.log(JSON.stringify({ok:false,status:\"blocked\",error:\"remote-runner-exited-without-result\",remoteRunDir:dir,exitCode:Number(raw),stdoutTail:tail(`${dir}/runner.stdout.txt`),stderrTail:tail(`${dir}/runner.stderr.txt`)}));process.exit(0)}",
"console.log(JSON.stringify({ok:false,status:\"running\",remoteRunDir:dir}))",
].join(";");
return `run_dir="${runDir}"; node -e ${shellQuote(pollJs)} "$run_dir"`;
}
function buildRemoteCommand(options: HwlabCdOptions, runId: string): string {
return buildRemoteInlineCommand(options, runId);
}
function splitFixed(value: string, size: number): string[] {
const chunks: string[] = [];
for (let index = 0; index < value.length; index += size) chunks.push(value.slice(index, index + size));
return chunks;
}
function safeRemoteId(value: string): string {
return value.replace(/[^A-Za-z0-9_.-]/gu, "-");
}
function compactTail(value: string, maxLength = 1000): string {
return value.slice(Math.max(0, value.length - maxLength));
}
function uploadCommandShape(providerId = defaultProviderId): string[] {
return ["frontend", "/api/dispatch", providerId, "host.ssh", "upload remote runner in small chunks", "start nohup remote runner", "poll ~/.state/unidesk-hwlab-cd/<run-id>/result.compact.json"];
}
function buildUploadPlan(runId: string): { remoteDir: string; remotePath: string; commands: Array<{ id: string; command: string }> } {
const remoteDir = "/tmp/unidesk-hwlab-cd";
const remotePath = `${remoteDir}/remote-runner-${safeRemoteId(runId)}.cjs`;
const encoded = gzipSync(Buffer.from(remoteScriptSource, "utf8")).toString("base64");
const b64Path = `${remotePath}.gz.b64`;
const commands = [
{
id: "init",
command: `umask 077; mkdir -p ${shellQuote(remoteDir)}; rm -f ${shellQuote(remotePath)} ${shellQuote(b64Path)}; : > ${shellQuote(b64Path)}`,
},
...splitFixed(encoded, 2400).map((chunk, index) => ({
id: `chunk-${index + 1}`,
command: `printf %s ${shellQuote(chunk)} >> ${shellQuote(b64Path)}`,
})),
{
id: "finalize",
command: `base64 -d ${shellQuote(b64Path)} | gzip -d > ${shellQuote(remotePath)}; chmod 700 ${shellQuote(remotePath)}; rm -f ${shellQuote(b64Path)}; test -s ${shellQuote(remotePath)}; wc -c ${shellQuote(remotePath)}`,
},
];
return { remoteDir, remotePath, commands };
}
function uploadStepView(step: { id: string; command: string }, result: FrontendHostSshResult): Record<string, unknown> {
const remoteOptions = {
id: step.id,
ok: result.ok,
taskId: result.taskId,
taskStatus: result.taskStatus,
exitCode: result.exitCode,
commandBytes: step.command.length,
stdoutTail: compactTail(result.stdout, 500),
stderrTail: compactTail(result.stderr, 500),
error: result.error,
};
return remoteOptions;
}
export function buildHwlabCdRemoteCommandForTest(args: string[]): string {
return buildRemoteCommand(parseOptions(args), "test-run-id");
}
@@ -325,7 +444,7 @@ function parseRemoteStdout(stdout: string): Record<string, unknown> | null {
}
function commandShape(providerId = defaultProviderId): string[] {
return ["frontend", "/api/dispatch", providerId, "host.ssh", "UNIDESK_HWLAB_CD_OPTIONS_B64=... node <<'UNIDESK_HWLAB_CD_JS'"];
return uploadCommandShape(providerId);
}
async function runLocalTransport(options: HwlabCdOptions, remoteCommand: string, runId: string): Promise<Record<string, unknown>> {
@@ -370,65 +489,219 @@ async function runLocalTransport(options: HwlabCdOptions, remoteCommand: string,
};
}
async function runFrontendTransport(options: HwlabCdOptions, remoteCommand: string, config: UniDeskConfig, runId: string): Promise<Record<string, unknown>> {
const host = options.mainServerHost ?? config.network.publicHost;
const dumpDir = rootPath(".state", "hwlab-cd", runId);
mkdirSync(dumpDir, { recursive: true, mode: 0o700 });
const session = await loginFrontend(host, config);
async function dispatchFrontendHostSsh(
session: FrontendSession,
options: HwlabCdOptions,
command: string,
timeoutMs: number,
source = "cli-hwlab-cd",
): Promise<FrontendHostSshResult> {
const dispatch = await frontendJson(session, "/api/dispatch", {
method: "POST",
body: JSON.stringify({
providerId: options.providerId,
command: "host.ssh",
payload: { source: "cli-hwlab-cd", mode: "exec", command: remoteCommand, timeoutMs: options.timeoutMs },
payload: { source, mode: "exec", command, timeoutMs },
}),
}, 12_000);
const taskId = String(asRecord(dispatch.body)?.taskId ?? "");
if (!dispatch.ok || taskId.length === 0) {
return {
ok: false,
taskId: taskId || null,
taskStatus: null,
stdout: "",
stderr: "",
exitCode: null,
error: asRecord(dispatch.body)?.error as string ?? dispatch.error ?? "provider dispatch failed",
raw: dispatch,
};
}
const wait = await waitForFrontendTask(session, taskId, Math.min(timeoutMs + 5000, maxTimeoutMs));
const task = asRecord((wait as { task?: unknown }).task);
const result = asRecord(task?.result) ?? {};
const exitCode = typeof result.exitCode === "number" ? result.exitCode : null;
const stdout = typeof result.stdout === "string" ? result.stdout : "";
const stderr = typeof result.stderr === "string" ? result.stderr : "";
const error = typeof result.error === "string" ? result.error : "";
return {
ok: task?.status === "succeeded" && (exitCode === null || exitCode === 0),
taskId,
taskStatus: task?.status ?? null,
stdout,
stderr,
exitCode,
error,
raw: task,
};
}
async function uploadRemoteRunner(session: FrontendSession, options: HwlabCdOptions, runId: string, dumpDir: string): Promise<{ ok: boolean; path: string; steps: Record<string, unknown>[]; error: string }> {
const plan = buildUploadPlan(runId);
const steps: Record<string, unknown>[] = [];
for (const step of plan.commands) {
const result = await dispatchFrontendHostSsh(session, options, step.command, Math.min(options.timeoutMs, 20_000), "cli-hwlab-cd-upload");
steps.push(uploadStepView(step, result));
if (!result.ok) {
await Bun.write(join(dumpDir, "frontend-upload.json"), `${JSON.stringify({ ok: false, path: plan.remotePath, steps }, null, 2)}\n`);
return { ok: false, path: plan.remotePath, steps, error: result.error || result.stderr || `upload step ${step.id} failed` };
}
}
await Bun.write(join(dumpDir, "frontend-upload.json"), `${JSON.stringify({ ok: true, path: plan.remotePath, steps }, null, 2)}\n`);
return { ok: true, path: plan.remotePath, steps, error: "" };
}
function parseRemoteRunnerJob(stdout: string): RemoteRunnerJob | null {
const parsed = parseRemoteStdout(stdout);
if (parsed === null) return null;
return {
pid: typeof parsed.pid === "number" ? parsed.pid : null,
remoteRunDir: typeof parsed.remoteRunDir === "string" ? parsed.remoteRunDir : null,
remoteCompactResultPath: typeof parsed.remoteCompactResultPath === "string" ? parsed.remoteCompactResultPath : null,
remoteFullResultPath: typeof parsed.remoteFullResultPath === "string" ? parsed.remoteFullResultPath : null,
};
}
async function startRemoteRunner(
session: FrontendSession,
options: HwlabCdOptions,
runId: string,
remoteScriptPath: string,
dumpDir: string,
): Promise<{ ok: boolean; job: RemoteRunnerJob | null; result: FrontendHostSshResult; dump: string }> {
const result = await dispatchFrontendHostSsh(session, options, buildRemoteStartCommand(options, runId, remoteScriptPath), 8000, "cli-hwlab-cd-start");
const stdoutDump = join(dumpDir, "frontend-runner-start.stdout.txt");
const stderrDump = join(dumpDir, "frontend-runner-start.stderr.txt");
await Bun.write(stdoutDump, result.stdout);
await Bun.write(stderrDump, result.stderr);
const job = result.ok ? parseRemoteRunnerJob(result.stdout) : null;
const dump = join(dumpDir, "frontend-runner-start.json");
await Bun.write(dump, `${JSON.stringify({ ok: result.ok && job !== null, job, taskId: result.taskId, taskStatus: result.taskStatus, exitCode: result.exitCode, stdoutDump, stderrDump, error: result.error }, null, 2)}\n`);
return { ok: result.ok && job !== null, job, result, dump };
}
async function pollRemoteRunnerResult(
session: FrontendSession,
options: HwlabCdOptions,
runId: string,
dumpDir: string,
): Promise<{ parsed: Record<string, unknown> | null; stdout: string; stderr: string; result: FrontendHostSshResult | null; polls: Record<string, unknown>[]; timedOut: boolean; dump: string }> {
const startedAt = Date.now();
const polls: Record<string, unknown>[] = [];
const pollCommand = buildRemotePollCommand(runId);
let lastResult: FrontendHostSshResult | null = null;
while (Date.now() - startedAt < options.timeoutMs) {
const result = await dispatchFrontendHostSsh(session, options, pollCommand, 6000, "cli-hwlab-cd-poll");
lastResult = result;
const parsed = result.ok ? parseRemoteStdout(result.stdout) : null;
polls.push({
ok: result.ok,
taskId: result.taskId,
taskStatus: result.taskStatus,
exitCode: result.exitCode,
parsedStatus: parsed?.status ?? null,
parsedError: parsed?.error ?? null,
stdoutTail: compactTail(result.stdout, 500),
stderrTail: compactTail(result.stderr, 500),
error: result.error,
});
if (parsed !== null && parsed.status !== "running") {
const dump = join(dumpDir, "frontend-runner-poll.json");
await Bun.write(dump, `${JSON.stringify({ ok: true, polls }, null, 2)}\n`);
return { parsed, stdout: result.stdout, stderr: result.stderr, result, polls, timedOut: false, dump };
}
await Bun.sleep(1000);
}
const dump = join(dumpDir, "frontend-runner-poll.json");
await Bun.write(dump, `${JSON.stringify({ ok: false, timedOut: true, polls }, null, 2)}\n`);
return { parsed: null, stdout: lastResult?.stdout ?? "", stderr: lastResult?.stderr ?? "", result: lastResult, polls, timedOut: true, dump };
}
async function runFrontendTransport(options: HwlabCdOptions, config: UniDeskConfig, runId: string): Promise<Record<string, unknown>> {
const host = options.mainServerHost ?? config.network.publicHost;
const dumpDir = rootPath(".state", "hwlab-cd", runId);
mkdirSync(dumpDir, { recursive: true, mode: 0o700 });
const session = await loginFrontend(host, config);
const upload = await uploadRemoteRunner(session, options, runId, dumpDir);
if (!upload.ok) {
return {
ok: false,
env: options.environment,
status: "blocked",
error: "provider-dispatch-failed",
error: "remote-runner-upload-failed",
remote: {
host: session.baseUrl,
providerId: options.providerId,
transport: "frontend",
commandShape: commandShape(options.providerId),
commandCalls: ["scripts/dev-cd-apply.mjs"],
dispatch,
uploadPath: upload.path,
uploadDump: join(dumpDir, "frontend-upload.json"),
},
dumpPath: dumpDir,
nextSafeCommand: "restore UniDesk frontend/backend-core provider dispatch, then rerun bun scripts/cli.ts hwlab cd status --env dev",
upload,
nextSafeCommand: "restore D601 host.ssh command dispatch or upload permissions, then rerun bun scripts/cli.ts hwlab cd status --env dev",
};
}
const wait = await waitForFrontendTask(session, taskId, Math.min(options.timeoutMs + 5000, maxTimeoutMs));
const task = asRecord((wait as { task?: unknown }).task);
const result = asRecord(task?.result) ?? {};
const stdout = typeof result.stdout === "string" ? result.stdout : "";
const stderr = typeof result.stderr === "string" ? result.stderr : "";
const runner = await startRemoteRunner(session, options, runId, upload.path, dumpDir);
if (!runner.ok) {
return {
ok: false,
env: options.environment,
status: "blocked",
error: "remote-runner-start-failed",
remote: {
host: session.baseUrl,
providerId: options.providerId,
transport: "frontend",
taskId: runner.result.taskId,
taskStatus: runner.result.taskStatus,
commandShape: commandShape(options.providerId),
commandCalls: ["scripts/dev-cd-apply.mjs"],
uploadPath: upload.path,
uploadDump: join(dumpDir, "frontend-upload.json"),
runnerStartDump: runner.dump,
exitCode: runner.result.exitCode,
error: runner.result.error,
},
dumpPath: dumpDir,
nextSafeCommand: "restore D601 host.ssh background job dispatch, then rerun bun scripts/cli.ts hwlab cd status --env dev",
};
}
const poll = await pollRemoteRunnerResult(session, options, runId, dumpDir);
const stdout = poll.stdout;
const stderr = poll.stderr;
const stdoutDump = join(dumpDir, "frontend-task.stdout.txt");
const stderrDump = join(dumpDir, "frontend-task.stderr.txt");
Bun.write(stdoutDump, stdout);
Bun.write(stderrDump, stderr);
const parsed = parseRemoteStdout(stdout);
await Bun.write(stdoutDump, stdout);
await Bun.write(stderrDump, stderr);
const parsed = poll.parsed ?? parseRemoteStdout(stdout);
if (parsed === null) {
return {
ok: false,
env: options.environment,
status: "blocked",
error: "remote-empty-or-invalid-json",
error: poll.timedOut ? "remote-runner-timeout" : "remote-empty-or-invalid-json",
remote: {
host: session.baseUrl,
providerId: options.providerId,
transport: "frontend",
taskId,
taskStatus: task?.status ?? null,
taskId: poll.result?.taskId ?? null,
taskStatus: poll.result?.taskStatus ?? null,
commandShape: commandShape(options.providerId),
commandCalls: ["scripts/dev-cd-apply.mjs"],
uploadPath: upload.path,
uploadDump: join(dumpDir, "frontend-upload.json"),
runnerStartDump: runner.dump,
runnerPollDump: poll.dump,
remoteRunDir: runner.job?.remoteRunDir ?? null,
remoteFullResultPath: runner.job?.remoteFullResultPath ?? null,
stdoutDump,
stderrDump,
exitCode: typeof result.exitCode === "number" ? result.exitCode : null,
exitCode: poll.result?.exitCode ?? null,
error: poll.result?.error ?? "",
},
dumpPath: dumpDir,
};
@@ -439,13 +712,19 @@ async function runFrontendTransport(options: HwlabCdOptions, remoteCommand: stri
host: session.baseUrl,
providerId: options.providerId,
transport: "frontend",
taskId,
taskStatus: task?.status ?? null,
taskId: poll.result?.taskId ?? null,
taskStatus: poll.result?.taskStatus ?? null,
commandShape: commandShape(options.providerId),
commandCalls: ["scripts/dev-cd-apply.mjs"],
uploadPath: upload.path,
uploadDump: join(dumpDir, "frontend-upload.json"),
runnerStartDump: runner.dump,
runnerPollDump: poll.dump,
remoteRunDir: runner.job?.remoteRunDir ?? null,
remoteFullResultPath: runner.job?.remoteFullResultPath ?? null,
stdoutDump,
stderrDump,
exitCode: typeof result.exitCode === "number" ? result.exitCode : null,
exitCode: poll.result?.exitCode ?? null,
},
localDumpPath: dumpDir,
};
@@ -502,14 +781,17 @@ export function hwlabHelp(): Record<string, unknown> {
"bun scripts/cli.ts hwlab cd preflight --env dev",
"bun scripts/cli.ts hwlab cd apply --env dev --dry-run",
],
description: "Bounded UniDesk wrapper for the HWLAB DEV CD path. The default transport dispatches a D601 provider host.ssh command and the D601 side calls HWLAB repo-owned scripts/dev-cd-apply.mjs.",
description: "Bounded UniDesk wrapper for the HWLAB DEV CD path. The default transport uploads a small remote runner to D601 through short host.ssh commands, creates a one-run HWLAB clone owned by the host.ssh user, then calls HWLAB repo-owned scripts/dev-cd-apply.mjs from the D601 side.",
boundary: [
`KUBECONFIG is forced to ${d601NativeKubeconfig}`,
"docker-desktop, desktop-control-plane, and 127.0.0.1:11700 are refusal signals for the explicit D601 target",
`default HWLAB CD repo is ${defaultHwlabCdRepoPath}; ${rejectedRunnerHistoryRepoPath} is rejected as runner history`,
`default HWLAB CD repo is ${defaultHwlabCdRepoPath}, materialized from dedicated full bare cache ${defaultHwlabCdCachePath}; ${rejectedRunnerHistoryRepoPath} is rejected as runner history`,
"deploy/deploy.json remains the authoritative desired-state source",
"the default HWLAB repo is an ephemeral one-run clone under the remote dump directory; --hwlab-repo is reserved for explicitly supplied clean diagnostic clones",
"the dedicated cache is owner-only and HWLAB-CD-only; it must keep full repo history so deploy/deploy.json promotion commits resolve, and one-run repos are copied with independent object stores rather than --shared alternates",
"preflight/apply --dry-run check required SecretRef object/key metadata without reading or printing Secret values",
"audit/status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs read-only/status paths with --skip-live-verify; no apply, rollout, lock mutation, DEV acceptance live verification, DB write, or secret read is executed",
"frontend transport keeps each host.ssh command below the provider-gateway short-command limit by uploading the remote runner in chunks, starting it as a remote background job, and polling compact result files",
"audit adds bounded read-only kubectl get/curl health probes for blocker classification; full stdout/stderr stays in temp dump paths, not reports/",
"real apply is structured refused and must remain with the host commander or unique CD runner",
],
@@ -524,5 +806,5 @@ export async function runHwlabCdCommand(args: string[]): Promise<Record<string,
const remoteCommand = buildRemoteCommand(options, runId);
if (options.transport === "local") return runLocalTransport(options, remoteCommand, runId);
const config = readConfig();
return runFrontendTransport(options, remoteCommand, config, runId);
return runFrontendTransport(options, config, runId);
}