pikasTech-unidesk/scripts/src/hwlab-cd.ts

import { spawn } from "node:child_process";
import { randomBytes } from "node:crypto";
import { createWriteStream, mkdirSync, readFileSync } from "node:fs";
import { mkdir } from "node:fs/promises";
import { join } from "node:path";
import { gzipSync } from "node:zlib";
import { readConfig, repoRoot, rootPath, type UniDeskConfig } from "./config";
import { d601NativeKubeconfig } from "./d601-k3s-guard";

type HwlabCdAction = "status" | "preflight" | "audit" | "apply";
type HwlabCdEnvironment = "dev";
type HwlabCdTransport = "frontend" | "local";

interface HwlabCdOptions {
  action: HwlabCdAction;
  environment: HwlabCdEnvironment;
  dryRun: boolean;
  repoPath: string | null;
  kubeconfig: string;
  providerId: string;
  transport: HwlabCdTransport;
  mainServerHost: string | null;
  timeoutMs: number;
}

interface CapturedCommand {
  id: string;
  command: string[];
  cwd: string;
  ok: boolean;
  exitCode: number | null;
  signal: NodeJS.Signals | null;
  timedOut: boolean;
  durationMs: number;
  stdoutText: string;
  stderrText: string;
  dump: {
    stdout: string;
    stderr: string;
    stdoutBytes: number;
    stderrBytes: number;
  };
}

interface FrontendSession {
  baseUrl: string;
  cookie: string;
}

interface FrontendHostSshResult {
  ok: boolean;
  taskId: string | null;
  taskStatus: unknown;
  stdout: string;
  stderr: string;
  exitCode: number | null;
  error: string;
  raw: unknown;
}

interface RemoteRunnerJob {
  pid: number | null;
  remoteRunDir: string | null;
  remoteCompactResultPath: string | null;
  remoteFullResultPath: string | null;
}

interface FetchJsonResult {
  ok: boolean;
  status?: number;
  body?: unknown;
  error?: string;
}

const defaultProviderId = "D601";
const defaultHwlabCdRepoPath = "~/.state/unidesk-hwlab-cd/<run-id>/ephemeral-repo/HWLAB";
const defaultHwlabCdCachePath = "~/.cache/unidesk/hwlab-cd/git-cache/HWLAB.git";
const rejectedRunnerHistoryRepoPath = "/home/ubuntu/hwlab";
const namespace = "hwlab-dev";
const lockName = "hwlab-dev-cd-lock";
const maxTimeoutMs = 60_000;
const defaultTimeoutMs = 45_000;
const remoteScriptSource = readFileSync(rootPath("scripts", "src", "hwlab-cd-remote-runner.cjs"), "utf8");

function isHelpArg(value: string | undefined): boolean {
  return value === "help" || value === "--help" || value === "-h";
}

function readOption(argv: string[], index: number, option: string): string {
  const value = argv[index];
  if (value === undefined || value.length === 0 || value.startsWith("--")) throw new Error(`${option} requires a value`);
  return value;
}

function parsePositiveInteger(value: string, option: string): number {
  const parsed = Number(value);
  if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`);
  return Math.min(parsed, maxTimeoutMs);
}

function parseTransport(value: string, option: string): HwlabCdTransport {
  if (value === "frontend" || value === "local") return value;
  throw new Error(`${option} must be one of: frontend, local`);
}

function envTransport(): HwlabCdTransport {
  return process.env.UNIDESK_HWLAB_CD_TRANSPORT === "local" ? "local" : "frontend";
}

function parseOptions(args: string[]): HwlabCdOptions {
  const [scope, actionArg] = args;
  if (scope !== "cd") throw new Error("hwlab usage: bun scripts/cli.ts hwlab cd status|preflight|audit|apply --env dev");
  if (actionArg !== "status" && actionArg !== "preflight" && actionArg !== "audit" && actionArg !== "apply") throw new Error("hwlab cd usage: status|preflight|audit|apply");

  const options: HwlabCdOptions = {
    action: actionArg,
    environment: "dev",
    dryRun: false,
    repoPath: null,
    kubeconfig: d601NativeKubeconfig,
    providerId: defaultProviderId,
    transport: envTransport(),
    mainServerHost: process.env.UNIDESK_MAIN_SERVER_IP ?? process.env.UNIDESK_MAIN_SERVER_HOST ?? null,
    timeoutMs: defaultTimeoutMs,
  };
  let envSeen = false;

  for (let index = 2; index < args.length; index += 1) {
    const arg = args[index] ?? "";
    if (arg === "--env") {
      const value = readOption(args, index + 1, arg);
      if (value !== "dev") throw new Error("hwlab cd only supports --env dev");
      options.environment = value;
      envSeen = true;
      index += 1;
    } else if (arg === "--dry-run") {
      options.dryRun = true;
    } else if (arg === "--hwlab-repo" || arg === "--repo") {
      options.repoPath = readOption(args, index + 1, arg);
      index += 1;
    } else if (arg === "--kubeconfig") {
      const value = readOption(args, index + 1, arg);
      if (value !== d601NativeKubeconfig) throw new Error(`hwlab cd requires the D601 native k3s kubeconfig: ${d601NativeKubeconfig}`);
      options.kubeconfig = value;
      index += 1;
    } else if (arg === "--provider-id") {
      options.providerId = readOption(args, index + 1, arg);
      index += 1;
    } else if (arg === "--main-server-ip" || arg === "--main-server-host") {
      options.mainServerHost = readOption(args, index + 1, arg);
      index += 1;
    } else if (arg === "--transport") {
      options.transport = parseTransport(readOption(args, index + 1, arg), arg);
      index += 1;
    } else if (arg === "--timeout-ms") {
      options.timeoutMs = parsePositiveInteger(readOption(args, index + 1, arg), arg);
      index += 1;
    } else if (!isHelpArg(arg)) {
      throw new Error(`unknown hwlab cd option: ${arg}`);
    }
  }

  if (!envSeen) throw new Error("hwlab cd requires --env dev");
  if (options.action === "apply" && !options.dryRun) {
    return options;
  }
  return options;
}

function makeRunId(): string {
  const timestamp = new Date().toISOString().replace(/[:.]/gu, "-");
  return `${timestamp}-${process.pid}-${randomBytes(4).toString("hex")}`;
}

function shellQuote(value: string): string {
  return `'${value.replace(/'/gu, "'\\''")}'`;
}

function b64Json(value: unknown): string {
  return Buffer.from(JSON.stringify(value), "utf8").toString("base64");
}

function buildRemoteOptions(options: HwlabCdOptions, runId: string): Record<string, unknown> {
  return {
    action: options.action,
    environment: options.environment,
    dryRun: options.dryRun,
    repoPath: options.repoPath,
    kubeconfig: options.kubeconfig,
    timeoutMs: options.timeoutMs,
    runId,
    compactStdout: options.transport !== "local",
  };
}

function buildRemoteInlineCommand(options: HwlabCdOptions, runId: string): string {
  return [
    `UNIDESK_HWLAB_CD_OPTIONS_B64=${shellQuote(b64Json(buildRemoteOptions(options, runId)))} node <<'UNIDESK_HWLAB_CD_JS'`,
    remoteScriptSource,
    "UNIDESK_HWLAB_CD_JS",
  ].join("\n");
}

function buildRemoteUploadedCommand(options: HwlabCdOptions, runId: string, remoteScriptPath: string): string {
  return `UNIDESK_HWLAB_CD_OPTIONS_B64=${shellQuote(b64Json(buildRemoteOptions(options, runId)))} node ${shellQuote(remoteScriptPath)}`;
}

function remoteRunnerDir(runId: string): string {
  return `$HOME/.state/unidesk-hwlab-cd/${safeRemoteId(runId)}`;
}

function buildRemoteStartCommand(options: HwlabCdOptions, runId: string, remoteScriptPath: string): string {
  const runDir = remoteRunnerDir(runId);
  const encodedOptions = shellQuote(b64Json(buildRemoteOptions(options, runId)));
  return [
    "umask 077",
    `run_dir="${runDir}"`,
    `mkdir -p "$run_dir"`,
    `rm -f "$run_dir/result.full.json" "$run_dir/result.compact.json" "$run_dir/runner.stdout.txt" "$run_dir/runner.stderr.txt" "$run_dir/runner.exit"`,
    `nohup sh -c 'UNIDESK_HWLAB_CD_OPTIONS_B64="$1" node "$2" > "$3" 2> "$4"; printf "%s\\n" "$?" > "$5"' sh ${encodedOptions} ${shellQuote(remoteScriptPath)} "$run_dir/runner.stdout.txt" "$run_dir/runner.stderr.txt" "$run_dir/runner.exit" >/dev/null 2>&1 & pid=$!`,
    `printf '{"pid":%s,"remoteRunDir":"%s","remoteCompactResultPath":"%s/result.compact.json","remoteFullResultPath":"%s/result.full.json"}\\n' "$pid" "$run_dir" "$run_dir" "$run_dir"`,
  ].join("; ");
}

function buildRemotePollCommand(runId: string): string {
  const runDir = remoteRunnerDir(runId);
  const pollJs = [
    "const fs=require(\"fs\")",
    "const dir=process.argv[1]",
    "const compact=`${dir}/result.compact.json`",
    "const exitPath=`${dir}/runner.exit`",
    "const tail=(p)=>{try{const b=fs.readFileSync(p);return b.slice(Math.max(0,b.length-500)).toString(\"utf8\")}catch{return \"\"}}",
    "if(fs.existsSync(compact)&&fs.statSync(compact).size>0){process.stdout.write(fs.readFileSync(compact,\"utf8\"));process.exit(0)}",
    "if(fs.existsSync(exitPath)){const raw=fs.readFileSync(exitPath,\"utf8\").trim();console.log(JSON.stringify({ok:false,status:\"blocked\",error:\"remote-runner-exited-without-result\",remoteRunDir:dir,exitCode:Number(raw),stdoutTail:tail(`${dir}/runner.stdout.txt`),stderrTail:tail(`${dir}/runner.stderr.txt`)}));process.exit(0)}",
    "console.log(JSON.stringify({ok:false,status:\"running\",remoteRunDir:dir}))",
  ].join(";");
  return `run_dir="${runDir}"; node -e ${shellQuote(pollJs)} "$run_dir"`;
}

function buildRemoteCommand(options: HwlabCdOptions, runId: string): string {
  return buildRemoteInlineCommand(options, runId);
}

function splitFixed(value: string, size: number): string[] {
  const chunks: string[] = [];
  for (let index = 0; index < value.length; index += size) chunks.push(value.slice(index, index + size));
  return chunks;
}

function safeRemoteId(value: string): string {
  return value.replace(/[^A-Za-z0-9_.-]/gu, "-");
}

function compactTail(value: string, maxLength = 1000): string {
  return value.slice(Math.max(0, value.length - maxLength));
}

function uploadCommandShape(providerId = defaultProviderId): string[] {
  return ["frontend", "/api/dispatch", providerId, "host.ssh", "upload remote runner in small chunks", "start nohup remote runner", "poll ~/.state/unidesk-hwlab-cd/<run-id>/result.compact.json"];
}

function buildUploadPlan(runId: string): { remoteDir: string; remotePath: string; commands: Array<{ id: string; command: string }> } {
  const remoteDir = "/tmp/unidesk-hwlab-cd";
  const remotePath = `${remoteDir}/remote-runner-${safeRemoteId(runId)}.cjs`;
  const encoded = gzipSync(Buffer.from(remoteScriptSource, "utf8")).toString("base64");
  const b64Path = `${remotePath}.gz.b64`;
  const commands = [
    {
      id: "init",
      command: `umask 077; mkdir -p ${shellQuote(remoteDir)}; rm -f ${shellQuote(remotePath)} ${shellQuote(b64Path)}; : > ${shellQuote(b64Path)}`,
    },
    ...splitFixed(encoded, 2400).map((chunk, index) => ({
      id: `chunk-${index + 1}`,
      command: `printf %s ${shellQuote(chunk)} >> ${shellQuote(b64Path)}`,
    })),
    {
      id: "finalize",
      command: `base64 -d ${shellQuote(b64Path)} | gzip -d > ${shellQuote(remotePath)}; chmod 700 ${shellQuote(remotePath)}; rm -f ${shellQuote(b64Path)}; test -s ${shellQuote(remotePath)}; wc -c ${shellQuote(remotePath)}`,
    },
  ];
  return { remoteDir, remotePath, commands };
}

function uploadStepView(step: { id: string; command: string }, result: FrontendHostSshResult): Record<string, unknown> {
  const remoteOptions = {
    id: step.id,
    ok: result.ok,
    taskId: result.taskId,
    taskStatus: result.taskStatus,
    exitCode: result.exitCode,
    commandBytes: step.command.length,
    stdoutTail: compactTail(result.stdout, 500),
    stderrTail: compactTail(result.stderr, 500),
    error: result.error,
  };
  return remoteOptions;
}

export function buildHwlabCdRemoteCommandForTest(args: string[]): string {
  return buildRemoteCommand(parseOptions(args), "test-run-id");
}

function frontendBaseUrl(host: string, config: UniDeskConfig): string {
  if (host.startsWith("http://") || host.startsWith("https://")) return host.replace(/\/+$/u, "");
  if (/:\d+$/u.test(host)) return `http://${host}`;
  return `http://${host}:${config.network.frontend.port}`;
}

async function readJson(url: string, init?: RequestInit, timeoutMs = 8000): Promise<FetchJsonResult> {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const response = await fetch(url, { ...init, signal: controller.signal });
    const text = await response.text();
    let body: unknown = text;
    try {
      body = text.length > 0 ? JSON.parse(text) : null;
    } catch {
      body = { text: text.slice(0, 2000) };
    }
    return { ok: response.ok, status: response.status, body };
  } catch (error) {
    return { ok: false, error: error instanceof Error ? error.message : String(error) };
  } finally {
    clearTimeout(timer);
  }
}

async function loginFrontend(host: string, config: UniDeskConfig): Promise<FrontendSession> {
  const baseUrl = frontendBaseUrl(host, config);
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), 8000);
  try {
    const raw = await fetch(`${baseUrl}/login`, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ username: config.auth.username, password: config.auth.password }),
      signal: controller.signal,
    });
    if (!raw.ok) throw new Error(`frontend login failed via ${baseUrl}: status=${raw.status}`);
    const cookie = raw.headers.get("set-cookie")?.split(";")[0] ?? "";
    if (cookie.length === 0) throw new Error(`frontend login via ${baseUrl} did not return a session cookie`);
    return { baseUrl, cookie };
  } finally {
    clearTimeout(timer);
  }
}

async function frontendJson(session: FrontendSession, path: string, init?: RequestInit, timeoutMs = 8000): Promise<FetchJsonResult> {
  const headers = new Headers(init?.headers);
  headers.set("cookie", session.cookie);
  if (init?.body !== undefined && !headers.has("content-type")) headers.set("content-type", "application/json");
  return readJson(`${session.baseUrl}${path}`, { ...init, headers }, timeoutMs);
}

function asRecord(value: unknown): Record<string, unknown> | null {
  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}

async function waitForFrontendTask(session: FrontendSession, taskId: string, timeoutMs: number): Promise<unknown> {
  const startedAt = Date.now();
  let latest: unknown = null;
  while (Date.now() - startedAt < timeoutMs) {
    latest = await frontendJson(session, `/api/tasks/${encodeURIComponent(taskId)}`, undefined, 8000);
    const task = asRecord(asRecord((latest as { body?: unknown }).body)?.task);
    const status = typeof task?.status === "string" ? task.status : null;
    if (status === "succeeded" || status === "failed") return { ok: true, task };
    await Bun.sleep(500);
  }
  return { ok: false, timeoutMs, latest };
}

async function runCaptured(command: string[], cwd: string, dumpDir: string, id: string, timeoutMs: number, env: NodeJS.ProcessEnv = process.env): Promise<CapturedCommand> {
  await mkdir(dumpDir, { recursive: true });
  const stdoutPath = join(dumpDir, `${id}.stdout.txt`);
  const stderrPath = join(dumpDir, `${id}.stderr.txt`);
  const stdout = createWriteStream(stdoutPath, { flags: "w", mode: 0o600 });
  const stderr = createWriteStream(stderrPath, { flags: "w", mode: 0o600 });
  const startedAt = Date.now();
  let stdoutText = "";
  let stderrText = "";
  let stdoutBytes = 0;
  let stderrBytes = 0;
  let timedOut = false;
  let spawnErrorMessage = "";
  const child = spawn(command[0] ?? "", command.slice(1), { cwd, env, stdio: ["ignore", "pipe", "pipe"] });
  const timer = setTimeout(() => {
    timedOut = true;
    child.kill("SIGTERM");
    setTimeout(() => child.kill("SIGKILL"), 1000).unref();
  }, timeoutMs);
  child.stdout.on("data", (chunk: Buffer) => {
    stdoutBytes += chunk.byteLength;
    stdout.write(chunk);
    if (stdoutText.length < 1_000_000) stdoutText += chunk.toString("utf8");
  });
  child.stderr.on("data", (chunk: Buffer) => {
    stderrBytes += chunk.byteLength;
    stderr.write(chunk);
    if (stderrText.length < 1_000_000) stderrText += chunk.toString("utf8");
  });
  const closed = await new Promise<{ exitCode: number | null; signal: NodeJS.Signals | null }>((resolveClose) => {
    child.on("error", (error) => {
      spawnErrorMessage = error.message;
    });
    child.on("close", (exitCode, signal) => resolveClose({ exitCode, signal }));
  });
  clearTimeout(timer);
  await new Promise<void>((resolveEnd) => stdout.end(resolveEnd));
  await new Promise<void>((resolveEnd) => stderr.end(resolveEnd));
  const exitCode = spawnErrorMessage.length > 0 && closed.exitCode === null ? 127 : closed.exitCode;
  return {
    id,
    command,
    cwd,
    ok: exitCode === 0 && !timedOut,
    exitCode,
    signal: closed.signal,
    timedOut,
    durationMs: Date.now() - startedAt,
    stdoutText,
    stderrText: stderrText || spawnErrorMessage,
    dump: { stdout: stdoutPath, stderr: stderrPath, stdoutBytes, stderrBytes },
  };
}

function parseRemoteStdout(stdout: string): Record<string, unknown> | null {
  const trimmed = stdout.trim();
  if (trimmed.length === 0) return null;
  try {
    return JSON.parse(trimmed) as Record<string, unknown>;
  } catch {
    const start = trimmed.indexOf("{");
    const end = trimmed.lastIndexOf("}");
    if (start >= 0 && end > start) {
      try {
        return JSON.parse(trimmed.slice(start, end + 1)) as Record<string, unknown>;
      } catch {
        return null;
      }
    }
    return null;
  }
}

function commandShape(providerId = defaultProviderId): string[] {
  return uploadCommandShape(providerId);
}

async function runLocalTransport(options: HwlabCdOptions, remoteCommand: string, runId: string): Promise<Record<string, unknown>> {
  const dumpDir = rootPath(".state", "hwlab-cd", runId);
  mkdirSync(dumpDir, { recursive: true, mode: 0o700 });
  const result = await runCaptured(["bash", "-c", remoteCommand], repoRoot, dumpDir, "local-remote-wrapper", options.timeoutMs, process.env);
  const parsed = parseRemoteStdout(result.stdoutText);
  if (parsed === null) {
    return {
      ok: false,
      env: options.environment,
      status: "blocked",
      error: "remote-empty-or-invalid-json",
      remote: {
        host: "local-test",
        providerId: options.providerId,
        transport: "local",
        commandShape: commandShape(options.providerId),
        commandCalls: ["scripts/dev-cd-apply.mjs"],
        stdoutDump: result.dump.stdout,
        stderrDump: result.dump.stderr,
        exitCode: result.exitCode,
        timedOut: result.timedOut,
      },
      dumpPath: dumpDir,
    };
  }
  return {
    ...parsed,
    remote: {
      host: "local-test",
      providerId: options.providerId,
      transport: "local",
      commandShape: commandShape(options.providerId),
      commandCalls: ["scripts/dev-cd-apply.mjs"],
      stdoutDump: result.dump.stdout,
      stderrDump: result.dump.stderr,
      exitCode: result.exitCode,
      timedOut: result.timedOut,
    },
    localDumpPath: dumpDir,
  };
}

async function dispatchFrontendHostSsh(
  session: FrontendSession,
  options: HwlabCdOptions,
  command: string,
  timeoutMs: number,
  source = "cli-hwlab-cd",
): Promise<FrontendHostSshResult> {
  const dispatch = await frontendJson(session, "/api/dispatch", {
    method: "POST",
    body: JSON.stringify({
      providerId: options.providerId,
      command: "host.ssh",
      payload: { source, mode: "exec", command, timeoutMs },
    }),
  }, 12_000);
  const taskId = String(asRecord(dispatch.body)?.taskId ?? "");
  if (!dispatch.ok || taskId.length === 0) {
    return {
      ok: false,
      taskId: taskId || null,
      taskStatus: null,
      stdout: "",
      stderr: "",
      exitCode: null,
      error: asRecord(dispatch.body)?.error as string ?? dispatch.error ?? "provider dispatch failed",
      raw: dispatch,
    };
  }
  const wait = await waitForFrontendTask(session, taskId, Math.min(timeoutMs + 5000, maxTimeoutMs));
  const task = asRecord((wait as { task?: unknown }).task);
  const result = asRecord(task?.result) ?? {};
  const exitCode = typeof result.exitCode === "number" ? result.exitCode : null;
  const stdout = typeof result.stdout === "string" ? result.stdout : "";
  const stderr = typeof result.stderr === "string" ? result.stderr : "";
  const error = typeof result.error === "string" ? result.error : "";
  return {
    ok: task?.status === "succeeded" && (exitCode === null || exitCode === 0),
    taskId,
    taskStatus: task?.status ?? null,
    stdout,
    stderr,
    exitCode,
    error,
    raw: task,
  };
}

async function uploadRemoteRunner(session: FrontendSession, options: HwlabCdOptions, runId: string, dumpDir: string): Promise<{ ok: boolean; path: string; steps: Record<string, unknown>[]; error: string }> {
  const plan = buildUploadPlan(runId);
  const steps: Record<string, unknown>[] = [];
  for (const step of plan.commands) {
    const result = await dispatchFrontendHostSsh(session, options, step.command, Math.min(options.timeoutMs, 20_000), "cli-hwlab-cd-upload");
    steps.push(uploadStepView(step, result));
    if (!result.ok) {
      await Bun.write(join(dumpDir, "frontend-upload.json"), `${JSON.stringify({ ok: false, path: plan.remotePath, steps }, null, 2)}\n`);
      return { ok: false, path: plan.remotePath, steps, error: result.error || result.stderr || `upload step ${step.id} failed` };
    }
  }
  await Bun.write(join(dumpDir, "frontend-upload.json"), `${JSON.stringify({ ok: true, path: plan.remotePath, steps }, null, 2)}\n`);
  return { ok: true, path: plan.remotePath, steps, error: "" };
}

function parseRemoteRunnerJob(stdout: string): RemoteRunnerJob | null {
  const parsed = parseRemoteStdout(stdout);
  if (parsed === null) return null;
  return {
    pid: typeof parsed.pid === "number" ? parsed.pid : null,
    remoteRunDir: typeof parsed.remoteRunDir === "string" ? parsed.remoteRunDir : null,
    remoteCompactResultPath: typeof parsed.remoteCompactResultPath === "string" ? parsed.remoteCompactResultPath : null,
    remoteFullResultPath: typeof parsed.remoteFullResultPath === "string" ? parsed.remoteFullResultPath : null,
  };
}

async function startRemoteRunner(
  session: FrontendSession,
  options: HwlabCdOptions,
  runId: string,
  remoteScriptPath: string,
  dumpDir: string,
): Promise<{ ok: boolean; job: RemoteRunnerJob | null; result: FrontendHostSshResult; dump: string }> {
  const result = await dispatchFrontendHostSsh(session, options, buildRemoteStartCommand(options, runId, remoteScriptPath), 8000, "cli-hwlab-cd-start");
  const stdoutDump = join(dumpDir, "frontend-runner-start.stdout.txt");
  const stderrDump = join(dumpDir, "frontend-runner-start.stderr.txt");
  await Bun.write(stdoutDump, result.stdout);
  await Bun.write(stderrDump, result.stderr);
  const job = result.ok ? parseRemoteRunnerJob(result.stdout) : null;
  const dump = join(dumpDir, "frontend-runner-start.json");
  await Bun.write(dump, `${JSON.stringify({ ok: result.ok && job !== null, job, taskId: result.taskId, taskStatus: result.taskStatus, exitCode: result.exitCode, stdoutDump, stderrDump, error: result.error }, null, 2)}\n`);
  return { ok: result.ok && job !== null, job, result, dump };
}

async function pollRemoteRunnerResult(
  session: FrontendSession,
  options: HwlabCdOptions,
  runId: string,
  dumpDir: string,
): Promise<{ parsed: Record<string, unknown> | null; stdout: string; stderr: string; result: FrontendHostSshResult | null; polls: Record<string, unknown>[]; timedOut: boolean; dump: string }> {
  const startedAt = Date.now();
  const polls: Record<string, unknown>[] = [];
  const pollCommand = buildRemotePollCommand(runId);
  let lastResult: FrontendHostSshResult | null = null;
  while (Date.now() - startedAt < options.timeoutMs) {
    const result = await dispatchFrontendHostSsh(session, options, pollCommand, 6000, "cli-hwlab-cd-poll");
    lastResult = result;
    const parsed = result.ok ? parseRemoteStdout(result.stdout) : null;
    polls.push({
      ok: result.ok,
      taskId: result.taskId,
      taskStatus: result.taskStatus,
      exitCode: result.exitCode,
      parsedStatus: parsed?.status ?? null,
      parsedError: parsed?.error ?? null,
      stdoutTail: compactTail(result.stdout, 500),
      stderrTail: compactTail(result.stderr, 500),
      error: result.error,
    });
    if (parsed !== null && parsed.status !== "running") {
      const dump = join(dumpDir, "frontend-runner-poll.json");
      await Bun.write(dump, `${JSON.stringify({ ok: true, polls }, null, 2)}\n`);
      return { parsed, stdout: result.stdout, stderr: result.stderr, result, polls, timedOut: false, dump };
    }
    await Bun.sleep(1000);
  }
  const dump = join(dumpDir, "frontend-runner-poll.json");
  await Bun.write(dump, `${JSON.stringify({ ok: false, timedOut: true, polls }, null, 2)}\n`);
  return { parsed: null, stdout: lastResult?.stdout ?? "", stderr: lastResult?.stderr ?? "", result: lastResult, polls, timedOut: true, dump };
}

async function runFrontendTransport(options: HwlabCdOptions, config: UniDeskConfig, runId: string): Promise<Record<string, unknown>> {
  const host = options.mainServerHost ?? config.network.publicHost;
  const dumpDir = rootPath(".state", "hwlab-cd", runId);
  mkdirSync(dumpDir, { recursive: true, mode: 0o700 });
  const session = await loginFrontend(host, config);
  const upload = await uploadRemoteRunner(session, options, runId, dumpDir);
  if (!upload.ok) {
    return {
      ok: false,
      env: options.environment,
      status: "blocked",
      error: "remote-runner-upload-failed",
      remote: {
        host: session.baseUrl,
        providerId: options.providerId,
        transport: "frontend",
        commandShape: commandShape(options.providerId),
        commandCalls: ["scripts/dev-cd-apply.mjs"],
        uploadPath: upload.path,
        uploadDump: join(dumpDir, "frontend-upload.json"),
      },
      dumpPath: dumpDir,
      upload,
      nextSafeCommand: "restore D601 host.ssh command dispatch or upload permissions, then rerun bun scripts/cli.ts hwlab cd status --env dev",
    };
  }

  const runner = await startRemoteRunner(session, options, runId, upload.path, dumpDir);
  if (!runner.ok) {
    return {
      ok: false,
      env: options.environment,
      status: "blocked",
      error: "remote-runner-start-failed",
      remote: {
        host: session.baseUrl,
        providerId: options.providerId,
        transport: "frontend",
        taskId: runner.result.taskId,
        taskStatus: runner.result.taskStatus,
        commandShape: commandShape(options.providerId),
        commandCalls: ["scripts/dev-cd-apply.mjs"],
        uploadPath: upload.path,
        uploadDump: join(dumpDir, "frontend-upload.json"),
        runnerStartDump: runner.dump,
        exitCode: runner.result.exitCode,
        error: runner.result.error,
      },
      dumpPath: dumpDir,
      nextSafeCommand: "restore D601 host.ssh background job dispatch, then rerun bun scripts/cli.ts hwlab cd status --env dev",
    };
  }

  const poll = await pollRemoteRunnerResult(session, options, runId, dumpDir);
  const stdout = poll.stdout;
  const stderr = poll.stderr;
  const stdoutDump = join(dumpDir, "frontend-task.stdout.txt");
  const stderrDump = join(dumpDir, "frontend-task.stderr.txt");
  await Bun.write(stdoutDump, stdout);
  await Bun.write(stderrDump, stderr);
  const parsed = poll.parsed ?? parseRemoteStdout(stdout);
  if (parsed === null) {
    return {
      ok: false,
      env: options.environment,
      status: "blocked",
      error: poll.timedOut ? "remote-runner-timeout" : "remote-empty-or-invalid-json",
      remote: {
        host: session.baseUrl,
        providerId: options.providerId,
        transport: "frontend",
        taskId: poll.result?.taskId ?? null,
        taskStatus: poll.result?.taskStatus ?? null,
        commandShape: commandShape(options.providerId),
        commandCalls: ["scripts/dev-cd-apply.mjs"],
        uploadPath: upload.path,
        uploadDump: join(dumpDir, "frontend-upload.json"),
        runnerStartDump: runner.dump,
        runnerPollDump: poll.dump,
        remoteRunDir: runner.job?.remoteRunDir ?? null,
        remoteFullResultPath: runner.job?.remoteFullResultPath ?? null,
        stdoutDump,
        stderrDump,
        exitCode: poll.result?.exitCode ?? null,
        error: poll.result?.error ?? "",
      },
      dumpPath: dumpDir,
    };
  }
  return {
    ...parsed,
    remote: {
      host: session.baseUrl,
      providerId: options.providerId,
      transport: "frontend",
      taskId: poll.result?.taskId ?? null,
      taskStatus: poll.result?.taskStatus ?? null,
      commandShape: commandShape(options.providerId),
      commandCalls: ["scripts/dev-cd-apply.mjs"],
      uploadPath: upload.path,
      uploadDump: join(dumpDir, "frontend-upload.json"),
      runnerStartDump: runner.dump,
      runnerPollDump: poll.dump,
      remoteRunDir: runner.job?.remoteRunDir ?? null,
      remoteFullResultPath: runner.job?.remoteFullResultPath ?? null,
      stdoutDump,
      stderrDump,
      exitCode: poll.result?.exitCode ?? null,
    },
    localDumpPath: dumpDir,
  };
}

function realApplyRefusal(options: HwlabCdOptions): Record<string, unknown> {
  return {
    ok: false,
    env: options.environment,
    status: "refused",
    action: "apply",
    dryRun: false,
    mutation: false,
    remote: {
      host: options.mainServerHost ?? "configured-main-server",
      providerId: options.providerId,
      transport: options.transport,
      commandShape: commandShape(options.providerId),
      commandCalls: ["scripts/dev-cd-apply.mjs"],
    },
    error: "host-commander-only-real-apply",
    summary: "UniDesk HWLAB CD wrapper does not execute live DEV apply, rollout, live verification, or CD lock mutation from this runner path.",
    controlledEntrypoint: "scripts/dev-cd-apply.mjs",
    controlledCommandShape: [
      "node",
      "scripts/dev-cd-apply.mjs",
      "--apply",
      "--confirm-dev",
      "--confirmed-non-production",
      "--write-report",
      "--kubeconfig",
      options.kubeconfig,
    ],
    nextSafeCommand: "bun scripts/cli.ts hwlab cd apply --env dev --dry-run",
    safety: {
      kubectlApplyExecuted: false,
      rolloutExecuted: false,
      liveVerifyExecuted: false,
      cdLockMutated: false,
      prodTouched: false,
      secretValuesRead: false,
      secretValuesPrinted: false,
    },
  };
}

export function hwlabHelp(): Record<string, unknown> {
  return {
    command: "hwlab cd",
    output: "json",
    usage: [
      "bun scripts/cli.ts hwlab cd status --env dev",
      "bun scripts/cli.ts hwlab cd audit --env dev",
      "bun scripts/cli.ts hwlab cd preflight --env dev",
      "bun scripts/cli.ts hwlab cd apply --env dev --dry-run",
    ],
    description: "Bounded UniDesk wrapper for the HWLAB DEV CD path. The default transport uploads a small remote runner to D601 through short host.ssh commands, creates a one-run HWLAB clone owned by the host.ssh user, then calls HWLAB repo-owned scripts/dev-cd-apply.mjs from the D601 side.",
    boundary: [
      `KUBECONFIG is forced to ${d601NativeKubeconfig}`,
      "docker-desktop, desktop-control-plane, and 127.0.0.1:11700 are refusal signals for the explicit D601 target",
      `default HWLAB CD repo is ${defaultHwlabCdRepoPath}, materialized from dedicated full bare cache ${defaultHwlabCdCachePath}; ${rejectedRunnerHistoryRepoPath} is rejected as runner history`,
      "deploy/deploy.json remains the authoritative desired-state source",
      "the default HWLAB repo is an ephemeral one-run clone under the remote dump directory; --hwlab-repo is reserved for explicitly supplied clean diagnostic clones",
      "the dedicated cache is owner-only and HWLAB-CD-only; it must keep full repo history so deploy/deploy.json promotion commits resolve, and one-run repos are copied with independent object stores rather than --shared alternates",
      "preflight/apply --dry-run check required SecretRef object/key metadata without reading or printing Secret values",
      "audit/status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs read-only/status paths with --skip-live-verify; no apply, rollout, lock mutation, DEV acceptance live verification, DB write, or secret read is executed",
      "frontend transport keeps each host.ssh command below the provider-gateway short-command limit by uploading the remote runner in chunks, starting it as a remote background job, and polling compact result files",
      "audit adds bounded read-only kubectl get/curl health probes for blocker classification; full stdout/stderr stays in temp dump paths, not reports/",
      "real apply is structured refused and must remain with the host commander or unique CD runner",
    ],
  };
}

export async function runHwlabCdCommand(args: string[]): Promise<Record<string, unknown>> {
  if (args.length === 0 || args.some(isHelpArg)) return { ok: true, ...hwlabHelp() };
  const options = parseOptions(args);
  if (options.action === "apply" && !options.dryRun) return realApplyRefusal(options);
  const runId = makeRunId();
  const remoteCommand = buildRemoteCommand(options, runId);
  if (options.transport === "local") return runLocalTransport(options, remoteCommand, runId);
  const config = readConfig();
  return runFrontendTransport(options, config, runId);
}