// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote module for scripts/src/hwlab-legacy-runtime.ts. // Moved mechanically from scripts/src/hwlab-legacy-runtime.ts:639-1204 for #903. import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand } from "../command"; import { cancelJob, listJobs, readJob, startJob } from "../jobs"; import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes"; import type { CommandJsonResult, LegacyRuntimeMonitorLane, OpenPullRequest, RemoteAsyncCommandSpec } from "./types"; import { tailText } from "./cleanup"; import { LEGACY_RUNTIME_PROVIDER, LEGACY_RUNTIME_SOURCE_BRANCH, LEGACY_RUNTIME_WORKSPACE, DEFAULT_RUNTIME_LANE_MASTER_ADMIN_API_KEY_LOCAL_ENV } from "./types"; export function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const index = args.indexOf(name); if (index === -1) return defaultValue; const raw = args[index + 1]; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return Math.min(value, maxValue); } export function shellQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; } export function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult { const startedAtMs = Date.now(); const result = runCommand(command, repoRoot, { timeoutMs }); const durationMs = Date.now() - startedAtMs; let parsed: unknown | null = null; if (result.stdout.trim().length > 0) { try { parsed = JSON.parse(result.stdout) as unknown; } catch { parsed = null; } } return { ok: result.exitCode === 0, command, exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr, parsed, durationMs, timedOut: result.timedOut, stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), stderrBytes: Buffer.byteLength(result.stderr, "utf8"), }; } export function commandJsonWithInput(command: string[], input: string, timeoutMs = 60_000): CommandJsonResult { const startedAtMs = Date.now(); const result = runCommand(command, repoRoot, { timeoutMs, input }); const durationMs = Date.now() - startedAtMs; let parsed: unknown | null = null; if (result.stdout.trim().length > 0) { try { parsed = JSON.parse(result.stdout) as unknown; } catch { parsed = null; } } return { ok: result.exitCode === 0, command, exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr, parsed, durationMs, timedOut: result.timedOut, stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), stderrBytes: Buffer.byteLength(result.stderr, "utf8"), }; } export function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult { return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs); } export function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } export function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } export function fieldBoolean(value: unknown): boolean | null { if (value === true || value === false) return value; if (typeof value !== "string") return null; const normalized = value.trim().toLowerCase(); if (normalized === "true" || normalized === "yes" || normalized === "1") return true; if (normalized === "false" || normalized === "no" || normalized === "0") return false; return null; } export function nested(value: unknown, keys: string[]): unknown { let current = value; for (const key of keys) current = record(current)[key]; return current; } export function expandedParsedRoot(result: CommandJsonResult): Record { const dumpPath = nested(result.parsed, ["data", "dump", "path"]); if (typeof dumpPath === "string" && existsSync(dumpPath)) { try { return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown); } catch { return record(result.parsed); } } return record(result.parsed); } export function commandData(result: CommandJsonResult): Record { return record(sanitizeCliOutput(record(expandedParsedRoot(result).data))); } export function compactCommandMetadata(result: CommandJsonResult, includeTail = false, includeCommand = false): Record { return { ok: result.ok, exitCode: result.exitCode, command: includeCommand ? redactedCommand(result.command) : undefined, stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), stderrBytes: Buffer.byteLength(result.stderr, "utf8"), stdoutTail: includeTail ? compactInlineText(result.stdout, 4000) : undefined, stderrTail: includeTail ? compactInlineText(result.stderr, 4000) : undefined, }; } export function textHash(value: string): string { return createHash("sha256").update(value).digest("hex").slice(0, 12); } export function generateHwlabApiKey(): string { return `hwl_live_${randomBytes(32).toString("base64url")}`; } export function hwlabApiKeyPrefix(value: string): string { const text = value.trim(); if (!text.startsWith("hwl_live_")) return ""; const remainder = text.slice("hwl_live_".length); const dot = remainder.indexOf("."); const segment = dot === -1 ? remainder : remainder.slice(0, dot); return `hwl_live_${segment}`.slice(0, 24); } export function runtimeLaneSecretFieldManager(spec: HwlabRuntimeLaneSpec): string { return `unidesk-hwlab-${spec.lane}-secret`; } export function runtimeLanePostgresSecretName(spec: HwlabRuntimeLaneSpec): string { return `${spec.runtimeNamespace}-postgres`; } export function runtimeLanePrimaryDbName(spec: HwlabRuntimeLaneSpec): string { return `hwlab_${spec.lane}`; } export function runtimeLaneOpenFgaSecretName(spec: HwlabRuntimeLaneSpec): string { return `hwlab-${spec.lane}-openfga`; } export function runtimeLaneMasterAdminApiKeySecretName(spec: HwlabRuntimeLaneSpec): string { return `hwlab-${spec.lane}-master-server-admin-api-key`; } export function runtimeLaneMasterAdminApiKeyLocalEnv(spec: HwlabRuntimeLaneSpec): string { if (spec.lane === "v02") return DEFAULT_RUNTIME_LANE_MASTER_ADMIN_API_KEY_LOCAL_ENV; return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`; } export function localMasterAdminApiKeyStatus(spec: HwlabRuntimeLaneSpec): Record { const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec); if (!existsSync(envPath)) { return { exists: false, path: envPath, mode: null, valueBytes: 0, keyPrefix: null }; } const stat = statSync(envPath); const content = readFileSync(envPath, "utf8"); const match = content.match(/^HWLAB_API_KEY=(.+)$/mu); const value = match?.[1]?.trim() ?? ""; return { exists: true, path: envPath, mode: `0${(stat.mode & 0o777).toString(8)}`, valueBytes: Buffer.byteLength(value, "utf8"), keyPrefix: value ? hwlabApiKeyPrefix(value) : null, validPrefix: value.startsWith("hwl_live_"), }; } export function readOrCreateLocalMasterAdminApiKey(spec: HwlabRuntimeLaneSpec, dryRun: boolean): { key: string | null; created: boolean; status: Record } { const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec); const existing = localMasterAdminApiKeyStatus(spec); if (existing.exists === true) { const content = readFileSync(envPath, "utf8"); const match = content.match(/^HWLAB_API_KEY=(.+)$/mu); const key = match?.[1]?.trim() ?? ""; if (!key.startsWith("hwl_live_")) throw new Error(`${envPath} exists but does not contain a hwl_live_ HWLAB_API_KEY`); if (!dryRun && existing.mode !== "0600") chmodSync(envPath, 0o600); return { key, created: false, status: localMasterAdminApiKeyStatus(spec) }; } if (dryRun) return { key: null, created: false, status: existing }; const key = generateHwlabApiKey(); mkdirSync(dirname(envPath), { recursive: true, mode: 0o700 }); writeFileSync(envPath, `# HWLAB ${spec.version} master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 }); return { key, created: true, status: localMasterAdminApiKeyStatus(spec) }; } export function redactLargePayloads(value: string): string { return value .replace(/hwl_live_[A-Za-z0-9._:-]{12,}/gu, (payload: string) => { return `hwl_live_`; }) .replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => { return `manifest_b64=''`; }) .replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => { return `manifest_b64=""`; }) .replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => { return `manifest_b64=`; }) .replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => { return ``; }); } export function compactInlineText(value: string, maxChars = 4000): string { const redacted = redactLargePayloads(value); if (redacted.length <= maxChars) return redacted; const headChars = Math.floor(maxChars * 0.35); const tailChars = maxChars - headChars; const omittedChars = redacted.length - headChars - tailChars; return [ redacted.slice(0, headChars), ``, redacted.slice(-tailChars), ].join("\n"); } export function sanitizeCliOutput(value: unknown): unknown { if (typeof value === "string") return compactInlineText(value); if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item)); if (typeof value !== "object" || value === null) return value; const output: Record = {}; for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item); return output; } export function redactedCommand(command: string[]): string[] { return command.map((arg) => compactInlineText(arg, 1200)); } export function compactCommandResult(result: CommandJsonResult): Record { const data = record(expandedParsedRoot(result).data); const stdout = String(data.stdout ?? result.stdout ?? ""); const stderr = String(data.stderr ?? result.stderr ?? ""); return { ok: isCommandSuccess(result), exitCode: result.exitCode, command: redactedCommand(result.command), stdout: compactInlineText(stdout.trim(), 4000), stderr: compactInlineText(stderr.trim(), 4000), stdoutBytes: Buffer.byteLength(stdout), stderrBytes: Buffer.byteLength(stderr), }; } export function compactControlPlaneRefresh(value: Record): Record { return { ok: value.ok === true, phase: value.phase ?? null, sourceCommit: value.sourceCommit ?? null, renderDir: value.renderDir ?? null, applyOk: nested(value, ["apply", "ok"]) ?? null, cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null, applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200), degradedReason: value.degradedReason ?? null, }; } export function compactTriggerCommandResult(value: Record | null): Record | null { if (value === null) return null; return { ok: value.ok ?? null, exitCode: value.exitCode ?? null, stdout: tailText(value.stdout, 1200), stderr: tailText(value.stderr, 1200), stdoutBytes: value.stdoutBytes ?? null, stderrBytes: value.stderrBytes ?? null, }; } export function isCommandSuccess(result: CommandJsonResult): boolean { if (!result.ok) return false; const topOk = record(result.parsed).ok; if (topOk === false) return false; const dataOk = nested(result.parsed, ["data", "ok"]); return dataOk !== false; } export function monitorBaseBranch(lane: LegacyRuntimeMonitorLane): string { return lane === "g14" ? LEGACY_RUNTIME_SOURCE_BRANCH : hwlabRuntimeLaneSpec(lane).sourceBranch; } export function extractPullRequests(result: CommandJsonResult, baseBranch = LEGACY_RUNTIME_SOURCE_BRANCH): OpenPullRequest[] { const prs = nested(result.parsed, ["data", "pullRequests"]); if (!Array.isArray(prs)) return []; return prs .map((item) => record(item)) .filter((item) => item.baseRefName === baseBranch || record(item.base).ref === baseBranch) .map((item) => ({ number: Number(item.number), title: typeof item.title === "string" ? item.title : undefined, url: typeof item.url === "string" ? item.url : undefined, baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined, headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined, mergeable: typeof item.mergeable === "string" ? item.mergeable : null, mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null, draft: item.draft === true, })) .filter((item) => Number.isInteger(item.number) && item.number > 0); } export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } export function printEvent(event: string, data: Record = {}): void { process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`); } export function printProgressEvent(event: string, data: Record = {}): void { process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`); } export function printDefaultRuntimeLanePrMonitorProgress(data: Record = {}): void { printProgressEvent("hwlab.v02.pr-monitor.progress", data); } export function printRuntimeLanePrMonitorProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { printProgressEvent("hwlab.runtime-lane.pr-monitor.progress", { lane: spec.lane, node: spec.nodeId, ...data, }); } export function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { printProgressEvent("hwlab.runtime-lane.trigger.progress", { lane: spec.lane, node: spec.nodeId, ...data, }); } export function shortSha(sha: string): string { return sha.slice(0, 12); } export function precheckWorkspace(): CommandJsonResult { return cliJson(["ssh", `${LEGACY_RUNTIME_PROVIDER}:${LEGACY_RUNTIME_WORKSPACE}`, "sh", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000); } export function legacyRuntimeHostScript(script: string, timeoutMs = 120_000): CommandJsonResult { return cliJson(["ssh", LEGACY_RUNTIME_PROVIDER, "sh", "--", script], timeoutMs); } export function legacyRuntimeK3s(args: string[], timeoutMs = 60_000): CommandJsonResult { return cliJson(["ssh", `${LEGACY_RUNTIME_PROVIDER}:k3s`, ...args], timeoutMs); } export function legacyRuntimeK3sInlineScriptWithInput(script: string, input: string, timeoutMs = 60_000): CommandJsonResult { return commandJsonWithInput(["bun", "scripts/cli.ts", "ssh", `${LEGACY_RUNTIME_PROVIDER}:k3s`, "sh", "--", script], input, timeoutMs); } export function remoteAsyncStatusDir(label: string): string { return `/tmp/hwlab-${label}-async`; } export function remoteAsyncStatusPath(label: string, token: string): string { return `${remoteAsyncStatusDir(label)}/${token}.json`; } export function remoteAsyncStartScript(spec: RemoteAsyncCommandSpec): string { const statusPath = remoteAsyncStatusPath(spec.label, spec.token); const donePath = `${statusPath}.done`; const stdoutPath = `${statusPath}.stdout`; const stderrPath = `${statusPath}.stderr`; const commandB64 = Buffer.from(spec.script, "utf8").toString("base64"); return [ "set -eu", `status_dir=${shellQuote(remoteAsyncStatusDir(spec.label))}`, `status_path=${shellQuote(statusPath)}`, `done_path=${shellQuote(donePath)}`, `stdout_path=${shellQuote(stdoutPath)}`, `stderr_path=${shellQuote(stderrPath)}`, `command_b64=${shellQuote(commandB64)}`, "mkdir -p \"$status_dir\"", "rm -f \"$status_path\" \"$done_path\" \"$stdout_path\" \"$stderr_path\"", "command_path=\"$status_path.command.sh\"", "printf '%s' \"$command_b64\" | base64 -d > \"$command_path\"", "chmod +x \"$command_path\"", "node -e 'const fs=require(\"fs\"); const p=process.argv[1]; const payload={ok:null,state:\"running\",pid:Number(process.argv[2]),startedAt:new Date().toISOString(),finishedAt:null,exitCode:null,stdout:\"\",stderr:\"\"}; fs.writeFileSync(p, JSON.stringify(payload));' \"$status_path\" $$", "(", " set +e", " sh \"$command_path\" >\"$stdout_path\" 2>\"$stderr_path\"", " code=$?", " node - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'NODE'", "const fs = require('fs');", "const [statusPath, stdoutPath, stderrPath, codeRaw] = process.argv.slice(2);", "const code = Number(codeRaw);", "const read = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };", "fs.writeFileSync(statusPath, JSON.stringify({", " ok: code === 0,", " state: 'finished',", " pid: null,", " startedAt: null,", " finishedAt: new Date().toISOString(),", " exitCode: code,", " stdout: read(stdoutPath),", " stderr: read(stderrPath),", "}));", "NODE", " touch \"$done_path\"", ") >/dev/null 2>&1 &", "pid=$!", "node - \"$status_path\" \"$pid\" <<'NODE'", "const fs = require('fs');", "const [statusPath, pidRaw] = process.argv.slice(2);", "const current = JSON.parse(fs.readFileSync(statusPath, 'utf8'));", "current.pid = Number(pidRaw);", "fs.writeFileSync(statusPath, JSON.stringify(current));", "console.log(JSON.stringify({ ok: true, state: current.state, pid: current.pid, statusPath }));", "NODE", ].join("\n"); } export function remoteAsyncStatusScript(label: string, token: string): string { const statusPath = remoteAsyncStatusPath(label, token); const donePath = `${statusPath}.done`; return [ "set -eu", `status_path=${shellQuote(statusPath)}`, `done_path=${shellQuote(donePath)}`, "if [ ! -f \"$status_path\" ]; then", " printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing\"}'", " exit 0", "fi", "if [ -f \"$done_path\" ]; then", " cat \"$status_path\"", " exit 0", "fi", "node - \"$status_path\" <<'NODE'", "const fs = require('fs');", "const statusPath = process.argv[2];", "const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));", "payload.state = payload.state || 'running';", "payload.ok = payload.ok ?? null;", "payload.stdout = payload.stdout || '';", "payload.stderr = payload.stderr || '';", "console.log(JSON.stringify(payload));", "NODE", ].join("\n"); } export function remoteAsyncKillScript(label: string, token: string): string { const statusPath = remoteAsyncStatusPath(label, token); const donePath = `${statusPath}.done`; return [ "set -eu", `status_path=${shellQuote(statusPath)}`, `done_path=${shellQuote(donePath)}`, "if [ ! -f \"$status_path\" ]; then", " printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing during kill\"}'", " exit 0", "fi", "node - \"$status_path\" <<'NODE'", "const fs = require('fs');", "const statusPath = process.argv[2];", "const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));", "if (Number.isInteger(payload.pid) && payload.pid > 0) {", " try { process.kill(payload.pid, 'TERM'); } catch {}", "}", "payload.ok = false;", "payload.state = 'timeout-killed';", "payload.exitCode = 124;", "payload.finishedAt = new Date().toISOString();", "payload.stderr = `${payload.stderr || ''}\\nremote async command exceeded local poll timeout and was terminated`;", "fs.writeFileSync(statusPath, JSON.stringify(payload));", "console.log(JSON.stringify(payload));", "NODE", "touch \"$done_path\"", ].join("\n"); } export function parseRemoteAsyncPayload(result: CommandJsonResult): Record { const text = result.stdout.trim(); if (text.length === 0) return {}; try { return record(JSON.parse(text) as unknown); } catch { return {}; } } export function runLegacyRuntimeK3sRemoteAsync(spec: RemoteAsyncCommandSpec): CommandJsonResult { const startedAtMs = Date.now(); const start = legacyRuntimeK3s(["sh", "--", remoteAsyncStartScript(spec)], 55_000); const startPayload = parseRemoteAsyncPayload(start); if (!isCommandSuccess(start) || startPayload.ok !== true) return start; const deadlineMs = startedAtMs + Math.max(1, spec.timeoutSeconds) * 1000; let attempts = 0; let lastStatus: CommandJsonResult | null = null; while (Date.now() <= deadlineMs) { attempts += 1; const wait = runCommand(["sleep", attempts <= 1 ? "1" : "2"], repoRoot, { timeoutMs: 3_000 }); if (wait.exitCode !== 0 && wait.timedOut) break; const status = legacyRuntimeK3s(["sh", "--", remoteAsyncStatusScript(spec.label, spec.token)], 55_000); lastStatus = status; const payload = parseRemoteAsyncPayload(status); const state = typeof payload.state === "string" ? payload.state : null; if (state === "finished" || payload.ok === true || payload.ok === false) { return { ok: payload.ok === true, command: spec.command, exitCode: typeof payload.exitCode === "number" ? payload.exitCode : status.exitCode, stdout: typeof payload.stdout === "string" ? payload.stdout : status.stdout, stderr: typeof payload.stderr === "string" ? payload.stderr : status.stderr, parsed: { ok: payload.ok === true, data: { mode: "remote-async", label: spec.label, token: spec.token, statusPath: remoteAsyncStatusPath(spec.label, spec.token), attempts, elapsedMs: Date.now() - startedAtMs, state, pid: payload.pid ?? null, }, }, }; } } const killed = legacyRuntimeK3s(["sh", "--", remoteAsyncKillScript(spec.label, spec.token)], 55_000); const payload = parseRemoteAsyncPayload(killed); return { ok: false, command: spec.command, exitCode: 124, stdout: typeof payload.stdout === "string" ? payload.stdout : "", stderr: [ `remote async command timed out after ${spec.timeoutSeconds}s`, typeof payload.stderr === "string" ? payload.stderr : "", lastStatus === null ? "" : `last status stderr: ${lastStatus.stderr.trim()}`, killed.stderr.trim(), ].filter(Boolean).join("\n"), parsed: { ok: false, data: { mode: "remote-async", label: spec.label, token: spec.token, statusPath: remoteAsyncStatusPath(spec.label, spec.token), attempts, elapsedMs: Date.now() - startedAtMs, state: typeof payload.state === "string" ? payload.state : "timeout", pid: payload.pid ?? null, killed: compactCommandMetadata(killed, true, false), }, }, }; }