Codex
582 lines
24 KiB
TypeScript
582 lines
24 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote module for scripts/src/hwlab-legacy-runtime.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-legacy-runtime.ts:639-1204 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult, LegacyRuntimeMonitorLane, OpenPullRequest, RemoteAsyncCommandSpec } from "./types";
|
|
import { tailText } from "./cleanup";
|
|
import { LEGACY_RUNTIME_PROVIDER, LEGACY_RUNTIME_SOURCE_BRANCH, LEGACY_RUNTIME_WORKSPACE, DEFAULT_RUNTIME_LANE_MASTER_ADMIN_API_KEY_LOCAL_ENV } from "./types";
|
|
|
|
export function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return defaultValue;
|
|
const raw = args[index + 1];
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
export function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
|
}
|
|
|
|
export function commandJson(command: string[], timeoutMs = 60_000): CommandJsonResult {
|
|
const startedAtMs = Date.now();
|
|
const result = runCommand(command, repoRoot, { timeoutMs });
|
|
const durationMs = Date.now() - startedAtMs;
|
|
let parsed: unknown | null = null;
|
|
if (result.stdout.trim().length > 0) {
|
|
try {
|
|
parsed = JSON.parse(result.stdout) as unknown;
|
|
} catch {
|
|
parsed = null;
|
|
}
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
command,
|
|
exitCode: result.exitCode,
|
|
stdout: result.stdout,
|
|
stderr: result.stderr,
|
|
parsed,
|
|
durationMs,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
};
|
|
}
|
|
|
|
export function commandJsonWithInput(command: string[], input: string, timeoutMs = 60_000): CommandJsonResult {
|
|
const startedAtMs = Date.now();
|
|
const result = runCommand(command, repoRoot, { timeoutMs, input });
|
|
const durationMs = Date.now() - startedAtMs;
|
|
let parsed: unknown | null = null;
|
|
if (result.stdout.trim().length > 0) {
|
|
try {
|
|
parsed = JSON.parse(result.stdout) as unknown;
|
|
} catch {
|
|
parsed = null;
|
|
}
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
command,
|
|
exitCode: result.exitCode,
|
|
stdout: result.stdout,
|
|
stderr: result.stderr,
|
|
parsed,
|
|
durationMs,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
};
|
|
}
|
|
|
|
export function cliJson(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
|
return commandJson(["bun", "scripts/cli.ts", ...args], timeoutMs);
|
|
}
|
|
|
|
export function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
export function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
export function fieldBoolean(value: unknown): boolean | null {
|
|
if (value === true || value === false) return value;
|
|
if (typeof value !== "string") return null;
|
|
const normalized = value.trim().toLowerCase();
|
|
if (normalized === "true" || normalized === "yes" || normalized === "1") return true;
|
|
if (normalized === "false" || normalized === "no" || normalized === "0") return false;
|
|
return null;
|
|
}
|
|
|
|
export function nested(value: unknown, keys: string[]): unknown {
|
|
let current = value;
|
|
for (const key of keys) current = record(current)[key];
|
|
return current;
|
|
}
|
|
|
|
export function expandedParsedRoot(result: CommandJsonResult): Record<string, unknown> {
|
|
const dumpPath = nested(result.parsed, ["data", "dump", "path"]);
|
|
if (typeof dumpPath === "string" && existsSync(dumpPath)) {
|
|
try {
|
|
return record(JSON.parse(readFileSync(dumpPath, "utf8")) as unknown);
|
|
} catch {
|
|
return record(result.parsed);
|
|
}
|
|
}
|
|
return record(result.parsed);
|
|
}
|
|
|
|
export function commandData(result: CommandJsonResult): Record<string, unknown> {
|
|
return record(sanitizeCliOutput(record(expandedParsedRoot(result).data)));
|
|
}
|
|
|
|
export function compactCommandMetadata(result: CommandJsonResult, includeTail = false, includeCommand = false): Record<string, unknown> {
|
|
return {
|
|
ok: result.ok,
|
|
exitCode: result.exitCode,
|
|
command: includeCommand ? redactedCommand(result.command) : undefined,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
stdoutTail: includeTail ? compactInlineText(result.stdout, 4000) : undefined,
|
|
stderrTail: includeTail ? compactInlineText(result.stderr, 4000) : undefined,
|
|
};
|
|
}
|
|
|
|
export function textHash(value: string): string {
|
|
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
}
|
|
|
|
export function generateHwlabApiKey(): string {
|
|
return `hwl_live_${randomBytes(32).toString("base64url")}`;
|
|
}
|
|
|
|
export function hwlabApiKeyPrefix(value: string): string {
|
|
const text = value.trim();
|
|
if (!text.startsWith("hwl_live_")) return "";
|
|
const remainder = text.slice("hwl_live_".length);
|
|
const dot = remainder.indexOf(".");
|
|
const segment = dot === -1 ? remainder : remainder.slice(0, dot);
|
|
return `hwl_live_${segment}`.slice(0, 24);
|
|
}
|
|
|
|
export function runtimeLaneSecretFieldManager(spec: HwlabRuntimeLaneSpec): string {
|
|
return `unidesk-hwlab-${spec.lane}-secret`;
|
|
}
|
|
|
|
export function runtimeLanePostgresSecretName(spec: HwlabRuntimeLaneSpec): string {
|
|
return `${spec.runtimeNamespace}-postgres`;
|
|
}
|
|
|
|
export function runtimeLanePrimaryDbName(spec: HwlabRuntimeLaneSpec): string {
|
|
return `hwlab_${spec.lane}`;
|
|
}
|
|
|
|
export function runtimeLaneOpenFgaSecretName(spec: HwlabRuntimeLaneSpec): string {
|
|
return `hwlab-${spec.lane}-openfga`;
|
|
}
|
|
|
|
export function runtimeLaneMasterAdminApiKeySecretName(spec: HwlabRuntimeLaneSpec): string {
|
|
return `hwlab-${spec.lane}-master-server-admin-api-key`;
|
|
}
|
|
|
|
export function runtimeLaneMasterAdminApiKeyLocalEnv(spec: HwlabRuntimeLaneSpec): string {
|
|
if (spec.lane === "v02") return DEFAULT_RUNTIME_LANE_MASTER_ADMIN_API_KEY_LOCAL_ENV;
|
|
return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`;
|
|
}
|
|
|
|
export function localMasterAdminApiKeyStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
|
|
if (!existsSync(envPath)) {
|
|
return { exists: false, path: envPath, mode: null, valueBytes: 0, keyPrefix: null };
|
|
}
|
|
const stat = statSync(envPath);
|
|
const content = readFileSync(envPath, "utf8");
|
|
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
|
|
const value = match?.[1]?.trim() ?? "";
|
|
return {
|
|
exists: true,
|
|
path: envPath,
|
|
mode: `0${(stat.mode & 0o777).toString(8)}`,
|
|
valueBytes: Buffer.byteLength(value, "utf8"),
|
|
keyPrefix: value ? hwlabApiKeyPrefix(value) : null,
|
|
validPrefix: value.startsWith("hwl_live_"),
|
|
};
|
|
}
|
|
|
|
export function readOrCreateLocalMasterAdminApiKey(spec: HwlabRuntimeLaneSpec, dryRun: boolean): { key: string | null; created: boolean; status: Record<string, unknown> } {
|
|
const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec);
|
|
const existing = localMasterAdminApiKeyStatus(spec);
|
|
if (existing.exists === true) {
|
|
const content = readFileSync(envPath, "utf8");
|
|
const match = content.match(/^HWLAB_API_KEY=(.+)$/mu);
|
|
const key = match?.[1]?.trim() ?? "";
|
|
if (!key.startsWith("hwl_live_")) throw new Error(`${envPath} exists but does not contain a hwl_live_ HWLAB_API_KEY`);
|
|
if (!dryRun && existing.mode !== "0600") chmodSync(envPath, 0o600);
|
|
return { key, created: false, status: localMasterAdminApiKeyStatus(spec) };
|
|
}
|
|
if (dryRun) return { key: null, created: false, status: existing };
|
|
const key = generateHwlabApiKey();
|
|
mkdirSync(dirname(envPath), { recursive: true, mode: 0o700 });
|
|
writeFileSync(envPath, `# HWLAB ${spec.version} master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 });
|
|
return { key, created: true, status: localMasterAdminApiKeyStatus(spec) };
|
|
}
|
|
|
|
export function redactLargePayloads(value: string): string {
|
|
return value
|
|
.replace(/hwl_live_[A-Za-z0-9._:-]{12,}/gu, (payload: string) => {
|
|
return `hwl_live_<redacted chars=${payload.length} sha256=${textHash(payload)}>`;
|
|
})
|
|
.replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => {
|
|
return `manifest_b64='<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>'`;
|
|
})
|
|
.replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => {
|
|
return `manifest_b64="<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>"`;
|
|
})
|
|
.replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => {
|
|
return `manifest_b64=<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
|
})
|
|
.replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => {
|
|
return `<omitted base64 chars=${payload.length} sha256=${textHash(payload)}>`;
|
|
});
|
|
}
|
|
|
|
export function compactInlineText(value: string, maxChars = 4000): string {
|
|
const redacted = redactLargePayloads(value);
|
|
if (redacted.length <= maxChars) return redacted;
|
|
const headChars = Math.floor(maxChars * 0.35);
|
|
const tailChars = maxChars - headChars;
|
|
const omittedChars = redacted.length - headChars - tailChars;
|
|
return [
|
|
redacted.slice(0, headChars),
|
|
`<truncated chars=${omittedChars} sha256=${textHash(redacted)}>`,
|
|
redacted.slice(-tailChars),
|
|
].join("\n");
|
|
}
|
|
|
|
export function sanitizeCliOutput(value: unknown): unknown {
|
|
if (typeof value === "string") return compactInlineText(value);
|
|
if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item));
|
|
if (typeof value !== "object" || value === null) return value;
|
|
const output: Record<string, unknown> = {};
|
|
for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item);
|
|
return output;
|
|
}
|
|
|
|
export function redactedCommand(command: string[]): string[] {
|
|
return command.map((arg) => compactInlineText(arg, 1200));
|
|
}
|
|
|
|
export function compactCommandResult(result: CommandJsonResult): Record<string, unknown> {
|
|
const data = record(expandedParsedRoot(result).data);
|
|
const stdout = String(data.stdout ?? result.stdout ?? "");
|
|
const stderr = String(data.stderr ?? result.stderr ?? "");
|
|
return {
|
|
ok: isCommandSuccess(result),
|
|
exitCode: result.exitCode,
|
|
command: redactedCommand(result.command),
|
|
stdout: compactInlineText(stdout.trim(), 4000),
|
|
stderr: compactInlineText(stderr.trim(), 4000),
|
|
stdoutBytes: Buffer.byteLength(stdout),
|
|
stderrBytes: Buffer.byteLength(stderr),
|
|
};
|
|
}
|
|
|
|
export function compactControlPlaneRefresh(value: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
ok: value.ok === true,
|
|
phase: value.phase ?? null,
|
|
sourceCommit: value.sourceCommit ?? null,
|
|
renderDir: value.renderDir ?? null,
|
|
applyOk: nested(value, ["apply", "ok"]) ?? null,
|
|
cleanupOk: nested(value, ["cleanupObsoleteCronJobs", "ok"]) ?? null,
|
|
applyStdout: tailText(nested(value, ["apply", "stdout"]), 1200),
|
|
degradedReason: value.degradedReason ?? null,
|
|
};
|
|
}
|
|
|
|
export function compactTriggerCommandResult(value: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (value === null) return null;
|
|
return {
|
|
ok: value.ok ?? null,
|
|
exitCode: value.exitCode ?? null,
|
|
stdout: tailText(value.stdout, 1200),
|
|
stderr: tailText(value.stderr, 1200),
|
|
stdoutBytes: value.stdoutBytes ?? null,
|
|
stderrBytes: value.stderrBytes ?? null,
|
|
};
|
|
}
|
|
|
|
export function isCommandSuccess(result: CommandJsonResult): boolean {
|
|
if (!result.ok) return false;
|
|
const topOk = record(result.parsed).ok;
|
|
if (topOk === false) return false;
|
|
const dataOk = nested(result.parsed, ["data", "ok"]);
|
|
return dataOk !== false;
|
|
}
|
|
|
|
export function monitorBaseBranch(lane: LegacyRuntimeMonitorLane): string {
|
|
return lane === "g14" ? LEGACY_RUNTIME_SOURCE_BRANCH : hwlabRuntimeLaneSpec(lane).sourceBranch;
|
|
}
|
|
|
|
export function extractPullRequests(result: CommandJsonResult, baseBranch = LEGACY_RUNTIME_SOURCE_BRANCH): OpenPullRequest[] {
|
|
const prs = nested(result.parsed, ["data", "pullRequests"]);
|
|
if (!Array.isArray(prs)) return [];
|
|
return prs
|
|
.map((item) => record(item))
|
|
.filter((item) => item.baseRefName === baseBranch || record(item.base).ref === baseBranch)
|
|
.map((item) => ({
|
|
number: Number(item.number),
|
|
title: typeof item.title === "string" ? item.title : undefined,
|
|
url: typeof item.url === "string" ? item.url : undefined,
|
|
baseRefName: typeof item.baseRefName === "string" ? item.baseRefName : typeof record(item.base).ref === "string" ? record(item.base).ref as string : undefined,
|
|
headRefName: typeof item.headRefName === "string" ? item.headRefName : typeof record(item.head).ref === "string" ? record(item.head).ref as string : undefined,
|
|
mergeable: typeof item.mergeable === "string" ? item.mergeable : null,
|
|
mergeStateStatus: typeof item.mergeStateStatus === "string" ? item.mergeStateStatus : null,
|
|
draft: item.draft === true,
|
|
}))
|
|
.filter((item) => Number.isInteger(item.number) && item.number > 0);
|
|
}
|
|
|
|
export function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
export function printEvent(event: string, data: Record<string, unknown> = {}): void {
|
|
process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
|
}
|
|
|
|
export function printProgressEvent(event: string, data: Record<string, unknown> = {}): void {
|
|
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...data })}\n`);
|
|
}
|
|
|
|
export function printDefaultRuntimeLanePrMonitorProgress(data: Record<string, unknown> = {}): void {
|
|
printProgressEvent("hwlab.v02.pr-monitor.progress", data);
|
|
}
|
|
|
|
export function printRuntimeLanePrMonitorProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
|
printProgressEvent("hwlab.runtime-lane.pr-monitor.progress", {
|
|
lane: spec.lane,
|
|
node: spec.nodeId,
|
|
...data,
|
|
});
|
|
}
|
|
|
|
export function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
|
printProgressEvent("hwlab.runtime-lane.trigger.progress", {
|
|
lane: spec.lane,
|
|
node: spec.nodeId,
|
|
...data,
|
|
});
|
|
}
|
|
|
|
export function shortSha(sha: string): string {
|
|
return sha.slice(0, 12);
|
|
}
|
|
|
|
export function precheckWorkspace(): CommandJsonResult {
|
|
return cliJson(["ssh", `${LEGACY_RUNTIME_PROVIDER}:${LEGACY_RUNTIME_WORKSPACE}`, "sh", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
|
|
}
|
|
|
|
export function legacyRuntimeHostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
|
|
return cliJson(["ssh", LEGACY_RUNTIME_PROVIDER, "sh", "--", script], timeoutMs);
|
|
}
|
|
|
|
export function legacyRuntimeK3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
|
|
return cliJson(["ssh", `${LEGACY_RUNTIME_PROVIDER}:k3s`, ...args], timeoutMs);
|
|
}
|
|
|
|
export function legacyRuntimeK3sInlineScriptWithInput(script: string, input: string, timeoutMs = 60_000): CommandJsonResult {
|
|
return commandJsonWithInput(["bun", "scripts/cli.ts", "ssh", `${LEGACY_RUNTIME_PROVIDER}:k3s`, "sh", "--", script], input, timeoutMs);
|
|
}
|
|
|
|
export function remoteAsyncStatusDir(label: string): string {
|
|
return `/tmp/hwlab-${label}-async`;
|
|
}
|
|
|
|
export function remoteAsyncStatusPath(label: string, token: string): string {
|
|
return `${remoteAsyncStatusDir(label)}/${token}.json`;
|
|
}
|
|
|
|
export function remoteAsyncStartScript(spec: RemoteAsyncCommandSpec): string {
|
|
const statusPath = remoteAsyncStatusPath(spec.label, spec.token);
|
|
const donePath = `${statusPath}.done`;
|
|
const stdoutPath = `${statusPath}.stdout`;
|
|
const stderrPath = `${statusPath}.stderr`;
|
|
const commandB64 = Buffer.from(spec.script, "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`status_dir=${shellQuote(remoteAsyncStatusDir(spec.label))}`,
|
|
`status_path=${shellQuote(statusPath)}`,
|
|
`done_path=${shellQuote(donePath)}`,
|
|
`stdout_path=${shellQuote(stdoutPath)}`,
|
|
`stderr_path=${shellQuote(stderrPath)}`,
|
|
`command_b64=${shellQuote(commandB64)}`,
|
|
"mkdir -p \"$status_dir\"",
|
|
"rm -f \"$status_path\" \"$done_path\" \"$stdout_path\" \"$stderr_path\"",
|
|
"command_path=\"$status_path.command.sh\"",
|
|
"printf '%s' \"$command_b64\" | base64 -d > \"$command_path\"",
|
|
"chmod +x \"$command_path\"",
|
|
"node -e 'const fs=require(\"fs\"); const p=process.argv[1]; const payload={ok:null,state:\"running\",pid:Number(process.argv[2]),startedAt:new Date().toISOString(),finishedAt:null,exitCode:null,stdout:\"\",stderr:\"\"}; fs.writeFileSync(p, JSON.stringify(payload));' \"$status_path\" $$",
|
|
"(",
|
|
" set +e",
|
|
" sh \"$command_path\" >\"$stdout_path\" 2>\"$stderr_path\"",
|
|
" code=$?",
|
|
" node - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const [statusPath, stdoutPath, stderrPath, codeRaw] = process.argv.slice(2);",
|
|
"const code = Number(codeRaw);",
|
|
"const read = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };",
|
|
"fs.writeFileSync(statusPath, JSON.stringify({",
|
|
" ok: code === 0,",
|
|
" state: 'finished',",
|
|
" pid: null,",
|
|
" startedAt: null,",
|
|
" finishedAt: new Date().toISOString(),",
|
|
" exitCode: code,",
|
|
" stdout: read(stdoutPath),",
|
|
" stderr: read(stderrPath),",
|
|
"}));",
|
|
"NODE",
|
|
" touch \"$done_path\"",
|
|
") >/dev/null 2>&1 &",
|
|
"pid=$!",
|
|
"node - \"$status_path\" \"$pid\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const [statusPath, pidRaw] = process.argv.slice(2);",
|
|
"const current = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
|
|
"current.pid = Number(pidRaw);",
|
|
"fs.writeFileSync(statusPath, JSON.stringify(current));",
|
|
"console.log(JSON.stringify({ ok: true, state: current.state, pid: current.pid, statusPath }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function remoteAsyncStatusScript(label: string, token: string): string {
|
|
const statusPath = remoteAsyncStatusPath(label, token);
|
|
const donePath = `${statusPath}.done`;
|
|
return [
|
|
"set -eu",
|
|
`status_path=${shellQuote(statusPath)}`,
|
|
`done_path=${shellQuote(donePath)}`,
|
|
"if [ ! -f \"$status_path\" ]; then",
|
|
" printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing\"}'",
|
|
" exit 0",
|
|
"fi",
|
|
"if [ -f \"$done_path\" ]; then",
|
|
" cat \"$status_path\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node - \"$status_path\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const statusPath = process.argv[2];",
|
|
"const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
|
|
"payload.state = payload.state || 'running';",
|
|
"payload.ok = payload.ok ?? null;",
|
|
"payload.stdout = payload.stdout || '';",
|
|
"payload.stderr = payload.stderr || '';",
|
|
"console.log(JSON.stringify(payload));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function remoteAsyncKillScript(label: string, token: string): string {
|
|
const statusPath = remoteAsyncStatusPath(label, token);
|
|
const donePath = `${statusPath}.done`;
|
|
return [
|
|
"set -eu",
|
|
`status_path=${shellQuote(statusPath)}`,
|
|
`done_path=${shellQuote(donePath)}`,
|
|
"if [ ! -f \"$status_path\" ]; then",
|
|
" printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing during kill\"}'",
|
|
" exit 0",
|
|
"fi",
|
|
"node - \"$status_path\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const statusPath = process.argv[2];",
|
|
"const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
|
|
"if (Number.isInteger(payload.pid) && payload.pid > 0) {",
|
|
" try { process.kill(payload.pid, 'TERM'); } catch {}",
|
|
"}",
|
|
"payload.ok = false;",
|
|
"payload.state = 'timeout-killed';",
|
|
"payload.exitCode = 124;",
|
|
"payload.finishedAt = new Date().toISOString();",
|
|
"payload.stderr = `${payload.stderr || ''}\\nremote async command exceeded local poll timeout and was terminated`;",
|
|
"fs.writeFileSync(statusPath, JSON.stringify(payload));",
|
|
"console.log(JSON.stringify(payload));",
|
|
"NODE",
|
|
"touch \"$done_path\"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function parseRemoteAsyncPayload(result: CommandJsonResult): Record<string, unknown> {
|
|
const text = result.stdout.trim();
|
|
if (text.length === 0) return {};
|
|
try {
|
|
return record(JSON.parse(text) as unknown);
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
export function runLegacyRuntimeK3sRemoteAsync(spec: RemoteAsyncCommandSpec): CommandJsonResult {
|
|
const startedAtMs = Date.now();
|
|
const start = legacyRuntimeK3s(["sh", "--", remoteAsyncStartScript(spec)], 55_000);
|
|
const startPayload = parseRemoteAsyncPayload(start);
|
|
if (!isCommandSuccess(start) || startPayload.ok !== true) return start;
|
|
const deadlineMs = startedAtMs + Math.max(1, spec.timeoutSeconds) * 1000;
|
|
let attempts = 0;
|
|
let lastStatus: CommandJsonResult | null = null;
|
|
while (Date.now() <= deadlineMs) {
|
|
attempts += 1;
|
|
const wait = runCommand(["sleep", attempts <= 1 ? "1" : "2"], repoRoot, { timeoutMs: 3_000 });
|
|
if (wait.exitCode !== 0 && wait.timedOut) break;
|
|
const status = legacyRuntimeK3s(["sh", "--", remoteAsyncStatusScript(spec.label, spec.token)], 55_000);
|
|
lastStatus = status;
|
|
const payload = parseRemoteAsyncPayload(status);
|
|
const state = typeof payload.state === "string" ? payload.state : null;
|
|
if (state === "finished" || payload.ok === true || payload.ok === false) {
|
|
return {
|
|
ok: payload.ok === true,
|
|
command: spec.command,
|
|
exitCode: typeof payload.exitCode === "number" ? payload.exitCode : status.exitCode,
|
|
stdout: typeof payload.stdout === "string" ? payload.stdout : status.stdout,
|
|
stderr: typeof payload.stderr === "string" ? payload.stderr : status.stderr,
|
|
parsed: {
|
|
ok: payload.ok === true,
|
|
data: {
|
|
mode: "remote-async",
|
|
label: spec.label,
|
|
token: spec.token,
|
|
statusPath: remoteAsyncStatusPath(spec.label, spec.token),
|
|
attempts,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
state,
|
|
pid: payload.pid ?? null,
|
|
},
|
|
},
|
|
};
|
|
}
|
|
}
|
|
const killed = legacyRuntimeK3s(["sh", "--", remoteAsyncKillScript(spec.label, spec.token)], 55_000);
|
|
const payload = parseRemoteAsyncPayload(killed);
|
|
return {
|
|
ok: false,
|
|
command: spec.command,
|
|
exitCode: 124,
|
|
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
|
|
stderr: [
|
|
`remote async command timed out after ${spec.timeoutSeconds}s`,
|
|
typeof payload.stderr === "string" ? payload.stderr : "",
|
|
lastStatus === null ? "" : `last status stderr: ${lastStatus.stderr.trim()}`,
|
|
killed.stderr.trim(),
|
|
].filter(Boolean).join("\n"),
|
|
parsed: {
|
|
ok: false,
|
|
data: {
|
|
mode: "remote-async",
|
|
label: spec.label,
|
|
token: spec.token,
|
|
statusPath: remoteAsyncStatusPath(spec.label, spec.token),
|
|
attempts,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
state: typeof payload.state === "string" ? payload.state : "timeout",
|
|
pid: payload.pid ?? null,
|
|
killed: compactCommandMetadata(killed, true, false),
|
|
},
|
|
},
|
|
};
|
|
}
|