Codex
819 lines
46 KiB
TypeScript
819 lines
46 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-d518-yaml-first-cutover.
|
|
// Responsibility: YAML-first D518/D601 HWLAB source workspace status and bootstrap for web-probe clients.
|
|
import { createHash } from "node:crypto";
|
|
import { posix as posixPath } from "node:path";
|
|
import { hwlabNodeControlPlaneSourceWorkspaceBootstrap } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, type HwlabRuntimeLaneSpec, type HwlabRuntimeSourceWorkspaceHostDependenciesSpec, type HwlabRuntimeSourceWorkspaceSpec } from "../hwlab-node-lanes";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { runTransHostScript, runTransScript } from "./public-exposure";
|
|
import { compactRuntimeCommand, parseLastJsonLineObject } from "./runtime-common";
|
|
import { commaListField, keyValueLinesFromText, numericField, shellQuote, statusText } from "./utils";
|
|
|
|
type ScopedNodeOptions = ReturnType<typeof parseNodeScopedDelegatedOptions>;
|
|
type SourceWorkspaceAction = "status" | "bootstrap" | "host-deps";
|
|
type SourceWorkspaceHostDependenciesAction = "status" | "sync";
|
|
|
|
export function nodeRuntimeSourceWorkspaceCommand(scoped: ScopedNodeOptions): Record<string, unknown> {
|
|
const action = sourceWorkspaceAction(scoped);
|
|
const sourceWorkspace = scoped.spec.sourceWorkspace;
|
|
if (sourceWorkspace === undefined) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane source-workspace ${action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mutation: false,
|
|
degradedReason: "source-workspace-yaml-missing",
|
|
message: "lanes.<lane>.targets.<node>.sourceWorkspace must declare requiredCommands, requiredFiles and install commands before this controlled entry can run.",
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
};
|
|
}
|
|
if (action === "host-deps") return nodeRuntimeSourceWorkspaceHostDependencies(scoped, sourceWorkspace);
|
|
if (action === "status") return nodeRuntimeSourceWorkspaceStatus(scoped, sourceWorkspace);
|
|
return nodeRuntimeSourceWorkspaceBootstrap(scoped, sourceWorkspace);
|
|
}
|
|
|
|
function sourceWorkspaceAction(scoped: ScopedNodeOptions): SourceWorkspaceAction {
|
|
const value = scoped.originalArgs[1];
|
|
if (value !== "status" && value !== "bootstrap" && value !== "host-deps") {
|
|
throw new Error("control-plane source-workspace usage: source-workspace status|bootstrap|host-deps --node NODE --lane vNN [--dry-run|--confirm]");
|
|
}
|
|
if (value === "status" && (scoped.confirm || scoped.dryRun)) {
|
|
throw new Error("control-plane source-workspace status is read-only and does not accept --dry-run or --confirm");
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function sourceWorkspaceHostDependenciesAction(scoped: ScopedNodeOptions): SourceWorkspaceHostDependenciesAction {
|
|
const value = scoped.originalArgs[2];
|
|
if (value !== "status" && value !== "sync") {
|
|
throw new Error("control-plane source-workspace host-deps usage: source-workspace host-deps status|sync --node NODE --lane vNN [--dry-run|--confirm] [--wait]");
|
|
}
|
|
if (value === "status" && (scoped.confirm || scoped.dryRun || scoped.wait)) {
|
|
throw new Error("control-plane source-workspace host-deps status is read-only and does not accept --dry-run, --confirm or --wait");
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function nodeRuntimeSourceWorkspaceStatus(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
|
const result = runTransHostScript(scoped.node, sourceWorkspaceStatusScript(scoped.spec, sourceWorkspace), "", scoped.timeoutSeconds);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const status = sourceWorkspaceStatusFromFields(fields, result.exitCode);
|
|
return {
|
|
ok: status.ok === true,
|
|
command: `hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "read-only",
|
|
mutation: false,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
|
status,
|
|
probe: compactRuntimeCommand(result),
|
|
degradedReason: status.ok === true ? undefined : "source-workspace-not-ready",
|
|
next: status.ok === true
|
|
? { webProbe: `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}` }
|
|
: { bootstrap: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
|
if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace bootstrap accepts only one of --dry-run or --confirm");
|
|
const dryRun = scoped.dryRun || !scoped.confirm;
|
|
const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane);
|
|
const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
|
|
const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.max(scoped.timeoutSeconds, sourceWorkspace.install.timeoutSeconds + 60));
|
|
const payload = parseLastJsonLineObject(statusText(result));
|
|
const after = dryRun ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
|
|
const afterOk = after === null ? null : after.ok === true;
|
|
const payloadOk = payload.ok === true;
|
|
const ok = result.exitCode === 0 && payloadOk && (dryRun || afterOk === true);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: dryRun ? "dry-run" : "confirmed-bootstrap",
|
|
mutation: !dryRun && ok,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
controlPlaneConfigPath: controlPlane.configPath,
|
|
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
|
controlPlane,
|
|
before,
|
|
bootstrap: payload,
|
|
result: compactRuntimeCommand(result),
|
|
after,
|
|
degradedReason: ok ? undefined : dryRun ? "source-workspace-bootstrap-dry-run-failed" : "source-workspace-bootstrap-failed",
|
|
next: ok
|
|
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
|
|
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
|
const action = sourceWorkspaceHostDependenciesAction(scoped);
|
|
const hostDependencies = sourceWorkspace.hostDependencies;
|
|
if (hostDependencies === undefined) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane source-workspace host-deps ${action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mutation: false,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
degradedReason: "source-workspace-host-dependencies-yaml-missing",
|
|
message: "lanes.<lane>.targets.<node>.sourceWorkspace.hostDependencies must declare checkCommands, stateDir and install before this controlled entry can run.",
|
|
};
|
|
}
|
|
if (action === "status") return nodeRuntimeSourceWorkspaceHostDependenciesStatus(scoped, hostDependencies);
|
|
return nodeRuntimeSourceWorkspaceHostDependenciesSync(scoped, sourceWorkspace, hostDependencies);
|
|
}
|
|
|
|
function nodeRuntimeSourceWorkspaceHostDependenciesStatus(scoped: ScopedNodeOptions, hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
|
|
const result = runTransHostScript(scoped.node, sourceWorkspaceHostDependenciesStatusScript(hostDependencies), "", scoped.timeoutSeconds);
|
|
const payload = parseLastJsonLineObject(statusText(result));
|
|
const ok = result.exitCode === 0 && payload.ok === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane source-workspace host-deps status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "read-only",
|
|
mutation: false,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
|
|
status: payload,
|
|
probe: compactRuntimeCommand(result),
|
|
degradedReason: ok ? undefined : "source-workspace-host-dependencies-not-ready",
|
|
next: ok ? undefined : { sync: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeSourceWorkspaceHostDependenciesSync(
|
|
scoped: ScopedNodeOptions,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec,
|
|
): Record<string, unknown> {
|
|
if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace host-deps sync accepts only one of --dry-run or --confirm");
|
|
const dryRun = scoped.dryRun || !scoped.confirm;
|
|
const before = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
|
|
if (before.ok === true) {
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: dryRun ? "dry-run" : "already-ready",
|
|
mutation: false,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
|
|
before,
|
|
sync: { ok: true, status: "skipped", reason: "host-dependencies-already-ready", valuesPrinted: false },
|
|
after: before,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
if (dryRun) {
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
|
|
before,
|
|
sync: {
|
|
ok: true,
|
|
status: "dry-run",
|
|
stateDir: hostDependencies.stateDir,
|
|
checkCommands: hostDependencies.checkCommands,
|
|
installCommandSha256: createHash("sha256").update(hostDependencies.install.command).digest("hex"),
|
|
timeoutSeconds: hostDependencies.install.timeoutSeconds,
|
|
valuesPrinted: false,
|
|
},
|
|
after: null,
|
|
next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
const submit = runTransHostScript(scoped.node, sourceWorkspaceHostDependenciesSubmitScript(scoped.spec, sourceWorkspace, hostDependencies), "", Math.min(scoped.timeoutSeconds, 55));
|
|
const submitted = parseLastJsonLineObject(statusText(submit));
|
|
if (submit.exitCode !== 0 || submitted.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "confirmed-submit",
|
|
mutation: false,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
|
|
before,
|
|
sync: submitted,
|
|
result: compactRuntimeCommand(submit),
|
|
after: null,
|
|
degradedReason: "source-workspace-host-dependencies-submit-failed",
|
|
next: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
if (!scoped.wait) {
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "submitted",
|
|
mutation: true,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
|
|
before,
|
|
sync: submitted,
|
|
result: compactRuntimeCommand(submit),
|
|
after: null,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const waited = waitForSourceWorkspaceHostDependencies(scoped, hostDependencies);
|
|
const after = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
|
|
const ok = waited.ok === true && after.ok === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "confirmed-wait",
|
|
mutation: true,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
|
|
before,
|
|
sync: submitted,
|
|
result: compactRuntimeCommand(submit),
|
|
wait: waited,
|
|
after,
|
|
degradedReason: ok ? undefined : "source-workspace-host-dependencies-sync-failed",
|
|
next: ok
|
|
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
|
|
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
function waitForSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
|
|
const startedAt = Date.now();
|
|
const deadline = startedAt + (hostDependencies.install.timeoutSeconds + 60) * 1000;
|
|
let polls = 0;
|
|
let lastPayload: Record<string, unknown> = { ok: false, status: "not-polled", valuesPrinted: false };
|
|
while (Date.now() <= deadline) {
|
|
polls += 1;
|
|
const status = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
|
|
lastPayload = typeof status.status === "object" && status.status !== null ? status.status as Record<string, unknown> : lastPayload;
|
|
if (status.ok === true) return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
|
|
const lastInstall = typeof lastPayload.lastInstall === "object" && lastPayload.lastInstall !== null ? lastPayload.lastInstall as Record<string, unknown> : {};
|
|
if (lastInstall.status === "failed") return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
|
|
sleepMs(5000);
|
|
}
|
|
return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
|
|
}
|
|
|
|
function sleepMs(ms: number): void {
|
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
}
|
|
|
|
function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
|
return {
|
|
workspace: spec.workspace,
|
|
sourceBranch: spec.sourceBranch,
|
|
git: {
|
|
url: spec.gitUrl,
|
|
readUrl: spec.gitReadUrl,
|
|
writeUrl: spec.gitWriteUrl,
|
|
},
|
|
requiredCommands: sourceWorkspace.requiredCommands,
|
|
requiredFiles: sourceWorkspace.requiredFiles,
|
|
hostDependencies: sourceWorkspace.hostDependencies === undefined ? null : sourceWorkspaceHostDependenciesExpected(sourceWorkspace.hostDependencies),
|
|
install: sourceWorkspace.install,
|
|
downloadProfile: {
|
|
id: spec.downloadProfileId,
|
|
npm: spec.downloadProfile.npm,
|
|
git: spec.downloadProfile.git,
|
|
},
|
|
networkProfile: {
|
|
id: spec.networkProfileId,
|
|
proxy: spec.networkProfile.proxy,
|
|
},
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceHostDependenciesExpected(hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
|
|
return {
|
|
checkCommands: hostDependencies.checkCommands,
|
|
stateDir: hostDependencies.stateDir,
|
|
install: {
|
|
commandSha256: createHash("sha256").update(hostDependencies.install.command).digest("hex"),
|
|
timeoutSeconds: hostDependencies.install.timeoutSeconds,
|
|
},
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceStatusFromFields(fields: Record<string, string>, exitCode: number | null): Record<string, unknown> {
|
|
const missingCommands = commaListField(fields.missingCommands);
|
|
const missingFiles = commaListField(fields.missingFiles);
|
|
const ok = exitCode === 0
|
|
&& fields.workspaceExists === "yes"
|
|
&& fields.gitDirExists === "yes"
|
|
&& fields.workspaceClean === "yes"
|
|
&& fields.branch === fields.expectedBranch
|
|
&& fields.remoteMatchesYaml === "yes"
|
|
&& missingCommands.length === 0
|
|
&& missingFiles.length === 0
|
|
&& fields.playwrightPackagePresent === "yes"
|
|
&& fields.browserExecutablePresent === "yes"
|
|
&& fields.launcherImportOk === "yes";
|
|
return {
|
|
ok,
|
|
workspace: fields.workspace || null,
|
|
expectedBranch: fields.expectedBranch || null,
|
|
workspaceExists: fields.workspaceExists === "yes",
|
|
gitDirExists: fields.gitDirExists === "yes",
|
|
workspaceClean: fields.workspaceClean === "yes",
|
|
branch: fields.branch || null,
|
|
detached: fields.detached === "yes",
|
|
localHead: fields.localHead || null,
|
|
remoteUrl: fields.remoteUrl || null,
|
|
remoteMatchesYaml: fields.remoteMatchesYaml === "yes",
|
|
requiredCommands: commaListField(fields.requiredCommands),
|
|
missingCommands,
|
|
requiredFiles: commaListField(fields.requiredFiles),
|
|
missingFiles,
|
|
nodeVersion: fields.nodeVersion || null,
|
|
npmVersion: fields.npmVersion || null,
|
|
npxVersion: fields.npxVersion || null,
|
|
playwrightPackagePresent: fields.playwrightPackagePresent === "yes",
|
|
browserCachePresent: fields.browserCachePresent === "yes",
|
|
browserExecutablePresent: fields.browserExecutablePresent === "yes",
|
|
browserExecutableExitCode: numericField(fields.browserExecutableExitCode),
|
|
browserExecutablePath: fields.browserExecutablePath || null,
|
|
browserExecutableErrorTail: fields.browserExecutableErrorTail || null,
|
|
launcherImportOk: fields.launcherImportOk === "yes",
|
|
launcherImportExitCode: numericField(fields.launcherImportExitCode),
|
|
statusShortLines: numericField(fields.statusShortLines),
|
|
statusShortPreview: fields.statusShortPreview || null,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceHostDependenciesStatusScript(hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): string {
|
|
const checkCommands = hostDependencies.checkCommands.join("\n");
|
|
return [
|
|
"set +e",
|
|
`state_dir=${shellQuote(hostDependencies.stateDir)}`,
|
|
`check_commands_b64=${shellQuote(Buffer.from(checkCommands, "utf8").toString("base64"))}`,
|
|
"mkdir -p \"$state_dir\" 2>/tmp/hwlab-source-workspace-host-deps-mkdir.err",
|
|
"mkdir_exit=$?",
|
|
"commands_file=$(mktemp)",
|
|
"trap 'rm -f \"$commands_file\"' EXIT",
|
|
"printf '%s' \"$check_commands_b64\" | base64 -d >\"$commands_file\"",
|
|
"missing_commands=",
|
|
"present_commands=",
|
|
"while IFS= read -r command_name; do",
|
|
" [ -z \"$command_name\" ] && continue",
|
|
" if command -v \"$command_name\" >/dev/null 2>&1; then",
|
|
" present_commands=\"$present_commands${present_commands:+,}$command_name\"",
|
|
" else",
|
|
" missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"",
|
|
" fi",
|
|
"done <\"$commands_file\"",
|
|
"status_file=\"$state_dir/install-status.json\"",
|
|
"last_install_json='{}'",
|
|
"if [ -f \"$status_file\" ]; then last_install_json=$(tail -c 20000 \"$status_file\" 2>/dev/null || printf '{}'); fi",
|
|
"STATE_DIR=\"$state_dir\" MKDIR_EXIT=\"$mkdir_exit\" MISSING_COMMANDS=\"$missing_commands\" PRESENT_COMMANDS=\"$present_commands\" STATUS_FILE=\"$status_file\" LAST_INSTALL_JSON=\"$last_install_json\" node <<'NODE'",
|
|
"const lastRaw = process.env.LAST_INSTALL_JSON || '{}';",
|
|
"let lastInstall = {};",
|
|
"try { lastInstall = JSON.parse(lastRaw); } catch { lastInstall = { ok: false, status: 'unparseable', valuesPrinted: false }; }",
|
|
"const missing = (process.env.MISSING_COMMANDS || '').split(',').filter(Boolean);",
|
|
"const present = (process.env.PRESENT_COMMANDS || '').split(',').filter(Boolean);",
|
|
"const mkdirExit = Number(process.env.MKDIR_EXIT || '1');",
|
|
"console.log(JSON.stringify({ ok: mkdirExit === 0 && missing.length === 0, status: mkdirExit === 0 && missing.length === 0 ? 'ready' : 'missing', stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, presentCommands: present, missingCommands: missing, mkdirExitCode: mkdirExit, lastInstall, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceHostDependenciesSubmitScript(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec,
|
|
): string {
|
|
const checkCommands = hostDependencies.checkCommands.join("\n");
|
|
const noProxy = spec.networkProfile.proxy.noProxy.join(",");
|
|
const commandSha = createHash("sha256").update(hostDependencies.install.command).digest("hex");
|
|
const requiredCommandsSha = createHash("sha256").update(sourceWorkspace.requiredCommands.join("\n")).digest("hex");
|
|
return [
|
|
"set +e",
|
|
`state_dir=${shellQuote(hostDependencies.stateDir)}`,
|
|
`install_command_b64=${shellQuote(Buffer.from(hostDependencies.install.command, "utf8").toString("base64"))}`,
|
|
`check_commands_b64=${shellQuote(Buffer.from(checkCommands, "utf8").toString("base64"))}`,
|
|
`timeout_seconds=${String(hostDependencies.install.timeoutSeconds)}`,
|
|
`http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
|
|
`https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
|
|
`all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
|
|
`no_proxy_value=${shellQuote(noProxy)}`,
|
|
`command_sha=${shellQuote(commandSha)}`,
|
|
`required_commands_sha=${shellQuote(requiredCommandsSha)}`,
|
|
"mkdir -p \"$state_dir\"",
|
|
"mkdir_exit=$?",
|
|
"if [ \"$mkdir_exit\" -ne 0 ]; then",
|
|
" MKDIR_EXIT=\"$mkdir_exit\" STATE_DIR=\"$state_dir\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: false, status: 'submit-failed', stateDir: process.env.STATE_DIR, mkdirExitCode: Number(process.env.MKDIR_EXIT || '1'), valuesPrinted: false }));",
|
|
"NODE",
|
|
" exit \"$mkdir_exit\"",
|
|
"fi",
|
|
"job_id=\"$(date -u +%Y%m%dT%H%M%SZ)-$$\"",
|
|
"status_file=\"$state_dir/install-status.json\"",
|
|
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
|
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
|
"command_file=\"$state_dir/$job_id.install.sh\"",
|
|
"commands_file=\"$state_dir/$job_id.commands.txt\"",
|
|
"printf '%s' \"$install_command_b64\" | base64 -d >\"$command_file\"",
|
|
"printf '%s' \"$check_commands_b64\" | base64 -d >\"$commands_file\"",
|
|
"chmod 700 \"$command_file\"",
|
|
"JOB_ID=\"$job_id\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" node <<'NODE' >\"$status_file\"",
|
|
"console.log(JSON.stringify({ ok: false, status: 'running', jobId: process.env.JOB_ID, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, startedAt: new Date().toISOString(), valuesPrinted: false }));",
|
|
"NODE",
|
|
"(",
|
|
" set +e",
|
|
" export HTTP_PROXY=\"$http_proxy_value\" HTTPS_PROXY=\"$https_proxy_value\" ALL_PROXY=\"$all_proxy_value\" NO_PROXY=\"$no_proxy_value\"",
|
|
" export http_proxy=\"$http_proxy_value\" https_proxy=\"$https_proxy_value\" all_proxy=\"$all_proxy_value\" no_proxy=\"$no_proxy_value\"",
|
|
" timeout \"$timeout_seconds\" /bin/bash \"$command_file\" >\"$stdout_file\" 2>\"$stderr_file\"",
|
|
" install_exit=$?",
|
|
" missing_commands=",
|
|
" present_commands=",
|
|
" while IFS= read -r command_name; do",
|
|
" [ -z \"$command_name\" ] && continue",
|
|
" if command -v \"$command_name\" >/dev/null 2>&1; then",
|
|
" present_commands=\"$present_commands${present_commands:+,}$command_name\"",
|
|
" else",
|
|
" missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"",
|
|
" fi",
|
|
" done <\"$commands_file\"",
|
|
" stdout_tail=$(tail -c 4000 \"$stdout_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000)",
|
|
" stderr_tail=$(tail -c 4000 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000)",
|
|
" JOB_ID=\"$job_id\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" INSTALL_EXIT=\"$install_exit\" PRESENT_COMMANDS=\"$present_commands\" MISSING_COMMANDS=\"$missing_commands\" STDOUT_TAIL=\"$stdout_tail\" STDERR_TAIL=\"$stderr_tail\" node <<'NODE' >\"$status_file\"",
|
|
"const missing = (process.env.MISSING_COMMANDS || '').split(',').filter(Boolean);",
|
|
"const present = (process.env.PRESENT_COMMANDS || '').split(',').filter(Boolean);",
|
|
"const exitCode = Number(process.env.INSTALL_EXIT || '1');",
|
|
"const ok = exitCode === 0 && missing.length === 0;",
|
|
"console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, installExitCode: exitCode, presentCommands: present, missingCommands: missing, stdoutTail: process.env.STDOUT_TAIL || '', stderrTail: process.env.STDERR_TAIL || '', finishedAt: new Date().toISOString(), valuesPrinted: false }));",
|
|
"NODE",
|
|
") >/dev/null 2>&1 &",
|
|
"pid=$!",
|
|
"JOB_ID=\"$job_id\" PID=\"$pid\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.PID || '0'), stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string {
|
|
const requiredCommands = sourceWorkspace.requiredCommands.join("\n");
|
|
const requiredFiles = sourceWorkspace.requiredFiles.join("\n");
|
|
const remoteUrls = [spec.gitUrl, spec.gitReadUrl, spec.gitWriteUrl].join("\n");
|
|
return [
|
|
"set +e",
|
|
`workspace=${shellQuote(spec.workspace)}`,
|
|
`expected_branch=${shellQuote(spec.sourceBranch)}`,
|
|
`required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`,
|
|
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
|
|
`remote_urls_b64=${shellQuote(Buffer.from(remoteUrls, "utf8").toString("base64"))}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"",
|
|
"printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"",
|
|
"printf '%s' \"$remote_urls_b64\" | base64 -d >\"$tmp_dir/remotes.txt\"",
|
|
"workspace_exists=no",
|
|
"git_dir_exists=no",
|
|
"workspace_clean=no",
|
|
"branch=",
|
|
"detached=no",
|
|
"local_head=",
|
|
"remote_url=",
|
|
"remote_matches_yaml=no",
|
|
"status_short=",
|
|
"if [ -d \"$workspace\" ]; then",
|
|
" workspace_exists=yes",
|
|
" if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
|
|
" git_dir_exists=yes",
|
|
" branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
" [ \"$branch\" = HEAD ] && detached=yes",
|
|
" local_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
|
" remote_url=$(git -C \"$workspace\" remote get-url origin 2>/dev/null || true)",
|
|
" status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
|
" [ -z \"$status_short\" ] && workspace_clean=yes",
|
|
" while IFS= read -r expected_remote; do [ \"$remote_url\" = \"$expected_remote\" ] && remote_matches_yaml=yes; done <\"$tmp_dir/remotes.txt\"",
|
|
" fi",
|
|
"fi",
|
|
"missing_commands=",
|
|
"while IFS= read -r command_name; do",
|
|
" [ -z \"$command_name\" ] && continue",
|
|
" if ! command -v \"$command_name\" >/dev/null 2>&1; then missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"; fi",
|
|
"done <\"$tmp_dir/commands.txt\"",
|
|
"missing_files=",
|
|
"if [ \"$workspace_exists\" = yes ]; then",
|
|
" while IFS= read -r file_path; do",
|
|
" [ -z \"$file_path\" ] && continue",
|
|
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
|
|
" done <\"$tmp_dir/files.txt\"",
|
|
"else",
|
|
" missing_files=$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')",
|
|
"fi",
|
|
"node_version=$(node --version 2>/dev/null || true)",
|
|
"npm_version=$(npm --version 2>/dev/null || true)",
|
|
"npx_version=$(npx --version 2>/dev/null || true)",
|
|
"playwright_package_present=no",
|
|
"[ -f \"$workspace/node_modules/playwright/package.json\" ] && playwright_package_present=yes",
|
|
"browser_cache_present=no",
|
|
"if find \"$workspace/node_modules\" -maxdepth 5 -type d -path '*playwright*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi",
|
|
"for cache_dir in \"${HOME:-}/.cache/ms-playwright\" /root/.cache/ms-playwright /home/ubuntu/.cache/ms-playwright; do",
|
|
" [ -d \"$cache_dir\" ] || continue",
|
|
" if find \"$cache_dir\" -maxdepth 1 -type d -name '*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi",
|
|
"done",
|
|
"browser_executable_present=no",
|
|
"browser_executable_exit=",
|
|
"browser_executable_path=",
|
|
"browser_executable_error_tail=",
|
|
"if command -v node >/dev/null 2>&1 && [ \"$playwright_package_present\" = yes ]; then",
|
|
" (cd \"$workspace\" && PLAYWRIGHT_BROWSERS_PATH=0 timeout 20s node --input-type=module - <<'NODE'",
|
|
"import { existsSync } from 'node:fs';",
|
|
"import { chromium } from 'playwright';",
|
|
"const executablePath = chromium.executablePath();",
|
|
"console.log(executablePath);",
|
|
"process.exit(existsSync(executablePath) ? 0 : 2);",
|
|
"NODE",
|
|
" ) >/tmp/hwlab-source-workspace-browser-executable.out 2>/tmp/hwlab-source-workspace-browser-executable.err",
|
|
" browser_executable_exit=$?",
|
|
" browser_executable_path=$(head -n 1 /tmp/hwlab-source-workspace-browser-executable.out 2>/dev/null | cut -c1-1000)",
|
|
" browser_executable_error_tail=$(tail -n 20 /tmp/hwlab-source-workspace-browser-executable.err 2>/dev/null | tr '\\n\\t' ' ' | cut -c1-1000)",
|
|
" [ \"$browser_executable_exit\" = 0 ] && browser_executable_present=yes",
|
|
"fi",
|
|
"launcher_import_ok=no",
|
|
"launcher_import_exit=",
|
|
"if command -v node >/dev/null 2>&1 && [ -f \"$workspace/scripts/src/browser-launcher.mjs\" ]; then",
|
|
" (cd \"$workspace\" && timeout 20s node -e \"import('./scripts/src/browser-launcher.mjs').then(()=>process.exit(0)).catch(()=>process.exit(1))\") >/tmp/hwlab-source-workspace-launcher.out 2>/tmp/hwlab-source-workspace-launcher.err",
|
|
" launcher_import_exit=$?",
|
|
" [ \"$launcher_import_exit\" = 0 ] && launcher_import_ok=yes",
|
|
"fi",
|
|
"status_short_lines=$(printf '%s' \"$status_short\" | sed '/^$/d' | wc -l | tr -d ' ')",
|
|
"status_short_preview=$(printf '%s' \"$status_short\" | tr '\\n\\t' ' ' | cut -c1-1000)",
|
|
"printf 'workspace\\t%s\\n' \"$workspace\"",
|
|
"printf 'expectedBranch\\t%s\\n' \"$expected_branch\"",
|
|
"printf 'workspaceExists\\t%s\\n' \"$workspace_exists\"",
|
|
"printf 'gitDirExists\\t%s\\n' \"$git_dir_exists\"",
|
|
"printf 'workspaceClean\\t%s\\n' \"$workspace_clean\"",
|
|
"printf 'branch\\t%s\\n' \"$branch\"",
|
|
"printf 'detached\\t%s\\n' \"$detached\"",
|
|
"printf 'localHead\\t%s\\n' \"$local_head\"",
|
|
"printf 'remoteUrl\\t%s\\n' \"$remote_url\"",
|
|
"printf 'remoteMatchesYaml\\t%s\\n' \"$remote_matches_yaml\"",
|
|
"printf 'requiredCommands\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/commands.txt\" | sed 's/,$//')\"",
|
|
"printf 'missingCommands\\t%s\\n' \"$missing_commands\"",
|
|
"printf 'requiredFiles\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')\"",
|
|
"printf 'missingFiles\\t%s\\n' \"$missing_files\"",
|
|
"printf 'nodeVersion\\t%s\\n' \"$node_version\"",
|
|
"printf 'npmVersion\\t%s\\n' \"$npm_version\"",
|
|
"printf 'npxVersion\\t%s\\n' \"$npx_version\"",
|
|
"printf 'playwrightPackagePresent\\t%s\\n' \"$playwright_package_present\"",
|
|
"printf 'browserCachePresent\\t%s\\n' \"$browser_cache_present\"",
|
|
"printf 'browserExecutablePresent\\t%s\\n' \"$browser_executable_present\"",
|
|
"printf 'browserExecutableExitCode\\t%s\\n' \"$browser_executable_exit\"",
|
|
"printf 'browserExecutablePath\\t%s\\n' \"$browser_executable_path\"",
|
|
"printf 'browserExecutableErrorTail\\t%s\\n' \"$browser_executable_error_tail\"",
|
|
"printf 'launcherImportOk\\t%s\\n' \"$launcher_import_ok\"",
|
|
"printf 'launcherImportExitCode\\t%s\\n' \"$launcher_import_exit\"",
|
|
"printf 'statusShortLines\\t%s\\n' \"$status_short_lines\"",
|
|
"printf 'statusShortPreview\\t%s\\n' \"$status_short_preview\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceBootstrapK8sScript(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
|
|
dryRun: boolean,
|
|
): string {
|
|
const jobName = sourceWorkspaceJobName(spec);
|
|
const manifest = sourceWorkspaceJobManifest(spec, sourceWorkspace, controlPlane, jobName);
|
|
const manifestYaml = Bun.YAML.stringify(manifest);
|
|
const manifestSha = createHash("sha256").update(manifestYaml).digest("hex");
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(controlPlane.ciNamespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
`manifest_sha=${shellQuote(manifestSha)}`,
|
|
`timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`,
|
|
`manifest_b64=${shellQuote(Buffer.from(manifestYaml, "utf8").toString("base64"))}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"manifest=\"$tmp_dir/job.yaml\"",
|
|
"apply_out=\"$tmp_dir/apply.out\"",
|
|
"apply_err=\"$tmp_dir/apply.err\"",
|
|
"wait_out=\"$tmp_dir/wait.out\"",
|
|
"wait_err=\"$tmp_dir/wait.err\"",
|
|
"logs_file=\"$tmp_dir/logs.txt\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d >\"$manifest\"",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" kubectl -n \"$namespace\" apply --dry-run=server -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
|
|
" apply_rc=$?",
|
|
" python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'",
|
|
"import json, pathlib, sys",
|
|
"apply_rc = int(sys.argv[1])",
|
|
"namespace, job, manifest_sha = sys.argv[2:5]",
|
|
"apply_out = pathlib.Path(sys.argv[5]).read_text(errors='replace')[-2000:]",
|
|
"apply_err = pathlib.Path(sys.argv[6]).read_text(errors='replace')[-2000:]",
|
|
"print(json.dumps({'ok': apply_rc == 0, 'status': 'dry-run', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'applyOutputTail': apply_out, 'applyErrorTail': apply_err, 'mutation': False, 'valuesPrinted': False}, ensure_ascii=False))",
|
|
"PY",
|
|
" exit \"$apply_rc\"",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true >/dev/null 2>&1",
|
|
"kubectl -n \"$namespace\" apply -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
|
|
"apply_rc=$?",
|
|
"wait_rc=1",
|
|
"if [ \"$apply_rc\" = 0 ]; then",
|
|
" kubectl -n \"$namespace\" wait --for=condition=complete \"job/$job\" --timeout=\"${timeout_seconds}s\" >\"$wait_out\" 2>\"$wait_err\"",
|
|
" wait_rc=$?",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true",
|
|
"python3 - \"$apply_rc\" \"$wait_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" \"$wait_out\" \"$wait_err\" \"$logs_file\" <<'PY'",
|
|
"import json, pathlib, sys",
|
|
"apply_rc = int(sys.argv[1])",
|
|
"wait_rc = int(sys.argv[2])",
|
|
"namespace, job, manifest_sha = sys.argv[3:6]",
|
|
"paths = [pathlib.Path(item) for item in sys.argv[6:]]",
|
|
"apply_out, apply_err, wait_out, wait_err, logs = [path.read_text(errors='replace') for path in paths]",
|
|
"job_payload = None",
|
|
"for line in reversed(logs.splitlines()):",
|
|
" text = line.strip()",
|
|
" if text.startswith('{') and text.endswith('}'):",
|
|
" try:",
|
|
" job_payload = json.loads(text)",
|
|
" break",
|
|
" except Exception:",
|
|
" pass",
|
|
"job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True",
|
|
"ok = apply_rc == 0 and wait_rc == 0 and job_ok",
|
|
"print(json.dumps({'ok': ok, 'status': 'succeeded' if ok else 'failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'waitExitCode': wait_rc, 'jobPayload': job_payload, 'applyOutputTail': apply_out[-2000:], 'applyErrorTail': apply_err[-2000:], 'waitOutputTail': wait_out[-2000:], 'waitErrorTail': wait_err[-2000:], 'logsTail': logs[-4000:], 'mutation': ok, 'valuesPrinted': False}, ensure_ascii=False))",
|
|
"PY",
|
|
"exit $([ \"$apply_rc\" = 0 ] && [ \"$wait_rc\" = 0 ] && printf 0 || printf 1)",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceJobManifest(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
|
|
jobName: string,
|
|
): Record<string, unknown> {
|
|
const parent = posixPath.dirname(spec.workspace);
|
|
const basename = posixPath.basename(spec.workspace);
|
|
if (!spec.workspace.startsWith("/") || parent === "." || basename.length === 0) {
|
|
throw new Error(`source workspace must be an absolute POSIX path; got ${spec.workspace}`);
|
|
}
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name: jobName,
|
|
namespace: controlPlane.ciNamespace,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/name": "hwlab-source-workspace-bootstrap",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
ttlSecondsAfterFinished: 3600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/name": "hwlab-source-workspace-bootstrap",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
},
|
|
},
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
serviceAccountName: controlPlane.serviceAccountName,
|
|
containers: [
|
|
{
|
|
name: "bootstrap",
|
|
image: controlPlane.toolsImage,
|
|
imagePullPolicy: controlPlane.imagePullPolicy,
|
|
command: ["/bin/sh", "-lc", sourceWorkspaceJobContainerScript(spec, sourceWorkspace, controlPlane, `/host-workspaces/${basename}`)],
|
|
volumeMounts: [{ name: "workspace-parent", mountPath: "/host-workspaces" }],
|
|
env: sourceWorkspaceJobEnv(spec),
|
|
},
|
|
],
|
|
volumes: [{ name: "workspace-parent", hostPath: { path: parent, type: "DirectoryOrCreate" } }],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceJobEnv(spec: HwlabRuntimeLaneSpec): Array<Record<string, string>> {
|
|
const proxy = spec.networkProfile.proxy;
|
|
const npm = spec.downloadProfile.npm;
|
|
return [
|
|
{ name: "HTTP_PROXY", value: proxy.http },
|
|
{ name: "HTTPS_PROXY", value: proxy.https },
|
|
{ name: "ALL_PROXY", value: proxy.all },
|
|
{ name: "NO_PROXY", value: proxy.noProxy.join(",") },
|
|
{ name: "http_proxy", value: proxy.http },
|
|
{ name: "https_proxy", value: proxy.https },
|
|
{ name: "all_proxy", value: proxy.all },
|
|
{ name: "no_proxy", value: proxy.noProxy.join(",") },
|
|
{ name: "npm_config_registry", value: npm.registry },
|
|
{ name: "npm_config_fetch_retries", value: String(npm.retries) },
|
|
{ name: "npm_config_fetch_timeout", value: String(npm.fetchTimeoutSeconds * 1000) },
|
|
{ name: "BUN_CONFIG_REGISTRY", value: npm.registry },
|
|
];
|
|
}
|
|
|
|
function sourceWorkspaceJobContainerScript(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
|
|
workspaceInContainer: string,
|
|
): string {
|
|
const requiredCommands = sourceWorkspace.requiredCommands.join("\n");
|
|
const requiredFiles = sourceWorkspace.requiredFiles.join("\n");
|
|
return [
|
|
"set -eu",
|
|
`workspace=${shellQuote(workspaceInContainer)}`,
|
|
`remote=${shellQuote(controlPlane.gitReadUrl)}`,
|
|
`branch=${shellQuote(spec.sourceBranch)}`,
|
|
`dependency_command=${shellQuote(sourceWorkspace.install.dependencyCommand)}`,
|
|
`browser_command=${shellQuote(sourceWorkspace.install.browserCommand)}`,
|
|
`required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`,
|
|
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"stdout_log=\"$tmp_dir/bootstrap.stdout\"",
|
|
"stderr_log=\"$tmp_dir/bootstrap.stderr\"",
|
|
"failure() {",
|
|
" code=$?",
|
|
" if [ \"$code\" -ne 0 ]; then",
|
|
" tail_text=$(cat \"$stderr_log\" 2>/dev/null | tail -n 80 | tr '\\n' ' ' | cut -c1-3000 || true)",
|
|
" CODE=\"$code\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" ERROR_TAIL=\"$tail_text\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: Number(process.env.CODE || '1'), workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" fi",
|
|
" rm -rf \"$tmp_dir\"",
|
|
" exit \"$code\"",
|
|
"}",
|
|
"trap failure EXIT",
|
|
"printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"",
|
|
"printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"",
|
|
"while IFS= read -r command_name; do",
|
|
" [ -z \"$command_name\" ] && continue",
|
|
" command -v \"$command_name\" >/dev/null 2>>\"$stderr_log\"",
|
|
"done <\"$tmp_dir/commands.txt\"",
|
|
"mkdir -p \"$(dirname \"$workspace\")\"",
|
|
"git config --global --add safe.directory \"$workspace\" >>\"$stdout_log\" 2>>\"$stderr_log\" || true",
|
|
"if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
|
|
" status_short=$(git -C \"$workspace\" status --short)",
|
|
" if [ -n \"$status_short\" ]; then echo \"source workspace is dirty; refusing to overwrite\" >>\"$stderr_log\"; exit 42; fi",
|
|
" git -C \"$workspace\" remote set-url origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\" || git -C \"$workspace\" remote add origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
" git -C \"$workspace\" fetch --depth=1 origin \"$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
" git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"else",
|
|
" if [ -e \"$workspace\" ] && [ \"$(find \"$workspace\" -mindepth 1 -maxdepth 1 2>/dev/null | head -n 1)\" ]; then echo \"source workspace path exists but is not a git repo\" >>\"$stderr_log\"; exit 43; fi",
|
|
" rm -rf \"$workspace\"",
|
|
" git clone --depth=1 --branch \"$branch\" \"$remote\" \"$workspace\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"fi",
|
|
"cd \"$workspace\"",
|
|
"sh -lc \"$dependency_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"sh -lc \"$browser_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"missing_files=",
|
|
"while IFS= read -r file_path; do",
|
|
" [ -z \"$file_path\" ] && continue",
|
|
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
|
|
"done <\"$tmp_dir/files.txt\"",
|
|
"if [ -n \"$missing_files\" ]; then echo \"missing required files: $missing_files\" >>\"$stderr_log\"; exit 44; fi",
|
|
"source_commit=$(git rev-parse HEAD)",
|
|
"status_short=$(git status --short)",
|
|
"node_version=$(node --version 2>/dev/null || true)",
|
|
"npm_version=$(npm --version 2>/dev/null || true)",
|
|
"STATUS_SHORT=\"$status_short\" SOURCE_COMMIT=\"$source_commit\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" node <<'NODE'",
|
|
"const clean = !process.env.STATUS_SHORT;",
|
|
"console.log(JSON.stringify({ ok: clean, status: clean ? 'succeeded' : 'dirty-after-bootstrap', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, workspaceClean: clean, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
"rm -rf \"$tmp_dir\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceJobName(spec: HwlabRuntimeLaneSpec): string {
|
|
return `hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-workspace`.slice(0, 63).replace(/-$/u, "");
|
|
}
|