feat: allow host source workspace bootstrap
This commit is contained in:
Codex
@@ -529,6 +529,7 @@ lanes:
|
||||
- scripts/src/browser-launcher.mjs
|
||||
- scripts/web-live-dom-probe.mjs
|
||||
install:
|
||||
executor: k3s-job
|
||||
dependencyCommand: npm ci
|
||||
browserCommand: PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium
|
||||
timeoutSeconds: 900
|
||||
@@ -821,8 +822,10 @@ lanes:
|
||||
npm --version
|
||||
npx --version
|
||||
install:
|
||||
executor: host
|
||||
dependencyCommand: npm install --package-lock=false --ignore-scripts --no-audit --no-fund playwright@1.59.1
|
||||
browserCommand: PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium
|
||||
stateDir: /var/lib/unidesk/hwlab-source-workspace-install
|
||||
timeoutSeconds: 900
|
||||
cicdRepo: /root/workspace/hwlab-v03-cicd.git
|
||||
cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock
|
||||
|
||||
@@ -356,8 +356,10 @@ export interface HwlabRuntimeSourceWorkspaceSpec {
|
||||
readonly requiredFiles: readonly string[];
|
||||
readonly hostDependencies?: HwlabRuntimeSourceWorkspaceHostDependenciesSpec;
|
||||
readonly install: {
|
||||
readonly executor: "k3s-job" | "host";
|
||||
readonly dependencyCommand: string;
|
||||
readonly browserCommand: string;
|
||||
readonly stateDir?: string;
|
||||
readonly timeoutSeconds: number;
|
||||
};
|
||||
}
|
||||
@@ -881,8 +883,10 @@ function sourceWorkspaceConfig(value: unknown, path: string): HwlabRuntimeSource
|
||||
.map((item, index) => relativeWorkspacePathField(item, `${path}.requiredFiles[${index}]`)),
|
||||
hostDependencies: sourceWorkspaceHostDependenciesConfig(raw.hostDependencies, `${path}.hostDependencies`),
|
||||
install: {
|
||||
executor: enumStringField(install, "executor", `${path}.install`, ["k3s-job", "host"]),
|
||||
dependencyCommand: stringField(install, "dependencyCommand", `${path}.install`),
|
||||
browserCommand: stringField(install, "browserCommand", `${path}.install`),
|
||||
...(install.stateDir === undefined ? {} : { stateDir: absoluteHostPathField(stringField(install, "stateDir", `${path}.install`), `${path}.install.stateDir`) }),
|
||||
timeoutSeconds: numberField(install, "timeoutSeconds", `${path}.install`),
|
||||
},
|
||||
};
|
||||
|
||||
@@ -79,6 +79,7 @@ function nodeRuntimeSourceWorkspaceStatus(scoped: ScopedNodeOptions, sourceWorks
|
||||
|
||||
function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
||||
if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace bootstrap accepts only one of --dry-run or --confirm");
|
||||
if (sourceWorkspace.install.executor === "host") return nodeRuntimeSourceWorkspaceHostBootstrap(scoped, sourceWorkspace);
|
||||
const dryRun = scoped.dryRun || !scoped.confirm;
|
||||
const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane);
|
||||
const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
|
||||
@@ -115,6 +116,113 @@ function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWo
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeSourceWorkspaceHostBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
||||
const stateDir = sourceWorkspace.install.stateDir;
|
||||
if (stateDir === undefined) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "host-bootstrap",
|
||||
mutation: false,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
degradedReason: "source-workspace-host-bootstrap-state-dir-missing",
|
||||
message: "sourceWorkspace.install.stateDir is required when sourceWorkspace.install.executor=host.",
|
||||
};
|
||||
}
|
||||
const dryRun = scoped.dryRun || !scoped.confirm;
|
||||
const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
|
||||
if (before.ok === true) {
|
||||
return {
|
||||
ok: true,
|
||||
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: dryRun ? "dry-run" : "already-ready",
|
||||
mutation: false,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
||||
before,
|
||||
bootstrap: { ok: true, status: "skipped", reason: "source-workspace-already-ready", valuesPrinted: false },
|
||||
result: null,
|
||||
wait: null,
|
||||
after: before,
|
||||
next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` },
|
||||
};
|
||||
}
|
||||
if (dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "host-dry-run",
|
||||
mutation: false,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
||||
before,
|
||||
bootstrap: {
|
||||
ok: true,
|
||||
status: "dry-run",
|
||||
executor: "host",
|
||||
stateDir,
|
||||
dependencyCommandSha256: createHash("sha256").update(sourceWorkspace.install.dependencyCommand).digest("hex"),
|
||||
browserCommandSha256: createHash("sha256").update(sourceWorkspace.install.browserCommand).digest("hex"),
|
||||
timeoutSeconds: sourceWorkspace.install.timeoutSeconds,
|
||||
valuesPrinted: false,
|
||||
},
|
||||
result: null,
|
||||
wait: null,
|
||||
after: null,
|
||||
next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
||||
};
|
||||
}
|
||||
const submit = runTransHostScript(scoped.node, sourceWorkspaceHostBootstrapSubmitScript(scoped.spec, sourceWorkspace, stateDir), "", Math.min(scoped.timeoutSeconds, 55));
|
||||
const submitted = parseLastJsonLineObject(statusText(submit));
|
||||
if (submit.exitCode !== 0 || submitted.ok !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "host-submit",
|
||||
mutation: false,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
||||
before,
|
||||
bootstrap: submitted,
|
||||
result: compactRuntimeCommand(submit),
|
||||
wait: null,
|
||||
after: null,
|
||||
degradedReason: "source-workspace-host-bootstrap-submit-failed",
|
||||
next: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
||||
};
|
||||
}
|
||||
const wait = waitForSourceWorkspaceHostBootstrap(scoped, stateDir, sourceWorkspace.install.timeoutSeconds);
|
||||
const after = wait.ok === true ? nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace) : null;
|
||||
const ok = wait.ok === true && after?.ok === true;
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "host-confirmed-wait",
|
||||
mutation: true,
|
||||
configPath: hwlabRuntimeLaneConfigPath(),
|
||||
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
||||
before,
|
||||
bootstrap: submitted,
|
||||
result: compactRuntimeCommand(submit),
|
||||
wait,
|
||||
after,
|
||||
degradedReason: ok ? undefined : "source-workspace-host-bootstrap-failed",
|
||||
next: ok
|
||||
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
|
||||
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
||||
const action = sourceWorkspaceHostDependenciesAction(scoped);
|
||||
const hostDependencies = sourceWorkspace.hostDependencies;
|
||||
@@ -305,6 +413,143 @@ function waitForSourceWorkspaceBootstrapJob(scoped: ScopedNodeOptions, namespace
|
||||
return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
|
||||
}
|
||||
|
||||
function waitForSourceWorkspaceHostBootstrap(scoped: ScopedNodeOptions, stateDir: string, timeoutSeconds: number): Record<string, unknown> {
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + (timeoutSeconds + 60) * 1000;
|
||||
let polls = 0;
|
||||
let lastPayload: Record<string, unknown> = { ok: false, status: "not-polled", valuesPrinted: false };
|
||||
while (Date.now() <= deadline) {
|
||||
polls += 1;
|
||||
const result = runTransHostScript(scoped.node, sourceWorkspaceHostBootstrapStatusScript(stateDir), "", Math.min(scoped.timeoutSeconds, 55));
|
||||
const payload = parseLastJsonLineObject(statusText(result));
|
||||
lastPayload = { ...payload, probe: compactRuntimeCommand(result) };
|
||||
if (result.exitCode === 0 && payload.status === "succeeded" && payload.ok === true) {
|
||||
return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
|
||||
}
|
||||
if (payload.status === "failed") {
|
||||
return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
|
||||
}
|
||||
sleepMs(5000);
|
||||
}
|
||||
return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
|
||||
}
|
||||
|
||||
function sourceWorkspaceHostBootstrapStatusScript(stateDir: string): string {
|
||||
return [
|
||||
"set +e",
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
"status_file=\"$state_dir/bootstrap-status.json\"",
|
||||
"if [ -f \"$status_file\" ]; then",
|
||||
" cat \"$status_file\"",
|
||||
"else",
|
||||
" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: false, status: 'missing', stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
"fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function sourceWorkspaceHostBootstrapSubmitScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec, stateDir: string): string {
|
||||
const requiredFiles = shellLinesText(sourceWorkspace.requiredFiles);
|
||||
const commandSha = createHash("sha256").update(`${sourceWorkspace.install.dependencyCommand}\n${sourceWorkspace.install.browserCommand}`).digest("hex");
|
||||
return [
|
||||
"set +e",
|
||||
`workspace=${shellQuote(spec.workspace)}`,
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
`dependency_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.dependencyCommand, "utf8").toString("base64"))}`,
|
||||
`browser_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.browserCommand, "utf8").toString("base64"))}`,
|
||||
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
|
||||
`timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`,
|
||||
`http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
|
||||
`https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
|
||||
`all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
|
||||
`no_proxy_value=${shellQuote(spec.networkProfile.proxy.noProxy.join(","))}`,
|
||||
`command_sha=${shellQuote(commandSha)}`,
|
||||
"mkdir -p \"$state_dir\"",
|
||||
"mkdir_exit=$?",
|
||||
"if [ \"$mkdir_exit\" -ne 0 ]; then",
|
||||
" MKDIR_EXIT=\"$mkdir_exit\" STATE_DIR=\"$state_dir\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: false, status: 'submit-failed', stateDir: process.env.STATE_DIR, mkdirExitCode: Number(process.env.MKDIR_EXIT || '1'), valuesPrinted: false }));",
|
||||
"NODE",
|
||||
" exit \"$mkdir_exit\"",
|
||||
"fi",
|
||||
"job_id=\"$(date -u +%Y%m%dT%H%M%SZ)-$$\"",
|
||||
"status_file=\"$state_dir/bootstrap-status.json\"",
|
||||
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
||||
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
||||
"dependency_file=\"$state_dir/$job_id.dependency.sh\"",
|
||||
"browser_file=\"$state_dir/$job_id.browser.sh\"",
|
||||
"required_files_file=\"$state_dir/$job_id.required-files.txt\"",
|
||||
"printf '%s' \"$dependency_command_b64\" | base64 -d >\"$dependency_file\"",
|
||||
"printf '%s' \"$browser_command_b64\" | base64 -d >\"$browser_file\"",
|
||||
"printf '%s' \"$required_files_b64\" | base64 -d >\"$required_files_file\"",
|
||||
"chmod 700 \"$dependency_file\" \"$browser_file\"",
|
||||
"JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" node <<'NODE' >\"$status_file\"",
|
||||
"console.log(JSON.stringify({ ok: false, status: 'running', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, startedAt: new Date().toISOString(), valuesPrinted: false }));",
|
||||
"NODE",
|
||||
"(",
|
||||
" set +e",
|
||||
" export HTTP_PROXY=\"$http_proxy_value\" HTTPS_PROXY=\"$https_proxy_value\" ALL_PROXY=\"$all_proxy_value\" NO_PROXY=\"$no_proxy_value\"",
|
||||
" export http_proxy=\"$http_proxy_value\" https_proxy=\"$https_proxy_value\" all_proxy=\"$all_proxy_value\" no_proxy=\"$no_proxy_value\"",
|
||||
" failure_kind=",
|
||||
" dependency_exit=1",
|
||||
" browser_exit=1",
|
||||
" missing_files=",
|
||||
" source_commit=",
|
||||
" status_short=",
|
||||
" node_version=",
|
||||
" npm_version=",
|
||||
" npx_version=",
|
||||
" if [ ! -d \"$workspace/.git\" ]; then",
|
||||
" failure_kind=workspace-git-missing",
|
||||
" printf '%s\\n' \"source workspace git repo is missing: $workspace\" >>\"$stderr_file\"",
|
||||
" elif ! cd \"$workspace\"; then",
|
||||
" failure_kind=workspace-cd-failed",
|
||||
" else",
|
||||
" status_short=$(git status --short 2>/dev/null || true)",
|
||||
" if [ -n \"$status_short\" ]; then",
|
||||
" failure_kind=workspace-dirty",
|
||||
" printf '%s\\n' \"source workspace is dirty; refusing to install dependencies\" >>\"$stderr_file\"",
|
||||
" else",
|
||||
" timeout \"$timeout_seconds\" /bin/bash \"$dependency_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
|
||||
" dependency_exit=$?",
|
||||
" if [ \"$dependency_exit\" -eq 0 ]; then",
|
||||
" timeout \"$timeout_seconds\" /bin/bash \"$browser_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
|
||||
" browser_exit=$?",
|
||||
" else",
|
||||
" failure_kind=dependency-command-failed",
|
||||
" fi",
|
||||
" if [ \"$dependency_exit\" -eq 0 ] && [ \"$browser_exit\" -ne 0 ]; then failure_kind=browser-command-failed; fi",
|
||||
" while IFS= read -r file_path; do",
|
||||
" [ -z \"$file_path\" ] && continue",
|
||||
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
|
||||
" done <\"$required_files_file\"",
|
||||
" if [ -n \"$missing_files\" ] && [ -z \"$failure_kind\" ]; then failure_kind=missing-required-files; fi",
|
||||
" source_commit=$(git rev-parse HEAD 2>/dev/null || true)",
|
||||
" status_short=$(git status --short 2>/dev/null || true)",
|
||||
" node_version=$(node --version 2>/dev/null || true)",
|
||||
" npm_version=$(npm --version 2>/dev/null || true)",
|
||||
" npx_version=$(npx --version 2>/dev/null || true)",
|
||||
" fi",
|
||||
" fi",
|
||||
" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" FAILURE_KIND=\"$failure_kind\" DEPENDENCY_EXIT=\"$dependency_exit\" BROWSER_EXIT=\"$browser_exit\" MISSING_FILES=\"$missing_files\" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" NPX_VERSION=\"$npx_version\" node <<'NODE' >\"$status_file\"",
|
||||
"const fs = require('fs');",
|
||||
"const tail = (file) => { try { return fs.readFileSync(file, 'utf8').slice(-4000); } catch { return ''; } };",
|
||||
"const missingFiles = (process.env.MISSING_FILES || '').split(',').filter(Boolean);",
|
||||
"const dependencyExit = Number(process.env.DEPENDENCY_EXIT || '1');",
|
||||
"const browserExit = Number(process.env.BROWSER_EXIT || '1');",
|
||||
"const statusShort = process.env.STATUS_SHORT || '';",
|
||||
"const ok = dependencyExit === 0 && browserExit === 0 && missingFiles.length === 0 && !statusShort && !process.env.FAILURE_KIND;",
|
||||
"console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, failureKind: process.env.FAILURE_KIND || null, dependencyExitCode: dependencyExit, browserExitCode: browserExit, missingFiles, sourceCommit: process.env.SOURCE_COMMIT || null, workspaceClean: !statusShort, statusShort: statusShort || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, npxVersion: process.env.NPX_VERSION || null, stdoutTail: tail(process.env.STDOUT_FILE), stderrTail: tail(process.env.STDERR_FILE), finishedAt: new Date().toISOString(), valuesPrinted: false }));",
|
||||
"NODE",
|
||||
") >/dev/null 2>&1 &",
|
||||
"pid=$!",
|
||||
"JOB_ID=\"$job_id\" PID=\"$pid\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.PID || '0'), workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function sourceWorkspaceBootstrapJobStatusScript(namespace: string, jobName: string): string {
|
||||
return [
|
||||
"set +e",
|
||||
|
||||
Reference in New Issue
Block a user