From 04297283302c19dddb652f6a947c661345d4963e Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 29 Jun 2026 09:30:38 +0000 Subject: [PATCH] feat: allow host source workspace bootstrap --- config/hwlab-node-lanes.yaml | 3 + scripts/src/hwlab-node-lanes.ts | 4 + scripts/src/hwlab-node/source-workspace.ts | 245 +++++++++++++++++++++ 3 files changed, 252 insertions(+) diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index f9caceef..6b67056e 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -529,6 +529,7 @@ lanes: - scripts/src/browser-launcher.mjs - scripts/web-live-dom-probe.mjs install: + executor: k3s-job dependencyCommand: npm ci browserCommand: PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium timeoutSeconds: 900 @@ -821,8 +822,10 @@ lanes: npm --version npx --version install: + executor: host dependencyCommand: npm install --package-lock=false --ignore-scripts --no-audit --no-fund playwright@1.59.1 browserCommand: PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium + stateDir: /var/lib/unidesk/hwlab-source-workspace-install timeoutSeconds: 900 cicdRepo: /root/workspace/hwlab-v03-cicd.git cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 99ecfb19..9367006f 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -356,8 +356,10 @@ export interface HwlabRuntimeSourceWorkspaceSpec { readonly requiredFiles: readonly string[]; readonly hostDependencies?: HwlabRuntimeSourceWorkspaceHostDependenciesSpec; readonly install: { + readonly executor: "k3s-job" | "host"; readonly dependencyCommand: string; readonly browserCommand: string; + readonly stateDir?: string; readonly timeoutSeconds: number; }; } @@ -881,8 +883,10 @@ function sourceWorkspaceConfig(value: unknown, path: string): HwlabRuntimeSource .map((item, index) => relativeWorkspacePathField(item, `${path}.requiredFiles[${index}]`)), hostDependencies: sourceWorkspaceHostDependenciesConfig(raw.hostDependencies, `${path}.hostDependencies`), install: { + executor: enumStringField(install, "executor", `${path}.install`, ["k3s-job", "host"]), dependencyCommand: stringField(install, "dependencyCommand", `${path}.install`), browserCommand: stringField(install, "browserCommand", `${path}.install`), + ...(install.stateDir === undefined ? {} : { stateDir: absoluteHostPathField(stringField(install, "stateDir", `${path}.install`), `${path}.install.stateDir`) }), timeoutSeconds: numberField(install, "timeoutSeconds", `${path}.install`), }, }; diff --git a/scripts/src/hwlab-node/source-workspace.ts b/scripts/src/hwlab-node/source-workspace.ts index fee7fddb..f95ae2ba 100644 --- a/scripts/src/hwlab-node/source-workspace.ts +++ b/scripts/src/hwlab-node/source-workspace.ts @@ -79,6 +79,7 @@ function nodeRuntimeSourceWorkspaceStatus(scoped: ScopedNodeOptions, sourceWorks function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record { if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace bootstrap accepts only one of --dry-run or --confirm"); + if (sourceWorkspace.install.executor === "host") return nodeRuntimeSourceWorkspaceHostBootstrap(scoped, sourceWorkspace); const dryRun = scoped.dryRun || !scoped.confirm; const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane); const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace); @@ -115,6 +116,113 @@ function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWo }; } +function nodeRuntimeSourceWorkspaceHostBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record { + const stateDir = sourceWorkspace.install.stateDir; + if (stateDir === undefined) { + return { + ok: false, + command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "host-bootstrap", + mutation: false, + configPath: hwlabRuntimeLaneConfigPath(), + degradedReason: "source-workspace-host-bootstrap-state-dir-missing", + message: "sourceWorkspace.install.stateDir is required when sourceWorkspace.install.executor=host.", + }; + } + const dryRun = scoped.dryRun || !scoped.confirm; + const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace); + if (before.ok === true) { + return { + ok: true, + command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: dryRun ? "dry-run" : "already-ready", + mutation: false, + configPath: hwlabRuntimeLaneConfigPath(), + expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace), + before, + bootstrap: { ok: true, status: "skipped", reason: "source-workspace-already-ready", valuesPrinted: false }, + result: null, + wait: null, + after: before, + next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }, + }; + } + if (dryRun) { + return { + ok: true, + command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "host-dry-run", + mutation: false, + configPath: hwlabRuntimeLaneConfigPath(), + expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace), + before, + bootstrap: { + ok: true, + status: "dry-run", + executor: "host", + stateDir, + dependencyCommandSha256: createHash("sha256").update(sourceWorkspace.install.dependencyCommand).digest("hex"), + browserCommandSha256: createHash("sha256").update(sourceWorkspace.install.browserCommand).digest("hex"), + timeoutSeconds: sourceWorkspace.install.timeoutSeconds, + valuesPrinted: false, + }, + result: null, + wait: null, + after: null, + next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; + } + const submit = runTransHostScript(scoped.node, sourceWorkspaceHostBootstrapSubmitScript(scoped.spec, sourceWorkspace, stateDir), "", Math.min(scoped.timeoutSeconds, 55)); + const submitted = parseLastJsonLineObject(statusText(submit)); + if (submit.exitCode !== 0 || submitted.ok !== true) { + return { + ok: false, + command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "host-submit", + mutation: false, + configPath: hwlabRuntimeLaneConfigPath(), + expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace), + before, + bootstrap: submitted, + result: compactRuntimeCommand(submit), + wait: null, + after: null, + degradedReason: "source-workspace-host-bootstrap-submit-failed", + next: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; + } + const wait = waitForSourceWorkspaceHostBootstrap(scoped, stateDir, sourceWorkspace.install.timeoutSeconds); + const after = wait.ok === true ? nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace) : null; + const ok = wait.ok === true && after?.ok === true; + return { + ok, + command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "host-confirmed-wait", + mutation: true, + configPath: hwlabRuntimeLaneConfigPath(), + expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace), + before, + bootstrap: submitted, + result: compactRuntimeCommand(submit), + wait, + after, + degradedReason: ok ? undefined : "source-workspace-host-bootstrap-failed", + next: ok + ? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` } + : { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; +} + function nodeRuntimeSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record { const action = sourceWorkspaceHostDependenciesAction(scoped); const hostDependencies = sourceWorkspace.hostDependencies; @@ -305,6 +413,143 @@ function waitForSourceWorkspaceBootstrapJob(scoped: ScopedNodeOptions, namespace return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false }; } +function waitForSourceWorkspaceHostBootstrap(scoped: ScopedNodeOptions, stateDir: string, timeoutSeconds: number): Record { + const startedAt = Date.now(); + const deadline = startedAt + (timeoutSeconds + 60) * 1000; + let polls = 0; + let lastPayload: Record = { ok: false, status: "not-polled", valuesPrinted: false }; + while (Date.now() <= deadline) { + polls += 1; + const result = runTransHostScript(scoped.node, sourceWorkspaceHostBootstrapStatusScript(stateDir), "", Math.min(scoped.timeoutSeconds, 55)); + const payload = parseLastJsonLineObject(statusText(result)); + lastPayload = { ...payload, probe: compactRuntimeCommand(result) }; + if (result.exitCode === 0 && payload.status === "succeeded" && payload.ok === true) { + return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false }; + } + if (payload.status === "failed") { + return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false }; + } + sleepMs(5000); + } + return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false }; +} + +function sourceWorkspaceHostBootstrapStatusScript(stateDir: string): string { + return [ + "set +e", + `state_dir=${shellQuote(stateDir)}`, + "status_file=\"$state_dir/bootstrap-status.json\"", + "if [ -f \"$status_file\" ]; then", + " cat \"$status_file\"", + "else", + " STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" node <<'NODE'", + "console.log(JSON.stringify({ ok: false, status: 'missing', stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, valuesPrinted: false }));", + "NODE", + "fi", + ].join("\n"); +} + +function sourceWorkspaceHostBootstrapSubmitScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec, stateDir: string): string { + const requiredFiles = shellLinesText(sourceWorkspace.requiredFiles); + const commandSha = createHash("sha256").update(`${sourceWorkspace.install.dependencyCommand}\n${sourceWorkspace.install.browserCommand}`).digest("hex"); + return [ + "set +e", + `workspace=${shellQuote(spec.workspace)}`, + `state_dir=${shellQuote(stateDir)}`, + `dependency_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.dependencyCommand, "utf8").toString("base64"))}`, + `browser_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.browserCommand, "utf8").toString("base64"))}`, + `required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`, + `timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`, + `http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`, + `https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`, + `all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`, + `no_proxy_value=${shellQuote(spec.networkProfile.proxy.noProxy.join(","))}`, + `command_sha=${shellQuote(commandSha)}`, + "mkdir -p \"$state_dir\"", + "mkdir_exit=$?", + "if [ \"$mkdir_exit\" -ne 0 ]; then", + " MKDIR_EXIT=\"$mkdir_exit\" STATE_DIR=\"$state_dir\" node <<'NODE'", + "console.log(JSON.stringify({ ok: false, status: 'submit-failed', stateDir: process.env.STATE_DIR, mkdirExitCode: Number(process.env.MKDIR_EXIT || '1'), valuesPrinted: false }));", + "NODE", + " exit \"$mkdir_exit\"", + "fi", + "job_id=\"$(date -u +%Y%m%dT%H%M%SZ)-$$\"", + "status_file=\"$state_dir/bootstrap-status.json\"", + "stdout_file=\"$state_dir/$job_id.stdout.log\"", + "stderr_file=\"$state_dir/$job_id.stderr.log\"", + "dependency_file=\"$state_dir/$job_id.dependency.sh\"", + "browser_file=\"$state_dir/$job_id.browser.sh\"", + "required_files_file=\"$state_dir/$job_id.required-files.txt\"", + "printf '%s' \"$dependency_command_b64\" | base64 -d >\"$dependency_file\"", + "printf '%s' \"$browser_command_b64\" | base64 -d >\"$browser_file\"", + "printf '%s' \"$required_files_b64\" | base64 -d >\"$required_files_file\"", + "chmod 700 \"$dependency_file\" \"$browser_file\"", + "JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" node <<'NODE' >\"$status_file\"", + "console.log(JSON.stringify({ ok: false, status: 'running', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, startedAt: new Date().toISOString(), valuesPrinted: false }));", + "NODE", + "(", + " set +e", + " export HTTP_PROXY=\"$http_proxy_value\" HTTPS_PROXY=\"$https_proxy_value\" ALL_PROXY=\"$all_proxy_value\" NO_PROXY=\"$no_proxy_value\"", + " export http_proxy=\"$http_proxy_value\" https_proxy=\"$https_proxy_value\" all_proxy=\"$all_proxy_value\" no_proxy=\"$no_proxy_value\"", + " failure_kind=", + " dependency_exit=1", + " browser_exit=1", + " missing_files=", + " source_commit=", + " status_short=", + " node_version=", + " npm_version=", + " npx_version=", + " if [ ! -d \"$workspace/.git\" ]; then", + " failure_kind=workspace-git-missing", + " printf '%s\\n' \"source workspace git repo is missing: $workspace\" >>\"$stderr_file\"", + " elif ! cd \"$workspace\"; then", + " failure_kind=workspace-cd-failed", + " else", + " status_short=$(git status --short 2>/dev/null || true)", + " if [ -n \"$status_short\" ]; then", + " failure_kind=workspace-dirty", + " printf '%s\\n' \"source workspace is dirty; refusing to install dependencies\" >>\"$stderr_file\"", + " else", + " timeout \"$timeout_seconds\" /bin/bash \"$dependency_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " dependency_exit=$?", + " if [ \"$dependency_exit\" -eq 0 ]; then", + " timeout \"$timeout_seconds\" /bin/bash \"$browser_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " browser_exit=$?", + " else", + " failure_kind=dependency-command-failed", + " fi", + " if [ \"$dependency_exit\" -eq 0 ] && [ \"$browser_exit\" -ne 0 ]; then failure_kind=browser-command-failed; fi", + " while IFS= read -r file_path; do", + " [ -z \"$file_path\" ] && continue", + " if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi", + " done <\"$required_files_file\"", + " if [ -n \"$missing_files\" ] && [ -z \"$failure_kind\" ]; then failure_kind=missing-required-files; fi", + " source_commit=$(git rev-parse HEAD 2>/dev/null || true)", + " status_short=$(git status --short 2>/dev/null || true)", + " node_version=$(node --version 2>/dev/null || true)", + " npm_version=$(npm --version 2>/dev/null || true)", + " npx_version=$(npx --version 2>/dev/null || true)", + " fi", + " fi", + " JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" FAILURE_KIND=\"$failure_kind\" DEPENDENCY_EXIT=\"$dependency_exit\" BROWSER_EXIT=\"$browser_exit\" MISSING_FILES=\"$missing_files\" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" NPX_VERSION=\"$npx_version\" node <<'NODE' >\"$status_file\"", + "const fs = require('fs');", + "const tail = (file) => { try { return fs.readFileSync(file, 'utf8').slice(-4000); } catch { return ''; } };", + "const missingFiles = (process.env.MISSING_FILES || '').split(',').filter(Boolean);", + "const dependencyExit = Number(process.env.DEPENDENCY_EXIT || '1');", + "const browserExit = Number(process.env.BROWSER_EXIT || '1');", + "const statusShort = process.env.STATUS_SHORT || '';", + "const ok = dependencyExit === 0 && browserExit === 0 && missingFiles.length === 0 && !statusShort && !process.env.FAILURE_KIND;", + "console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, failureKind: process.env.FAILURE_KIND || null, dependencyExitCode: dependencyExit, browserExitCode: browserExit, missingFiles, sourceCommit: process.env.SOURCE_COMMIT || null, workspaceClean: !statusShort, statusShort: statusShort || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, npxVersion: process.env.NPX_VERSION || null, stdoutTail: tail(process.env.STDOUT_FILE), stderrTail: tail(process.env.STDERR_FILE), finishedAt: new Date().toISOString(), valuesPrinted: false }));", + "NODE", + ") >/dev/null 2>&1 &", + "pid=$!", + "JOB_ID=\"$job_id\" PID=\"$pid\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.PID || '0'), workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + function sourceWorkspaceBootstrapJobStatusScript(namespace: string, jobName: string): string { return [ "set +e",