Codex
515 lines
27 KiB
TypeScript
515 lines
27 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-d518-yaml-first-cutover.
|
|
// Responsibility: YAML-first D518/D601 HWLAB source workspace status and bootstrap for web-probe clients.
|
|
import { createHash } from "node:crypto";
|
|
import { posix as posixPath } from "node:path";
|
|
import { hwlabNodeControlPlaneSourceWorkspaceBootstrap } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, type HwlabRuntimeLaneSpec, type HwlabRuntimeSourceWorkspaceSpec } from "../hwlab-node-lanes";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { runTransHostScript, runTransScript } from "./public-exposure";
|
|
import { compactRuntimeCommand, parseLastJsonLineObject } from "./runtime-common";
|
|
import { commaListField, keyValueLinesFromText, numericField, shellQuote, statusText } from "./utils";
|
|
|
|
type ScopedNodeOptions = ReturnType<typeof parseNodeScopedDelegatedOptions>;
|
|
type SourceWorkspaceAction = "status" | "bootstrap";
|
|
|
|
export function nodeRuntimeSourceWorkspaceCommand(scoped: ScopedNodeOptions): Record<string, unknown> {
|
|
const action = sourceWorkspaceAction(scoped);
|
|
const sourceWorkspace = scoped.spec.sourceWorkspace;
|
|
if (sourceWorkspace === undefined) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane source-workspace ${action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mutation: false,
|
|
degradedReason: "source-workspace-yaml-missing",
|
|
message: "lanes.<lane>.targets.<node>.sourceWorkspace must declare requiredCommands, requiredFiles and install commands before this controlled entry can run.",
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
};
|
|
}
|
|
if (action === "status") return nodeRuntimeSourceWorkspaceStatus(scoped, sourceWorkspace);
|
|
return nodeRuntimeSourceWorkspaceBootstrap(scoped, sourceWorkspace);
|
|
}
|
|
|
|
function sourceWorkspaceAction(scoped: ScopedNodeOptions): SourceWorkspaceAction {
|
|
const value = scoped.originalArgs[1];
|
|
if (value !== "status" && value !== "bootstrap") {
|
|
throw new Error("control-plane source-workspace usage: source-workspace status|bootstrap --node NODE --lane vNN [--dry-run|--confirm]");
|
|
}
|
|
if (value === "status" && (scoped.confirm || scoped.dryRun)) {
|
|
throw new Error("control-plane source-workspace status is read-only and does not accept --dry-run or --confirm");
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function nodeRuntimeSourceWorkspaceStatus(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
|
const result = runTransHostScript(scoped.node, sourceWorkspaceStatusScript(scoped.spec, sourceWorkspace), "", scoped.timeoutSeconds);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const status = sourceWorkspaceStatusFromFields(fields, result.exitCode);
|
|
return {
|
|
ok: status.ok === true,
|
|
command: `hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "read-only",
|
|
mutation: false,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
|
status,
|
|
probe: compactRuntimeCommand(result),
|
|
degradedReason: status.ok === true ? undefined : "source-workspace-not-ready",
|
|
next: status.ok === true
|
|
? { webProbe: `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}` }
|
|
: { bootstrap: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
|
if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace bootstrap accepts only one of --dry-run or --confirm");
|
|
const dryRun = scoped.dryRun || !scoped.confirm;
|
|
const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane);
|
|
const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
|
|
const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.max(scoped.timeoutSeconds, sourceWorkspace.install.timeoutSeconds + 60));
|
|
const payload = parseLastJsonLineObject(statusText(result));
|
|
const after = dryRun ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
|
|
const afterOk = after === null ? null : after.ok === true;
|
|
const payloadOk = payload.ok === true;
|
|
const ok = result.exitCode === 0 && payloadOk && (dryRun || afterOk === true);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: dryRun ? "dry-run" : "confirmed-bootstrap",
|
|
mutation: !dryRun && ok,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
controlPlaneConfigPath: controlPlane.configPath,
|
|
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
|
|
controlPlane,
|
|
before,
|
|
bootstrap: payload,
|
|
result: compactRuntimeCommand(result),
|
|
after,
|
|
degradedReason: ok ? undefined : dryRun ? "source-workspace-bootstrap-dry-run-failed" : "source-workspace-bootstrap-failed",
|
|
next: ok
|
|
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
|
|
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
|
|
return {
|
|
workspace: spec.workspace,
|
|
sourceBranch: spec.sourceBranch,
|
|
git: {
|
|
url: spec.gitUrl,
|
|
readUrl: spec.gitReadUrl,
|
|
writeUrl: spec.gitWriteUrl,
|
|
},
|
|
requiredCommands: sourceWorkspace.requiredCommands,
|
|
requiredFiles: sourceWorkspace.requiredFiles,
|
|
install: sourceWorkspace.install,
|
|
downloadProfile: {
|
|
id: spec.downloadProfileId,
|
|
npm: spec.downloadProfile.npm,
|
|
git: spec.downloadProfile.git,
|
|
},
|
|
networkProfile: {
|
|
id: spec.networkProfileId,
|
|
proxy: spec.networkProfile.proxy,
|
|
},
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceStatusFromFields(fields: Record<string, string>, exitCode: number | null): Record<string, unknown> {
|
|
const missingCommands = commaListField(fields.missingCommands);
|
|
const missingFiles = commaListField(fields.missingFiles);
|
|
const ok = exitCode === 0
|
|
&& fields.workspaceExists === "yes"
|
|
&& fields.gitDirExists === "yes"
|
|
&& fields.workspaceClean === "yes"
|
|
&& fields.branch === fields.expectedBranch
|
|
&& fields.remoteMatchesYaml === "yes"
|
|
&& missingCommands.length === 0
|
|
&& missingFiles.length === 0
|
|
&& fields.playwrightPackagePresent === "yes"
|
|
&& fields.browserExecutablePresent === "yes"
|
|
&& fields.launcherImportOk === "yes";
|
|
return {
|
|
ok,
|
|
workspace: fields.workspace || null,
|
|
expectedBranch: fields.expectedBranch || null,
|
|
workspaceExists: fields.workspaceExists === "yes",
|
|
gitDirExists: fields.gitDirExists === "yes",
|
|
workspaceClean: fields.workspaceClean === "yes",
|
|
branch: fields.branch || null,
|
|
detached: fields.detached === "yes",
|
|
localHead: fields.localHead || null,
|
|
remoteUrl: fields.remoteUrl || null,
|
|
remoteMatchesYaml: fields.remoteMatchesYaml === "yes",
|
|
requiredCommands: commaListField(fields.requiredCommands),
|
|
missingCommands,
|
|
requiredFiles: commaListField(fields.requiredFiles),
|
|
missingFiles,
|
|
nodeVersion: fields.nodeVersion || null,
|
|
npmVersion: fields.npmVersion || null,
|
|
npxVersion: fields.npxVersion || null,
|
|
playwrightPackagePresent: fields.playwrightPackagePresent === "yes",
|
|
browserCachePresent: fields.browserCachePresent === "yes",
|
|
browserExecutablePresent: fields.browserExecutablePresent === "yes",
|
|
browserExecutableExitCode: numericField(fields.browserExecutableExitCode),
|
|
browserExecutablePath: fields.browserExecutablePath || null,
|
|
browserExecutableErrorTail: fields.browserExecutableErrorTail || null,
|
|
launcherImportOk: fields.launcherImportOk === "yes",
|
|
launcherImportExitCode: numericField(fields.launcherImportExitCode),
|
|
statusShortLines: numericField(fields.statusShortLines),
|
|
statusShortPreview: fields.statusShortPreview || null,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string {
|
|
const requiredCommands = sourceWorkspace.requiredCommands.join("\n");
|
|
const requiredFiles = sourceWorkspace.requiredFiles.join("\n");
|
|
const remoteUrls = [spec.gitUrl, spec.gitReadUrl, spec.gitWriteUrl].join("\n");
|
|
return [
|
|
"set +e",
|
|
`workspace=${shellQuote(spec.workspace)}`,
|
|
`expected_branch=${shellQuote(spec.sourceBranch)}`,
|
|
`required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`,
|
|
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
|
|
`remote_urls_b64=${shellQuote(Buffer.from(remoteUrls, "utf8").toString("base64"))}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"",
|
|
"printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"",
|
|
"printf '%s' \"$remote_urls_b64\" | base64 -d >\"$tmp_dir/remotes.txt\"",
|
|
"workspace_exists=no",
|
|
"git_dir_exists=no",
|
|
"workspace_clean=no",
|
|
"branch=",
|
|
"detached=no",
|
|
"local_head=",
|
|
"remote_url=",
|
|
"remote_matches_yaml=no",
|
|
"status_short=",
|
|
"if [ -d \"$workspace\" ]; then",
|
|
" workspace_exists=yes",
|
|
" if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
|
|
" git_dir_exists=yes",
|
|
" branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
" [ \"$branch\" = HEAD ] && detached=yes",
|
|
" local_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
|
" remote_url=$(git -C \"$workspace\" remote get-url origin 2>/dev/null || true)",
|
|
" status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
|
" [ -z \"$status_short\" ] && workspace_clean=yes",
|
|
" while IFS= read -r expected_remote; do [ \"$remote_url\" = \"$expected_remote\" ] && remote_matches_yaml=yes; done <\"$tmp_dir/remotes.txt\"",
|
|
" fi",
|
|
"fi",
|
|
"missing_commands=",
|
|
"while IFS= read -r command_name; do",
|
|
" [ -z \"$command_name\" ] && continue",
|
|
" if ! command -v \"$command_name\" >/dev/null 2>&1; then missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"; fi",
|
|
"done <\"$tmp_dir/commands.txt\"",
|
|
"missing_files=",
|
|
"if [ \"$workspace_exists\" = yes ]; then",
|
|
" while IFS= read -r file_path; do",
|
|
" [ -z \"$file_path\" ] && continue",
|
|
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
|
|
" done <\"$tmp_dir/files.txt\"",
|
|
"else",
|
|
" missing_files=$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')",
|
|
"fi",
|
|
"node_version=$(node --version 2>/dev/null || true)",
|
|
"npm_version=$(npm --version 2>/dev/null || true)",
|
|
"npx_version=$(npx --version 2>/dev/null || true)",
|
|
"playwright_package_present=no",
|
|
"[ -f \"$workspace/node_modules/playwright/package.json\" ] && playwright_package_present=yes",
|
|
"browser_cache_present=no",
|
|
"if find \"$workspace/node_modules\" -maxdepth 5 -type d -path '*playwright*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi",
|
|
"for cache_dir in \"${HOME:-}/.cache/ms-playwright\" /root/.cache/ms-playwright /home/ubuntu/.cache/ms-playwright; do",
|
|
" [ -d \"$cache_dir\" ] || continue",
|
|
" if find \"$cache_dir\" -maxdepth 1 -type d -name '*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi",
|
|
"done",
|
|
"browser_executable_present=no",
|
|
"browser_executable_exit=",
|
|
"browser_executable_path=",
|
|
"browser_executable_error_tail=",
|
|
"if command -v node >/dev/null 2>&1 && [ \"$playwright_package_present\" = yes ]; then",
|
|
" (cd \"$workspace\" && PLAYWRIGHT_BROWSERS_PATH=0 timeout 20s node --input-type=module - <<'NODE'",
|
|
"import { existsSync } from 'node:fs';",
|
|
"import { chromium } from 'playwright';",
|
|
"const executablePath = chromium.executablePath();",
|
|
"console.log(executablePath);",
|
|
"process.exit(existsSync(executablePath) ? 0 : 2);",
|
|
"NODE",
|
|
" ) >/tmp/hwlab-source-workspace-browser-executable.out 2>/tmp/hwlab-source-workspace-browser-executable.err",
|
|
" browser_executable_exit=$?",
|
|
" browser_executable_path=$(head -n 1 /tmp/hwlab-source-workspace-browser-executable.out 2>/dev/null | cut -c1-1000)",
|
|
" browser_executable_error_tail=$(tail -n 20 /tmp/hwlab-source-workspace-browser-executable.err 2>/dev/null | tr '\\n\\t' ' ' | cut -c1-1000)",
|
|
" [ \"$browser_executable_exit\" = 0 ] && browser_executable_present=yes",
|
|
"fi",
|
|
"launcher_import_ok=no",
|
|
"launcher_import_exit=",
|
|
"if command -v node >/dev/null 2>&1 && [ -f \"$workspace/scripts/src/browser-launcher.mjs\" ]; then",
|
|
" (cd \"$workspace\" && timeout 20s node -e \"import('./scripts/src/browser-launcher.mjs').then(()=>process.exit(0)).catch(()=>process.exit(1))\") >/tmp/hwlab-source-workspace-launcher.out 2>/tmp/hwlab-source-workspace-launcher.err",
|
|
" launcher_import_exit=$?",
|
|
" [ \"$launcher_import_exit\" = 0 ] && launcher_import_ok=yes",
|
|
"fi",
|
|
"status_short_lines=$(printf '%s' \"$status_short\" | sed '/^$/d' | wc -l | tr -d ' ')",
|
|
"status_short_preview=$(printf '%s' \"$status_short\" | tr '\\n\\t' ' ' | cut -c1-1000)",
|
|
"printf 'workspace\\t%s\\n' \"$workspace\"",
|
|
"printf 'expectedBranch\\t%s\\n' \"$expected_branch\"",
|
|
"printf 'workspaceExists\\t%s\\n' \"$workspace_exists\"",
|
|
"printf 'gitDirExists\\t%s\\n' \"$git_dir_exists\"",
|
|
"printf 'workspaceClean\\t%s\\n' \"$workspace_clean\"",
|
|
"printf 'branch\\t%s\\n' \"$branch\"",
|
|
"printf 'detached\\t%s\\n' \"$detached\"",
|
|
"printf 'localHead\\t%s\\n' \"$local_head\"",
|
|
"printf 'remoteUrl\\t%s\\n' \"$remote_url\"",
|
|
"printf 'remoteMatchesYaml\\t%s\\n' \"$remote_matches_yaml\"",
|
|
"printf 'requiredCommands\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/commands.txt\" | sed 's/,$//')\"",
|
|
"printf 'missingCommands\\t%s\\n' \"$missing_commands\"",
|
|
"printf 'requiredFiles\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')\"",
|
|
"printf 'missingFiles\\t%s\\n' \"$missing_files\"",
|
|
"printf 'nodeVersion\\t%s\\n' \"$node_version\"",
|
|
"printf 'npmVersion\\t%s\\n' \"$npm_version\"",
|
|
"printf 'npxVersion\\t%s\\n' \"$npx_version\"",
|
|
"printf 'playwrightPackagePresent\\t%s\\n' \"$playwright_package_present\"",
|
|
"printf 'browserCachePresent\\t%s\\n' \"$browser_cache_present\"",
|
|
"printf 'browserExecutablePresent\\t%s\\n' \"$browser_executable_present\"",
|
|
"printf 'browserExecutableExitCode\\t%s\\n' \"$browser_executable_exit\"",
|
|
"printf 'browserExecutablePath\\t%s\\n' \"$browser_executable_path\"",
|
|
"printf 'browserExecutableErrorTail\\t%s\\n' \"$browser_executable_error_tail\"",
|
|
"printf 'launcherImportOk\\t%s\\n' \"$launcher_import_ok\"",
|
|
"printf 'launcherImportExitCode\\t%s\\n' \"$launcher_import_exit\"",
|
|
"printf 'statusShortLines\\t%s\\n' \"$status_short_lines\"",
|
|
"printf 'statusShortPreview\\t%s\\n' \"$status_short_preview\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceBootstrapK8sScript(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
|
|
dryRun: boolean,
|
|
): string {
|
|
const jobName = sourceWorkspaceJobName(spec);
|
|
const manifest = sourceWorkspaceJobManifest(spec, sourceWorkspace, controlPlane, jobName);
|
|
const manifestYaml = Bun.YAML.stringify(manifest);
|
|
const manifestSha = createHash("sha256").update(manifestYaml).digest("hex");
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(controlPlane.ciNamespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
`manifest_sha=${shellQuote(manifestSha)}`,
|
|
`timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`,
|
|
`manifest_b64=${shellQuote(Buffer.from(manifestYaml, "utf8").toString("base64"))}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"manifest=\"$tmp_dir/job.yaml\"",
|
|
"apply_out=\"$tmp_dir/apply.out\"",
|
|
"apply_err=\"$tmp_dir/apply.err\"",
|
|
"wait_out=\"$tmp_dir/wait.out\"",
|
|
"wait_err=\"$tmp_dir/wait.err\"",
|
|
"logs_file=\"$tmp_dir/logs.txt\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d >\"$manifest\"",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" kubectl -n \"$namespace\" apply --dry-run=server -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
|
|
" apply_rc=$?",
|
|
" python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'",
|
|
"import json, pathlib, sys",
|
|
"apply_rc = int(sys.argv[1])",
|
|
"namespace, job, manifest_sha = sys.argv[2:5]",
|
|
"apply_out = pathlib.Path(sys.argv[5]).read_text(errors='replace')[-2000:]",
|
|
"apply_err = pathlib.Path(sys.argv[6]).read_text(errors='replace')[-2000:]",
|
|
"print(json.dumps({'ok': apply_rc == 0, 'status': 'dry-run', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'applyOutputTail': apply_out, 'applyErrorTail': apply_err, 'mutation': False, 'valuesPrinted': False}, ensure_ascii=False))",
|
|
"PY",
|
|
" exit \"$apply_rc\"",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true >/dev/null 2>&1",
|
|
"kubectl -n \"$namespace\" apply -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
|
|
"apply_rc=$?",
|
|
"wait_rc=1",
|
|
"if [ \"$apply_rc\" = 0 ]; then",
|
|
" kubectl -n \"$namespace\" wait --for=condition=complete \"job/$job\" --timeout=\"${timeout_seconds}s\" >\"$wait_out\" 2>\"$wait_err\"",
|
|
" wait_rc=$?",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true",
|
|
"python3 - \"$apply_rc\" \"$wait_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" \"$wait_out\" \"$wait_err\" \"$logs_file\" <<'PY'",
|
|
"import json, pathlib, sys",
|
|
"apply_rc = int(sys.argv[1])",
|
|
"wait_rc = int(sys.argv[2])",
|
|
"namespace, job, manifest_sha = sys.argv[3:6]",
|
|
"paths = [pathlib.Path(item) for item in sys.argv[6:]]",
|
|
"apply_out, apply_err, wait_out, wait_err, logs = [path.read_text(errors='replace') for path in paths]",
|
|
"job_payload = None",
|
|
"for line in reversed(logs.splitlines()):",
|
|
" text = line.strip()",
|
|
" if text.startswith('{') and text.endswith('}'):",
|
|
" try:",
|
|
" job_payload = json.loads(text)",
|
|
" break",
|
|
" except Exception:",
|
|
" pass",
|
|
"job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True",
|
|
"ok = apply_rc == 0 and wait_rc == 0 and job_ok",
|
|
"print(json.dumps({'ok': ok, 'status': 'succeeded' if ok else 'failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'waitExitCode': wait_rc, 'jobPayload': job_payload, 'applyOutputTail': apply_out[-2000:], 'applyErrorTail': apply_err[-2000:], 'waitOutputTail': wait_out[-2000:], 'waitErrorTail': wait_err[-2000:], 'logsTail': logs[-4000:], 'mutation': ok, 'valuesPrinted': False}, ensure_ascii=False))",
|
|
"PY",
|
|
"exit $([ \"$apply_rc\" = 0 ] && [ \"$wait_rc\" = 0 ] && printf 0 || printf 1)",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceJobManifest(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
|
|
jobName: string,
|
|
): Record<string, unknown> {
|
|
const parent = posixPath.dirname(spec.workspace);
|
|
const basename = posixPath.basename(spec.workspace);
|
|
if (!spec.workspace.startsWith("/") || parent === "." || basename.length === 0) {
|
|
throw new Error(`source workspace must be an absolute POSIX path; got ${spec.workspace}`);
|
|
}
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name: jobName,
|
|
namespace: controlPlane.ciNamespace,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/name": "hwlab-source-workspace-bootstrap",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
ttlSecondsAfterFinished: 3600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/name": "hwlab-source-workspace-bootstrap",
|
|
"unidesk.ai/node": spec.nodeId,
|
|
"unidesk.ai/lane": spec.lane,
|
|
},
|
|
},
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
serviceAccountName: controlPlane.serviceAccountName,
|
|
containers: [
|
|
{
|
|
name: "bootstrap",
|
|
image: controlPlane.toolsImage,
|
|
imagePullPolicy: controlPlane.imagePullPolicy,
|
|
command: ["/bin/sh", "-lc", sourceWorkspaceJobContainerScript(spec, sourceWorkspace, controlPlane, `/host-workspaces/${basename}`)],
|
|
volumeMounts: [{ name: "workspace-parent", mountPath: "/host-workspaces" }],
|
|
env: sourceWorkspaceJobEnv(spec),
|
|
},
|
|
],
|
|
volumes: [{ name: "workspace-parent", hostPath: { path: parent, type: "DirectoryOrCreate" } }],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sourceWorkspaceJobEnv(spec: HwlabRuntimeLaneSpec): Array<Record<string, string>> {
|
|
const proxy = spec.networkProfile.proxy;
|
|
const npm = spec.downloadProfile.npm;
|
|
return [
|
|
{ name: "HTTP_PROXY", value: proxy.http },
|
|
{ name: "HTTPS_PROXY", value: proxy.https },
|
|
{ name: "ALL_PROXY", value: proxy.all },
|
|
{ name: "NO_PROXY", value: proxy.noProxy.join(",") },
|
|
{ name: "http_proxy", value: proxy.http },
|
|
{ name: "https_proxy", value: proxy.https },
|
|
{ name: "all_proxy", value: proxy.all },
|
|
{ name: "no_proxy", value: proxy.noProxy.join(",") },
|
|
{ name: "npm_config_registry", value: npm.registry },
|
|
{ name: "npm_config_fetch_retries", value: String(npm.retries) },
|
|
{ name: "npm_config_fetch_timeout", value: String(npm.fetchTimeoutSeconds * 1000) },
|
|
{ name: "BUN_CONFIG_REGISTRY", value: npm.registry },
|
|
];
|
|
}
|
|
|
|
function sourceWorkspaceJobContainerScript(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
|
|
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
|
|
workspaceInContainer: string,
|
|
): string {
|
|
const requiredCommands = sourceWorkspace.requiredCommands.join("\n");
|
|
const requiredFiles = sourceWorkspace.requiredFiles.join("\n");
|
|
return [
|
|
"set -eu",
|
|
`workspace=${shellQuote(workspaceInContainer)}`,
|
|
`remote=${shellQuote(controlPlane.gitReadUrl)}`,
|
|
`branch=${shellQuote(spec.sourceBranch)}`,
|
|
`dependency_command=${shellQuote(sourceWorkspace.install.dependencyCommand)}`,
|
|
`browser_command=${shellQuote(sourceWorkspace.install.browserCommand)}`,
|
|
`required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`,
|
|
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"stdout_log=\"$tmp_dir/bootstrap.stdout\"",
|
|
"stderr_log=\"$tmp_dir/bootstrap.stderr\"",
|
|
"failure() {",
|
|
" code=$?",
|
|
" if [ \"$code\" -ne 0 ]; then",
|
|
" tail_text=$(cat \"$stderr_log\" 2>/dev/null | tail -n 80 | tr '\\n' ' ' | cut -c1-3000 || true)",
|
|
" CODE=\"$code\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" ERROR_TAIL=\"$tail_text\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: Number(process.env.CODE || '1'), workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" fi",
|
|
" rm -rf \"$tmp_dir\"",
|
|
" exit \"$code\"",
|
|
"}",
|
|
"trap failure EXIT",
|
|
"printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"",
|
|
"printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"",
|
|
"while IFS= read -r command_name; do",
|
|
" [ -z \"$command_name\" ] && continue",
|
|
" command -v \"$command_name\" >/dev/null 2>>\"$stderr_log\"",
|
|
"done <\"$tmp_dir/commands.txt\"",
|
|
"mkdir -p \"$(dirname \"$workspace\")\"",
|
|
"git config --global --add safe.directory \"$workspace\" >>\"$stdout_log\" 2>>\"$stderr_log\" || true",
|
|
"if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
|
|
" status_short=$(git -C \"$workspace\" status --short)",
|
|
" if [ -n \"$status_short\" ]; then echo \"source workspace is dirty; refusing to overwrite\" >>\"$stderr_log\"; exit 42; fi",
|
|
" git -C \"$workspace\" remote set-url origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\" || git -C \"$workspace\" remote add origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
" git -C \"$workspace\" fetch --depth=1 origin \"$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
" git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"else",
|
|
" if [ -e \"$workspace\" ] && [ \"$(find \"$workspace\" -mindepth 1 -maxdepth 1 2>/dev/null | head -n 1)\" ]; then echo \"source workspace path exists but is not a git repo\" >>\"$stderr_log\"; exit 43; fi",
|
|
" rm -rf \"$workspace\"",
|
|
" git clone --depth=1 --branch \"$branch\" \"$remote\" \"$workspace\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"fi",
|
|
"cd \"$workspace\"",
|
|
"sh -lc \"$dependency_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"sh -lc \"$browser_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
|
|
"missing_files=",
|
|
"while IFS= read -r file_path; do",
|
|
" [ -z \"$file_path\" ] && continue",
|
|
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
|
|
"done <\"$tmp_dir/files.txt\"",
|
|
"if [ -n \"$missing_files\" ]; then echo \"missing required files: $missing_files\" >>\"$stderr_log\"; exit 44; fi",
|
|
"source_commit=$(git rev-parse HEAD)",
|
|
"status_short=$(git status --short)",
|
|
"node_version=$(node --version 2>/dev/null || true)",
|
|
"npm_version=$(npm --version 2>/dev/null || true)",
|
|
"STATUS_SHORT=\"$status_short\" SOURCE_COMMIT=\"$source_commit\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" node <<'NODE'",
|
|
"const clean = !process.env.STATUS_SHORT;",
|
|
"console.log(JSON.stringify({ ok: clean, status: clean ? 'succeeded' : 'dirty-after-bootstrap', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, workspaceClean: clean, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
"rm -rf \"$tmp_dir\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function sourceWorkspaceJobName(spec: HwlabRuntimeLaneSpec): string {
|
|
return `hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-workspace`.slice(0, 63).replace(/-$/u, "");
|
|
}
|