mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
4a16a1f5a0982e1affded145847f9911243869d4
pikasTech-unidesk/scripts/src/hwlab-node/source-workspace.ts
T

1535 lines
90 KiB
TypeScript
Raw Blame History

// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-d518-yaml-first-cutover.
// Responsibility: YAML-first D518/D601 HWLAB source workspace status and bootstrap for web-probe clients.
import { createHash } from "node:crypto";
import { posix as posixPath } from "node:path";
import type { UniDeskConfig } from "../config";
import { ensureGithubSshIdentityForProvider } from "../deploy-ssh-identity";
import { hwlabNodeControlPlaneSourceWorkspaceBootstrap } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, type HwlabRuntimeLaneSpec, type HwlabRuntimeSourceWorkspaceHostDependenciesSpec, type HwlabRuntimeSourceWorkspaceSpec } from "../hwlab-node-lanes";
import { parseNodeScopedDelegatedOptions } from "./plan";
import { runTransHostScript, runTransScript } from "./public-exposure";
import { compactRuntimeCommand, parseLastJsonLineObject } from "./runtime-common";
import { commaListField, keyValueLinesFromText, numericField, shellQuote, statusText } from "./utils";
type ScopedNodeOptions = ReturnType<typeof parseNodeScopedDelegatedOptions>;
type SourceWorkspaceAction = "status" | "sync" | "bootstrap" | "host-deps";
type SourceWorkspaceHostDependenciesAction = "status" | "sync";
export async function nodeRuntimeSourceWorkspaceCommand(config: UniDeskConfig, scoped: ScopedNodeOptions): Promise<Record<string, unknown>> {
const action = sourceWorkspaceAction(scoped);
const sourceWorkspace = scoped.spec.sourceWorkspace;
if (sourceWorkspace === undefined) {
return {
ok: false,
command: `hwlab nodes control-plane source-workspace ${action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mutation: false,
degradedReason: "source-workspace-yaml-missing",
message: "lanes.<lane>.targets.<node>.sourceWorkspace must declare requiredCommands, requiredFiles and install commands before this controlled entry can run.",
configPath: hwlabRuntimeLaneConfigPath(),
};
}
if (action === "host-deps") return nodeRuntimeSourceWorkspaceHostDependencies(scoped, sourceWorkspace);
if (action === "sync") return await nodeRuntimeSourceWorkspaceSync(config, scoped, sourceWorkspace);
if (action === "status") return nodeRuntimeSourceWorkspaceStatus(scoped, sourceWorkspace);
return nodeRuntimeSourceWorkspaceBootstrap(scoped, sourceWorkspace);
}
function sourceWorkspaceAction(scoped: ScopedNodeOptions): SourceWorkspaceAction {
const value = scoped.originalArgs[1];
if (value !== "status" && value !== "sync" && value !== "bootstrap" && value !== "host-deps") {
throw new Error("control-plane source-workspace usage: source-workspace status|sync|bootstrap|host-deps --node NODE --lane vNN [--dry-run|--confirm]");
}
if (value === "status" && (scoped.confirm || scoped.dryRun)) {
throw new Error("control-plane source-workspace status is read-only and does not accept --dry-run or --confirm");
}
if (value === "sync" && scoped.wait) {
throw new Error("control-plane source-workspace sync is synchronous and does not accept --wait");
}
return value;
}
function sourceWorkspaceHostDependenciesAction(scoped: ScopedNodeOptions): SourceWorkspaceHostDependenciesAction {
const value = scoped.originalArgs[2];
if (value !== "status" && value !== "sync") {
throw new Error("control-plane source-workspace host-deps usage: source-workspace host-deps status|sync --node NODE --lane vNN [--dry-run|--confirm] [--wait]");
}
if (value === "status" && (scoped.confirm || scoped.dryRun || scoped.wait)) {
throw new Error("control-plane source-workspace host-deps status is read-only and does not accept --dry-run, --confirm or --wait");
}
return value;
}
function nodeRuntimeSourceWorkspaceStatus(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
const result = runTransHostScript(scoped.node, sourceWorkspaceStatusScript(scoped.spec, sourceWorkspace), "", scoped.timeoutSeconds);
const fields = keyValueLinesFromText(statusText(result));
const status = sourceWorkspaceStatusFromFields(fields, result.exitCode);
return {
ok: status.ok === true,
command: `hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "read-only",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
status,
probe: compactRuntimeCommand(result),
degradedReason: status.ok === true ? undefined : "source-workspace-not-ready",
next: status.ok === true
? { webProbe: `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}` }
: { bootstrap: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace bootstrap accepts only one of --dry-run or --confirm");
if (sourceWorkspace.install.executor === "host") return nodeRuntimeSourceWorkspaceHostBootstrap(scoped, sourceWorkspace);
const dryRun = scoped.dryRun || !scoped.confirm;
const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane);
const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.min(scoped.timeoutSeconds, 55));
const payload = parseLastJsonLineObject(statusText(result));
const wait = dryRun || result.exitCode !== 0 || payload.ok !== true
? null
: waitForSourceWorkspaceBootstrapJob(scoped, controlPlane.ciNamespace, sourceWorkspaceJobName(scoped.spec), sourceWorkspace.install.timeoutSeconds);
const after = dryRun || wait?.ok !== true ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
const afterOk = after === null ? null : after.ok === true;
const payloadOk = payload.ok === true;
const waitOk = dryRun ? true : wait?.ok === true;
const ok = result.exitCode === 0 && payloadOk && waitOk && (dryRun || afterOk === true);
return {
ok,
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: dryRun ? "dry-run" : "confirmed-bootstrap",
mutation: !dryRun && ok,
configPath: hwlabRuntimeLaneConfigPath(),
controlPlaneConfigPath: controlPlane.configPath,
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
controlPlane,
before,
bootstrap: payload,
result: compactRuntimeCommand(result),
wait,
after,
degradedReason: ok ? undefined : dryRun ? "source-workspace-bootstrap-dry-run-failed" : "source-workspace-bootstrap-failed",
next: ok
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
async function nodeRuntimeSourceWorkspaceSync(config: UniDeskConfig, scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Promise<Record<string, unknown>> {
if (sourceWorkspace.git === undefined) {
return {
ok: false,
command: `hwlab nodes control-plane source-workspace sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "yaml-missing",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
degradedReason: "source-workspace-git-yaml-missing",
message: "sourceWorkspace.git must declare remoteName, remoteUrl, identityTarget/identityId, proxyEnvPath and up-to-date policy before sync can run.",
};
}
const dryRun = scoped.dryRun || !scoped.confirm;
const before = nodeRuntimeSourceWorkspaceGitStatus(scoped, sourceWorkspace);
const identityPlan = sourceWorkspaceIdentityPlan(scoped, sourceWorkspace);
if (dryRun) {
return {
ok: true,
command: `hwlab nodes control-plane source-workspace sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "dry-run",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
before,
identity: identityPlan,
sync: {
ok: true,
status: "dry-run",
remoteName: sourceWorkspace.git.remoteName,
remoteUrl: sourceWorkspace.git.remoteUrl,
sourceBranch: scoped.spec.sourceBranch,
valuesPrinted: false,
},
after: null,
next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane source-workspace sync --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
const identity = await sourceWorkspaceEnsureIdentity(config, scoped, sourceWorkspace);
if (identity.ok === false) {
return {
ok: false,
command: `hwlab nodes control-plane source-workspace sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "identity-distribution-failed",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
before,
identity,
sync: null,
after: null,
degradedReason: "source-workspace-identity-distribution-failed",
next: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace sync --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
const result = runTransHostScript(scoped.node, sourceWorkspaceSyncScript(scoped.spec, sourceWorkspace), "", scoped.timeoutSeconds);
const payload = parseLastJsonLineObject(statusText(result));
const after = nodeRuntimeSourceWorkspaceGitStatus(scoped, sourceWorkspace);
const ok = result.exitCode === 0 && payload.ok === true && after.ok === true;
return {
ok,
command: `hwlab nodes control-plane source-workspace sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "confirmed-sync",
mutation: true,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
before,
identity,
sync: payload,
result: compactRuntimeCommand(result),
after,
degradedReason: ok ? undefined : "source-workspace-sync-failed",
next: ok
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace sync --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
function sourceWorkspaceIdentityPlan(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
const git = sourceWorkspace.git;
if (git === undefined) return { required: false, valuesPrinted: false };
const remoteNeedsSsh = remoteUrlNeedsSshIdentity(git.remoteUrl);
return {
required: remoteNeedsSsh,
source: "sourceWorkspace.git",
identityTarget: git.identityTarget ?? null,
identityId: git.identityId ?? null,
configRefs: {
lane: `${hwlabRuntimeLaneConfigPath()}#lanes.${scoped.lane}.targets.${scoped.node}.sourceWorkspace.git`,
identity: git.identityTarget === undefined || git.identityId === undefined ? null : `config/deploy-ssh-identities.yaml#targets.${git.identityTarget}.identities[${git.identityId}]`,
},
valuesPrinted: false,
};
}
async function sourceWorkspaceEnsureIdentity(config: UniDeskConfig, scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Promise<Record<string, unknown>> {
const git = sourceWorkspace.git;
if (git === undefined || !remoteUrlNeedsSshIdentity(git.remoteUrl)) {
return { ok: true, required: false, status: "not-required", valuesPrinted: false };
}
if (git.identityTarget === undefined || git.identityId === undefined) {
return {
ok: false,
required: true,
status: "yaml-missing",
degradedReason: "source-workspace-git-identity-yaml-missing",
message: `sourceWorkspace.git for node=${scoped.node} lane=${scoped.lane} uses SSH remoteUrl and must declare identityTarget and identityId.`,
configPath: hwlabRuntimeLaneConfigPath(),
valuesPrinted: false,
};
}
const result = await ensureGithubSshIdentityForProvider(config, git.identityTarget, git.identityId);
return {
ok: result.ok,
required: true,
status: result.ok ? "distributed" : "failed",
identityTarget: result.targetId ?? git.identityTarget,
identityId: result.identityId ?? git.identityId,
fingerprint: result.fingerprint,
seededFromLocal: result.seededFromLocal,
detail: result.ok ? result.detail : "identity distribution failed; inspect the returned redacted raw tail for details",
configTruth: result.configTruth,
raw: result.ok ? null : result.raw,
valuesPrinted: false,
};
}
function remoteUrlNeedsSshIdentity(remoteUrl: string): boolean {
return remoteUrl.startsWith("git@") || remoteUrl.startsWith("ssh://");
}
function nodeRuntimeSourceWorkspaceGitStatus(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
if (sourceWorkspace.git === undefined) {
return { ok: false, status: "yaml-missing", degradedReason: "source-workspace-git-yaml-missing", valuesPrinted: false };
}
const result = runTransHostScript(scoped.node, sourceWorkspaceGitStatusScript(scoped.spec, sourceWorkspace), "", scoped.timeoutSeconds);
const payload = parseLastJsonLineObject(statusText(result));
return {
ok: result.exitCode === 0 && payload.ok === true,
...payload,
probe: compactRuntimeCommand(result),
valuesPrinted: false,
};
}
function sourceWorkspaceGitStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string {
const git = sourceWorkspace.git;
if (git === undefined) throw new Error("sourceWorkspace.git is required");
return [
"set +e",
`workspace=${shellQuote(spec.workspace)}`,
`expected_branch=${shellQuote(spec.sourceBranch)}`,
`remote_name=${shellQuote(git.remoteName)}`,
`expected_remote_url=${shellQuote(git.remoteUrl)}`,
`verify_remote=${shellQuote(git.verifyRemote ? "yes" : "no")}`,
`require_up_to_date=${shellQuote(git.requireUpToDate ? "yes" : "no")}`,
`proxy_env_path=${shellQuote(git.proxyEnvPath ?? "")}`,
`git_timeout=${String(spec.downloadProfile.git.timeoutSeconds)}`,
`http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
`https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
`all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
`no_proxy_value=${shellQuote(spec.networkProfile.proxy.noProxy.join(","))}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"export GIT_TERMINAL_PROMPT=0",
"workspace_exists=false",
"git_dir_exists=false",
"workspace_clean=false",
"branch=",
"local_head=",
"remote_url=",
"remote_matches=false",
"remote_reachable=null",
"remote_head=",
"remote_probe_exit=",
"remote_probe_error_tail=",
"status_short=",
"failure_kind=",
"if [ ! -d \"$workspace\" ]; then",
" failure_kind=workspace-missing",
"else",
" workspace_exists=true",
" if ! git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
" failure_kind=workspace-git-missing",
" else",
" git_dir_exists=true",
" branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
" local_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
" remote_url=$(git -C \"$workspace\" remote get-url \"$remote_name\" 2>/dev/null || true)",
" status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
" [ -z \"$status_short\" ] && workspace_clean=true",
" [ \"$remote_url\" = \"$expected_remote_url\" ] && remote_matches=true",
" if [ \"$verify_remote\" = yes ]; then",
" if [ -n \"$proxy_env_path\" ] && [ -f \"$proxy_env_path\" ]; then . \"$proxy_env_path\" 2>/dev/null || true; fi",
" export HTTP_PROXY=\"${HTTP_PROXY:-$http_proxy_value}\" HTTPS_PROXY=\"${HTTPS_PROXY:-$https_proxy_value}\" ALL_PROXY=\"${ALL_PROXY:-$all_proxy_value}\" NO_PROXY=\"${NO_PROXY:-$no_proxy_value}\"",
" export http_proxy=\"${http_proxy:-$HTTP_PROXY}\" https_proxy=\"${https_proxy:-$HTTPS_PROXY}\" all_proxy=\"${all_proxy:-$ALL_PROXY}\" no_proxy=\"${no_proxy:-$NO_PROXY}\"",
" timeout \"$git_timeout\" git -C \"$workspace\" ls-remote \"$expected_remote_url\" \"refs/heads/$expected_branch\" >\"$tmp_dir/remote.out\" 2>\"$tmp_dir/remote.err\"",
" remote_probe_exit=$?",
" remote_head=$(awk '{print $1}' \"$tmp_dir/remote.out\" | head -n 1)",
" remote_probe_error_tail=$(tail -c 1000 \"$tmp_dir/remote.err\" 2>/dev/null | tr '\\n\\t' ' ' | cut -c1-1000)",
" if [ \"$remote_probe_exit\" = 0 ] && [ -n \"$remote_head\" ]; then remote_reachable=true; else remote_reachable=false; fi",
" fi",
" fi",
"fi",
"WORKSPACE=\"$workspace\" EXPECTED_BRANCH=\"$expected_branch\" WORKSPACE_EXISTS=\"$workspace_exists\" GIT_DIR_EXISTS=\"$git_dir_exists\" WORKSPACE_CLEAN=\"$workspace_clean\" BRANCH=\"$branch\" LOCAL_HEAD=\"$local_head\" REMOTE_NAME=\"$remote_name\" EXPECTED_REMOTE_URL=\"$expected_remote_url\" REMOTE_URL=\"$remote_url\" REMOTE_MATCHES=\"$remote_matches\" VERIFY_REMOTE=\"$verify_remote\" REQUIRE_UP_TO_DATE=\"$require_up_to_date\" REMOTE_REACHABLE=\"$remote_reachable\" REMOTE_HEAD=\"$remote_head\" REMOTE_PROBE_EXIT=\"$remote_probe_exit\" REMOTE_PROBE_ERROR_TAIL=\"$remote_probe_error_tail\" STATUS_SHORT=\"$status_short\" FAILURE_KIND=\"$failure_kind\" node <<'NODE'",
"const statusShort = process.env.STATUS_SHORT || '';",
"const remoteHead = process.env.REMOTE_HEAD || '';",
"const remoteRequired = process.env.VERIFY_REMOTE === 'yes';",
"const upToDateRequired = process.env.REQUIRE_UP_TO_DATE === 'yes';",
"const remoteReachable = process.env.REMOTE_REACHABLE === 'null' ? null : process.env.REMOTE_REACHABLE === 'true';",
"const localHead = process.env.LOCAL_HEAD || '';",
"const ok = process.env.WORKSPACE_EXISTS === 'true' && process.env.GIT_DIR_EXISTS === 'true' && process.env.WORKSPACE_CLEAN === 'true' && process.env.BRANCH === process.env.EXPECTED_BRANCH && process.env.REMOTE_MATCHES === 'true' && (!remoteRequired || remoteReachable === true) && (!upToDateRequired || !remoteHead || localHead === remoteHead);",
"console.log(JSON.stringify({ ok, status: ok ? 'ready' : 'not-ready', workspace: process.env.WORKSPACE, expectedBranch: process.env.EXPECTED_BRANCH, workspaceExists: process.env.WORKSPACE_EXISTS === 'true', gitDirExists: process.env.GIT_DIR_EXISTS === 'true', workspaceClean: process.env.WORKSPACE_CLEAN === 'true', branch: process.env.BRANCH || null, localHead: localHead || null, remoteName: process.env.REMOTE_NAME, expectedRemoteUrl: process.env.EXPECTED_REMOTE_URL, remoteUrl: process.env.REMOTE_URL || null, remoteMatchesYaml: process.env.REMOTE_MATCHES === 'true', remoteReachabilityRequired: remoteRequired, remoteReachable, remoteHead: remoteHead || null, remoteProbeExitCode: process.env.REMOTE_PROBE_EXIT ? Number(process.env.REMOTE_PROBE_EXIT) : null, remoteProbeErrorTail: process.env.REMOTE_PROBE_ERROR_TAIL || null, remoteUpToDateRequired: upToDateRequired, remoteUpToDate: remoteHead ? localHead === remoteHead : null, statusShortLines: statusShort ? statusShort.split(/\\n/).filter(Boolean).length : 0, statusShortPreview: statusShort.replace(/[\\n\\t]+/g, ' ').slice(0, 1000) || null, failureKind: process.env.FAILURE_KIND || null, valuesPrinted: false }));",
"NODE",
].join("\n");
}
function sourceWorkspaceSyncScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string {
const git = sourceWorkspace.git;
if (git === undefined) throw new Error("sourceWorkspace.git is required");
return [
"set +e",
`workspace=${shellQuote(spec.workspace)}`,
`expected_branch=${shellQuote(spec.sourceBranch)}`,
`remote_name=${shellQuote(git.remoteName)}`,
`remote_url=${shellQuote(git.remoteUrl)}`,
`proxy_env_path=${shellQuote(git.proxyEnvPath ?? "")}`,
`git_timeout=${String(spec.downloadProfile.git.timeoutSeconds)}`,
`http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
`https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
`all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
`no_proxy_value=${shellQuote(spec.networkProfile.proxy.noProxy.join(","))}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"export GIT_TERMINAL_PROMPT=0",
"if [ -n \"$proxy_env_path\" ] && [ -f \"$proxy_env_path\" ]; then . \"$proxy_env_path\" 2>/dev/null || true; fi",
"export HTTP_PROXY=\"${HTTP_PROXY:-$http_proxy_value}\" HTTPS_PROXY=\"${HTTPS_PROXY:-$https_proxy_value}\" ALL_PROXY=\"${ALL_PROXY:-$all_proxy_value}\" NO_PROXY=\"${NO_PROXY:-$no_proxy_value}\"",
"export http_proxy=\"${http_proxy:-$HTTP_PROXY}\" https_proxy=\"${https_proxy:-$HTTPS_PROXY}\" all_proxy=\"${all_proxy:-$ALL_PROXY}\" no_proxy=\"${no_proxy:-$NO_PROXY}\"",
"failure_kind=",
"fetch_exit=1",
"update_exit=1",
"remote_url_after=",
"remote_head=",
"local_head=",
"branch=",
"status_short=",
"stderr_tail=",
"if [ ! -d \"$workspace/.git\" ]; then",
" failure_kind=workspace-git-missing",
"elif ! cd \"$workspace\"; then",
" failure_kind=workspace-cd-failed",
"else",
" status_short=$(git status --short 2>/dev/null || true)",
" branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
" if [ -n \"$status_short\" ]; then",
" failure_kind=workspace-dirty",
" elif [ \"$branch\" != \"$expected_branch\" ]; then",
" failure_kind=branch-mismatch",
" else",
" git remote set-url \"$remote_name\" \"$remote_url\" >\"$tmp_dir/remote.out\" 2>\"$tmp_dir/remote.err\" || git remote add \"$remote_name\" \"$remote_url\" >>\"$tmp_dir/remote.out\" 2>>\"$tmp_dir/remote.err\"",
" remote_set_exit=$?",
" remote_url_after=$(git remote get-url \"$remote_name\" 2>/dev/null || true)",
" if [ \"$remote_set_exit\" -ne 0 ]; then",
" failure_kind=remote-config-failed",
" else",
" timeout \"$git_timeout\" git fetch \"$remote_name\" \"$expected_branch:refs/remotes/$remote_name/$expected_branch\" >\"$tmp_dir/fetch.out\" 2>\"$tmp_dir/fetch.err\"",
" fetch_exit=$?",
" if [ \"$fetch_exit\" -ne 0 ]; then",
" failure_kind=git-fetch-failed",
" else",
" remote_head=$(git rev-parse \"refs/remotes/$remote_name/$expected_branch\" 2>/dev/null || true)",
" git merge --ff-only \"refs/remotes/$remote_name/$expected_branch\" >\"$tmp_dir/update.out\" 2>\"$tmp_dir/update.err\"",
" update_exit=$?",
" if [ \"$update_exit\" -ne 0 ]; then failure_kind=git-ff-only-failed; fi",
" fi",
" fi",
" fi",
" local_head=$(git rev-parse HEAD 2>/dev/null || true)",
" status_short=$(git status --short 2>/dev/null || true)",
"fi",
"stderr_tail=$(cat \"$tmp_dir\"/*.err 2>/dev/null | tail -c 2000 | tr '\\n\\t' ' ' | cut -c1-2000)",
"WORKSPACE=\"$workspace\" EXPECTED_BRANCH=\"$expected_branch\" REMOTE_NAME=\"$remote_name\" REMOTE_URL=\"$remote_url\" REMOTE_URL_AFTER=\"$remote_url_after\" BRANCH=\"$branch\" LOCAL_HEAD=\"$local_head\" REMOTE_HEAD=\"$remote_head\" FETCH_EXIT=\"$fetch_exit\" UPDATE_EXIT=\"$update_exit\" STATUS_SHORT=\"$status_short\" FAILURE_KIND=\"$failure_kind\" STDERR_TAIL=\"$stderr_tail\" node <<'NODE'",
"const statusShort = process.env.STATUS_SHORT || '';",
"const ok = !process.env.FAILURE_KIND && !statusShort && process.env.LOCAL_HEAD && (!process.env.REMOTE_HEAD || process.env.LOCAL_HEAD === process.env.REMOTE_HEAD);",
"console.log(JSON.stringify({ ok, status: ok ? 'synced' : 'failed', workspace: process.env.WORKSPACE, branch: process.env.BRANCH || null, expectedBranch: process.env.EXPECTED_BRANCH, remoteName: process.env.REMOTE_NAME, remoteUrl: process.env.REMOTE_URL_AFTER || process.env.REMOTE_URL, localHead: process.env.LOCAL_HEAD || null, remoteHead: process.env.REMOTE_HEAD || null, remoteUpToDate: process.env.REMOTE_HEAD ? process.env.LOCAL_HEAD === process.env.REMOTE_HEAD : null, fetchExitCode: Number(process.env.FETCH_EXIT || '1'), updateExitCode: Number(process.env.UPDATE_EXIT || '1'), failureKind: process.env.FAILURE_KIND || null, statusShortLines: statusShort ? statusShort.split(/\\n/).filter(Boolean).length : 0, statusShortPreview: statusShort.replace(/[\\n\\t]+/g, ' ').slice(0, 1000) || null, stderrTail: process.env.STDERR_TAIL || null, valuesPrinted: false }));",
"NODE",
].join("\n");
}
function nodeRuntimeSourceWorkspaceHostBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
const stateDir = sourceWorkspace.install.stateDir;
if (stateDir === undefined) {
return {
ok: false,
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "host-bootstrap",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
degradedReason: "source-workspace-host-bootstrap-state-dir-missing",
message: "sourceWorkspace.install.stateDir is required when sourceWorkspace.install.executor=host.",
};
}
const dryRun = scoped.dryRun || !scoped.confirm;
const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
if (before.ok === true) {
return {
ok: true,
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: dryRun ? "dry-run" : "already-ready",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
before,
bootstrap: { ok: true, status: "skipped", reason: "source-workspace-already-ready", valuesPrinted: false },
result: null,
wait: null,
after: before,
next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
if (dryRun) {
return {
ok: true,
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "host-dry-run",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
before,
bootstrap: {
ok: true,
status: "dry-run",
executor: "host",
stateDir,
dependencyCommandSha256: createHash("sha256").update(sourceWorkspace.install.dependencyCommand).digest("hex"),
browserCommandSha256: createHash("sha256").update(sourceWorkspace.install.browserCommand).digest("hex"),
timeoutSeconds: sourceWorkspace.install.timeoutSeconds,
valuesPrinted: false,
},
result: null,
wait: null,
after: null,
next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
const submit = runTransHostScript(scoped.node, sourceWorkspaceHostBootstrapSubmitScript(scoped.spec, sourceWorkspace, stateDir), "", Math.min(scoped.timeoutSeconds, 55));
const submitted = parseLastJsonLineObject(statusText(submit));
if (submit.exitCode !== 0 || submitted.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "host-submit",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
before,
bootstrap: submitted,
result: compactRuntimeCommand(submit),
wait: null,
after: null,
degradedReason: "source-workspace-host-bootstrap-submit-failed",
next: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
const wait = waitForSourceWorkspaceHostBootstrap(scoped, stateDir, sourceWorkspace.install.timeoutSeconds);
const after = wait.ok === true ? nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace) : null;
const ok = wait.ok === true && after?.ok === true;
return {
ok,
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "host-confirmed-wait",
mutation: true,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace),
before,
bootstrap: submitted,
result: compactRuntimeCommand(submit),
wait,
after,
degradedReason: ok ? undefined : "source-workspace-host-bootstrap-failed",
next: ok
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
function nodeRuntimeSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
const action = sourceWorkspaceHostDependenciesAction(scoped);
const hostDependencies = sourceWorkspace.hostDependencies;
if (hostDependencies === undefined) {
return {
ok: false,
command: `hwlab nodes control-plane source-workspace host-deps ${action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
degradedReason: "source-workspace-host-dependencies-yaml-missing",
message: "lanes.<lane>.targets.<node>.sourceWorkspace.hostDependencies must declare checkCommands, stateDir and install before this controlled entry can run.",
};
}
if (action === "status") return nodeRuntimeSourceWorkspaceHostDependenciesStatus(scoped, hostDependencies);
return nodeRuntimeSourceWorkspaceHostDependenciesSync(scoped, sourceWorkspace, hostDependencies);
}
function nodeRuntimeSourceWorkspaceHostDependenciesStatus(scoped: ScopedNodeOptions, hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
const result = runTransHostScript(scoped.node, sourceWorkspaceHostDependenciesStatusScript(hostDependencies), "", scoped.timeoutSeconds);
const payload = parseLastJsonLineObject(statusText(result));
const ok = result.exitCode === 0 && payload.ok === true;
return {
ok,
command: `hwlab nodes control-plane source-workspace host-deps status --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "read-only",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
status: payload,
probe: compactRuntimeCommand(result),
degradedReason: ok ? undefined : "source-workspace-host-dependencies-not-ready",
next: ok ? undefined : { sync: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
};
}
function nodeRuntimeSourceWorkspaceHostDependenciesSync(
scoped: ScopedNodeOptions,
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec,
): Record<string, unknown> {
if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace host-deps sync accepts only one of --dry-run or --confirm");
const dryRun = scoped.dryRun || !scoped.confirm;
const before = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
if (before.ok === true) {
return {
ok: true,
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: dryRun ? "dry-run" : "already-ready",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
before,
sync: { ok: true, status: "skipped", reason: "host-dependencies-already-ready", valuesPrinted: false },
after: before,
next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
if (dryRun) {
return {
ok: true,
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "dry-run",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
before,
sync: {
ok: true,
status: "dry-run",
stateDir: hostDependencies.stateDir,
checkCommands: hostDependencies.checkCommands,
installCommandSha256: createHash("sha256").update(hostDependencies.install.command).digest("hex"),
timeoutSeconds: hostDependencies.install.timeoutSeconds,
valuesPrinted: false,
},
after: null,
next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
};
}
const submit = runTransHostScript(scoped.node, sourceWorkspaceHostDependenciesSubmitScript(scoped.spec, sourceWorkspace, hostDependencies), "", Math.min(scoped.timeoutSeconds, 55));
const submitted = parseLastJsonLineObject(statusText(submit));
if (submit.exitCode !== 0 || submitted.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "confirmed-submit",
mutation: false,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
before,
sync: submitted,
result: compactRuntimeCommand(submit),
after: null,
degradedReason: "source-workspace-host-dependencies-submit-failed",
next: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
};
}
if (!scoped.wait) {
return {
ok: true,
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "submitted",
mutation: true,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
before,
sync: submitted,
result: compactRuntimeCommand(submit),
after: null,
next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
const waited = waitForSourceWorkspaceHostDependencies(scoped, hostDependencies);
const after = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
const ok = waited.ok === true && after.ok === true;
return {
ok,
command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "confirmed-wait",
mutation: true,
configPath: hwlabRuntimeLaneConfigPath(),
expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
before,
sync: submitted,
result: compactRuntimeCommand(submit),
wait: waited,
after,
degradedReason: ok ? undefined : "source-workspace-host-dependencies-sync-failed",
next: ok
? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
};
}
function waitForSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
const startedAt = Date.now();
const deadline = startedAt + (hostDependencies.install.timeoutSeconds + 60) * 1000;
let polls = 0;
let lastPayload: Record<string, unknown> = { ok: false, status: "not-polled", valuesPrinted: false };
while (Date.now() <= deadline) {
polls += 1;
const status = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
lastPayload = typeof status.status === "object" && status.status !== null ? status.status as Record<string, unknown> : lastPayload;
if (status.ok === true) return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
const lastInstall = typeof lastPayload.lastInstall === "object" && lastPayload.lastInstall !== null ? lastPayload.lastInstall as Record<string, unknown> : {};
if (lastInstall.status === "failed") return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
sleepMs(5000);
}
return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
function sleepMs(ms: number): void {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
}
function waitForSourceWorkspaceBootstrapJob(scoped: ScopedNodeOptions, namespace: string, jobName: string, timeoutSeconds: number): Record<string, unknown> {
const startedAt = Date.now();
const deadline = startedAt + (timeoutSeconds + 60) * 1000;
let polls = 0;
let lastPayload: Record<string, unknown> = { ok: false, status: "not-polled", valuesPrinted: false };
while (Date.now() <= deadline) {
polls += 1;
const result = runTransScript(scoped.node, sourceWorkspaceBootstrapJobStatusScript(namespace, jobName), "", Math.min(scoped.timeoutSeconds, 55));
const payload = parseLastJsonLineObject(statusText(result));
lastPayload = { ...payload, probe: compactRuntimeCommand(result) };
if (result.exitCode === 0 && payload.status === "succeeded" && payload.ok === true) {
return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
if (payload.status === "failed") {
return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
sleepMs(5000);
}
return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
function waitForSourceWorkspaceHostBootstrap(scoped: ScopedNodeOptions, stateDir: string, timeoutSeconds: number): Record<string, unknown> {
const startedAt = Date.now();
const deadline = startedAt + (timeoutSeconds + 60) * 1000;
let polls = 0;
let lastPayload: Record<string, unknown> = { ok: false, status: "not-polled", valuesPrinted: false };
while (Date.now() <= deadline) {
polls += 1;
const result = runTransHostScript(scoped.node, sourceWorkspaceHostBootstrapStatusScript(stateDir), "", Math.min(scoped.timeoutSeconds, 55));
const payload = parseLastJsonLineObject(statusText(result));
lastPayload = { ...payload, probe: compactRuntimeCommand(result) };
if (result.exitCode === 0 && payload.status === "succeeded" && payload.ok === true) {
return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
if (payload.status === "failed") {
return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
sleepMs(5000);
}
return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
function sourceWorkspaceHostBootstrapStatusScript(stateDir: string): string {
return [
"set +e",
`state_dir=${shellQuote(stateDir)}`,
"status_file=\"$state_dir/bootstrap-status.json\"",
"if [ -f \"$status_file\" ]; then",
" cat \"$status_file\"",
"else",
" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" node <<'NODE'",
"console.log(JSON.stringify({ ok: false, status: 'missing', stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, valuesPrinted: false }));",
"NODE",
"fi",
].join("\n");
}
function sourceWorkspaceHostBootstrapSubmitScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec, stateDir: string): string {
const requiredFiles = shellLinesText(sourceWorkspace.requiredFiles);
const commandSha = createHash("sha256").update(`${sourceWorkspace.install.dependencyCommand}\n${sourceWorkspace.install.browserCommand}`).digest("hex");
return [
"set +e",
`workspace=${shellQuote(spec.workspace)}`,
`expected_branch=${shellQuote(spec.sourceBranch)}`,
`state_dir=${shellQuote(stateDir)}`,
`dependency_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.dependencyCommand, "utf8").toString("base64"))}`,
`browser_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.browserCommand, "utf8").toString("base64"))}`,
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
`timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`,
`remote_name=${shellQuote(sourceWorkspace.git?.remoteName ?? "origin")}`,
`remote_url=${shellQuote(sourceWorkspace.git?.remoteUrl ?? "")}`,
`proxy_env_path=${shellQuote(sourceWorkspace.git?.proxyEnvPath ?? "")}`,
`git_timeout=${String(spec.downloadProfile.git.timeoutSeconds)}`,
`http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
`https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
`all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
`no_proxy_value=${shellQuote(spec.networkProfile.proxy.noProxy.join(","))}`,
`command_sha=${shellQuote(commandSha)}`,
"mkdir -p \"$state_dir\"",
"mkdir_exit=$?",
"if [ \"$mkdir_exit\" -ne 0 ]; then",
" MKDIR_EXIT=\"$mkdir_exit\" STATE_DIR=\"$state_dir\" node <<'NODE'",
"console.log(JSON.stringify({ ok: false, status: 'submit-failed', stateDir: process.env.STATE_DIR, mkdirExitCode: Number(process.env.MKDIR_EXIT || '1'), valuesPrinted: false }));",
"NODE",
" exit \"$mkdir_exit\"",
"fi",
"job_id=\"$(date -u +%Y%m%dT%H%M%SZ)-$$\"",
"status_file=\"$state_dir/bootstrap-status.json\"",
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
"dependency_file=\"$state_dir/$job_id.dependency.sh\"",
"browser_file=\"$state_dir/$job_id.browser.sh\"",
"required_files_file=\"$state_dir/$job_id.required-files.txt\"",
"printf '%s' \"$dependency_command_b64\" | base64 -d >\"$dependency_file\"",
"printf '%s' \"$browser_command_b64\" | base64 -d >\"$browser_file\"",
"printf '%s' \"$required_files_b64\" | base64 -d >\"$required_files_file\"",
"chmod 700 \"$dependency_file\" \"$browser_file\"",
"JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" node <<'NODE' >\"$status_file\"",
"console.log(JSON.stringify({ ok: false, status: 'running', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, startedAt: new Date().toISOString(), valuesPrinted: false }));",
"NODE",
"(",
" set +e",
" if [ -n \"$proxy_env_path\" ] && [ -f \"$proxy_env_path\" ]; then . \"$proxy_env_path\" 2>/dev/null || true; fi",
" export HTTP_PROXY=\"${HTTP_PROXY:-$http_proxy_value}\" HTTPS_PROXY=\"${HTTPS_PROXY:-$https_proxy_value}\" ALL_PROXY=\"${ALL_PROXY:-$all_proxy_value}\" NO_PROXY=\"${NO_PROXY:-$no_proxy_value}\"",
" export http_proxy=\"${http_proxy:-$HTTP_PROXY}\" https_proxy=\"${https_proxy:-$HTTPS_PROXY}\" all_proxy=\"${all_proxy:-$ALL_PROXY}\" no_proxy=\"${no_proxy:-$NO_PROXY}\"",
" failure_kind=",
" dependency_exit=1",
" browser_exit=1",
" git_fetch_exit=0",
" git_update_exit=0",
" remote_url_after=",
" missing_files=",
" source_commit=",
" status_short=",
" node_version=",
" npm_version=",
" npx_version=",
" if [ ! -d \"$workspace/.git\" ]; then",
" failure_kind=workspace-git-missing",
" printf '%s\\n' \"source workspace git repo is missing: $workspace\" >>\"$stderr_file\"",
" elif ! cd \"$workspace\"; then",
" failure_kind=workspace-cd-failed",
" else",
" status_short=$(git status --short 2>/dev/null || true)",
" if [ -n \"$status_short\" ]; then",
" failure_kind=workspace-dirty",
" printf '%s\\n' \"source workspace is dirty; refusing to install dependencies\" >>\"$stderr_file\"",
" else",
" if [ -n \"$remote_url\" ]; then",
" git remote set-url \"$remote_name\" \"$remote_url\" >>\"$stdout_file\" 2>>\"$stderr_file\" || git remote add \"$remote_name\" \"$remote_url\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
" remote_url_after=$(git remote get-url \"$remote_name\" 2>/dev/null || true)",
" timeout \"$git_timeout\" git fetch \"$remote_name\" \"$expected_branch:refs/remotes/$remote_name/$expected_branch\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
" git_fetch_exit=$?",
" if [ \"$git_fetch_exit\" -ne 0 ]; then",
" failure_kind=git-fetch-failed",
" else",
" current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
" if [ \"$current_branch\" = \"$expected_branch\" ]; then",
" git merge --ff-only \"refs/remotes/$remote_name/$expected_branch\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
" git_update_exit=$?",
" else",
" git checkout -B \"$expected_branch\" \"refs/remotes/$remote_name/$expected_branch\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
" git_update_exit=$?",
" fi",
" if [ \"$git_update_exit\" -ne 0 ]; then failure_kind=git-update-failed; fi",
" fi",
" fi",
" if [ -z \"$failure_kind\" ]; then",
" timeout \"$timeout_seconds\" /bin/bash \"$dependency_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
" dependency_exit=$?",
" if [ \"$dependency_exit\" -eq 0 ]; then",
" timeout \"$timeout_seconds\" /bin/bash \"$browser_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"",
" browser_exit=$?",
" else",
" failure_kind=dependency-command-failed",
" fi",
" if [ \"$dependency_exit\" -eq 0 ] && [ \"$browser_exit\" -ne 0 ]; then failure_kind=browser-command-failed; fi",
" fi",
" while IFS= read -r file_path; do",
" [ -z \"$file_path\" ] && continue",
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
" done <\"$required_files_file\"",
" if [ -n \"$missing_files\" ] && [ -z \"$failure_kind\" ]; then failure_kind=missing-required-files; fi",
" source_commit=$(git rev-parse HEAD 2>/dev/null || true)",
" status_short=$(git status --short 2>/dev/null || true)",
" node_version=$(node --version 2>/dev/null || true)",
" npm_version=$(npm --version 2>/dev/null || true)",
" npx_version=$(npx --version 2>/dev/null || true)",
" fi",
" fi",
" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" FAILURE_KIND=\"$failure_kind\" DEPENDENCY_EXIT=\"$dependency_exit\" BROWSER_EXIT=\"$browser_exit\" GIT_FETCH_EXIT=\"$git_fetch_exit\" GIT_UPDATE_EXIT=\"$git_update_exit\" REMOTE_URL_AFTER=\"$remote_url_after\" MISSING_FILES=\"$missing_files\" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" NPX_VERSION=\"$npx_version\" node <<'NODE' >\"$status_file\"",
"const fs = require('fs');",
"const tail = (file) => { try { return fs.readFileSync(file, 'utf8').slice(-4000); } catch { return ''; } };",
"const missingFiles = (process.env.MISSING_FILES || '').split(',').filter(Boolean);",
"const dependencyExit = Number(process.env.DEPENDENCY_EXIT || '1');",
"const browserExit = Number(process.env.BROWSER_EXIT || '1');",
"const gitFetchExit = Number(process.env.GIT_FETCH_EXIT || '0');",
"const gitUpdateExit = Number(process.env.GIT_UPDATE_EXIT || '0');",
"const statusShort = process.env.STATUS_SHORT || '';",
"const ok = dependencyExit === 0 && browserExit === 0 && missingFiles.length === 0 && !statusShort && !process.env.FAILURE_KIND;",
"console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, failureKind: process.env.FAILURE_KIND || null, dependencyExitCode: dependencyExit, browserExitCode: browserExit, gitFetchExitCode: gitFetchExit, gitUpdateExitCode: gitUpdateExit, remoteUrlAfter: process.env.REMOTE_URL_AFTER || null, missingFiles, sourceCommit: process.env.SOURCE_COMMIT || null, workspaceClean: !statusShort, statusShort: statusShort || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, npxVersion: process.env.NPX_VERSION || null, stdoutTail: tail(process.env.STDOUT_FILE), stderrTail: tail(process.env.STDERR_FILE), finishedAt: new Date().toISOString(), valuesPrinted: false }));",
"NODE",
") >/dev/null 2>&1 &",
"pid=$!",
"JOB_ID=\"$job_id\" PID=\"$pid\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.PID || '0'), workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, valuesPrinted: false }));",
"NODE",
].join("\n");
}
function sourceWorkspaceBootstrapJobStatusScript(namespace: string, jobName: string): string {
return [
"set +e",
`namespace=${shellQuote(namespace)}`,
`job=${shellQuote(jobName)}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"job_json=\"$tmp_dir/job.json\"",
"job_err=\"$tmp_dir/job.err\"",
"logs_file=\"$tmp_dir/logs.txt\"",
"kubectl -n \"$namespace\" get job \"$job\" -o json >\"$job_json\" 2>\"$job_err\"",
"job_exit=$?",
"kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true",
"python3 - \"$job_exit\" \"$namespace\" \"$job\" \"$job_json\" \"$job_err\" \"$logs_file\" <<'PY'",
"import json, pathlib, sys",
"job_exit = int(sys.argv[1])",
"namespace, job = sys.argv[2:4]",
"job_text = pathlib.Path(sys.argv[4]).read_text(errors='replace') if pathlib.Path(sys.argv[4]).exists() else ''",
"job_err = pathlib.Path(sys.argv[5]).read_text(errors='replace') if pathlib.Path(sys.argv[5]).exists() else ''",
"logs = pathlib.Path(sys.argv[6]).read_text(errors='replace') if pathlib.Path(sys.argv[6]).exists() else ''",
"job_obj = None",
"if job_exit == 0 and job_text.strip():",
" try:",
" job_obj = json.loads(job_text)",
" except Exception:",
" job_obj = None",
"conditions = (((job_obj or {}).get('status') or {}).get('conditions') or [])",
"complete = any((c or {}).get('type') == 'Complete' and (c or {}).get('status') == 'True' for c in conditions)",
"failed = any((c or {}).get('type') == 'Failed' and (c or {}).get('status') == 'True' for c in conditions)",
"job_payload = None",
"for line in reversed(logs.splitlines()):",
" text = line.strip()",
" if text.startswith('{') and text.endswith('}'):",
" try:",
" job_payload = json.loads(text)",
" break",
" except Exception:",
" pass",
"job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True",
"status = 'succeeded' if complete and job_ok else 'failed' if failed or (complete and not job_ok) else 'missing' if job_exit != 0 else 'running'",
"print(json.dumps({'ok': status == 'succeeded', 'status': status, 'namespace': namespace, 'job': job, 'jobExitCode': job_exit, 'complete': complete, 'failed': failed, 'active': ((job_obj or {}).get('status') or {}).get('active'), 'succeeded': ((job_obj or {}).get('status') or {}).get('succeeded'), 'failedCount': ((job_obj or {}).get('status') or {}).get('failed'), 'jobPayload': job_payload, 'jobErrorTail': job_err[-2000:], 'logsTail': logs[-4000:], 'valuesPrinted': False}, ensure_ascii=False))",
"PY",
].join("\n");
}
function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
return {
workspace: spec.workspace,
sourceBranch: spec.sourceBranch,
sourceWorkspaceGit: sourceWorkspace.git === undefined ? null : {
remoteName: sourceWorkspace.git.remoteName,
remoteUrl: sourceWorkspace.git.remoteUrl,
identityTarget: sourceWorkspace.git.identityTarget ?? null,
identityId: sourceWorkspace.git.identityId ?? null,
proxyEnvPath: sourceWorkspace.git.proxyEnvPath ?? null,
verifyRemote: sourceWorkspace.git.verifyRemote,
requireUpToDate: sourceWorkspace.git.requireUpToDate,
},
git: {
url: spec.gitUrl,
readUrl: spec.gitReadUrl,
writeUrl: spec.gitWriteUrl,
},
requiredCommands: sourceWorkspace.requiredCommands,
requiredFiles: sourceWorkspace.requiredFiles,
hostDependencies: sourceWorkspace.hostDependencies === undefined ? null : sourceWorkspaceHostDependenciesExpected(sourceWorkspace.hostDependencies),
install: sourceWorkspace.install,
downloadProfile: {
id: spec.downloadProfileId,
npm: spec.downloadProfile.npm,
git: spec.downloadProfile.git,
},
networkProfile: {
id: spec.networkProfileId,
proxy: spec.networkProfile.proxy,
},
};
}
function sourceWorkspaceHostDependenciesExpected(hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
return {
checkCommands: hostDependencies.checkCommands,
stateDir: hostDependencies.stateDir,
install: {
commandSha256: createHash("sha256").update(hostDependencies.install.command).digest("hex"),
timeoutSeconds: hostDependencies.install.timeoutSeconds,
},
};
}
function sourceWorkspaceStatusFromFields(fields: Record<string, string>, exitCode: number | null): Record<string, unknown> {
const missingCommands = commaListField(fields.missingCommands);
const missingFiles = commaListField(fields.missingFiles);
const remoteReachabilityRequired = fields.remoteReachabilityRequired === "yes";
const remoteUpToDateRequired = fields.remoteUpToDateRequired === "yes";
const remoteReachable = fields.remoteReachable === "yes";
const remoteHead = fields.remoteHead || "";
const ok = exitCode === 0
&& fields.workspaceExists === "yes"
&& fields.gitDirExists === "yes"
&& fields.workspaceClean === "yes"
&& fields.branch === fields.expectedBranch
&& fields.remoteMatchesYaml === "yes"
&& (!remoteReachabilityRequired || remoteReachable)
&& (!remoteUpToDateRequired || remoteHead.length === 0 || fields.localHead === remoteHead)
&& missingCommands.length === 0
&& missingFiles.length === 0
&& fields.playwrightPackagePresent === "yes"
&& fields.browserExecutablePresent === "yes"
&& fields.launcherImportOk === "yes"
&& fields.browserLaunchOk === "yes";
return {
ok,
workspace: fields.workspace || null,
expectedBranch: fields.expectedBranch || null,
workspaceExists: fields.workspaceExists === "yes",
gitDirExists: fields.gitDirExists === "yes",
workspaceClean: fields.workspaceClean === "yes",
branch: fields.branch || null,
detached: fields.detached === "yes",
localHead: fields.localHead || null,
expectedRemoteUrl: fields.expectedRemoteUrl || null,
remoteUrl: fields.remoteUrl || null,
remoteMatchesYaml: fields.remoteMatchesYaml === "yes",
remoteReachabilityRequired,
remoteUpToDateRequired,
remoteReachable,
remoteHead: remoteHead || null,
remoteProbeExitCode: numericField(fields.remoteProbeExitCode),
remoteProbeErrorTail: fields.remoteProbeErrorTail || null,
remoteUpToDate: remoteHead.length === 0 ? null : fields.localHead === remoteHead,
requiredCommands: commaListField(fields.requiredCommands),
missingCommands,
requiredFiles: commaListField(fields.requiredFiles),
missingFiles,
nodeVersion: fields.nodeVersion || null,
npmVersion: fields.npmVersion || null,
npxVersion: fields.npxVersion || null,
playwrightPackagePresent: fields.playwrightPackagePresent === "yes",
browserCachePresent: fields.browserCachePresent === "yes",
browserExecutablePresent: fields.browserExecutablePresent === "yes",
browserExecutableExitCode: numericField(fields.browserExecutableExitCode),
browserExecutablePath: fields.browserExecutablePath || null,
browserExecutableErrorTail: fields.browserExecutableErrorTail || null,
launcherImportOk: fields.launcherImportOk === "yes",
launcherImportExitCode: numericField(fields.launcherImportExitCode),
browserLaunchOk: fields.browserLaunchOk === "yes",
browserLaunchExitCode: numericField(fields.browserLaunchExitCode),
browserLaunchErrorTail: fields.browserLaunchErrorTail || null,
statusShortLines: numericField(fields.statusShortLines),
statusShortPreview: fields.statusShortPreview || null,
valuesPrinted: false,
};
}
function sourceWorkspaceHostDependenciesStatusScript(hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): string {
const checkCommands = shellLinesText(hostDependencies.checkCommands);
return [
"set +e",
`state_dir=${shellQuote(hostDependencies.stateDir)}`,
`check_commands_b64=${shellQuote(Buffer.from(checkCommands, "utf8").toString("base64"))}`,
"mkdir -p \"$state_dir\" 2>/tmp/hwlab-source-workspace-host-deps-mkdir.err",
"mkdir_exit=$?",
"commands_file=$(mktemp)",
"trap 'rm -f \"$commands_file\"' EXIT",
"printf '%s' \"$check_commands_b64\" | base64 -d >\"$commands_file\"",
"missing_commands=",
"present_commands=",
"while IFS= read -r command_name; do",
" [ -z \"$command_name\" ] && continue",
" if command -v \"$command_name\" >/dev/null 2>&1; then",
" present_commands=\"$present_commands${present_commands:+,}$command_name\"",
" else",
" missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"",
" fi",
"done <\"$commands_file\"",
"status_file=\"$state_dir/install-status.json\"",
"last_install_json='{}'",
"if [ -f \"$status_file\" ]; then last_install_json=$(tail -c 20000 \"$status_file\" 2>/dev/null || printf '{}'); fi",
"STATE_DIR=\"$state_dir\" MKDIR_EXIT=\"$mkdir_exit\" MISSING_COMMANDS=\"$missing_commands\" PRESENT_COMMANDS=\"$present_commands\" STATUS_FILE=\"$status_file\" LAST_INSTALL_JSON=\"$last_install_json\" node <<'NODE'",
"const lastRaw = process.env.LAST_INSTALL_JSON || '{}';",
"let lastInstall = {};",
"try { lastInstall = JSON.parse(lastRaw); } catch { lastInstall = { ok: false, status: 'unparseable', valuesPrinted: false }; }",
"const missing = (process.env.MISSING_COMMANDS || '').split(',').filter(Boolean);",
"const present = (process.env.PRESENT_COMMANDS || '').split(',').filter(Boolean);",
"const mkdirExit = Number(process.env.MKDIR_EXIT || '1');",
"console.log(JSON.stringify({ ok: mkdirExit === 0 && missing.length === 0, status: mkdirExit === 0 && missing.length === 0 ? 'ready' : 'missing', stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, presentCommands: present, missingCommands: missing, mkdirExitCode: mkdirExit, lastInstall, valuesPrinted: false }));",
"NODE",
].join("\n");
}
function sourceWorkspaceHostDependenciesSubmitScript(
spec: HwlabRuntimeLaneSpec,
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec,
): string {
const checkCommands = shellLinesText(hostDependencies.checkCommands);
const noProxy = spec.networkProfile.proxy.noProxy.join(",");
const commandSha = createHash("sha256").update(hostDependencies.install.command).digest("hex");
const requiredCommandsSha = createHash("sha256").update(sourceWorkspace.requiredCommands.join("\n")).digest("hex");
return [
"set +e",
`state_dir=${shellQuote(hostDependencies.stateDir)}`,
`install_command_b64=${shellQuote(Buffer.from(hostDependencies.install.command, "utf8").toString("base64"))}`,
`check_commands_b64=${shellQuote(Buffer.from(checkCommands, "utf8").toString("base64"))}`,
`timeout_seconds=${String(hostDependencies.install.timeoutSeconds)}`,
`http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
`https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
`all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
`no_proxy_value=${shellQuote(noProxy)}`,
`command_sha=${shellQuote(commandSha)}`,
`required_commands_sha=${shellQuote(requiredCommandsSha)}`,
"mkdir -p \"$state_dir\"",
"mkdir_exit=$?",
"if [ \"$mkdir_exit\" -ne 0 ]; then",
" MKDIR_EXIT=\"$mkdir_exit\" STATE_DIR=\"$state_dir\" node <<'NODE'",
"console.log(JSON.stringify({ ok: false, status: 'submit-failed', stateDir: process.env.STATE_DIR, mkdirExitCode: Number(process.env.MKDIR_EXIT || '1'), valuesPrinted: false }));",
"NODE",
" exit \"$mkdir_exit\"",
"fi",
"job_id=\"$(date -u +%Y%m%dT%H%M%SZ)-$$\"",
"status_file=\"$state_dir/install-status.json\"",
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
"command_file=\"$state_dir/$job_id.install.sh\"",
"commands_file=\"$state_dir/$job_id.commands.txt\"",
"printf '%s' \"$install_command_b64\" | base64 -d >\"$command_file\"",
"printf '%s' \"$check_commands_b64\" | base64 -d >\"$commands_file\"",
"chmod 700 \"$command_file\"",
"JOB_ID=\"$job_id\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" node <<'NODE' >\"$status_file\"",
"console.log(JSON.stringify({ ok: false, status: 'running', jobId: process.env.JOB_ID, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, startedAt: new Date().toISOString(), valuesPrinted: false }));",
"NODE",
"(",
" set +e",
" export HTTP_PROXY=\"$http_proxy_value\" HTTPS_PROXY=\"$https_proxy_value\" ALL_PROXY=\"$all_proxy_value\" NO_PROXY=\"$no_proxy_value\"",
" export http_proxy=\"$http_proxy_value\" https_proxy=\"$https_proxy_value\" all_proxy=\"$all_proxy_value\" no_proxy=\"$no_proxy_value\"",
" timeout \"$timeout_seconds\" /bin/bash \"$command_file\" >\"$stdout_file\" 2>\"$stderr_file\"",
" install_exit=$?",
" missing_commands=",
" present_commands=",
" while IFS= read -r command_name; do",
" [ -z \"$command_name\" ] && continue",
" if command -v \"$command_name\" >/dev/null 2>&1; then",
" present_commands=\"$present_commands${present_commands:+,}$command_name\"",
" else",
" missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"",
" fi",
" done <\"$commands_file\"",
" stdout_tail=$(tail -c 4000 \"$stdout_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000)",
" stderr_tail=$(tail -c 4000 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000)",
" JOB_ID=\"$job_id\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" INSTALL_EXIT=\"$install_exit\" PRESENT_COMMANDS=\"$present_commands\" MISSING_COMMANDS=\"$missing_commands\" STDOUT_TAIL=\"$stdout_tail\" STDERR_TAIL=\"$stderr_tail\" node <<'NODE' >\"$status_file\"",
"const missing = (process.env.MISSING_COMMANDS || '').split(',').filter(Boolean);",
"const present = (process.env.PRESENT_COMMANDS || '').split(',').filter(Boolean);",
"const exitCode = Number(process.env.INSTALL_EXIT || '1');",
"const ok = exitCode === 0 && missing.length === 0;",
"console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, installExitCode: exitCode, presentCommands: present, missingCommands: missing, stdoutTail: process.env.STDOUT_TAIL || '', stderrTail: process.env.STDERR_TAIL || '', finishedAt: new Date().toISOString(), valuesPrinted: false }));",
"NODE",
") >/dev/null 2>&1 &",
"pid=$!",
"JOB_ID=\"$job_id\" PID=\"$pid\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.PID || '0'), stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, valuesPrinted: false }));",
"NODE",
].join("\n");
}
function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string {
const requiredCommands = shellLinesText(sourceWorkspace.requiredCommands);
const requiredFiles = shellLinesText(sourceWorkspace.requiredFiles);
const remoteUrls = shellLinesText(sourceWorkspace.git === undefined ? [spec.gitUrl, spec.gitReadUrl, spec.gitWriteUrl] : [sourceWorkspace.git.remoteUrl]);
return [
"set +e",
`workspace=${shellQuote(spec.workspace)}`,
`expected_branch=${shellQuote(spec.sourceBranch)}`,
`remote_name=${shellQuote(sourceWorkspace.git?.remoteName ?? "origin")}`,
`verify_remote=${shellQuote(sourceWorkspace.git?.verifyRemote === true ? "yes" : "no")}`,
`require_up_to_date=${shellQuote(sourceWorkspace.git?.requireUpToDate === true ? "yes" : "no")}`,
`proxy_env_path=${shellQuote(sourceWorkspace.git?.proxyEnvPath ?? "")}`,
`git_timeout=${String(spec.downloadProfile.git.timeoutSeconds)}`,
`http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
`https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
`all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
`no_proxy_value=${shellQuote(spec.networkProfile.proxy.noProxy.join(","))}`,
`required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`,
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
`remote_urls_b64=${shellQuote(Buffer.from(remoteUrls, "utf8").toString("base64"))}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"",
"printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"",
"printf '%s' \"$remote_urls_b64\" | base64 -d >\"$tmp_dir/remotes.txt\"",
"expected_remote_url=$(head -n 1 \"$tmp_dir/remotes.txt\" 2>/dev/null || true)",
"workspace_exists=no",
"git_dir_exists=no",
"workspace_clean=no",
"branch=",
"detached=no",
"local_head=",
"remote_url=",
"remote_matches_yaml=no",
"remote_reachable=skipped",
"remote_head=",
"remote_probe_exit=",
"remote_probe_error_tail=",
"status_short=",
"if [ -d \"$workspace\" ]; then",
" workspace_exists=yes",
" if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
" git_dir_exists=yes",
" branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
" [ \"$branch\" = HEAD ] && detached=yes",
" local_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
" remote_url=$(git -C \"$workspace\" remote get-url \"$remote_name\" 2>/dev/null || true)",
" status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
" [ -z \"$status_short\" ] && workspace_clean=yes",
" while IFS= read -r expected_remote; do [ \"$remote_url\" = \"$expected_remote\" ] && remote_matches_yaml=yes; done <\"$tmp_dir/remotes.txt\"",
" if [ \"$verify_remote\" = yes ] && [ -n \"$expected_remote_url\" ]; then",
" if [ -n \"$proxy_env_path\" ] && [ -f \"$proxy_env_path\" ]; then . \"$proxy_env_path\" 2>/dev/null || true; fi",
" export HTTP_PROXY=\"${HTTP_PROXY:-$http_proxy_value}\" HTTPS_PROXY=\"${HTTPS_PROXY:-$https_proxy_value}\" ALL_PROXY=\"${ALL_PROXY:-$all_proxy_value}\" NO_PROXY=\"${NO_PROXY:-$no_proxy_value}\"",
" export http_proxy=\"${http_proxy:-$HTTP_PROXY}\" https_proxy=\"${https_proxy:-$HTTPS_PROXY}\" all_proxy=\"${all_proxy:-$ALL_PROXY}\" no_proxy=\"${no_proxy:-$NO_PROXY}\"",
" timeout \"$git_timeout\" git -C \"$workspace\" ls-remote \"$expected_remote_url\" \"refs/heads/$expected_branch\" >\"$tmp_dir/remote.out\" 2>\"$tmp_dir/remote.err\"",
" remote_probe_exit=$?",
" remote_head=$(awk '{print $1}' \"$tmp_dir/remote.out\" | head -n 1)",
" remote_probe_error_tail=$(tail -c 1000 \"$tmp_dir/remote.err\" 2>/dev/null | tr '\\n\\t' ' ' | cut -c1-1000)",
" if [ \"$remote_probe_exit\" = 0 ] && [ -n \"$remote_head\" ]; then remote_reachable=yes; else remote_reachable=no; fi",
" fi",
" fi",
"fi",
"missing_commands=",
"while IFS= read -r command_name; do",
" [ -z \"$command_name\" ] && continue",
" if ! command -v \"$command_name\" >/dev/null 2>&1; then missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"; fi",
"done <\"$tmp_dir/commands.txt\"",
"missing_files=",
"if [ \"$workspace_exists\" = yes ]; then",
" while IFS= read -r file_path; do",
" [ -z \"$file_path\" ] && continue",
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
" done <\"$tmp_dir/files.txt\"",
"else",
" missing_files=$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')",
"fi",
"node_version=$(node --version 2>/dev/null || true)",
"npm_version=$(npm --version 2>/dev/null || true)",
"npx_version=$(npx --version 2>/dev/null || true)",
"playwright_package_present=no",
"[ -f \"$workspace/node_modules/playwright/package.json\" ] && playwright_package_present=yes",
"browser_cache_present=no",
"if find \"$workspace/node_modules\" -maxdepth 5 -type d -path '*playwright*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi",
"for cache_dir in \"${HOME:-}/.cache/ms-playwright\" /root/.cache/ms-playwright /home/ubuntu/.cache/ms-playwright; do",
" [ -d \"$cache_dir\" ] || continue",
" if find \"$cache_dir\" -maxdepth 1 -type d -name '*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi",
"done",
"browser_executable_present=no",
"browser_executable_exit=",
"browser_executable_path=",
"browser_executable_error_tail=",
"if command -v node >/dev/null 2>&1 && [ \"$playwright_package_present\" = yes ]; then",
" (cd \"$workspace\" && PLAYWRIGHT_BROWSERS_PATH=0 timeout 20s node --input-type=module - <<'NODE'",
"import { existsSync } from 'node:fs';",
"import { chromium } from 'playwright';",
"const executablePath = chromium.executablePath();",
"console.log(executablePath);",
"process.exit(existsSync(executablePath) ? 0 : 2);",
"NODE",
" ) >/tmp/hwlab-source-workspace-browser-executable.out 2>/tmp/hwlab-source-workspace-browser-executable.err",
" browser_executable_exit=$?",
" browser_executable_path=$(head -n 1 /tmp/hwlab-source-workspace-browser-executable.out 2>/dev/null | cut -c1-1000)",
" browser_executable_error_tail=$(tail -n 20 /tmp/hwlab-source-workspace-browser-executable.err 2>/dev/null | tr '\\n\\t' ' ' | cut -c1-1000)",
" [ \"$browser_executable_exit\" = 0 ] && browser_executable_present=yes",
"fi",
"launcher_import_ok=no",
"launcher_import_exit=",
"if command -v node >/dev/null 2>&1 && [ -f \"$workspace/scripts/src/browser-launcher.mjs\" ]; then",
" (cd \"$workspace\" && timeout 20s node -e \"import('./scripts/src/browser-launcher.mjs').then(()=>process.exit(0)).catch(()=>process.exit(1))\") >/tmp/hwlab-source-workspace-launcher.out 2>/tmp/hwlab-source-workspace-launcher.err",
" launcher_import_exit=$?",
" [ \"$launcher_import_exit\" = 0 ] && launcher_import_ok=yes",
"fi",
"browser_launch_ok=no",
"browser_launch_exit=",
"browser_launch_error_tail=",
"if command -v node >/dev/null 2>&1 && [ \"$launcher_import_ok\" = yes ]; then",
" (cd \"$workspace\" && PLAYWRIGHT_BROWSERS_PATH=0 timeout 30s node --input-type=module - <<'NODE'",
"const doctor = await import('./scripts/src/browser-launcher.mjs').then((mod) => mod.browserDoctor());",
"if (doctor.status !== 'pass') {",
" console.error(JSON.stringify({ failureCode: doctor.failureCode, error: doctor.error, browser: doctor.browser, managedBrowser: doctor.managedBrowser, remediation: doctor.remediation }));",
" process.exit(1);",
"}",
"console.log(JSON.stringify({ status: doctor.status, browser: doctor.browser, managedBrowser: doctor.managedBrowser }));",
"NODE",
" ) >/tmp/hwlab-source-workspace-browser-launch.out 2>/tmp/hwlab-source-workspace-browser-launch.err",
" browser_launch_exit=$?",
" browser_launch_error_tail=$(tail -n 20 /tmp/hwlab-source-workspace-browser-launch.err 2>/dev/null | tr '\\n\\t' ' ' | cut -c1-1000)",
" [ \"$browser_launch_exit\" = 0 ] && browser_launch_ok=yes",
"fi",
"status_short_lines=$(printf '%s' \"$status_short\" | sed '/^$/d' | wc -l | tr -d ' ')",
"status_short_preview=$(printf '%s' \"$status_short\" | tr '\\n\\t' ' ' | cut -c1-1000)",
"printf 'workspace\\t%s\\n' \"$workspace\"",
"printf 'expectedBranch\\t%s\\n' \"$expected_branch\"",
"printf 'workspaceExists\\t%s\\n' \"$workspace_exists\"",
"printf 'gitDirExists\\t%s\\n' \"$git_dir_exists\"",
"printf 'workspaceClean\\t%s\\n' \"$workspace_clean\"",
"printf 'branch\\t%s\\n' \"$branch\"",
"printf 'detached\\t%s\\n' \"$detached\"",
"printf 'localHead\\t%s\\n' \"$local_head\"",
"printf 'expectedRemoteUrl\\t%s\\n' \"$expected_remote_url\"",
"printf 'remoteUrl\\t%s\\n' \"$remote_url\"",
"printf 'remoteMatchesYaml\\t%s\\n' \"$remote_matches_yaml\"",
"printf 'remoteReachabilityRequired\\t%s\\n' \"$verify_remote\"",
"printf 'remoteUpToDateRequired\\t%s\\n' \"$require_up_to_date\"",
"printf 'remoteReachable\\t%s\\n' \"$remote_reachable\"",
"printf 'remoteHead\\t%s\\n' \"$remote_head\"",
"printf 'remoteProbeExitCode\\t%s\\n' \"$remote_probe_exit\"",
"printf 'remoteProbeErrorTail\\t%s\\n' \"$remote_probe_error_tail\"",
"printf 'requiredCommands\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/commands.txt\" | sed 's/,$//')\"",
"printf 'missingCommands\\t%s\\n' \"$missing_commands\"",
"printf 'requiredFiles\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')\"",
"printf 'missingFiles\\t%s\\n' \"$missing_files\"",
"printf 'nodeVersion\\t%s\\n' \"$node_version\"",
"printf 'npmVersion\\t%s\\n' \"$npm_version\"",
"printf 'npxVersion\\t%s\\n' \"$npx_version\"",
"printf 'playwrightPackagePresent\\t%s\\n' \"$playwright_package_present\"",
"printf 'browserCachePresent\\t%s\\n' \"$browser_cache_present\"",
"printf 'browserExecutablePresent\\t%s\\n' \"$browser_executable_present\"",
"printf 'browserExecutableExitCode\\t%s\\n' \"$browser_executable_exit\"",
"printf 'browserExecutablePath\\t%s\\n' \"$browser_executable_path\"",
"printf 'browserExecutableErrorTail\\t%s\\n' \"$browser_executable_error_tail\"",
"printf 'launcherImportOk\\t%s\\n' \"$launcher_import_ok\"",
"printf 'launcherImportExitCode\\t%s\\n' \"$launcher_import_exit\"",
"printf 'browserLaunchOk\\t%s\\n' \"$browser_launch_ok\"",
"printf 'browserLaunchExitCode\\t%s\\n' \"$browser_launch_exit\"",
"printf 'browserLaunchErrorTail\\t%s\\n' \"$browser_launch_error_tail\"",
"printf 'statusShortLines\\t%s\\n' \"$status_short_lines\"",
"printf 'statusShortPreview\\t%s\\n' \"$status_short_preview\"",
].join("\n");
}
function sourceWorkspaceBootstrapK8sScript(
spec: HwlabRuntimeLaneSpec,
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
dryRun: boolean,
): string {
const jobName = sourceWorkspaceJobName(spec);
const manifest = sourceWorkspaceJobManifest(spec, sourceWorkspace, controlPlane, jobName);
const manifestYaml = Bun.YAML.stringify(manifest);
const manifestSha = createHash("sha256").update(manifestYaml).digest("hex");
return [
"set +e",
`namespace=${shellQuote(controlPlane.ciNamespace)}`,
`job=${shellQuote(jobName)}`,
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
`manifest_sha=${shellQuote(manifestSha)}`,
`timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`,
`manifest_b64=${shellQuote(Buffer.from(manifestYaml, "utf8").toString("base64"))}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"manifest=\"$tmp_dir/job.yaml\"",
"apply_out=\"$tmp_dir/apply.out\"",
"apply_err=\"$tmp_dir/apply.err\"",
"wait_out=\"$tmp_dir/wait.out\"",
"wait_err=\"$tmp_dir/wait.err\"",
"logs_file=\"$tmp_dir/logs.txt\"",
"printf '%s' \"$manifest_b64\" | base64 -d >\"$manifest\"",
"if [ \"$dry_run\" = true ]; then",
" kubectl -n \"$namespace\" apply --dry-run=client -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
" apply_rc=$?",
" python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'",
"import json, pathlib, sys",
"apply_rc = int(sys.argv[1])",
"namespace, job, manifest_sha = sys.argv[2:5]",
"apply_out = pathlib.Path(sys.argv[5]).read_text(errors='replace')[-2000:]",
"apply_err = pathlib.Path(sys.argv[6]).read_text(errors='replace')[-2000:]",
"print(json.dumps({'ok': apply_rc == 0, 'status': 'dry-run', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'applyOutputTail': apply_out, 'applyErrorTail': apply_err, 'mutation': False, 'valuesPrinted': False}, ensure_ascii=False))",
"PY",
" exit \"$apply_rc\"",
"fi",
"kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true >/dev/null 2>&1",
"kubectl -n \"$namespace\" apply -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
"apply_rc=$?",
"python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'",
"import json, pathlib, sys",
"apply_rc = int(sys.argv[1])",
"namespace, job, manifest_sha = sys.argv[2:5]",
"apply_out = pathlib.Path(sys.argv[5]).read_text(errors='replace')[-2000:]",
"apply_err = pathlib.Path(sys.argv[6]).read_text(errors='replace')[-2000:]",
"print(json.dumps({'ok': apply_rc == 0, 'status': 'submitted' if apply_rc == 0 else 'submit-failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'applyOutputTail': apply_out, 'applyErrorTail': apply_err, 'mutation': apply_rc == 0, 'valuesPrinted': False}, ensure_ascii=False))",
"PY",
"exit \"$apply_rc\"",
].join("\n");
}
function sourceWorkspaceJobManifest(
spec: HwlabRuntimeLaneSpec,
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
jobName: string,
): Record<string, unknown> {
const parent = posixPath.dirname(spec.workspace);
const basename = posixPath.basename(spec.workspace);
if (!spec.workspace.startsWith("/") || parent === "." || basename.length === 0) {
throw new Error(`source workspace must be an absolute POSIX path; got ${spec.workspace}`);
}
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name: jobName,
namespace: controlPlane.ciNamespace,
labels: {
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
"app.kubernetes.io/name": "hwlab-source-workspace-bootstrap",
"unidesk.ai/node": spec.nodeId,
"unidesk.ai/lane": spec.lane,
},
},
spec: {
backoffLimit: 0,
ttlSecondsAfterFinished: 3600,
template: {
metadata: {
labels: {
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
"app.kubernetes.io/name": "hwlab-source-workspace-bootstrap",
"unidesk.ai/node": spec.nodeId,
"unidesk.ai/lane": spec.lane,
},
},
spec: {
restartPolicy: "Never",
serviceAccountName: controlPlane.serviceAccountName,
containers: [
{
name: "bootstrap",
image: controlPlane.toolsImage,
imagePullPolicy: controlPlane.imagePullPolicy,
command: ["/bin/sh", "-lc", sourceWorkspaceJobContainerScript(spec, sourceWorkspace, controlPlane, `/host-workspaces/${basename}`)],
volumeMounts: [{ name: "workspace-parent", mountPath: "/host-workspaces" }],
env: sourceWorkspaceJobEnv(spec),
},
],
volumes: [{ name: "workspace-parent", hostPath: { path: parent, type: "DirectoryOrCreate" } }],
},
},
},
};
}
function sourceWorkspaceJobEnv(spec: HwlabRuntimeLaneSpec): Array<Record<string, string>> {
const proxy = spec.networkProfile.proxy;
const npm = spec.downloadProfile.npm;
return [
{ name: "HTTP_PROXY", value: proxy.http },
{ name: "HTTPS_PROXY", value: proxy.https },
{ name: "ALL_PROXY", value: proxy.all },
{ name: "NO_PROXY", value: proxy.noProxy.join(",") },
{ name: "http_proxy", value: proxy.http },
{ name: "https_proxy", value: proxy.https },
{ name: "all_proxy", value: proxy.all },
{ name: "no_proxy", value: proxy.noProxy.join(",") },
{ name: "npm_config_registry", value: npm.registry },
{ name: "npm_config_fetch_retries", value: String(npm.retries) },
{ name: "npm_config_fetch_timeout", value: String(npm.fetchTimeoutSeconds * 1000) },
{ name: "BUN_CONFIG_REGISTRY", value: npm.registry },
];
}
function sourceWorkspaceJobContainerScript(
spec: HwlabRuntimeLaneSpec,
sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
controlPlane: ReturnType<typeof hwlabNodeControlPlaneSourceWorkspaceBootstrap>,
workspaceInContainer: string,
): string {
const requiredCommands = shellLinesText(sourceWorkspace.requiredCommands);
const requiredFiles = shellLinesText(sourceWorkspace.requiredFiles);
return [
"set -eu",
`workspace=${shellQuote(workspaceInContainer)}`,
`remote=${shellQuote(controlPlane.gitReadUrl)}`,
`branch=${shellQuote(spec.sourceBranch)}`,
`dependency_command=${shellQuote(sourceWorkspace.install.dependencyCommand)}`,
`browser_command=${shellQuote(sourceWorkspace.install.browserCommand)}`,
`required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`,
`required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`,
"tmp_dir=$(mktemp -d)",
"stdout_log=\"$tmp_dir/bootstrap.stdout\"",
"stderr_log=\"$tmp_dir/bootstrap.stderr\"",
"failure() {",
" code=$?",
" if [ \"$code\" -ne 0 ]; then",
" tail_text=$(cat \"$stderr_log\" 2>/dev/null | tail -n 80 | tr '\\n' ' ' | cut -c1-3000 || true)",
" CODE=\"$code\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" ERROR_TAIL=\"$tail_text\" node <<'NODE'",
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: Number(process.env.CODE || '1'), workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
"NODE",
" fi",
" rm -rf \"$tmp_dir\"",
" exit \"$code\"",
"}",
"trap failure EXIT",
"printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"",
"printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"",
"while IFS= read -r command_name; do",
" [ -z \"$command_name\" ] && continue",
" command -v \"$command_name\" >/dev/null 2>>\"$stderr_log\"",
"done <\"$tmp_dir/commands.txt\"",
"mkdir -p \"$(dirname \"$workspace\")\"",
"git config --global --add safe.directory \"$workspace\" >>\"$stdout_log\" 2>>\"$stderr_log\" || true",
"if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
" status_short=$(git -C \"$workspace\" status --short)",
" if [ -n \"$status_short\" ]; then echo \"source workspace is dirty; refusing to overwrite\" >>\"$stderr_log\"; exit 42; fi",
" git -C \"$workspace\" remote set-url origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\" || git -C \"$workspace\" remote add origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
" git -C \"$workspace\" fetch --depth=1 origin \"$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
" git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
"else",
" if [ -e \"$workspace\" ] && [ \"$(find \"$workspace\" -mindepth 1 -maxdepth 1 2>/dev/null | head -n 1)\" ]; then echo \"source workspace path exists but is not a git repo\" >>\"$stderr_log\"; exit 43; fi",
" rm -rf \"$workspace\"",
" git clone --depth=1 --branch \"$branch\" \"$remote\" \"$workspace\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
"fi",
"cd \"$workspace\"",
"sh -lc \"$dependency_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
"sh -lc \"$browser_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"",
"missing_files=",
"while IFS= read -r file_path; do",
" [ -z \"$file_path\" ] && continue",
" if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi",
"done <\"$tmp_dir/files.txt\"",
"if [ -n \"$missing_files\" ]; then echo \"missing required files: $missing_files\" >>\"$stderr_log\"; exit 44; fi",
"source_commit=$(git rev-parse HEAD)",
"status_short=$(git status --short)",
"node_version=$(node --version 2>/dev/null || true)",
"npm_version=$(npm --version 2>/dev/null || true)",
"STATUS_SHORT=\"$status_short\" SOURCE_COMMIT=\"$source_commit\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" node <<'NODE'",
"const clean = !process.env.STATUS_SHORT;",
"console.log(JSON.stringify({ ok: clean, status: clean ? 'succeeded' : 'dirty-after-bootstrap', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, workspaceClean: clean, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));",
"NODE",
"trap - EXIT",
"rm -rf \"$tmp_dir\"",
].join("\n");
}
function shellLinesText(values: readonly string[]): string {
return values.length === 0 ? "" : `${values.join("\n")}\n`;
}
function sourceWorkspaceJobName(spec: HwlabRuntimeLaneSpec): string {
return `hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-workspace`.slice(0, 63).replace(/-$/u, "");
}
Reference in New Issue View Git Blame Copy Permalink