diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 2ecf9114..29a687d1 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -344,8 +344,8 @@ targets: syncConfigMapName: git-mirror-sync-script syncJobPrefix: git-mirror-hwlab-d518-v03-sync-manual flushJobPrefix: git-mirror-hwlab-d518-v03-flush-manual - readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git - writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git egressProxy: mode: node-global required: true diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index eb5cb8e7..95e96940 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -509,6 +509,22 @@ lanes: D518: node: D518 workspace: /home/ubuntu/workspace/hwlab-v03 + sourceWorkspace: + requiredCommands: + - git + - node + - npm + - npx + requiredFiles: + - AGENTS.md + - package.json + - package-lock.json + - scripts/src/browser-launcher.mjs + - scripts/web-live-dom-probe.mjs + install: + dependencyCommand: npm ci + browserCommand: PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium + timeoutSeconds: 900 cicdRepo: /home/ubuntu/workspace/hwlab-v03-cicd.git cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock app: hwlab-node-v03 @@ -636,6 +652,11 @@ lanes: XDG_CONFIG_HOME: /tekton/home/.config observability: prometheusOperator: false + webProbe: + sentinels: + - id: workbench-dsflash-go-tool-call-10x + enabled: true + configRef: config/hwlab-web-probe-sentinels/d518-v03/workbench-dsflash-go-tool-call-10x.yaml#sentinel runtimeImageRewrites: - source: fatedier/frpc:v0.68.1 target: 127.0.0.1:5000/hwlab/frpc:v0.68.1 diff --git a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml new file mode 100644 index 00000000..05f65424 --- /dev/null +++ b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml @@ -0,0 +1,64 @@ +version: 1 +kind: HwlabWebProbeSentinelCicd +metadata: + id: d518-v03-web-probe-sentinel-cicd + owner: UniDesk + specRef: PJ2026-01060508 +sentinel: + cicd: + controlPlaneConfigRef: config/hwlab-node-control-plane.yaml#targets[1] + source: + repository: pikasTech/unidesk + branch: master + gitSshUrl: ssh://git@ssh.github.com:443/pikasTech/unidesk.git + gitMirrorReadUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/unidesk.git + buildContext: . + entrypoint: scripts/web-probe-sentinel-service.ts + checkoutPaths: + - scripts + - config + - package.json + - bun.lock + - bun.lockb + builder: + namespace: devops-infra + sourceMode: sparse-git-checkout + jobPrefix: web-probe-sentinel-publish + gitSshSecretName: git-mirror-github-ssh + dockerSocketPath: /var/run/docker.sock + activeDeadlineSeconds: 900 + ttlSecondsAfterFinished: 3600 + gitopsPath: deploy/gitops/node/d518/web-probe-sentinel + argo: + namespace: argocd + projectName: hwlab-d518 + applicationName: hwlab-web-probe-sentinel + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + targetRevision: v0.3-gitops + image: + repository: 127.0.0.1:5000/hwlab/web-probe-sentinel + tagSource: source-commit + baseImageRef: config/hwlab-node-control-plane.yaml#targets[1].tekton.toolsImage.output + envRecipeRef: config/hwlab-web-probe-sentinel/runtime.d518-v03.yaml#sentinel.runtime + maintenance: + startCommand: sentinel maintenance start + stopCommand: sentinel maintenance stop + monitorWeb: + frontendStack: vue3-vendored-browser-build + runtimeMode: runner-served-bridge + assetRoot: scripts/assets/web-probe-sentinel-monitor-web + envReuse: + mode: docker-layer-and-ci-node-deps + nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules + gitMirror: + source: source.gitMirrorReadUrl + preSync: required + postFlush: required + ciBudget: + maxSeconds: 120 + confirmWait: + maxSeconds: 120 + targetValidation: + scenarioId: workbench-dsflash-go-tool-call-10x + maxSeconds: 300 + serviceUnavailablePolicy: structured-failure diff --git a/config/hwlab-web-probe-sentinel/public-exposure.d518-v03.yaml b/config/hwlab-web-probe-sentinel/public-exposure.d518-v03.yaml new file mode 100644 index 00000000..a4e42376 --- /dev/null +++ b/config/hwlab-web-probe-sentinel/public-exposure.d518-v03.yaml @@ -0,0 +1,37 @@ +version: 1 +kind: HwlabWebProbeSentinelPublicExposure +metadata: + id: d518-v03-web-probe-sentinel-public-exposure + owner: UniDesk + specRef: PJ2026-01060508 +sentinel: + publicExposure: + enabled: true + mode: pk01-caddy-frp + publicBaseUrl: https://monitor.pikapython.com/sentinels/d518-workbench-dsflash-go-tool-call-10x + hostname: monitor.pikapython.com + routePrefix: /sentinels/d518-workbench-dsflash-go-tool-call-10x + expectedA: 82.156.23.220 + frpc: + deploymentName: hwlab-web-probe-sentinel-frpc + image: 127.0.0.1:5000/hwlab/frpc:v0.68.1 + serverAddr: 82.156.23.220 + serverPort: 22000 + tokenSourceRef: platform-infra/pk01-frp.env + tokenSourceKey: FRP_TOKEN + secretName: hwlab-web-probe-sentinel-frpc + secretKey: frpc.toml + tokenKey: token + httpProxy: + name: hwlab-d518-v03-web-probe-sentinel + remotePort: 22093 + localIP: hwlab-web-probe-sentinel.hwlab-v03.svc.cluster.local + localPort: 8080 + caddy: + route: PK01 + configPath: /etc/caddy/Caddyfile + serviceName: caddy + email: ops@pikapython.com + tls: auto + responseHeaderTimeoutSeconds: 600 + managedBlockOwner: hwlab-web-probe-sentinel-d518-v03 diff --git a/config/hwlab-web-probe-sentinel/runtime.d518-v03.yaml b/config/hwlab-web-probe-sentinel/runtime.d518-v03.yaml new file mode 100644 index 00000000..bd63e312 --- /dev/null +++ b/config/hwlab-web-probe-sentinel/runtime.d518-v03.yaml @@ -0,0 +1,33 @@ +version: 1 +kind: HwlabWebProbeSentinelRuntime +metadata: + id: d518-v03-web-probe-sentinel-runtime + owner: UniDesk + specRef: PJ2026-01060508 +sentinel: + runtime: + target: + node: D518 + lane: v03 + publicOriginRef: config/hwlab-node-lanes.yaml#lanes.v03.targets.D518.public.webUrl + observeWrapperRef: config/hwlab-node-lanes.yaml#lanes.v03.targets.D518.observability.webProbe.sentinels[0] + namespace: hwlab-v03 + serviceAccountName: hwlab-web-probe-sentinel + deploymentName: hwlab-web-probe-sentinel + serviceName: hwlab-web-probe-sentinel + listenHost: 0.0.0.0 + servicePort: 8080 + pvcName: hwlab-web-probe-sentinel-state + pvcStorage: 10Gi + stateRoot: /var/lib/web-probe-sentinel + imageRef: 127.0.0.1:5000/hwlab/web-probe-sentinel:p3-service + replicas: 1 + healthPath: /api/health + metricsPath: /metrics + scheduler: + intervalMs: 600000 + heartbeatStaleSeconds: 900 + maxConcurrentRuns: 1 + sqlite: + path: /var/lib/web-probe-sentinel/index.sqlite + busyTimeoutMs: 2000 diff --git a/config/hwlab-web-probe-sentinel/secrets.d518-v03.yaml b/config/hwlab-web-probe-sentinel/secrets.d518-v03.yaml new file mode 100644 index 00000000..979d0814 --- /dev/null +++ b/config/hwlab-web-probe-sentinel/secrets.d518-v03.yaml @@ -0,0 +1,34 @@ +version: 1 +kind: HwlabWebProbeSentinelSecrets +metadata: + id: d518-v03-web-probe-sentinel-secrets + owner: UniDesk + specRef: PJ2026-01060508 +sentinel: + secrets: + sources: + - purpose: bootstrap-admin + sourceRef: hwlab/d518-v03-bootstrap-admin.env + sourceKey: HWLAB_BOOTSTRAP_ADMIN_PASSWORD + - purpose: prompt-set + sourceRef: hwlab/web-probe-sentinel-dsflash-go.env + sourceKey: DSFLASH_GO_TOOL_CALL_10X_PROMPTS_JSON + - purpose: frp-token + sourceRef: platform-infra/pk01-frp.env + sourceKey: FRP_TOKEN + runtimeSecrets: + - name: hwlab-web-probe-sentinel-bootstrap + namespace: hwlab-v03 + data: + - sourcePurpose: bootstrap-admin + targetKey: bootstrap-admin-password + - name: hwlab-web-probe-sentinel-prompt-set + namespace: hwlab-v03 + data: + - sourcePurpose: prompt-set + targetKey: prompts.json + - name: hwlab-web-probe-sentinel-frpc + namespace: hwlab-v03 + data: + - sourcePurpose: frp-token + targetKey: token diff --git a/config/hwlab-web-probe-sentinels/d518-v03/workbench-dsflash-go-tool-call-10x.yaml b/config/hwlab-web-probe-sentinels/d518-v03/workbench-dsflash-go-tool-call-10x.yaml new file mode 100644 index 00000000..8076c855 --- /dev/null +++ b/config/hwlab-web-probe-sentinels/d518-v03/workbench-dsflash-go-tool-call-10x.yaml @@ -0,0 +1,18 @@ +version: 1 +kind: HwlabWebProbeSentinel +metadata: + id: d518-v03-workbench-dsflash-go-tool-call-10x + owner: UniDesk + specRef: PJ2026-01060508 +sentinel: + id: workbench-dsflash-go-tool-call-10x + enabled: true + mode: web-probe-observe-wrapper + configRefs: + runtime: config/hwlab-web-probe-sentinel/runtime.d518-v03.yaml#sentinel.runtime + scenarios: config/hwlab-web-probe-sentinel/scenarios.workbench.yaml#sentinel.scenarios + promptSet: config/hwlab-web-probe-sentinel/prompt-set.dsflash-go.yaml#sentinel.promptSet + reportViews: config/hwlab-web-probe-sentinel/report-views.yaml#sentinel.reportViews + publicExposure: config/hwlab-web-probe-sentinel/public-exposure.d518-v03.yaml#sentinel.publicExposure + cicd: config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml#sentinel.cicd + secrets: config/hwlab-web-probe-sentinel/secrets.d518-v03.yaml#sentinel.secrets diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 1af64cf9..52a980a7 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -269,6 +269,20 @@ interface ControlPlaneConfig { targets: readonly ControlPlaneTargetSpec[]; } +export interface HwlabNodeControlPlaneSourceWorkspaceBootstrapSpec { + readonly configPath: string; + readonly targetId: string; + readonly node: string; + readonly lane: string; + readonly ciNamespace: string; + readonly serviceAccountName: string; + readonly toolsImage: string; + readonly imagePullPolicy: "Always" | "IfNotPresent" | "Never"; + readonly gitReadUrl: string; + readonly gitWriteUrl: string; + readonly gitMirrorNamespace: string; +} + export function runHwlabNodeControlPlaneInfra(args: string[]): Record | RenderedCliResult { if (args[0] === "tools-image") { const options = parseToolsImageOptions(args.slice(1)); @@ -316,6 +330,23 @@ export function hwlabNodeControlPlaneCiGitWorkspaceSecret(nodeId: string, lane: }; } +export function hwlabNodeControlPlaneSourceWorkspaceBootstrap(nodeId: string, lane: string): HwlabNodeControlPlaneSourceWorkspaceBootstrapSpec { + const { target } = controlPlaneContext(nodeId, lane); + return { + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + targetId: target.id, + node: target.node, + lane: target.lane, + ciNamespace: target.ciNamespace, + serviceAccountName: target.tekton.serviceAccountName, + toolsImage: target.tekton.toolsImage.output, + imagePullPolicy: target.tekton.toolsImage.imagePullPolicy, + gitReadUrl: target.gitMirror.readUrl, + gitWriteUrl: target.gitMirror.writeUrl, + gitMirrorNamespace: target.gitMirror.namespace, + }; +} + export function hwlabNodeControlPlaneInfraHelp(): Record { return { ok: true, diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 5a6953b0..ce018fd8 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -313,6 +313,16 @@ export interface HwlabRuntimeCodeAgentProviderSpec { readonly opencodeSourceKey?: string; } +export interface HwlabRuntimeSourceWorkspaceSpec { + readonly requiredCommands: readonly string[]; + readonly requiredFiles: readonly string[]; + readonly install: { + readonly dependencyCommand: string; + readonly browserCommand: string; + readonly timeoutSeconds: number; + }; +} + export interface HwlabRuntimeLaneSpec { readonly lane: HwlabRuntimeLane; readonly nodeId: string; @@ -351,6 +361,7 @@ export interface HwlabRuntimeLaneSpec { readonly buildkit?: HwlabRuntimeBuildkitSpec; readonly bootstrapAdmin?: HwlabRuntimeBootstrapAdminSpec; readonly codeAgentProvider?: HwlabRuntimeCodeAgentProviderSpec; + readonly sourceWorkspace?: HwlabRuntimeSourceWorkspaceSpec; readonly externalPostgres?: HwlabRuntimeExternalPostgresSpec; readonly runtimeStore?: HwlabRuntimeStoreSpec; readonly webProbe?: HwlabRuntimeWebProbeSpec; @@ -401,6 +412,7 @@ interface HwlabLaneConfig { readonly buildkit?: HwlabRuntimeBuildkitSpec; readonly bootstrapAdmin?: HwlabRuntimeBootstrapAdminSpec; readonly codeAgentProvider?: HwlabRuntimeCodeAgentProviderSpec; + readonly sourceWorkspace?: HwlabRuntimeSourceWorkspaceSpec; readonly externalPostgres?: HwlabRuntimeExternalPostgresSpec; readonly runtimeStore?: HwlabRuntimeStoreSpec; readonly webProbe?: HwlabRuntimeWebProbeSpec; @@ -643,6 +655,7 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa buildkit: buildkitConfig(raw.buildkit, `lanes.${id}.buildkit`), bootstrapAdmin: bootstrapAdminConfig(raw.bootstrapAdmin, `lanes.${id}.bootstrapAdmin`), codeAgentProvider: codeAgentProviderConfig(raw.codeAgentProvider, `lanes.${id}.codeAgentProvider`), + sourceWorkspace: sourceWorkspaceConfig(raw.sourceWorkspace, `lanes.${id}.sourceWorkspace`), externalPostgres: externalPostgresConfig(raw.externalPostgres, `lanes.${id}.externalPostgres`), runtimeStore: runtimeStoreConfig(raw.runtimeStore, `lanes.${id}.runtimeStore`), webProbe: webProbeConfig(raw.webProbe, `lanes.${id}.webProbe`), @@ -666,6 +679,7 @@ function laneTargetConfig(id: HwlabRuntimeLane, nodeId: string, baseRaw: Record< buildkit: mergeOptionalRecord(baseRaw.buildkit, targetRaw.buildkit), bootstrapAdmin: mergeOptionalRecord(baseRaw.bootstrapAdmin, targetRaw.bootstrapAdmin), codeAgentProvider: mergeOptionalRecord(baseRaw.codeAgentProvider, targetRaw.codeAgentProvider), + sourceWorkspace: mergeOptionalRecord(baseRaw.sourceWorkspace, targetRaw.sourceWorkspace), externalPostgres: mergeOptionalRecord(baseRaw.externalPostgres, targetRaw.externalPostgres), runtimeStore: mergeOptionalRecord(baseRaw.runtimeStore, targetRaw.runtimeStore), webProbe: mergeOptionalRecord(baseRaw.webProbe, targetRaw.webProbe), @@ -732,6 +746,35 @@ function codeAgentProviderConfig(value: unknown, path: string): HwlabRuntimeCode }; } +function commandNameField(value: string, path: string): string { + if (!/^[A-Za-z0-9._+-]+$/u.test(value)) throw new Error(`${path} must be a simple command name`); + return value; +} + +function relativeWorkspacePathField(value: string, path: string): string { + if (value.startsWith("/") || value.includes("\0") || value.split("/").some((part) => part === "..")) { + throw new Error(`${path} must be a relative workspace path without ..`); + } + return value; +} + +function sourceWorkspaceConfig(value: unknown, path: string): HwlabRuntimeSourceWorkspaceSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + const install = asRecord(raw.install, `${path}.install`); + return { + requiredCommands: stringArrayField(raw, "requiredCommands", path) + .map((item, index) => commandNameField(item, `${path}.requiredCommands[${index}]`)), + requiredFiles: stringArrayField(raw, "requiredFiles", path) + .map((item, index) => relativeWorkspacePathField(item, `${path}.requiredFiles[${index}]`)), + install: { + dependencyCommand: stringField(install, "dependencyCommand", `${path}.install`), + browserCommand: stringField(install, "browserCommand", `${path}.install`), + timeoutSeconds: numberField(install, "timeoutSeconds", `${path}.install`), + }, + }; +} + function externalPostgresComponentConfig(value: unknown, path: string): HwlabRuntimeExternalPostgresComponentSpec { const raw = asRecord(value, path); return { @@ -1308,6 +1351,7 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec { ...(config.buildkit === undefined ? {} : { buildkit: config.buildkit }), ...(config.bootstrapAdmin === undefined ? {} : { bootstrapAdmin: config.bootstrapAdmin }), ...(config.codeAgentProvider === undefined ? {} : { codeAgentProvider: config.codeAgentProvider }), + ...(config.sourceWorkspace === undefined ? {} : { sourceWorkspace: config.sourceWorkspace }), ...(config.externalPostgres === undefined ? {} : { externalPostgres: config.externalPostgres }), ...(config.runtimeStore === undefined ? {} : { runtimeStore: config.runtimeStore }), ...(config.webProbe === undefined ? {} : { webProbe: config.webProbe }), diff --git a/scripts/src/hwlab-node/entry.ts b/scripts/src/hwlab-node/entry.ts index 8b9b5adc..0f9a7683 100644 --- a/scripts/src/hwlab-node/entry.ts +++ b/scripts/src/hwlab-node/entry.ts @@ -38,6 +38,7 @@ import { parseSecretOptions, runNodePublicExposure, runNodeSecret } from "./publ import { nodeRuntimeControlPlaneStatus } from "./render"; import { nodeRuntimeUnsupportedAction } from "./runtime-common"; import { runNodeEndpointBridge } from "./secret-scripts"; +import { nodeRuntimeSourceWorkspaceCommand } from "./source-workspace"; import { nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeScopedFullOutput, withNodeRuntimeGitMirrorRendered } from "./status"; import { assertNodeId, positiveIntegerOption, requiredOption, stripOption } from "./utils"; import { legacyHwlabNodeWebProbeUnsupported } from "../web-probe"; @@ -565,6 +566,9 @@ export async function runNodeDelegatedDomain(config: Config, domain: DelegatedNo if (domain === "control-plane" && scoped.action === "runtime-image") { return nodeRuntimeBaseImageCommand(scoped); } + if (domain === "control-plane" && scoped.action === "source-workspace") { + return nodeRuntimeSourceWorkspaceCommand(scoped); + } if (domain === "control-plane" && scoped.action === "plan") { const result = nodeRuntimeControlPlanePlan(scoped); return nodeScopedFullOutput(scoped) ? result : withNodeRuntimeControlPlanePlanRendered(result, scoped); diff --git a/scripts/src/hwlab-node/plan.ts b/scripts/src/hwlab-node/plan.ts index e8ff6124..ebca62d3 100644 --- a/scripts/src/hwlab-node/plan.ts +++ b/scripts/src/hwlab-node/plan.ts @@ -144,6 +144,11 @@ export function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record` }, }; } - const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds); - let secretFields = keyValueLinesFromText(statusText(secretApplyResult)); - let secretResult = secretApplyResult; - if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") { - const restartResult = runPublicExposureFrpcRecreate(options); - secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) }; - secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult); - } const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds); const caddyFields = keyValueLinesFromText(statusText(caddyResult)); - const secretStatus = publicExposureSecretStatus(secretFields, secretResult); + const secretStatus = options.dryRun + ? publicExposureSecretDryRunStatus(options, exposure, source.value?.length ?? 0) + : publicExposureSecretApplyStatus(options, exposure, source.value ?? ""); const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult); const ok = secretStatus.ok === true && caddyStatus.ok === true; return { @@ -510,7 +504,7 @@ export function runNodePublicExposure(options: NodePublicExposureOptions): Recor node: options.node, lane: options.lane, mode: options.dryRun ? "dry-run" : "confirmed", - mutation: !options.dryRun && (secretFields.mutation === "true" || caddyFields.mutation === "true"), + mutation: !options.dryRun && (secretStatus.mutation === true || caddyFields.mutation === "true"), configPath: hwlabRuntimeLaneConfigPath(), publicExposure: publicExposureSummary(exposure), source: { @@ -533,6 +527,53 @@ export function runNodePublicExposure(options: NodePublicExposureOptions): Recor }; } +function publicExposureSecretDryRunStatus(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec, tokenBytes: number): Record { + return { + ok: true, + dryRun: true, + mutation: false, + planned: true, + namespace: options.spec.runtimeNamespace, + secret: exposure.secretName, + deployment: `${options.spec.runtimeNamespace}-frpc`, + beforeExists: null, + afterExists: null, + tokenBytes, + applyExitCode: null, + restartMode: null, + strategyPatchExitCode: null, + rolloutRestartExitCode: null, + scaleDownExitCode: null, + scaleDownWaitExitCode: null, + scaleDownWaitPodCount: null, + scaleUpExitCode: null, + rolloutStatusExitCode: null, + readyReplicas: null, + availableReplicas: null, + desiredReplicas: null, + strategyPatchErrorPreview: null, + restartErrorPreview: null, + scaleDownErrorPreview: null, + scaleUpErrorPreview: null, + readyErrorPreview: null, + exitCode: 0, + stderr: "", + valuesRedacted: true, + }; +} + +function publicExposureSecretApplyStatus(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec, token: string): Record { + const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), token, options.timeoutSeconds); + let secretFields = keyValueLinesFromText(statusText(secretApplyResult)); + let secretResult = secretApplyResult; + if (secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") { + const restartResult = runPublicExposureFrpcRecreate(options); + secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) }; + secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult); + } + return publicExposureSecretStatus(secretFields, secretResult); +} + export function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } { const checkedPaths = publicExposureTokenSourcePaths(exposure); const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef); diff --git a/scripts/src/hwlab-node/runtime-common.ts b/scripts/src/hwlab-node/runtime-common.ts index 8eb60be3..70f9510b 100644 --- a/scripts/src/hwlab-node/runtime-common.ts +++ b/scripts/src/hwlab-node/runtime-common.ts @@ -209,7 +209,7 @@ export function nodeRuntimeUnsupportedAction(scoped: ReturnType; +type SourceWorkspaceAction = "status" | "bootstrap"; + +export function nodeRuntimeSourceWorkspaceCommand(scoped: ScopedNodeOptions): Record { + const action = sourceWorkspaceAction(scoped); + const sourceWorkspace = scoped.spec.sourceWorkspace; + if (sourceWorkspace === undefined) { + return { + ok: false, + command: `hwlab nodes control-plane source-workspace ${action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mutation: false, + degradedReason: "source-workspace-yaml-missing", + message: "lanes..targets..sourceWorkspace must declare requiredCommands, requiredFiles and install commands before this controlled entry can run.", + configPath: hwlabRuntimeLaneConfigPath(), + }; + } + if (action === "status") return nodeRuntimeSourceWorkspaceStatus(scoped, sourceWorkspace); + return nodeRuntimeSourceWorkspaceBootstrap(scoped, sourceWorkspace); +} + +function sourceWorkspaceAction(scoped: ScopedNodeOptions): SourceWorkspaceAction { + const value = scoped.originalArgs[1]; + if (value !== "status" && value !== "bootstrap") { + throw new Error("control-plane source-workspace usage: source-workspace status|bootstrap --node NODE --lane vNN [--dry-run|--confirm]"); + } + if (value === "status" && (scoped.confirm || scoped.dryRun)) { + throw new Error("control-plane source-workspace status is read-only and does not accept --dry-run or --confirm"); + } + return value; +} + +function nodeRuntimeSourceWorkspaceStatus(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record { + const result = runTransHostScript(scoped.node, sourceWorkspaceStatusScript(scoped.spec, sourceWorkspace), "", scoped.timeoutSeconds); + const fields = keyValueLinesFromText(statusText(result)); + const status = sourceWorkspaceStatusFromFields(fields, result.exitCode); + return { + ok: status.ok === true, + command: `hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "read-only", + mutation: false, + configPath: hwlabRuntimeLaneConfigPath(), + expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace), + status, + probe: compactRuntimeCommand(result), + degradedReason: status.ok === true ? undefined : "source-workspace-not-ready", + next: status.ok === true + ? { webProbe: `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}` } + : { bootstrap: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; +} + +function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record { + if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace bootstrap accepts only one of --dry-run or --confirm"); + const dryRun = scoped.dryRun || !scoped.confirm; + const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane); + const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace); + const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.max(scoped.timeoutSeconds, sourceWorkspace.install.timeoutSeconds + 60)); + const payload = parseLastJsonLineObject(statusText(result)); + const after = dryRun ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace); + const afterOk = after === null ? null : after.ok === true; + const payloadOk = payload.ok === true; + const ok = result.exitCode === 0 && payloadOk && (dryRun || afterOk === true); + return { + ok, + command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: dryRun ? "dry-run" : "confirmed-bootstrap", + mutation: !dryRun && ok, + configPath: hwlabRuntimeLaneConfigPath(), + controlPlaneConfigPath: controlPlane.configPath, + expected: sourceWorkspaceExpected(scoped.spec, sourceWorkspace), + controlPlane, + before, + bootstrap: payload, + result: compactRuntimeCommand(result), + after, + degradedReason: ok ? undefined : dryRun ? "source-workspace-bootstrap-dry-run-failed" : "source-workspace-bootstrap-failed", + next: ok + ? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` } + : { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; +} + +function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record { + return { + workspace: spec.workspace, + sourceBranch: spec.sourceBranch, + git: { + url: spec.gitUrl, + readUrl: spec.gitReadUrl, + writeUrl: spec.gitWriteUrl, + }, + requiredCommands: sourceWorkspace.requiredCommands, + requiredFiles: sourceWorkspace.requiredFiles, + install: sourceWorkspace.install, + downloadProfile: { + id: spec.downloadProfileId, + npm: spec.downloadProfile.npm, + git: spec.downloadProfile.git, + }, + networkProfile: { + id: spec.networkProfileId, + proxy: spec.networkProfile.proxy, + }, + }; +} + +function sourceWorkspaceStatusFromFields(fields: Record, exitCode: number | null): Record { + const missingCommands = commaListField(fields.missingCommands); + const missingFiles = commaListField(fields.missingFiles); + const ok = exitCode === 0 + && fields.workspaceExists === "yes" + && fields.gitDirExists === "yes" + && fields.workspaceClean === "yes" + && fields.branch === fields.expectedBranch + && fields.remoteMatchesYaml === "yes" + && missingCommands.length === 0 + && missingFiles.length === 0 + && fields.playwrightPackagePresent === "yes" + && fields.launcherImportOk === "yes"; + return { + ok, + workspace: fields.workspace || null, + expectedBranch: fields.expectedBranch || null, + workspaceExists: fields.workspaceExists === "yes", + gitDirExists: fields.gitDirExists === "yes", + workspaceClean: fields.workspaceClean === "yes", + branch: fields.branch || null, + detached: fields.detached === "yes", + localHead: fields.localHead || null, + remoteUrl: fields.remoteUrl || null, + remoteMatchesYaml: fields.remoteMatchesYaml === "yes", + requiredCommands: commaListField(fields.requiredCommands), + missingCommands, + requiredFiles: commaListField(fields.requiredFiles), + missingFiles, + nodeVersion: fields.nodeVersion || null, + npmVersion: fields.npmVersion || null, + npxVersion: fields.npxVersion || null, + playwrightPackagePresent: fields.playwrightPackagePresent === "yes", + browserCachePresent: fields.browserCachePresent === "yes", + launcherImportOk: fields.launcherImportOk === "yes", + launcherImportExitCode: numericField(fields.launcherImportExitCode), + statusShortLines: numericField(fields.statusShortLines), + statusShortPreview: fields.statusShortPreview || null, + valuesPrinted: false, + }; +} + +function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string { + const requiredCommands = sourceWorkspace.requiredCommands.join("\n"); + const requiredFiles = sourceWorkspace.requiredFiles.join("\n"); + const remoteUrls = [spec.gitUrl, spec.gitReadUrl, spec.gitWriteUrl].join("\n"); + return [ + "set +e", + `workspace=${shellQuote(spec.workspace)}`, + `expected_branch=${shellQuote(spec.sourceBranch)}`, + `required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`, + `required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`, + `remote_urls_b64=${shellQuote(Buffer.from(remoteUrls, "utf8").toString("base64"))}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"", + "printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"", + "printf '%s' \"$remote_urls_b64\" | base64 -d >\"$tmp_dir/remotes.txt\"", + "workspace_exists=no", + "git_dir_exists=no", + "workspace_clean=no", + "branch=", + "detached=no", + "local_head=", + "remote_url=", + "remote_matches_yaml=no", + "status_short=", + "if [ -d \"$workspace\" ]; then", + " workspace_exists=yes", + " if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then", + " git_dir_exists=yes", + " branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", + " [ \"$branch\" = HEAD ] && detached=yes", + " local_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", + " remote_url=$(git -C \"$workspace\" remote get-url origin 2>/dev/null || true)", + " status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", + " [ -z \"$status_short\" ] && workspace_clean=yes", + " while IFS= read -r expected_remote; do [ \"$remote_url\" = \"$expected_remote\" ] && remote_matches_yaml=yes; done <\"$tmp_dir/remotes.txt\"", + " fi", + "fi", + "missing_commands=", + "while IFS= read -r command_name; do", + " [ -z \"$command_name\" ] && continue", + " if ! command -v \"$command_name\" >/dev/null 2>&1; then missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"; fi", + "done <\"$tmp_dir/commands.txt\"", + "missing_files=", + "if [ \"$workspace_exists\" = yes ]; then", + " while IFS= read -r file_path; do", + " [ -z \"$file_path\" ] && continue", + " if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi", + " done <\"$tmp_dir/files.txt\"", + "else", + " missing_files=$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')", + "fi", + "node_version=$(node --version 2>/dev/null || true)", + "npm_version=$(npm --version 2>/dev/null || true)", + "npx_version=$(npx --version 2>/dev/null || true)", + "playwright_package_present=no", + "[ -f \"$workspace/node_modules/playwright/package.json\" ] && playwright_package_present=yes", + "browser_cache_present=no", + "if find \"$workspace/node_modules\" -maxdepth 5 -type d -path '*playwright*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi", + "for cache_dir in \"${HOME:-}/.cache/ms-playwright\" /root/.cache/ms-playwright /home/ubuntu/.cache/ms-playwright; do", + " [ -d \"$cache_dir\" ] || continue", + " if find \"$cache_dir\" -maxdepth 1 -type d -name '*chromium*' 2>/dev/null | grep -q .; then browser_cache_present=yes; fi", + "done", + "launcher_import_ok=no", + "launcher_import_exit=", + "if command -v node >/dev/null 2>&1 && [ -f \"$workspace/scripts/src/browser-launcher.mjs\" ]; then", + " (cd \"$workspace\" && timeout 20s node -e \"import('./scripts/src/browser-launcher.mjs').then(()=>process.exit(0)).catch(()=>process.exit(1))\") >/tmp/hwlab-source-workspace-launcher.out 2>/tmp/hwlab-source-workspace-launcher.err", + " launcher_import_exit=$?", + " [ \"$launcher_import_exit\" = 0 ] && launcher_import_ok=yes", + "fi", + "status_short_lines=$(printf '%s' \"$status_short\" | sed '/^$/d' | wc -l | tr -d ' ')", + "status_short_preview=$(printf '%s' \"$status_short\" | tr '\\n\\t' ' ' | cut -c1-1000)", + "printf 'workspace\\t%s\\n' \"$workspace\"", + "printf 'expectedBranch\\t%s\\n' \"$expected_branch\"", + "printf 'workspaceExists\\t%s\\n' \"$workspace_exists\"", + "printf 'gitDirExists\\t%s\\n' \"$git_dir_exists\"", + "printf 'workspaceClean\\t%s\\n' \"$workspace_clean\"", + "printf 'branch\\t%s\\n' \"$branch\"", + "printf 'detached\\t%s\\n' \"$detached\"", + "printf 'localHead\\t%s\\n' \"$local_head\"", + "printf 'remoteUrl\\t%s\\n' \"$remote_url\"", + "printf 'remoteMatchesYaml\\t%s\\n' \"$remote_matches_yaml\"", + "printf 'requiredCommands\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/commands.txt\" | sed 's/,$//')\"", + "printf 'missingCommands\\t%s\\n' \"$missing_commands\"", + "printf 'requiredFiles\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')\"", + "printf 'missingFiles\\t%s\\n' \"$missing_files\"", + "printf 'nodeVersion\\t%s\\n' \"$node_version\"", + "printf 'npmVersion\\t%s\\n' \"$npm_version\"", + "printf 'npxVersion\\t%s\\n' \"$npx_version\"", + "printf 'playwrightPackagePresent\\t%s\\n' \"$playwright_package_present\"", + "printf 'browserCachePresent\\t%s\\n' \"$browser_cache_present\"", + "printf 'launcherImportOk\\t%s\\n' \"$launcher_import_ok\"", + "printf 'launcherImportExitCode\\t%s\\n' \"$launcher_import_exit\"", + "printf 'statusShortLines\\t%s\\n' \"$status_short_lines\"", + "printf 'statusShortPreview\\t%s\\n' \"$status_short_preview\"", + ].join("\n"); +} + +function sourceWorkspaceBootstrapK8sScript( + spec: HwlabRuntimeLaneSpec, + sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec, + controlPlane: ReturnType, + dryRun: boolean, +): string { + const jobName = sourceWorkspaceJobName(spec); + const manifest = sourceWorkspaceJobManifest(spec, sourceWorkspace, controlPlane, jobName); + const manifestYaml = Bun.YAML.stringify(manifest); + const manifestSha = createHash("sha256").update(manifestYaml).digest("hex"); + return [ + "set +e", + `namespace=${shellQuote(controlPlane.ciNamespace)}`, + `job=${shellQuote(jobName)}`, + `dry_run=${shellQuote(dryRun ? "true" : "false")}`, + `manifest_sha=${shellQuote(manifestSha)}`, + `timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`, + `manifest_b64=${shellQuote(Buffer.from(manifestYaml, "utf8").toString("base64"))}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "manifest=\"$tmp_dir/job.yaml\"", + "apply_out=\"$tmp_dir/apply.out\"", + "apply_err=\"$tmp_dir/apply.err\"", + "wait_out=\"$tmp_dir/wait.out\"", + "wait_err=\"$tmp_dir/wait.err\"", + "logs_file=\"$tmp_dir/logs.txt\"", + "printf '%s' \"$manifest_b64\" | base64 -d >\"$manifest\"", + "if [ \"$dry_run\" = true ]; then", + " kubectl -n \"$namespace\" apply --dry-run=server -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"", + " apply_rc=$?", + " python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'", + "import json, pathlib, sys", + "apply_rc = int(sys.argv[1])", + "namespace, job, manifest_sha = sys.argv[2:5]", + "apply_out = pathlib.Path(sys.argv[5]).read_text(errors='replace')[-2000:]", + "apply_err = pathlib.Path(sys.argv[6]).read_text(errors='replace')[-2000:]", + "print(json.dumps({'ok': apply_rc == 0, 'status': 'dry-run', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'applyOutputTail': apply_out, 'applyErrorTail': apply_err, 'mutation': False, 'valuesPrinted': False}, ensure_ascii=False))", + "PY", + " exit \"$apply_rc\"", + "fi", + "kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true >/dev/null 2>&1", + "kubectl -n \"$namespace\" apply -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"", + "apply_rc=$?", + "wait_rc=1", + "if [ \"$apply_rc\" = 0 ]; then", + " kubectl -n \"$namespace\" wait --for=condition=complete \"job/$job\" --timeout=\"${timeout_seconds}s\" >\"$wait_out\" 2>\"$wait_err\"", + " wait_rc=$?", + "fi", + "kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true", + "python3 - \"$apply_rc\" \"$wait_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" \"$wait_out\" \"$wait_err\" \"$logs_file\" <<'PY'", + "import json, pathlib, sys", + "apply_rc = int(sys.argv[1])", + "wait_rc = int(sys.argv[2])", + "namespace, job, manifest_sha = sys.argv[3:6]", + "paths = [pathlib.Path(item) for item in sys.argv[6:]]", + "apply_out, apply_err, wait_out, wait_err, logs = [path.read_text(errors='replace') for path in paths]", + "job_payload = None", + "for line in reversed(logs.splitlines()):", + " text = line.strip()", + " if text.startswith('{') and text.endswith('}'):", + " try:", + " job_payload = json.loads(text)", + " break", + " except Exception:", + " pass", + "job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True", + "ok = apply_rc == 0 and wait_rc == 0 and job_ok", + "print(json.dumps({'ok': ok, 'status': 'succeeded' if ok else 'failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'waitExitCode': wait_rc, 'jobPayload': job_payload, 'applyOutputTail': apply_out[-2000:], 'applyErrorTail': apply_err[-2000:], 'waitOutputTail': wait_out[-2000:], 'waitErrorTail': wait_err[-2000:], 'logsTail': logs[-4000:], 'mutation': ok, 'valuesPrinted': False}, ensure_ascii=False))", + "PY", + "exit $([ \"$apply_rc\" = 0 ] && [ \"$wait_rc\" = 0 ] && printf 0 || printf 1)", + ].join("\n"); +} + +function sourceWorkspaceJobManifest( + spec: HwlabRuntimeLaneSpec, + sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec, + controlPlane: ReturnType, + jobName: string, +): Record { + const parent = posixPath.dirname(spec.workspace); + const basename = posixPath.basename(spec.workspace); + if (!spec.workspace.startsWith("/") || parent === "." || basename.length === 0) { + throw new Error(`source workspace must be an absolute POSIX path; got ${spec.workspace}`); + } + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName, + namespace: controlPlane.ciNamespace, + labels: { + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "app.kubernetes.io/name": "hwlab-source-workspace-bootstrap", + "unidesk.ai/node": spec.nodeId, + "unidesk.ai/lane": spec.lane, + }, + }, + spec: { + backoffLimit: 0, + ttlSecondsAfterFinished: 3600, + template: { + metadata: { + labels: { + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "app.kubernetes.io/name": "hwlab-source-workspace-bootstrap", + "unidesk.ai/node": spec.nodeId, + "unidesk.ai/lane": spec.lane, + }, + }, + spec: { + restartPolicy: "Never", + serviceAccountName: controlPlane.serviceAccountName, + containers: [ + { + name: "bootstrap", + image: controlPlane.toolsImage, + imagePullPolicy: controlPlane.imagePullPolicy, + command: ["/bin/sh", "-lc", sourceWorkspaceJobContainerScript(spec, sourceWorkspace, controlPlane, `/host-workspaces/${basename}`)], + volumeMounts: [{ name: "workspace-parent", mountPath: "/host-workspaces" }], + env: sourceWorkspaceJobEnv(spec), + }, + ], + volumes: [{ name: "workspace-parent", hostPath: { path: parent, type: "DirectoryOrCreate" } }], + }, + }, + }, + }; +} + +function sourceWorkspaceJobEnv(spec: HwlabRuntimeLaneSpec): Array> { + const proxy = spec.networkProfile.proxy; + const npm = spec.downloadProfile.npm; + return [ + { name: "HTTP_PROXY", value: proxy.http }, + { name: "HTTPS_PROXY", value: proxy.https }, + { name: "ALL_PROXY", value: proxy.all }, + { name: "NO_PROXY", value: proxy.noProxy.join(",") }, + { name: "http_proxy", value: proxy.http }, + { name: "https_proxy", value: proxy.https }, + { name: "all_proxy", value: proxy.all }, + { name: "no_proxy", value: proxy.noProxy.join(",") }, + { name: "npm_config_registry", value: npm.registry }, + { name: "npm_config_fetch_retries", value: String(npm.retries) }, + { name: "npm_config_fetch_timeout", value: String(npm.fetchTimeoutSeconds * 1000) }, + { name: "BUN_CONFIG_REGISTRY", value: npm.registry }, + ]; +} + +function sourceWorkspaceJobContainerScript( + spec: HwlabRuntimeLaneSpec, + sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec, + controlPlane: ReturnType, + workspaceInContainer: string, +): string { + const requiredCommands = sourceWorkspace.requiredCommands.join("\n"); + const requiredFiles = sourceWorkspace.requiredFiles.join("\n"); + return [ + "set -eu", + `workspace=${shellQuote(workspaceInContainer)}`, + `remote=${shellQuote(controlPlane.gitReadUrl)}`, + `branch=${shellQuote(spec.sourceBranch)}`, + `dependency_command=${shellQuote(sourceWorkspace.install.dependencyCommand)}`, + `browser_command=${shellQuote(sourceWorkspace.install.browserCommand)}`, + `required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`, + `required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`, + "tmp_dir=$(mktemp -d)", + "stdout_log=\"$tmp_dir/bootstrap.stdout\"", + "stderr_log=\"$tmp_dir/bootstrap.stderr\"", + "failure() {", + " code=$?", + " if [ \"$code\" -ne 0 ]; then", + " tail_text=$(cat \"$stderr_log\" 2>/dev/null | tail -n 80 | tr '\\n' ' ' | cut -c1-3000 || true)", + " CODE=\"$code\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" ERROR_TAIL=\"$tail_text\" node <<'NODE'", + "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: Number(process.env.CODE || '1'), workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", + "NODE", + " fi", + " rm -rf \"$tmp_dir\"", + " exit \"$code\"", + "}", + "trap failure EXIT", + "printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"", + "printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"", + "while IFS= read -r command_name; do", + " [ -z \"$command_name\" ] && continue", + " command -v \"$command_name\" >/dev/null 2>>\"$stderr_log\"", + "done <\"$tmp_dir/commands.txt\"", + "mkdir -p \"$(dirname \"$workspace\")\"", + "if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then", + " status_short=$(git -C \"$workspace\" status --short)", + " if [ -n \"$status_short\" ]; then echo \"source workspace is dirty; refusing to overwrite\" >>\"$stderr_log\"; exit 42; fi", + " git -C \"$workspace\" remote set-url origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\" || git -C \"$workspace\" remote add origin \"$remote\" >>\"$stdout_log\" 2>>\"$stderr_log\"", + " git -C \"$workspace\" fetch --depth=1 origin \"$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"", + " git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" >>\"$stdout_log\" 2>>\"$stderr_log\"", + "else", + " if [ -e \"$workspace\" ] && [ \"$(find \"$workspace\" -mindepth 1 -maxdepth 1 2>/dev/null | head -n 1)\" ]; then echo \"source workspace path exists but is not a git repo\" >>\"$stderr_log\"; exit 43; fi", + " rm -rf \"$workspace\"", + " git clone --depth=1 --branch \"$branch\" \"$remote\" \"$workspace\" >>\"$stdout_log\" 2>>\"$stderr_log\"", + "fi", + "cd \"$workspace\"", + "sh -lc \"$dependency_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"", + "sh -lc \"$browser_command\" >>\"$stdout_log\" 2>>\"$stderr_log\"", + "missing_files=", + "while IFS= read -r file_path; do", + " [ -z \"$file_path\" ] && continue", + " if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi", + "done <\"$tmp_dir/files.txt\"", + "if [ -n \"$missing_files\" ]; then echo \"missing required files: $missing_files\" >>\"$stderr_log\"; exit 44; fi", + "source_commit=$(git rev-parse HEAD)", + "status_short=$(git status --short)", + "node_version=$(node --version 2>/dev/null || true)", + "npm_version=$(npm --version 2>/dev/null || true)", + "STATUS_SHORT=\"$status_short\" SOURCE_COMMIT=\"$source_commit\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" node <<'NODE'", + "const clean = !process.env.STATUS_SHORT;", + "console.log(JSON.stringify({ ok: clean, status: clean ? 'succeeded' : 'dirty-after-bootstrap', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, workspaceClean: clean, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));", + "NODE", + "trap - EXIT", + "rm -rf \"$tmp_dir\"", + ].join("\n"); +} + +function sourceWorkspaceJobName(spec: HwlabRuntimeLaneSpec): string { + return `hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-workspace`.slice(0, 63).replace(/-$/u, ""); +}