// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-probe module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:6032-7426 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneCiGitWorkspaceSecret, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { BootstrapAdminPasswordMaterial, BootstrapAdminSecretMaterial, NodeRuntimeGitMirrorEgressProxySpec, NodeRuntimeGitMirrorGithubTransportSpec, NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult, NodeWebProbeOptions, RuntimeSecretSpec } from "./entry"; import { isCommandSuccess, nodeRuntimeGitopsRoot, runNodeK3sArgs, runNodeK3sScript } from "./cleanup"; import { nodeRuntimeCicdWaitWarning } from "./git-mirror"; import { parseNodeScopedDelegatedOptions } from "./plan"; import { nodeRuntimePipelineFailureSummary, nodeRuntimePipelineRunDiagnostics } from "./render"; import { compactRuntimeCommand, runNodeHostScript, runNodeHostScriptAsync } from "./runtime-common"; import { assertLane, assertNodeId, keyValueLinesFromText, numericField, optionValue, optionalStringValue, positiveIntegerOption, positiveIntegerValue, record, requiredOption, shellQuote, statusText, stringValue, stripOptions } from "./utils"; import { discoverWebObserveIndexEntry, readWebObserveIndexEntry } from "./web-observe-render"; import { assertKnownOptions, nodeWebProbeAutoCommandTimeoutSeconds, nodeWebProbeDefaultUrl, normalizeNodeWebProbeObserveArgs, parseNodeWebProbeObserveOptions, parseNodeWebProbeSentinelOptions, parseWebProbeBrowserProxyMode } from "./web-probe-observe"; import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes"; import { resolveEgressProxySourceRef } from "../egress-proxy-sources"; export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record { const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http); const gitMirror = nodeRuntimeGitMirrorTarget(spec); const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec); const renderGitMirror = { ...gitMirror, egressProxy: gitMirror.egressProxy.mode === "direct" ? { mode: "direct", required: false, } : gitMirror.egressProxy.mode === "host-route" ? { ...gitMirror.egressProxy, mode: "host-route", required: true, } : { ...gitMirror.egressProxy, mode: "node-global", required: true, }, }; const deployYamlGitMirror = { ...renderGitMirror, egressProxy: renderGitMirror.egressProxy.mode !== "host-route" ? renderGitMirror.egressProxy : { mode: "node-global", required: true, clientName: renderGitMirror.egressProxy.clientName, namespace: "platform-infra", serviceName: renderGitMirror.egressProxy.clientName, port: httpProxyEndpoint(renderGitMirror.egressProxy.proxyUrl)?.port ?? 10808, proxyUrl: renderGitMirror.egressProxy.proxyUrl, noProxy: renderGitMirror.egressProxy.noProxy, }, }; return { nodeId: spec.nodeId, lane: spec.lane, sourceBranch: spec.sourceBranch, gitopsBranch: spec.gitopsBranch, gitopsRoot: nodeRuntimeGitopsRoot(spec), runtimePath: spec.runtimePath, runtimeRenderDir: spec.runtimeRenderDir, runtimeNamespace: spec.runtimeNamespace, catalogPath: spec.catalogPath, tektonDir: spec.tektonDir, pipelineName: spec.pipeline, argoApplicationFile: spec.argoApplicationFile, argoRepoUrl: spec.argoRepoUrl, gitUrl: spec.gitUrl, gitReadUrl: spec.gitReadUrl, gitWriteUrl: spec.gitWriteUrl, toolsImage: gitMirror.toolsImage, toolsImagePullPolicy: gitMirror.toolsImagePullPolicy, gitMirror: renderGitMirror, deployYamlGitMirror, networkProfileId: spec.networkProfileId, downloadProfileId: spec.downloadProfileId, gitSshProxyHost: gitSshProxy?.host, gitSshProxyPort: gitSshProxy?.port, proxyHttp: spec.networkProfile.proxy.http, proxyHttps: spec.networkProfile.proxy.https, proxyAll: spec.networkProfile.proxy.all, noProxy: spec.networkProfile.proxy.noProxy.join(","), dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http, dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https, dockerProxyAll: spec.networkProfile.dockerBuildProxy.all, dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","), dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy, npmRegistry: spec.downloadProfile.npm.registry, npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000, npmRetries: spec.downloadProfile.npm.retries, stepEnv: spec.stepEnv, observability: spec.observability, runtimeStore: spec.runtimeStore, codeAgentRuntime: spec.codeAgentRuntime === undefined ? undefined : { enabled: spec.codeAgentRuntime.enabled, adapter: spec.codeAgentRuntime.adapter, managerUrl: spec.codeAgentRuntime.managerUrl, apiKeySecretName: spec.codeAgentRuntime.apiKeySecretName, apiKeySecretKey: spec.codeAgentRuntime.apiKeySecretKey, runnerNamespace: spec.codeAgentRuntime.runnerNamespace, secretNamespace: spec.codeAgentRuntime.secretNamespace, repoUrlFrom: spec.codeAgentRuntime.repoUrlFrom, repoUrl: spec.gitReadUrl, providerIdFrom: spec.codeAgentRuntime.providerIdFrom, providerId: spec.nodeId, defaultProviderProfile: spec.codeAgentRuntime.defaultProviderProfile, codexStdioSupervisor: spec.codeAgentRuntime.codexStdioSupervisor, kafkaShadowProducer: spec.codeAgentRuntime.kafkaShadowProducer ?? null, valuesPrinted: false, }, runtimeImageRewrites: spec.runtimeImageRewrites, runtimeImageBuilds: spec.runtimeImageBuilds, registryPrefix: spec.registryPrefix, buildkitSidecarImage: spec.buildkit?.sidecarImage, publicWebUrl: spec.publicWebUrl, publicApiUrl: spec.publicApiUrl, publicExposure: spec.publicExposure === null ? undefined : { enabled: spec.publicExposure.enabled, mode: spec.publicExposure.mode, publicBaseUrl: spec.publicExposure.publicBaseUrl, hostname: spec.publicExposure.hostname, expectedA: spec.publicExposure.expectedA, serverAddr: spec.publicExposure.serverAddr, serverPort: spec.publicExposure.serverPort, secretName: spec.publicExposure.secretName, secretKey: spec.publicExposure.secretKey, tokenKey: spec.publicExposure.tokenKey, webProxy: spec.publicExposure.webProxy, apiProxy: spec.publicExposure.apiProxy, extraProxies: spec.publicExposure.extraProxies, }, externalPostgres: activeExternalPostgres === undefined ? undefined : { enabled: true, serviceName: activeExternalPostgres.serviceName, endpointAddress: activeExternalPostgres.endpointAddress, port: activeExternalPostgres.port, runtimeAccess: activeExternalPostgres.runtimeAccess ?? null, sslmode: activeExternalPostgres.sslmode, }, }; } export function httpProxyEndpoint(value: string): { host: string; port: number } | null { let parsed: URL; try { parsed = new URL(value); } catch { return null; } if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10); if (!Number.isInteger(port) || port <= 0) return null; return { host: parsed.hostname, port }; } export function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] { return [ `${renderDir}/devops-infra/git-mirror.yaml`, `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`, `${renderDir}/${spec.tektonDir}/rbac.yaml`, `${renderDir}/${spec.tektonDir}/pipeline.yaml`, `${renderDir}/argocd/project.yaml`, `${renderDir}/argocd/${spec.argoApplicationFile}`, ]; } export function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { const files = nodeRuntimeControlPlaneFiles(spec, renderDir); const args = [ "kubectl", "apply", ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), ...files.flatMap((file) => ["-f", file]), ]; return runNodeK3sArgs(spec, args, timeoutSeconds); } export function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir) .map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`) .join(""); const args = [ "kubectl", "apply", ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), "-f", "-", ]; return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest); } export function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult { const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; const script = [ "set -eu", `render_dir=${shellQuote(renderDir)}`, `prefix=${shellQuote(prefix)}`, "case \"$render_dir\" in", " \"$prefix\"*) rm -rf \"$render_dir\" ;;", " *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;", "esac", ].join("\n"); return runNodeHostScript(spec, script, 60); } export function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult { const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`; if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) { return { command: ["rm", "-rf", ""], cwd: repoRoot, exitCode: 44, stdout: "", stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`, signal: null, timedOut: false, }; } return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 }); } export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record { const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane); return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, namespace: gitWorkspaceSecret.namespace, labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": spec.lane, "hwlab.pikastech.local/source-commit": sourceCommit, "hwlab.pikastech.local/trigger": "unidesk-node-cli", }, annotations: { "hwlab.pikastech.local/node": spec.nodeId, "hwlab.pikastech.local/source-branch": spec.sourceBranch, "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch, "hwlab.pikastech.local/runtime-path": spec.runtimePath, "hwlab.pikastech.local/network-profile": spec.networkProfileId, "hwlab.pikastech.local/download-profile": spec.downloadProfileId, }, }, spec: { pipelineRef: { name: spec.pipeline }, taskRunTemplate: { serviceAccountName: spec.serviceAccountName, podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, }, }, params: [ { name: "git-url", value: spec.gitUrl }, { name: "git-read-url", value: spec.gitReadUrl }, { name: "git-write-url", value: spec.gitWriteUrl }, { name: "source-branch", value: spec.sourceBranch }, { name: "gitops-branch", value: spec.gitopsBranch }, { name: "lane", value: spec.lane }, { name: "catalog-path", value: spec.catalogPath }, { name: "image-tag-mode", value: "full" }, { name: "runtime-path", value: spec.runtimePath }, { name: "source-commit", value: sourceCommit }, { name: "revision", value: sourceCommit }, { name: "registry-prefix", value: spec.registryPrefix }, { name: "services", value: spec.serviceIds.join(",") }, { name: "base-image", value: spec.baseImage }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, { name: "git-ssh", secret: { secretName: gitWorkspaceSecret.name } }, ], }, }; } export function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult { const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane); const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64"); const script = [ "set -eu", `manifest_b64=${shellQuote(manifestB64)}`, `pipeline_run=${shellQuote(pipelineRun)}`, `ci_namespace=${shellQuote(gitWorkspaceSecret.namespace)}`, `git_workspace_secret=${shellQuote(gitWorkspaceSecret.name)}`, `rerun=${rerun ? "true" : "false"}`, "manifest_path=\"/tmp/$pipeline_run.json\"", "if ! kubectl -n \"$ci_namespace\" get secret \"$git_workspace_secret\" >/dev/null 2>&1; then", " printf 'missing Tekton git workspace Secret %s/%s; run: bun scripts/cli.ts hwlab nodes control-plane infra apply --node %s --lane %s --confirm\\n' \"$ci_namespace\" \"$git_workspace_secret\" " + shellQuote(spec.nodeId) + " " + shellQuote(spec.lane) + " >&2", " exit 44", "fi", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", "existing_status=$(kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", "if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then", " kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "elif [ \"$existing_status\" = False ]; then", " kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "fi", "if kubectl create -f \"$manifest_path\"; then", " :", "else", " code=$?", " if kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" >/dev/null 2>&1; then", " printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2", " else", " exit \"$code\"", " fi", "fi", "kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'", ].join("\n"); return runNodeK3sScript(spec, script, timeoutSeconds); } export function waitForNodeRuntimePipelineRunTerminal( spec: HwlabRuntimeLaneSpec, pipelineRun: string, timeoutSeconds: number, options: { opportunisticPostFlush?: () => Record | null; opportunisticPostSync?: () => Record | null } = {}, ): Record { const severeWarningThresholdMs = 120_000; const startedAt = Date.now(); const deadline = startedAt + timeoutSeconds * 1000; let polls = 0; let last: Record = { exists: false, name: pipelineRun }; let lastOpportunisticPostFlushAt = 0; let severeWarningEmitted = false; let opportunisticPostSyncAttempted = false; const opportunisticPostFlushes: Record[] = []; const opportunisticPostSyncs: Record[] = []; printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds }); while (Date.now() <= deadline) { polls += 1; last = getNodeRuntimePipelineRun(spec, pipelineRun); const status = typeof last.status === "string" ? last.status : null; const reason = typeof last.reason === "string" ? last.reason : null; const elapsedMs = Date.now() - startedAt; printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs }); if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") { severeWarningEmitted = true; printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", warning: "pipelinerun-wait-severe-timeout", severity: "warning", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs, thresholdMs: severeWarningThresholdMs, message: "PipelineRun has been non-terminal for more than 120s; this is a severe timeout and requires further investigation of Tekton taskRuns/pods instead of treating it as normal wait time.", next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, }, }); } if (status === "True" || status === "False") { const ok = status === "True"; const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics); printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); return { ok, status: ok ? "succeeded" : "failed", pipelineRun: last, diagnostics, failureSummary, polls, elapsedMs: Date.now() - startedAt, opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined, opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined, degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed", }; } if ( options.opportunisticPostSync !== undefined && !opportunisticPostSyncAttempted && nodeRuntimePipelineRunSourceCloneCompleted(spec, pipelineRun) ) { opportunisticPostSyncAttempted = true; const sync = options.opportunisticPostSync(); if (sync !== null) opportunisticPostSyncs.push(sync); } if (options.opportunisticPostFlush !== undefined && Date.now() - lastOpportunisticPostFlushAt >= 15_000) { lastOpportunisticPostFlushAt = Date.now(); const flush = options.opportunisticPostFlush(); if (flush !== null) opportunisticPostFlushes.push(flush); last = getNodeRuntimePipelineRun(spec, pipelineRun); const refreshedStatus = typeof last.status === "string" ? last.status : null; const refreshedReason = typeof last.reason === "string" ? last.reason : null; const refreshedElapsedMs = Date.now() - startedAt; if (refreshedStatus === "True" || refreshedStatus === "False") { const ok = refreshedStatus === "True"; const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics); printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: refreshedStatus, reason: refreshedReason, polls, elapsedMs: refreshedElapsedMs, observedAfterPostFlush: true }); return { ok, status: ok ? "succeeded" : "failed", pipelineRun: last, diagnostics, failureSummary, polls, elapsedMs: refreshedElapsedMs, opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined, opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined, observedAfterPostFlush: true, degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed", }; } } sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now()))); } const elapsedMs = Date.now() - startedAt; const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs }); printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds }); return { ok: true, status: "pending", pipelineRun: last, polls, elapsedMs, waitLimitSeconds: timeoutSeconds, warning, opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined, opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined, next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` }, }; } export function nodeRuntimePipelineRunSourceCloneCompleted(spec: HwlabRuntimeLaneSpec, pipelineRun: string): boolean { const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60); if (result.exitCode !== 0) return false; let parsed: unknown; try { parsed = JSON.parse(result.stdout); } catch { return false; } const items = Array.isArray(record(parsed).items) ? record(parsed).items as unknown[] : []; return items.some((item) => { const itemRecord = record(item); const metadata = record(itemRecord.metadata); const labels = record(metadata.labels); const name = typeof metadata.name === "string" ? metadata.name : ""; const pipelineTask = typeof labels["tekton.dev/pipelineTask"] === "string" ? labels["tekton.dev/pipelineTask"] : ""; const taskName = `${pipelineTask} ${name}`; if (!/(clone|checkout|local-git|source[-_ ]*worktree)/iu.test(taskName)) return false; const status = record(itemRecord.status); const conditions = Array.isArray(status.conditions) ? status.conditions as unknown[] : []; return conditions.some((condition) => { const conditionRecord = record(condition); return conditionRecord.type === "Succeeded" && conditionRecord.status === "True"; }); }); } export function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`); } export function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record { const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60); if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) }; const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u); return { exists: true, name: pipelineRun, status: status || null, reason: reason || null, message: message || null, createdAt: createdAt || null, result: compactRuntimeCommand(result), }; } export function sleepSync(ms: number): void { const buffer = new SharedArrayBuffer(4); Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms)); } export function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return null; const secretRoot = externalPostgresSecretSourceRoot(spec); const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey); const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey); const missing = [ ...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]), ...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]), ]; if (missing.length > 0) { if (dryRun) { return { ok: true, mode: "dry-run", mutation: false, skipped: true, reason: "external-postgres-secret-source-missing", missing, sourceRoot: secretRoot, valuesPrinted: false, next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, }; } return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, degradedReason: "external-postgres-secret-source-missing", missing, sourceRoot: secretRoot, valuesPrinted: false, next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, }; } const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds); if (!isCommandSuccess(setup)) { return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, phase: "namespace-authn-setup", setup: compactRuntimeCommand(setup), valuesPrinted: false, }; } const manifest = { apiVersion: "v1", kind: "List", items: [ { apiVersion: "v1", kind: "Secret", metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace }, type: "Opaque", stringData: { [pg.cloudApi.secretKey]: cloudApi.value }, }, { apiVersion: "v1", kind: "Secret", metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace }, type: "Opaque", stringData: { [pg.openfga.secretKey]: openfga.value }, }, ], }; const applyScript = [ "set -eu", [ "kubectl", "apply", "--server-side", "--force-conflicts", "--field-manager=unidesk-hwlab-node-external-postgres-secret", ...(dryRun ? ["--dry-run=server"] : []), "-f", "-", ].map(shellQuote).join(" "), ].join("\n"); const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest)); return { ok: isCommandSuccess(apply), mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: !dryRun && isCommandSuccess(apply), namespace: spec.runtimeNamespace, sourceRoot: secretRoot, secrets: [ { name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey }, { name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null }, ], setup: compactRuntimeCommand(setup), apply: compactRuntimeCommand(apply), valuesPrinted: false, }; } export function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { const pg = spec.runtimeStore?.postgres; if (pg?.mode !== "local-k3s") return null; const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`; const sourceRef = pg.adminPasswordSourceRef ?? ""; const sourceKey = pg.adminPasswordSourceKey ?? ""; if (sourceRef.length === 0 || sourceKey.length === 0) { return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, namespace: spec.runtimeNamespace, secretName, secretKey: "POSTGRES_PASSWORD", degradedReason: "local-postgres-secret-source-not-configured", valuesPrinted: false, next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` }, }; } const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun }); if (material.ok !== true) { return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, namespace: spec.runtimeNamespace, secretName, secretKey: "POSTGRES_PASSWORD", source: material, degradedReason: material.error ?? "local-postgres-secret-source-missing", valuesPrinted: false, next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=` }, }; } const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? ""); const fields = keyValueLinesFromText(statusText(result)); const afterBytes = numericField(fields.afterPasswordBytes) ?? 0; const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0; return { ok, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: !dryRun && fields.mutation === "true", namespace: spec.runtimeNamespace, secretName, secretKey: "POSTGRES_PASSWORD", source: { ok: material.ok, sourceRef, sourceKey, sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath), generated: material.generated, fingerprint: material.fingerprint, valueBytes: material.value?.length ?? 0, valuesPrinted: false, }, before: { namespaceExists: fields.beforeNamespaceExists === "yes", secretExists: fields.beforeSecretExists === "yes", passwordBytes: numericField(fields.beforePasswordBytes), fingerprint: fields.beforePasswordFingerprint || null, }, after: { namespaceExists: fields.afterNamespaceExists === "yes", secretExists: fields.afterSecretExists === "yes", passwordBytes: numericField(fields.afterPasswordBytes), fingerprint: fields.afterPasswordFingerprint || null, }, action: fields.action || null, applyExitCode: numericField(fields.applyExitCode), namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode), degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed", result: compactRuntimeCommand(result), valuesPrinted: false, }; } export function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): { ok: boolean; sourceRef: string; sourceKey: string; sourcePath: string | null; checkedPaths: string[]; value: string | null; fingerprint: string | null; generated: boolean; error?: string; } { const checkedPaths = localSecretSourcePaths(input.sourceRef); const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null; const sourcePath = existingPath ?? checkedPaths[0] ?? null; if (sourcePath === null) { return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" }; } if (existingPath === null && input.dryRun) { return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" }; } if (existingPath === null) { const value = randomBytes(32).toString("base64url"); mkdirSync(dirname(sourcePath), { recursive: true }); writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 }); return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true }; } const text = readFileSync(existingPath, "utf8"); const values = parseEnvFile(text); const existingValue = values[input.sourceKey]; if (existingValue !== undefined && existingValue.length > 0) { return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false }; } if (input.dryRun) { return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" }; } const value = randomBytes(32).toString("base64url"); const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n"; writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 }); return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true }; } export function localSecretSourcePaths(sourceRef: string): string[] { if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef); const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); const paths = index >= 0 ? [ join(repoRoot.slice(0, index), ".state", "secrets", sourceRef), join(repoRoot, ".state", "secrets", sourceRef), ] : [join(repoRoot, ".state", "secrets", sourceRef)]; return [...new Set(paths)]; } export function shortSecretFingerprint(value: string): string { return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`; } export function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string { return [ "set +e", `namespace=${shellQuote(spec.runtimeNamespace)}`, `secret=${shellQuote(secretName)}`, "secret_key=POSTGRES_PASSWORD", `source_ref=${shellQuote(sourceRef)}`, `source_key=${shellQuote(sourceKey)}`, `source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, "field_manager=unidesk-hwlab-node-local-postgres-secret", "password=$(cat)", "password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')", "exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }", "before_namespace_exists=$(namespace_exists_flag)", "before_secret_exists=no", "before_b64=", "if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi", "before_password_bytes=$(decoded_length \"$before_b64\")", "before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")", "after_namespace_exists=$before_namespace_exists", "after_secret_exists=$before_secret_exists", "after_password_bytes=$before_password_bytes", "after_password_fingerprint=$before_password_fingerprint", "action=observed", "mutation=false", "namespace_create_exit=", "apply_exit=", "if [ \"$dry_run\" = true ]; then", " if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi", "elif [ \"$password_bytes\" -le 0 ]; then", " action=source-empty", " apply_exit=42", "elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then", " action=source-mismatch", " apply_exit=47", "else", " if [ \"$before_namespace_exists\" != yes ]; then", " kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err", " namespace_create_exit=$?", " else", " namespace_create_exit=0", " fi", " if [ \"$namespace_create_exit\" = 0 ]; then", " tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)", " password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')", " cat >\"$tmp\" </tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err", " apply_exit=$?", " rm -f \"$tmp\"", " if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi", " else", " action=namespace-create-failed", " apply_exit=$namespace_create_exit", " fi", "fi", "after_namespace_exists=$(namespace_exists_flag)", "if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi", "after_password_bytes=$(decoded_length \"$after_b64\")", "after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'key\\t%s\\n' \"$secret_key\"", "printf 'sourceRef\\t%s\\n' \"$source_ref\"", "printf 'sourceKey\\t%s\\n' \"$source_key\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", "printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"", "printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"", "printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", "printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"", "printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"", "printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi", ].join("\n"); } interface NodeRuntimeImageDependency { id: string; target: string; source: string; mode: "pull" | "build-moonbridge" | "build-opencode-git"; sourceRepo?: string; sourceRef?: string; builderImage?: string; goProxy?: string; dockerNetworkMode?: "default" | "host"; } function nodeRuntimeImageDependencies(spec: HwlabRuntimeLaneSpec): NodeRuntimeImageDependency[] { const images: NodeRuntimeImageDependency[] = [ { id: "base", target: spec.baseImage, source: spec.baseImageSource ?? "", mode: "pull" }, ]; if (spec.buildkit !== undefined) { images.push({ id: "buildkit", target: spec.buildkit.sidecarImage, source: spec.buildkit.sourceImage, mode: "pull" }); } spec.runtimeImageRewrites.forEach((rewrite, index) => { images.push({ id: `rewrite-${index + 1}`, target: rewrite.target, source: rewrite.source, mode: "pull" }); }); spec.runtimeImageBuilds.forEach((build) => { if (build.kind === "moonbridge") { images.push({ id: build.id, target: build.target, source: `${build.sourceRepo}#${build.sourceRef}`, mode: "build-moonbridge", sourceRepo: build.sourceRepo, sourceRef: build.sourceRef, builderImage: build.builderImage, goProxy: build.goProxy, dockerNetworkMode: build.dockerNetworkMode, }); return; } images.push({ id: build.id, target: build.target, source: build.sourceImage ?? "", mode: "build-opencode-git", dockerNetworkMode: build.dockerNetworkMode, }); }); return images; } function nodeRuntimeImageCalls(spec: HwlabRuntimeLaneSpec, functionName: string): string { return nodeRuntimeImageDependencies(spec) .map((image) => [ functionName, image.id, image.target, image.source, image.mode, image.sourceRepo ?? "", image.sourceRef ?? "", image.builderImage ?? "", image.goProxy ?? "", image.dockerNetworkMode ?? "", ].map(shellQuote).join(" ")) .join("\n"); } function parseNodeRuntimeImageRows(text: string): Record[] { return text.split(/\r?\n/u).flatMap((line) => { if (!line.startsWith("imageJson\t")) return []; try { return [JSON.parse(line.slice("imageJson\t".length)) as Record]; } catch { return []; } }); } export function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record { const script = [ "set -eu", "check_image() {", " id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"", " repo_tag=${target#*/}", " repo=${repo_tag%:*}", " tag=${repo_tag##*:}", " if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", " registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", " registry_present=false", " if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi", " target_present=false", " if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi", " source_present=false", " if [ \"$mode\" = build-moonbridge ]; then source_present=true; elif [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi", " source_configured=false", " if [ -n \"$source\" ]; then source_configured=true; fi", " python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$source_configured\" \"$registry_url\" \"$registry_present\" \"$target_present\" \"$source_present\" <<'PY'", "import json, sys", "keys=['id','target','source','mode','sourceConfigured','registryUrl','registryTagPresent','targetImagePresent','sourceImagePresent']", "data=dict(zip(keys, sys.argv[1:]))", "for key in ['sourceConfigured','registryTagPresent','targetImagePresent','sourceImagePresent']:", " data[key] = data[key] == 'true'", "print('imageJson\\t' + json.dumps(data, separators=(',', ':')))", "PY", "}", nodeRuntimeImageCalls(spec, "check_image"), ].join("\n"); const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); const imageRows = parseNodeRuntimeImageRows(statusText(result)); const base = imageRows.find((image) => image.id === "base") ?? {}; return { ok: isCommandSuccess(result) && imageRows.length === nodeRuntimeImageDependencies(spec).length && imageRows.every((image) => image.sourceConfigured === true && image.registryTagPresent === true), target: base.target ?? spec.baseImage, source: base.source || null, sourceConfigured: base.sourceConfigured === true, registryUrl: base.registryUrl ?? null, registryTagPresent: base.registryTagPresent === true, targetImagePresent: base.targetImagePresent === true, sourceImagePresent: base.sourceImagePresent === true, images: imageRows, result: compactRuntimeCommand(result), }; } export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { if (spec.baseImageSource === undefined) return null; const script = [ "set -eu", `dry_run=${shellQuote(dryRun ? "true" : "false")}`, `pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`, `build_http_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.http)}`, `build_https_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.https)}`, `build_all_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.all)}`, `build_no_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.noProxy.join(","))}`, "resolve_host_proxy_url() {", " raw_url=\"$1\"", " if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi", " info=$(python3 - \"$raw_url\" <<'PY'", "from urllib.parse import urlparse", "import sys", "url = sys.argv[1]", "parsed = urlparse(url)", "host = parsed.hostname or ''", "port = parsed.port", "scheme = parsed.scheme or 'http'", "parts = host.split('.')", "if len(parts) >= 4 and parts[2] == 'svc':", " print('\\t'.join(['service', scheme, parts[0], parts[1], str(port or 80)]))", "else:", " print('\\t'.join(['literal', url]))", "PY", " )", " kind=$(printf '%s' \"$info\" | cut -f1)", " service_cluster_ip() {", " service_namespace=\"$1\"; service_name=\"$2\"", " cluster_ip=\"\"", " if command -v kubectl >/dev/null 2>&1; then", " cluster_ip=$(kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)", " fi", " if { [ -z \"$cluster_ip\" ] || [ \"$cluster_ip\" = None ]; } && command -v k3s >/dev/null 2>&1; then", " cluster_ip=$(k3s kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)", " fi", " printf '%s\\n' \"$cluster_ip\"", " }", " if [ \"$kind\" = service ]; then", " scheme=$(printf '%s' \"$info\" | cut -f2)", " service_name=$(printf '%s' \"$info\" | cut -f3)", " service_namespace=$(printf '%s' \"$info\" | cut -f4)", " service_port=$(printf '%s' \"$info\" | cut -f5)", " cluster_ip=$(service_cluster_ip \"$service_namespace\" \"$service_name\")", " if [ -n \"$cluster_ip\" ] && [ \"$cluster_ip\" != None ]; then printf '%s://%s:%s\\n' \"$scheme\" \"$cluster_ip\" \"$service_port\"; else printf '%s\\n' \"$raw_url\"; fi", " else", " printf '%s\\n' \"$raw_url\"", " fi", "}", "build_http_proxy=$(resolve_host_proxy_url \"$build_http_proxy\")", "build_https_proxy=$(resolve_host_proxy_url \"$build_https_proxy\")", "build_all_proxy=$(resolve_host_proxy_url \"$build_all_proxy\")", "build_container_proxy_url() {", " raw_url=\"$1\"", " if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi", " python3 - \"$raw_url\" <<'PY'", "from urllib.parse import urlparse, urlunparse", "import sys", "url = sys.argv[1]", "parsed = urlparse(url)", "host = parsed.hostname or ''", "if host in {'127.0.0.1', 'localhost'}:", " netloc = 'host.docker.internal'", " if parsed.port is not None:", " netloc = f'{netloc}:{parsed.port}'", " print(urlunparse(parsed._replace(netloc=netloc)))", "else:", " print(url)", "PY", "}", "build_container_http_proxy=$(build_container_proxy_url \"$build_http_proxy\")", "build_container_https_proxy=$(build_container_proxy_url \"$build_https_proxy\")", "build_container_all_proxy=$(build_container_proxy_url \"$build_all_proxy\")", "docker_build_add_host_args=\"\"", "case \"$build_container_http_proxy $build_container_https_proxy $build_container_all_proxy\" in", " *host.docker.internal*) docker_build_add_host_args=\"--add-host host.docker.internal:host-gateway\" ;;", "esac", "failed=false", "ensure_image() {", " id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"; source_repo=\"$5\"; source_ref=\"$6\"; builder_image=\"$7\"; go_proxy=\"$8\"; docker_network_mode=\"$9\"", " repo_tag=${target#*/}", " repo=${repo_tag%:*}", " tag=${repo_tag##*:}", " if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", " registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", " present=false", " if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi", " action=none", " source_present_before=unknown", " pulled_source=false", " pull_attempts=0", " if [ \"$present\" = false ]; then", " action=seed", " if [ \"$dry_run\" = false ]; then", " if [ \"$mode\" = build-moonbridge ]; then", " action=build", " source_present_before=build-source", " effective_build_container_http_proxy=\"$build_container_http_proxy\"", " effective_build_container_https_proxy=\"$build_container_https_proxy\"", " effective_build_container_all_proxy=\"$build_container_all_proxy\"", " effective_docker_build_add_host_args=\"$docker_build_add_host_args\"", " if [ \"$docker_network_mode\" = host ]; then", " effective_build_container_http_proxy=\"$build_http_proxy\"", " effective_build_container_https_proxy=\"$build_https_proxy\"", " effective_build_container_all_proxy=\"$build_all_proxy\"", " effective_docker_build_add_host_args=\"\"", " fi", " tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)", " dockerfile=\"$tmpdir/Dockerfile\"", " cat > \"$dockerfile\" <<'DOCKERFILE'", "ARG BUILDER_IMAGE", "FROM ${BUILDER_IMAGE} AS builder", "ARG MOONBRIDGE_REPO", "ARG MOONBRIDGE_REF", "ARG GOPROXY_VALUE", "ARG HTTP_PROXY", "ARG HTTPS_PROXY", "ARG ALL_PROXY", "ARG NO_PROXY", "ENV GOPROXY=${GOPROXY_VALUE}", "ENV HTTP_PROXY=${HTTP_PROXY}", "ENV HTTPS_PROXY=${HTTPS_PROXY}", "ENV ALL_PROXY=${ALL_PROXY}", "ENV NO_PROXY=${NO_PROXY}", "ENV http_proxy=${HTTP_PROXY}", "ENV https_proxy=${HTTPS_PROXY}", "ENV all_proxy=${ALL_PROXY}", "ENV no_proxy=${NO_PROXY}", "WORKDIR /src", "RUN git clone --filter=blob:none \"${MOONBRIDGE_REPO}\" moon-bridge", "WORKDIR /src/moon-bridge", "RUN git checkout \"${MOONBRIDGE_REF}\" \\", " && go mod download \\", " && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags=\"-s -w\" -o /out/moonbridge ./cmd/moonbridge", "FROM scratch", "WORKDIR /app", "COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt", "COPY --from=builder /out/moonbridge /app/moonbridge", "EXPOSE 4001", "USER 65532:65532", "ENTRYPOINT [\"/app/moonbridge\"]", "DOCKERFILE", " if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then", " docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }", " else", " cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true", " failed=true", " fi", " rm -rf \"$tmpdir\"", " elif [ \"$mode\" = build-opencode-git ]; then", " action=build", " source_present_before=false", " if [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present_before=true; fi", " effective_build_container_http_proxy=\"$build_container_http_proxy\"", " effective_build_container_https_proxy=\"$build_container_https_proxy\"", " effective_build_container_all_proxy=\"$build_container_all_proxy\"", " effective_docker_build_add_host_args=\"$docker_build_add_host_args\"", " if [ \"$docker_network_mode\" = host ]; then", " effective_build_container_http_proxy=\"$build_http_proxy\"", " effective_build_container_https_proxy=\"$build_https_proxy\"", " effective_build_container_all_proxy=\"$build_all_proxy\"", " effective_docker_build_add_host_args=\"\"", " fi", " tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)", " dockerfile=\"$tmpdir/Dockerfile\"", " cat > \"$dockerfile\" <<'DOCKERFILE'", "ARG SOURCE_IMAGE", "FROM ${SOURCE_IMAGE}", "ARG HTTP_PROXY", "ARG HTTPS_PROXY", "ARG ALL_PROXY", "ARG NO_PROXY", "ENV HTTP_PROXY=${HTTP_PROXY}", "ENV HTTPS_PROXY=${HTTPS_PROXY}", "ENV ALL_PROXY=${ALL_PROXY}", "ENV NO_PROXY=${NO_PROXY}", "ENV http_proxy=${HTTP_PROXY}", "ENV https_proxy=${HTTPS_PROXY}", "ENV all_proxy=${ALL_PROXY}", "ENV no_proxy=${NO_PROXY}", "RUN apk add --no-cache git", "DOCKERFILE", " if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg SOURCE_IMAGE=\"$source\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then", " docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }", " else", " cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true", " failed=true", " fi", " rm -rf \"$tmpdir\"", " else", " source_present_before=true", " if ! docker image inspect \"$source\" >/dev/null 2>&1; then", " source_present_before=false", " attempt=1", " while [ \"$attempt\" -le \"$pull_retries\" ]; do", " pull_attempts=$attempt", " if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker pull \"$source\" >/tmp/hwlab-node-runtime-image-$id-pull.out 2>&1; then pulled_source=true; break; fi", " attempt=$((attempt + 1))", " sleep 2", " done", " if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-runtime-image-$id-pull.out >&2 2>/dev/null || true; failed=true; fi", " fi", " if [ \"$failed\" != true ]; then", " if docker image inspect \"$source\" >/dev/null 2>&1; then", " docker tag \"$source\" \"$target\"", " docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || failed=true", " else", " failed=true", " fi", " fi", " fi", " fi", " fi", " after=false", " if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi", " python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$dry_run\" \"$present\" \"$after\" \"$action\" \"$pull_retries\" \"$source_present_before\" \"$pulled_source\" \"$pull_attempts\" <<'PY'", "import json, sys", "keys=['id','target','source','mode','dryRun','presentBefore','presentAfter','action','pullRetries','sourcePresentBefore','pulledSource','pullAttempts']", "data=dict(zip(keys, sys.argv[1:]))", "for key in ['dryRun','presentBefore','presentAfter','pulledSource']:", " data[key] = data[key] == 'true'", "for key in ['pullRetries','pullAttempts']:", " data[key] = int(data[key]) if str(data[key]).isdigit() else None", "print('imageJson\\t' + json.dumps(data, separators=(',', ':')))", "PY", " if [ \"$dry_run\" = false ] && [ \"$after\" != true ]; then failed=true; fi", "}", nodeRuntimeImageCalls(spec, "ensure_image"), "if [ \"$failed\" = true ]; then exit 1; fi", ].join("\n"); const hasBuild = nodeRuntimeImageDependencies(spec).some((image) => image.mode === "build-moonbridge" || image.mode === "build-opencode-git"); const result = hasBuild && !dryRun ? runNodeHostScriptAsync(spec, script, Math.min(timeoutSeconds, 600), "runtime-image-build") : runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); const imageRows = parseNodeRuntimeImageRows(statusText(result)); const dependencyCount = nodeRuntimeImageDependencies(spec).length; const imageRowsComplete = imageRows.length === dependencyCount && imageRows.every((image) => image.presentAfter === true || image.registryTagPresent === true); const base = imageRows.find((image) => image.id === "base") ?? {}; return { ok: isCommandSuccess(result) || imageRowsComplete, dryRun, target: base.target ?? spec.baseImage, source: base.source ?? spec.baseImageSource, presentBefore: base.presentBefore === true, presentAfter: base.presentAfter === true, action: base.action ?? null, pullRetries: numericField(typeof base.pullRetries === "number" ? String(base.pullRetries) : undefined), sourcePresentBefore: typeof base.sourcePresentBefore === "string" ? base.sourcePresentBefore : null, pulledSource: base.pulledSource === true, pullAttempts: numericField(typeof base.pullAttempts === "number" ? String(base.pullAttempts) : undefined), images: imageRows, result: compactRuntimeCommand(result), }; } export function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return "true"; const authnKey = pg.openfga.authnKey ?? "authn-preshared-key"; return [ "set -eu", `namespace=${shellQuote(spec.runtimeNamespace)}`, `secret=${shellQuote(pg.openfga.secretName)}`, `authn_key=${shellQuote(authnKey)}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, "namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"", "if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi", "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -", "authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')", "if [ \"${authn_present:-0}\" -gt 0 ]; then", " action=kept", "elif [ \"$dry_run\" = true ]; then", " action=would-generate-authn-key", "else", " if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi", " kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null", " authn_value=", " action=generated-authn-key", "fi", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'authnKey\\t%s\\n' \"$authn_key\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'valuesPrinted\\tfalse\\n'", ].join("\n"); } export function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string { const configRef = spec.externalPostgres?.configRef; if (configRef === undefined) return join(repoRoot, ".state", "secrets"); const configPath = rootPath(configRef); const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown; if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`); const secrets = (parsed as Record).secrets; if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`); const root = (secrets as Record).root; if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`); return root.startsWith("/") ? root : rootPath(root); } export function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } { const sourcePath = join(secretRoot, sourceRef); if (!existsSync(sourcePath)) return { ok: false, sourcePath }; const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const value = values[key]; if (value === undefined || value.length === 0) return { ok: false, sourcePath }; return { ok: true, value, sourcePath }; } export function secretSourcePaths(sourceRef: string): string[] { if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef); const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef)); return [...new Set(paths)]; } function ownerFileSourcePaths(sourceRef: string): string[] { if (sourceRef.includes("..") || sourceRef.includes("\0")) return []; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot]; return [...new Set(roots.map((root) => join(root, sourceRef)))]; } export function displayRepoPath(path: string): string { const normalizedRoot = repoRoot.replace(/\/+$/u, ""); if (path === normalizedRoot) return "."; if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1); const marker = "/.worktree/"; const index = normalizedRoot.indexOf(marker); if (index >= 0) { const mainRoot = normalizedRoot.slice(0, index); if (path === mainRoot) return "."; if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1); } return path; } export function hwlabPasswordHash(password: string): string { const salt = randomBytes(16).toString("hex"); return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`; } export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial { const sourceRef = spec.bootstrapAdminPasswordSourceRef; const sourceKey = spec.bootstrapAdminPasswordSourceKey; const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null; if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { return bootstrapAdminSecretMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing"); } const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine); if (!password.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error); const username = readBootstrapAdminUsername(spec); if (!username.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error); return { ok: true, username: username.value, usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, usernameFingerprint: shortSecretFingerprint(username.value), sourceRef, sourceKey, sourceLine, sourcePath: password.sourcePath, sourcePresent: true, sourceFingerprint: shortSecretFingerprint(password.value), passwordHash: hwlabPasswordHash(password.value), error: null, }; } export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial { const sourceRef = spec.bootstrapAdminPasswordSourceRef; const sourceKey = spec.bootstrapAdminPasswordSourceKey; const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null; if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { return bootstrapAdminPasswordMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing"); } const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine); if (!password.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error); const username = readBootstrapAdminUsername(spec); if (!username.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error); return { ok: true, username: username.value, usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, usernameFingerprint: shortSecretFingerprint(username.value), sourceRef, sourceKey, sourceLine, sourcePath: password.sourcePath, sourcePresent: true, sourceFingerprint: shortSecretFingerprint(password.value), password: password.value, error: null, }; } function readBootstrapAdminUsername(spec: RuntimeSecretSpec): { ok: true; value: string } | { ok: false; error: string } { const ref = spec.bootstrapAdminUsernameSourceRef; if (ref === undefined) return { ok: true, value: spec.bootstrapAdminUsername }; const key = spec.bootstrapAdminUsernameSourceKey ?? "username"; const material = readSecretSourceScalar(ref, key, spec.bootstrapAdminUsernameSourceLine ?? null); if (!material.ok) return { ok: false, error: `username-${material.error}` }; return { ok: true, value: material.value }; } function readSecretSourceScalar(sourceRef: string, sourceKey: string, sourceLine: number | null): { ok: true; value: string; sourcePath: string; sourcePresent: true } | { ok: false; sourcePath: string; sourcePresent: boolean; error: string } { const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, sourcePath, sourcePresent: false, error: "secret-source-missing" }; const text = readFileSync(sourcePath, "utf8"); if (sourceLine !== null) { const line = text.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? ""; if (line.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-line-missing" }; return { ok: true, value: line, sourcePath, sourcePresent: true }; } const values = parseEnvFile(text); const runtimeValue = process.env[sourceKey]; const value = values[sourceKey] ?? runtimeValue; if (value === undefined || value.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-key-missing" }; return { ok: true, value, sourcePath, sourcePresent: true }; } function bootstrapAdminSecretMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminSecretMaterial { return { ok: false, username: null, usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, usernameFingerprint: null, sourceRef, sourceKey, sourceLine, sourcePath, sourcePresent: sourcePresent === true, sourceFingerprint: null, passwordHash: null, error, }; } function bootstrapAdminPasswordMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminPasswordMaterial { return { ok: false, username: null, usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, usernameFingerprint: null, sourceRef, sourceKey, sourceLine, sourcePath, sourcePresent: sourcePresent === true, sourceFingerprint: null, password: null, error, }; } export function parseEnvFile(text: string): Record { const values: Record = {}; for (const rawLine of text.split(/\r?\n/u)) { const line = rawLine.trim(); if (line.length === 0 || line.startsWith("#")) continue; const eq = line.indexOf("="); if (eq <= 0) continue; const key = line.slice(0, eq).trim(); const rawValue = line.slice(eq + 1).trim(); values[key] = rawValue.startsWith("'") && rawValue.endsWith("'") ? rawValue.slice(1, -1).replace(/'"'"'/gu, "'") : rawValue.startsWith("\"") && rawValue.endsWith("\"") ? rawValue.slice(1, -1) : rawValue; } return values; } export function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean { if (!/postgres|pgsql|pgdata/iu.test(name)) return false; const externalServiceName = spec.externalPostgres?.serviceName; if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false; return true; } export function nodeRuntimeCodeAgentRuntimeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { const runtime = spec.codeAgentRuntime; if (runtime === undefined || runtime.enabled === false) return { required: false, ready: true, enabled: runtime?.enabled ?? false }; if (!namespaceExists) return { required: true, ready: false, enabled: true, degradedReason: "runtime-namespace-missing" }; const managerTarget = nodeRuntimeCodeAgentManagerTarget(runtime.managerUrl, runtime.runnerNamespace); const runnerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.runnerNamespace, "-o", "name"], 60); const secretNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.secretNamespace, "-o", "name"], 60); const managerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", managerTarget.namespace, "-o", "name"], 60); const managerService = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "service", managerTarget.serviceName, "-o", "name"], 60); const managerDeployment = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "deployment", managerTarget.serviceName, "-o", "jsonpath={.status.readyReplicas}{\"\\n\"}{.spec.replicas}{\"\\n\"}"], 60); const [managerReadyReplicasRaw = "", managerDesiredReplicasRaw = ""] = managerDeployment.stdout.split(/\r?\n/u); const managerReadyReplicas = numericField(managerReadyReplicasRaw); const managerDesiredReplicas = numericField(managerDesiredReplicasRaw); const managerDeploymentReady = managerDeployment.exitCode === 0 && (managerReadyReplicas ?? 0) >= Math.max(1, managerDesiredReplicas ?? 1); const apiKey = secretKeyStatus(spec, runtime.apiKeySecretName, runtime.apiKeySecretKey); const cloudApiEnv = nodeRuntimeCodeAgentCloudApiEnvStatus(spec, runtime); const ready = runnerNamespace.exitCode === 0 && secretNamespace.exitCode === 0 && managerNamespace.exitCode === 0 && managerService.exitCode === 0 && managerDeploymentReady && apiKey.present === true && cloudApiEnv.ready === true; const degradedReason = ready ? undefined : runnerNamespace.exitCode !== 0 ? "agentrun-runner-namespace-missing" : secretNamespace.exitCode !== 0 ? "agentrun-secret-namespace-missing" : managerNamespace.exitCode !== 0 ? "agentrun-manager-namespace-missing" : managerService.exitCode !== 0 ? "agentrun-manager-service-missing" : !managerDeploymentReady ? "agentrun-manager-deployment-not-ready" : apiKey.present !== true ? "agentrun-api-key-secret-missing" : "hwlab-cloud-api-code-agent-env-mismatch"; return { required: true, enabled: true, ready, degradedReason, adapter: runtime.adapter, managerUrl: runtime.managerUrl, manager: { namespace: managerTarget.namespace, serviceName: managerTarget.serviceName, serviceExists: managerService.exitCode === 0, deploymentReady: managerDeploymentReady, readyReplicas: managerReadyReplicas, desiredReplicas: managerDesiredReplicas, namespaceResult: compactRuntimeCommand(managerNamespace), serviceResult: compactRuntimeCommand(managerService), deploymentResult: compactRuntimeCommand(managerDeployment), }, runnerNamespace: { name: runtime.runnerNamespace, exists: runnerNamespace.exitCode === 0, result: compactRuntimeCommand(runnerNamespace), }, secretNamespace: { name: runtime.secretNamespace, exists: secretNamespace.exitCode === 0, result: compactRuntimeCommand(secretNamespace), }, apiKey, cloudApiEnv, repoUrlFrom: runtime.repoUrlFrom, providerIdFrom: runtime.providerIdFrom, valuesPrinted: false, }; } function nodeRuntimeCodeAgentManagerTarget(managerUrl: string, fallbackNamespace: string): { namespace: string; serviceName: string } { let parsed: URL | null = null; try { parsed = new URL(managerUrl); } catch { parsed = null; } const host = parsed?.hostname ?? ""; const parts = host.split(".").filter(Boolean); if (parts.length >= 3 && parts[2] === "svc") return { serviceName: parts[0], namespace: parts[1] }; return { serviceName: parts[0] || "agentrun-mgr", namespace: parts[1] || fallbackNamespace }; } function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable): Record { const deployment = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deployment", "hwlab-cloud-api", "-o", "json"], 60); if (deployment.exitCode !== 0) { return { ready: false, deploymentExists: false, mismatches: [{ name: "hwlab-cloud-api", reason: "deployment-missing" }], result: compactCodeAgentDeploymentProbeResult(deployment), valuesPrinted: false, }; } let parsed: unknown; try { parsed = JSON.parse(deployment.stdout) as unknown; } catch (error) { return { ready: false, deploymentExists: true, mismatches: [{ name: "hwlab-cloud-api", reason: "deployment-json-invalid", message: error instanceof Error ? error.message : String(error) }], result: compactCodeAgentDeploymentProbeResult(deployment), valuesPrinted: false, }; } const deploy = record(parsed); const specRecord = record(record(record(deploy.spec).template).spec); const containers = Array.isArray(specRecord.containers) ? specRecord.containers.map(record) : []; const container = containers.find((item) => item.name === "hwlab-cloud-api") ?? containers[0]; const env = Array.isArray(container?.env) ? container.env.map(record) : []; const envByName = new Map(env.map((item) => [String(item.name || ""), item])); const mismatches: Array> = []; const expectValue = (name: string, expected: string): void => { const item = envByName.get(name); const value = typeof item?.value === "string" ? item.value : null; if (value !== expected) mismatches.push({ name, expected, value, kind: "value" }); }; const expectSecret = (name: string, expectedSecret: string, expectedKey: string): void => { const item = envByName.get(name); const secretRef = record(record(item?.valueFrom).secretKeyRef); if (secretRef.name !== expectedSecret || secretRef.key !== expectedKey) { mismatches.push({ name, expectedSecret, expectedKey, secret: secretRef.name ?? null, key: secretRef.key ?? null, kind: "secretKeyRef" }); } }; expectValue("HWLAB_CODE_AGENT_ADAPTER", runtime.adapter); expectValue("AGENTRUN_MGR_URL", runtime.managerUrl); expectSecret("AGENTRUN_API_KEY", runtime.apiKeySecretName, runtime.apiKeySecretKey); expectValue("HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE", runtime.runnerNamespace); expectValue("HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE", runtime.secretNamespace); expectValue("HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", spec.gitReadUrl); expectValue("HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", spec.nodeId); expectValue("HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE", runtime.defaultProviderProfile); expectValue("HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", runtime.codexStdioSupervisor); const kafkaShadowProducer = runtime.kafkaShadowProducer; if (kafkaShadowProducer !== undefined) { expectValue("HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED", String(kafkaShadowProducer.enabled)); expectValue("HWLAB_KAFKA_SHADOW_CONSUME_ENABLED", String(kafkaShadowProducer.consumeEnabled)); expectValue("HWLAB_KAFKA_BOOTSTRAP_SERVERS", kafkaShadowProducer.bootstrapServers); expectValue("HWLAB_KAFKA_COMMAND_TOPIC", kafkaShadowProducer.commandTopic); expectValue("HWLAB_KAFKA_CLIENT_ID", kafkaShadowProducer.clientId); } return { ready: mismatches.length === 0, deploymentExists: true, deploymentName: "hwlab-cloud-api", containerName: container?.name ?? null, checkedEnvCount: kafkaShadowProducer === undefined ? 9 : 14, mismatches, result: compactCodeAgentDeploymentProbeResult(deployment), valuesPrinted: false, }; } function compactCodeAgentDeploymentProbeResult(result: CommandResult): Record { return { exitCode: result.exitCode, stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), stderrBytes: Buffer.byteLength(result.stderr, "utf8"), stderrTail: result.stderr.slice(-600), timedOut: result.timedOut === true, }; } export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return { required: false, ready: true }; if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; const runtimeAccess = pg.runtimeAccess ?? { endpointAddress: pg.endpointAddress, port: pg.port }; const endpointIsIpv4 = /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/u.test(runtimeAccess.endpointAddress); const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "jsonpath={.spec.type}{\"\\n\"}{.spec.externalName}{\"\\n\"}{.spec.ports[0].port}{\"\\n\"}"], 60); const [serviceType = "", externalName = "", servicePort = ""] = service.stdout.split(/\r?\n/u); if (!endpointIsIpv4) { const ready = service.exitCode === 0 && serviceType === "ExternalName" && externalName === runtimeAccess.endpointAddress && Number(servicePort) === runtimeAccess.port; return { required: true, ready, serviceName: pg.serviceName, serviceType: serviceType || null, externalName: externalName || null, expectedExternalName: runtimeAccess.endpointAddress, port: numericField(servicePort), expectedPort: runtimeAccess.port, providerEndpointAddress: pg.endpointAddress, providerPort: pg.port, runtimeAccess: pg.runtimeAccess ?? null, service: compactRuntimeCommand(service), }; } const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60); const [addressType = "", address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u); return { required: true, ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port, serviceName: pg.serviceName, serviceType: serviceType || null, addressType: addressType || null, endpointAddress: address || null, expectedEndpointAddress: runtimeAccess.endpointAddress, port: numericField(port), expectedPort: runtimeAccess.port, providerEndpointAddress: pg.endpointAddress, providerPort: pg.port, runtimeAccess: pg.runtimeAccess ?? null, service: compactRuntimeCommand(service), endpointSlice: compactRuntimeCommand(endpointSlice), }; } export function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return { required: false, ready: true }; if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey); const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey); const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key"); return { required: true, ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present, valuesPrinted: false, cloudApi, openfga: { datastoreUri: openfgaUri, authnPresharedKey: openfgaAuthn, }, }; } export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec { const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown); const nodes = record(parsed.nodes); const node = record(nodes[spec.nodeId]); const targets = Array.isArray(parsed.targets) ? parsed.targets : []; const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane); if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); const gitMirror = record(target.gitMirror); const gitMirrorEgressProxy = record(gitMirror.egressProxy); const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode"); if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "host-route" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global, host-route, or direct for node=${spec.nodeId} lane=${spec.lane}`); const nodeEgressProxy = gitMirrorEgressMode === "direct" ? { mode: "direct" as const, required: false as const } : nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`); if ((gitMirrorEgressMode === "node-global" || gitMirrorEgressMode === "host-route") && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`); if (gitMirrorEgressMode === "node-global" && nodeEgressProxy.mode !== "k8s-service-cluster-ip") throw new Error(`gitMirror.egressProxy.mode=node-global requires nodes.${spec.nodeId}.egressProxy.mode=k8s-service-cluster-ip`); if (gitMirrorEgressMode === "host-route" && nodeEgressProxy.mode !== "host-route") throw new Error(`gitMirror.egressProxy.mode=host-route requires nodes.${spec.nodeId}.egressProxy.mode=host-route`); const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport"); const source = record(target.source); const gitops = record(target.gitops); const tekton = record(target.tekton); const toolsImage = record(tekton.toolsImage); const toolsImagePullPolicy = optionalStringValue(toolsImage.imagePullPolicy, "tekton.toolsImage.imagePullPolicy") ?? "IfNotPresent"; if (toolsImagePullPolicy !== "Always" && toolsImagePullPolicy !== "IfNotPresent" && toolsImagePullPolicy !== "Never") throw new Error("tekton.toolsImage.imagePullPolicy must be Always, IfNotPresent, or Never"); return { id: stringValue(target.id, "target.id"), node: stringValue(target.node, "target.node"), lane: stringValue(target.lane, "target.lane"), namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"), serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"), serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"), cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"), cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"), cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"), servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"), secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"), syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"), syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"), flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"), toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"), toolsImagePullPolicy, sourceRepository: stringValue(source.repository, "source.repository"), sourceBranch: stringValue(source.branch, "source.branch"), gitopsBranch: stringValue(gitops.branch, "gitops.branch"), egressProxy: nodeEgressProxy, githubTransport, }; } export function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record, path: string): NodeRuntimeGitMirrorGithubTransportSpec { const mode = stringValue(raw.mode, `${path}.mode`); if (mode === "ssh") { const knownHostsSecretKey = optionalStringValue(raw.knownHostsSecretKey, `${path}.knownHostsSecretKey`); const knownHostsSourceRef = optionalStringValue(raw.knownHostsSourceRef, `${path}.knownHostsSourceRef`); const knownHostsSourceKey = optionalStringValue(raw.knownHostsSourceKey, `${path}.knownHostsSourceKey`); const knownHostsSourceEncoding = raw.knownHostsSourceEncoding === undefined ? null : gitMirrorSecretSourceEncoding(raw.knownHostsSourceEncoding, `${path}.knownHostsSourceEncoding`); const knownHostsValues = [knownHostsSecretKey, knownHostsSourceRef, knownHostsSourceKey, knownHostsSourceEncoding]; if (knownHostsValues.some((value) => value !== null) && knownHostsValues.some((value) => value === null)) { throw new Error(`${path}.knownHostsSecretKey/sourceRef/sourceKey/sourceEncoding must be declared together`); } return { mode, privateKeySecretKey: stringValue(raw.privateKeySecretKey, `${path}.privateKeySecretKey`), privateKeySourceRef: stringValue(raw.privateKeySourceRef, `${path}.privateKeySourceRef`), privateKeySourceKey: stringValue(raw.privateKeySourceKey, `${path}.privateKeySourceKey`), privateKeySourceEncoding: gitMirrorSecretSourceEncoding(raw.privateKeySourceEncoding, `${path}.privateKeySourceEncoding`), knownHostsSecretKey, knownHostsSourceRef, knownHostsSourceKey, knownHostsSourceEncoding, }; } if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`); return { mode, username: stringValue(raw.username, `${path}.username`), tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`), tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`), tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`), tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`), }; } function gitMirrorSecretSourceEncoding(raw: unknown, path: string): "plain" | "base64" { const value = stringValue(raw, path); if (value !== "plain" && value !== "base64") throw new Error(`${path} must be plain or base64`); return value; } export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record, path: string): NodeRuntimeGitMirrorEgressProxySpec { const mode = stringValue(raw.mode, `${path}.mode`); if (mode === "host-route") { const noProxyRaw = raw.noProxy; if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`); return { mode, clientName: stringValue(raw.clientName, `${path}.clientName`), hostProxyConfigRef: stringValue(raw.hostProxyConfigRef, `${path}.hostProxyConfigRef`), proxyEnvPath: stringValue(raw.proxyEnvPath, `${path}.proxyEnvPath`), proxyUrl: stringValue(raw.proxyUrl, `${path}.proxyUrl`), noProxy: noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)), }; } if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip or host-route`); const port = Number(raw.port); if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`); const sourceConfigRef = optionalStringValue(raw.sourceConfigRef, `${path}.sourceConfigRef`) ?? null; const source = sourceConfigRef === null ? null : resolveEgressProxySourceRef(sourceConfigRef, `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.${path}.sourceConfigRef`); const sourceType = source?.sourceType ?? stringValue(raw.sourceType, `${path}.sourceType`); if (sourceType !== "subscription-url" && sourceType !== "master-shadowsocks") throw new Error(`${path}.sourceType must be subscription-url or master-shadowsocks`); if (raw.sourceType !== undefined && raw.sourceType !== sourceType) throw new Error(`${path}.sourceType must match sourceConfigRef value ${sourceType}`); const sourceRef = source?.sourceRef ?? stringValue(raw.sourceRef, `${path}.sourceRef`); const sourceKey = source?.sourceKey ?? stringValue(raw.sourceKey, `${path}.sourceKey`); if (source !== null) { if (raw.sourceRef !== undefined && raw.sourceRef !== source.sourceRef) throw new Error(`${path}.sourceRef must match sourceConfigRef value ${source.sourceRef}`); if (raw.sourceKey !== undefined && raw.sourceKey !== source.sourceKey) throw new Error(`${path}.sourceKey must match sourceConfigRef value ${source.sourceKey}`); } const preferredOutbound = source?.preferredOutbound ?? (raw.preferredOutbound === undefined ? null : stringValue(raw.preferredOutbound, `${path}.preferredOutbound`)); if (preferredOutbound !== null && preferredOutbound !== "vless-reality" && preferredOutbound !== "hysteria2") throw new Error(`${path}.preferredOutbound must be vless-reality or hysteria2`); if (source !== null && source.preferredOutbound !== null && raw.preferredOutbound !== undefined && raw.preferredOutbound !== source.preferredOutbound) { throw new Error(`${path}.preferredOutbound must match sourceConfigRef value ${source.preferredOutbound}`); } const noProxyRaw = raw.noProxy; if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`); const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)); return { mode, clientName: stringValue(raw.clientName, `${path}.clientName`), namespace: stringValue(raw.namespace, `${path}.namespace`), serviceName: stringValue(raw.serviceName, `${path}.serviceName`), port, sourceConfigRef, sourceRef, sourceKey, sourceType, preferredOutbound, sourceFingerprint: source?.fingerprint ?? null, noProxy, }; } export function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record { const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60); return { secret, key, present: result.exitCode === 0 && result.stdout.trim().length > 0, valueBytesBase64: result.stdout.trim().length, exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500), valuesPrinted: false, }; } export function startNodeDelegatedJob(options: ReturnType): Record { const commandArgs = [ "hwlab", "nodes", options.domain, ...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]), "--node", options.node, "--lane", options.lane, "--confirm", "--timeout-seconds", String(options.timeoutSeconds), "--wait", ]; const command = ["bun", "scripts/cli.ts", ...commandArgs]; const job = startJob( `hwlab_nodes_${options.lane}_${options.domain}_${options.action}`, command, `Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`, ); return { ok: true, command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`, node: options.node, lane: options.lane, mode: "async-job", reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking", job, statusCommand: `bun scripts/cli.ts job status ${job.id}`, waitCommand: command.join(" "), }; } export function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType): Record { const rewritten = rewriteDelegatedNodeValue(value, scoped); const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record : { value: rewritten }; return { ...result, node: scoped.node, commandFamily: "hwlab nodes", }; } export function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType): unknown { if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped); if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped)); if (typeof value !== "object" || value === null) return value; return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)])); } export function rewriteDelegatedNodeString(value: string, scoped: ReturnType): string { const replaceCommand = (text: string, domain: DelegatedNodeDomain) => { const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); return text .replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`) .replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`); }; return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror"); } export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions { const [actionRaw] = args; if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "screenshot" && actionRaw !== "observe" && actionRaw !== "sentinel" && actionRaw !== "opencode-smoke") throw new Error("web-probe usage: run|script|screenshot|observe|sentinel|opencode-smoke --node NODE --lane vNN [--url URL]"); if (actionRaw === "sentinel") return parseNodeWebProbeSentinelOptions(args.slice(1)); if (actionRaw === "observe") { const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1)); const explicitNode = optionValue(args, "--node"); const explicitLane = optionValue(args, "--lane"); const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null; const indexed = requestedId === null ? null : readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null); const node = explicitNode ?? indexed?.node; const lane = explicitLane ?? indexed?.lane; if (node === undefined || lane === undefined) { if (requestedId !== null) { throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`); } throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id"); } assertNodeId(node); assertLane(lane); if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`); const spec = hwlabRuntimeLaneSpecForNode(lane, node); return parseNodeWebProbeObserveOptions(normalized.args, node, lane, spec, normalized.id, indexed); } const node = requiredOption(args, "--node"); assertNodeId(node); const lane = requiredOption(args, "--lane"); assertLane(lane); if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`); const spec = hwlabRuntimeLaneSpecForNode(lane, node); if (actionRaw === "screenshot") { assertKnownOptions(args.slice(1), new Set([ "--node", "--lane", "--url", "--path", "--viewport", "--local-dir", "--name", "--timeout-ms", "--wait-until", "--selector", "--wait-timeout-ms", "--command-timeout-seconds", ]), new Set([ "--full-page", "--no-full-page", "--keep-remote", ])); const url = optionValue(args, "--url"); const targetPath = optionValue(args, "--path") ?? null; if (url !== undefined && targetPath !== null) throw new Error("web-probe screenshot accepts --url or --path, not both"); const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000); const waitTimeoutMs = positiveIntegerOption(args, "--wait-timeout-ms", Math.max(90000, timeoutMs + 30000), 600000); const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.ceil(waitTimeoutMs / 1000) + 45, 900); return { action: "screenshot", node, lane, url: url ?? resolveWebProbeScreenshotUrl(spec, targetPath ?? "/"), path: targetPath, viewport: parseWebProbeScreenshotViewport(optionValue(args, "--viewport") ?? "1440x900"), localDir: optionValue(args, "--local-dir") ?? "/tmp", name: parseWebProbeScreenshotName(optionValue(args, "--name") ?? `web-probe-${node.toLowerCase()}-${lane}.png`), timeoutMs, waitUntil: parseWebProbeScreenshotWaitUntil(optionValue(args, "--wait-until") ?? "networkidle"), fullPage: !args.includes("--no-full-page"), selector: optionValue(args, "--selector") ?? null, keepRemote: args.includes("--keep-remote"), waitTimeoutMs, commandTimeoutSeconds, }; } if (actionRaw === "script") { assertKnownOptions(args.slice(1), new Set([ "--node", "--lane", "--url", "--timeout-ms", "--viewport", "--browser-proxy-mode", "--script-file", "--command-timeout-seconds", ]), new Set([])); const scriptFile = optionValue(args, "--script-file"); if (scriptFile === undefined && process.stdin.isTTY) { throw new Error("web-probe script requires a stdin heredoc or --script-file "); } const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8"); if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script"); const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000); const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600); return { action: "script", node, lane, url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec), timeoutMs, viewport: optionValue(args, "--viewport") ?? "1440x900", browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode), commandTimeoutSeconds, scriptText, scriptSource: { kind: scriptFile === undefined ? "stdin" : "file", path: scriptFile ?? null, byteCount: Buffer.byteLength(scriptText), sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`, }, }; } if (actionRaw === "opencode-smoke") { assertKnownOptions(args.slice(1), new Set([ "--node", "--lane", "--url", "--path", "--message", "--expect-text", "--timeout-ms", "--response-timeout-ms", "--viewport", "--browser-proxy-mode", "--command-timeout-seconds", ]), new Set([])); const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 40000, 300000); const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600); const requestedResponseTimeoutMs = positiveIntegerOption(args, "--response-timeout-ms", Math.min(30000, timeoutMs), timeoutMs); const responseTimeoutMs = Math.min(requestedResponseTimeoutMs, Math.max(5000, (commandTimeoutSeconds * 1000) - 20000)); const path = optionValue(args, "--path") ?? "/opencode"; if (!path.startsWith("/") || path.includes("\0") || path.length > 200) throw new Error("web-probe opencode-smoke --path must be an absolute path up to 200 chars"); const message = optionValue(args, "--message") ?? "hi"; if (message.includes("\0") || message.length > 500) throw new Error("web-probe opencode-smoke --message must be 0-500 non-NUL chars"); const expectText = optionValue(args, "--expect-text") ?? null; if (expectText !== null && (expectText.includes("\0") || expectText.length > 500)) throw new Error("web-probe opencode-smoke --expect-text must be 0-500 non-NUL chars"); const scriptText = nodeWebProbeOpencodeSmokeScript({ path, message, expectText, responseTimeoutMs, }); const commandLabel = `web-probe opencode-smoke --node ${node} --lane ${lane}`; return { action: "script", node, lane, url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec), timeoutMs, viewport: optionValue(args, "--viewport") ?? "1440x900", browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode), commandTimeoutSeconds, scriptText, commandLabel, suppressAdHocWarning: true, generatedHints: [ "OpenCode smoke is a repo-owned typed web-probe command; it uses the same managed auth, remote browser, report recovery and artifact contract as web-probe script.", "Passing means the browser DOM reached assistant text and the OpenCode EventSource produced update, finish and idle evidence.", ], generatedPreferredCommands: { withExpectedText: `${commandLabel} --expect-text `, }, scriptSource: { kind: "generated", path: "builtin:opencode-smoke", byteCount: Buffer.byteLength(scriptText), sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`, }, }; } assertKnownOptions(args.slice(1), new Set([ "--node", "--lane", "--url", "--timeout-ms", "--wait-after-submit-ms", "--wait-messages-ms", "--wait-agent-terminal-ms", "--trace-sample-count", "--trace-sample-interval-ms", "--message", "--conversation-id", "--command-timeout-seconds", ]), new Set([ "--fresh-session", "--no-cancel-running", ])); const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000); const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000); const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000); const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000); const message = optionValue(args, "--message") ?? null; const hasMessage = message !== null; const traceSampleCountDefault = hasMessage ? 2 : 0; const traceSampleIntervalDefault = hasMessage ? 1500 : 0; const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200); const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000); const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({ timeoutMs, waitAfterSubmitMs, waitMessagesMs, waitAgentTerminalMs, traceSampleCount, traceSampleIntervalMs, freshSession: args.includes("--fresh-session"), hasMessage, }); const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds"); const commandTimeoutUserProvided = commandTimeoutRaw !== undefined; const commandTimeoutSeconds = commandTimeoutUserProvided ? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds) : commandTimeoutAutoSeconds; return { action: "run", node, lane, url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec), timeoutMs, waitAfterSubmitMs, waitMessagesMs, waitAgentTerminalMs, traceSampleCount, traceSampleIntervalMs, message, conversationId: optionValue(args, "--conversation-id") ?? null, freshSession: args.includes("--fresh-session"), cancelRunning: !args.includes("--no-cancel-running"), commandTimeoutSeconds, commandTimeoutAutoSeconds, commandTimeoutUserProvided, }; } function nodeWebProbeOpencodeSmokeScript(input: { path: string; message: string; expectText: string | null; responseTimeoutMs: number }): string { const config = { path: input.path, message: input.message, expectText: input.expectText, responseTimeoutMs: input.responseTimeoutMs, }; return patchOpenCodeSmokeScript(`const config = ${JSON.stringify(config)};\n\nexport default async function opencodeSmoke({ page, baseUrl, recordStep, wait, screenshot, jsonArtifact }) {\n await page.addInitScript(() => {\n const key = \"__unideskOpenCodeEvents\";\n window[key] = Array.isArray(window[key]) ? window[key] : [];\n if (window.__unideskOpenCodeEventSourceWrapped) return;\n window.__unideskOpenCodeEventSourceWrapped = true;\n const OriginalEventSource = window.EventSource;\n if (!OriginalEventSource) return;\n const push = (source, type, event) => {\n let parsedType = null;\n let parsedStatus = null;\n let dataBytes = 0;\n const rawData = typeof event?.data === \"string\" ? event.data : \"\";\n dataBytes = rawData.length;\n if (rawData) {\n try {\n const parsed = JSON.parse(rawData);\n parsedType = typeof parsed?.type === \"string\" ? parsed.type : typeof parsed?.event === \"string\" ? parsed.event : typeof parsed?.name === \"string\" ? parsed.name : null;\n parsedStatus = typeof parsed?.status === \"string\" ? parsed.status : typeof parsed?.part?.status === \"string\" ? parsed.part.status : null;\n } catch {\n parsedType = null;\n }\n }\n const label = parsedType || (parsedStatus ? type + \":\" + parsedStatus : type);\n window[key].push({ at: Date.now(), source, type, parsedType, status: parsedStatus, label, dataBytes });\n if (window[key].length > 240) window[key].splice(0, window[key].length - 240);\n };\n const originalAddEventListener = OriginalEventSource.prototype.addEventListener;\n OriginalEventSource.prototype.addEventListener = function wrappedAddEventListener(type, listener, options) {\n if (typeof listener !== \"function\") return originalAddEventListener.call(this, type, listener, options);\n const wrapped = function wrappedOpenCodeEvent(event) {\n push(String(this?.url || \"eventsource\"), String(type || \"message\"), event);\n return listener.call(this, event);\n };\n return originalAddEventListener.call(this, type, wrapped, options);\n };\n function WrappedEventSource(url, options) {\n const instance = new OriginalEventSource(url, options);\n originalAddEventListener.call(instance, \"message\", (event) => push(String(url || \"eventsource\"), \"message\", event));\n return instance;\n }\n WrappedEventSource.prototype = OriginalEventSource.prototype;\n WrappedEventSource.CONNECTING = OriginalEventSource.CONNECTING;\n WrappedEventSource.OPEN = OriginalEventSource.OPEN;\n WrappedEventSource.CLOSED = OriginalEventSource.CLOSED;\n window.EventSource = WrappedEventSource;\n });\n\n const targetUrl = new URL(config.path, baseUrl).toString();\n await page.goto(targetUrl, { waitUntil: \"domcontentloaded\", timeout: Math.min(Math.max(config.responseTimeoutMs, 1000), 120000) });\n recordStep(\"opencode-navigate\", { ok: true, targetPath: config.path, finalUrl: page.url() });\n\n const frame = await findOpenCodeFrame(page, config.responseTimeoutMs);\n const before = await collectDomState(frame, [], config);\n const beforeLines = before.lines;\n recordStep(\"opencode-frame\", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit });\n\n const prompt = await fillOpenCodePrompt(frame, config.message);\n recordStep(\"opencode-prompt-filled\", { ok: true, selector: prompt.selector, textBytes: config.message.length });\n\n const submit = frame.locator(\"[data-action='prompt-submit']\").first();\n await submit.waitFor({ state: \"visible\", timeout: Math.min(config.responseTimeoutMs, 10000) });\n await frame.waitForFunction(() => {\n const button = document.querySelector(\"[data-action='prompt-submit']\");\n return Boolean(button && !button.disabled && button.getAttribute(\"aria-disabled\") !== \"true\");\n }, null, { timeout: 5000 }).catch(() => null);\n await submit.click({ timeout: 10000 });\n recordStep(\"opencode-submit\", { ok: true, selector: \"[data-action='prompt-submit']\" });\n\n const final = await waitForOpenCodeFinal(page, frame, beforeLines, config);\n await jsonArtifact(\"opencode-smoke-events.json\", final.events);\n const finalScreenshot = await screenshot(\"opencode-smoke-final.png\").catch((error) => ({ error: error instanceof Error ? error.message : String(error) }));\n const ok = final.conditions.assistantText && final.conditions.notThinking && final.conditions.updateEvent && final.conditions.finishEvent && final.conditions.idleEvent && final.conditions.expectedText;\n recordStep(\"opencode-final\", { ok, conditions: final.conditions, assistantTextPreview: final.assistantTextPreview, eventTypes: final.events.eventTypes });\n return {\n ok,\n status: ok ? \"pass\" : \"blocked\",\n failedCondition: ok ? null : failedCondition(final.conditions),\n frameUrl: final.url,\n finalUrl: page.url(),\n assistantTextPreview: final.assistantTextPreview,\n bodyPreview: final.bodyPreview,\n eventTypes: final.events.eventTypes,\n eventCount: final.events.eventCount,\n conditions: final.conditions,\n screenshot: finalScreenshot,\n issueEvidence: {\n frameUrl: final.url,\n assistantTextPreview: final.assistantTextPreview,\n eventTypes: final.events.eventTypes,\n conditions: final.conditions,\n screenshotSha256: typeof finalScreenshot?.sha256 === \"string\" ? finalScreenshot.sha256 : null,\n valuesRedacted: true,\n },\n valuesRedacted: true,\n };\n}\n\nasync function findOpenCodeFrame(page, timeoutMs) {\n const deadline = Date.now() + timeoutMs;\n let best = null;\n while (Date.now() < deadline) {\n for (const frame of page.frames()) {\n const summary = await frameSummary(frame);\n const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0);\n if (best === null || score > best.score) best = { frame, summary, score };\n if (summary.hasSubmit && summary.textboxCount > 0) return frame;\n }\n await waitFor(500);\n }\n throw new Error(\"OpenCode frame with prompt submit button was not found; best=\" + JSON.stringify(best?.summary || null));\n}\n\nasync function frameSummary(frame) {\n return await frame.evaluate(() => ({\n url: location.href,\n title: document.title,\n hasSubmit: Boolean(document.querySelector(\"[data-action='prompt-submit']\")),\n textboxCount: document.querySelectorAll(\"[contenteditable='true'], textarea, [role='textbox'], input:not([type]), input[type='text']\").length,\n bodyPreview: (document.body?.innerText || \"\").replace(/\\s+/gu, \" \").trim().slice(0, 240),\n })).catch((error) => ({ url: frame.url(), title: \"\", hasSubmit: false, textboxCount: 0, bodyPreview: String(error).slice(0, 160) }));\n}\n\nasync function fillOpenCodePrompt(frame, message) {\n const selectors = [\n \"[contenteditable='true']\",\n \"textarea\",\n \"[role='textbox']\",\n \"input:not([type])\",\n \"input[type='text']\",\n ];\n let lastError = null;\n for (const selector of selectors) {\n const locator = frame.locator(selector).last();\n const count = await locator.count().catch(() => 0);\n if (count < 1) continue;\n try {\n await locator.click({ timeout: 5000 });\n await locator.fill(message, { timeout: 5000 });\n return { selector };\n } catch (error) {\n lastError = error;\n try {\n await locator.evaluate((element, text) => {\n if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {\n element.value = text;\n element.dispatchEvent(new InputEvent(\"input\", { bubbles: true, data: text, inputType: \"insertText\" }));\n element.dispatchEvent(new Event(\"change\", { bubbles: true }));\n return;\n }\n element.textContent = text;\n element.dispatchEvent(new InputEvent(\"input\", { bubbles: true, data: text, inputType: \"insertText\" }));\n }, message);\n return { selector, fallback: \"dom-input-event\" };\n } catch (fallbackError) {\n lastError = fallbackError;\n }\n }\n }\n throw new Error(\"OpenCode prompt input was not fillable: \" + (lastError instanceof Error ? lastError.message : String(lastError || \"not found\")));\n}\n\nasync function waitForOpenCodeFinal(page, frame, beforeLines, config) {\n const deadline = Date.now() + config.responseTimeoutMs;\n let latest = await collectState(page, frame, beforeLines, config);\n while (Date.now() < deadline) {\n latest = await collectState(page, frame, beforeLines, config);\n if (latest.conditions.assistantText && latest.conditions.notThinking && latest.conditions.updateEvent && latest.conditions.finishEvent && latest.conditions.idleEvent && latest.conditions.expectedText) return latest;\n await waitFor(750);\n }\n return latest;\n}\n\nasync function collectState(page, frame, beforeLines, config) {\n const dom = await collectDomState(frame, beforeLines, config);\n const events = await collectEventSummary(page);\n const labels = events.eventTypes.join(\"\\n\");\n const conditions = {\n assistantText: Boolean(dom.assistantTextPreview),\n notThinking: dom.thinkingVisible !== true,\n updateEvent: /message[._-]?part[._-]?updated|part[._-]?updated|delta|updated/iu.test(labels),\n finishEvent: /step[._-]?finish|finish|completed|done/iu.test(labels),\n idleEvent: /session[._-]?idle|session[._-]?status[:/._-]?idle|\\bidle\\b/iu.test(labels),\n expectedText: config.expectText === null || dom.expectedTextPresent === true,\n };\n return { ...dom, events, conditions };\n}\n\nasync function collectDomState(frame, beforeLines, config) {\n return await frame.evaluate(({ beforeLines, message, expectText }) => {\n const normalize = (value) => String(value || \"\").replace(/\\r/g, \"\").replace(/[ \\t]+/g, \" \").trim();\n const bodyText = normalize(document.body?.innerText || \"\");\n const lines = bodyText.split(\"\\n\").map((line) => normalize(line)).filter(Boolean).slice(-400);\n const beforeSet = new Set(Array.isArray(beforeLines) ? beforeLines : []);\n const prompt = normalize(message).toLowerCase();\n const blocked = /^(send|stop|cancel|thinking|loading|new session|settings)$/iu;\n const selectorTexts = Array.from(document.querySelectorAll(\"[data-role='assistant'], [data-message-role='assistant'], [data-testid*='assistant' i], [aria-label*='assistant' i], [class*='assistant' i], main article, main [role='article'], main [data-message-id]\")).map((element) => normalize(element.textContent || \"\")).filter((text) => text.length > 0 && text.toLowerCase() !== prompt && !blocked.test(text)).slice(-20);\n const newLines = lines.filter((line) => !beforeSet.has(line) && line.toLowerCase() !== prompt && !blocked.test(line) && !/^you\\b/iu.test(line)).slice(-40);\n const assistantText = selectorTexts.find((text) => text.length > 0) || newLines.find((line) => line.length > 0) || null;\n return {\n url: location.href,\n title: document.title,\n hasSubmit: Boolean(document.querySelector(\"[data-action='prompt-submit']\")),\n lines,\n lineCount: lines.length,\n newLineCount: newLines.length,\n assistantCandidateCount: selectorTexts.length,\n assistantTextPreview: assistantText === null ? null : assistantText.slice(0, 500),\n thinkingVisible: /(^|\\n|\\b)Thinking(\\b|\\n)/u.test(bodyText),\n expectedTextPresent: expectText === null ? true : bodyText.includes(expectText),\n bodyPreview: bodyText.slice(-1200),\n };\n }, { beforeLines, message: config.message, expectText: config.expectText });\n}\n\nasync function collectEventSummary(page) {\n const frameEvents = [];\n for (const frame of page.frames()) {\n const events = await frame.evaluate(() => Array.isArray(window.__unideskOpenCodeEvents) ? window.__unideskOpenCodeEvents.slice(-120) : []).catch(() => []);\n if (events.length > 0) frameEvents.push({ frameUrl: frame.url(), events });\n }\n const labels = [];\n for (const item of frameEvents) {\n for (const event of item.events) {\n const raw = [event.label, event.parsedType, event.status, event.type].filter(Boolean).join(\":\");\n if (raw && !labels.includes(raw)) labels.push(raw);\n }\n }\n return {\n eventCount: frameEvents.reduce((sum, item) => sum + item.events.length, 0),\n eventTypes: labels.slice(0, 40),\n frames: frameEvents.map((item) => ({ frameUrl: item.frameUrl, count: item.events.length, last: item.events.slice(-5) })),\n valuesRedacted: true,\n };\n}\n\nfunction failedCondition(conditions) {\n return Object.entries(conditions).filter(([, ok]) => ok !== true).map(([key]) => key).join(\",\") || \"unknown\";\n}\n\nfunction waitFor(ms) {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n`); } function patchOpenCodeSmokeScript(source: string): string { const oldFlow = ` const frame = await findOpenCodeFrame(page, config.responseTimeoutMs); const before = await collectDomState(frame, [], config); const beforeLines = before.lines; recordStep("opencode-frame", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit }); const prompt = await fillOpenCodePrompt(frame, config.message);`; const newFlow = ` const frame = await findOpenCodeFrame(page, config.responseTimeoutMs); const project = await ensureOpenCodeProjectOpen(frame, config.responseTimeoutMs); recordStep("opencode-project", project); const composer = await waitForOpenCodeComposer(frame, config.responseTimeoutMs); recordStep("opencode-composer", composer); const eventProbe = await startOpenCodeEventProbe(frame); recordStep("opencode-event-probe", eventProbe); const before = await collectDomState(frame, [], config); const beforeLines = before.lines; recordStep("opencode-frame", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit }); const prompt = await fillOpenCodePrompt(frame, config.message);`; const oldFrameFinder = `async function findOpenCodeFrame(page, timeoutMs) { const deadline = Date.now() + timeoutMs; let best = null; while (Date.now() < deadline) { for (const frame of page.frames()) { const summary = await frameSummary(frame); const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0); if (best === null || score > best.score) best = { frame, summary, score }; if (summary.hasSubmit && summary.textboxCount > 0) return frame; } await waitFor(500); } throw new Error("OpenCode frame with prompt submit button was not found; best=" + JSON.stringify(best?.summary || null)); }`; const newFrameFinder = `async function findOpenCodeFrame(page, timeoutMs) { const deadline = Date.now() + timeoutMs; let best = null; while (Date.now() < deadline) { for (const frame of page.frames()) { const summary = await frameSummary(frame); const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0) + (isOpenCodeFrame(frame, summary) ? 50 : 0); if (best === null || score > best.score) best = { frame, summary, score }; if (summary.hasSubmit && summary.textboxCount > 0) return frame; if (isOpenCodeFrame(frame, summary)) return frame; } await waitFor(500); } throw new Error("OpenCode frame was not found; best=" + JSON.stringify(best?.summary || null)); } function isOpenCodeFrame(frame, summary) { const rawUrl = String(summary?.url || frame.url() || ""); let hostname = ""; try { hostname = new URL(rawUrl).hostname; } catch { hostname = ""; } const hasParent = typeof frame.parentFrame === "function" && frame.parentFrame() !== null; if (hasParent && (/opencode/iu.test(rawUrl) || String(summary?.title || "") === "OpenCode")) return true; return /^opencode[-.]/iu.test(hostname); } async function ensureOpenCodeProjectOpen(frame, timeoutMs) { const deadline = Date.now() + Math.min(timeoutMs, 20000); let clickedProject = null; let dismissed = []; let lastSummary = null; while (Date.now() < deadline) { lastSummary = await frameSummary(frame); if (lastSummary.hasSubmit && lastSummary.textboxCount > 0) { return { ok: true, alreadyOpen: true, frameUrl: lastSummary.url, dismissed, clickedProject }; } const dismissedButton = await clickButtonMatching(frame, (text) => /^(not yet|skip|maybe later)$/iu.test(text), "dismiss-provider"); if (dismissedButton) { dismissed.push(dismissedButton.text); await waitFor(500); continue; } if (/No projects open|Recent projects|Open a project/iu.test(String(lastSummary.bodyPreview || ""))) { const clicked = await clickButtonMatching(frame, (text) => text === "/" || /^\\/\\s*\\d/iu.test(text) || /^\\/.*ago$/iu.test(text), "recent-project"); if (clicked) { clickedProject = clicked.text; await waitFor(1500); return { ok: true, alreadyOpen: false, frameUrl: lastSummary.url, dismissed, clickedProject }; } } await waitFor(500); } return { ok: false, alreadyOpen: false, frameUrl: lastSummary?.url ?? frame.url(), dismissed, clickedProject, bodyPreview: lastSummary?.bodyPreview ?? null }; } async function waitForOpenCodeComposer(frame, timeoutMs) { const deadline = Date.now() + timeoutMs; let lastSummary = null; while (Date.now() < deadline) { lastSummary = await frameSummary(frame); if (lastSummary.hasSubmit && lastSummary.textboxCount > 0) { return { ok: true, frameUrl: lastSummary.url, textboxCount: lastSummary.textboxCount, hasSubmit: true }; } await waitFor(750); } throw new Error("OpenCode composer was not ready; last=" + JSON.stringify(lastSummary)); } async function startOpenCodeEventProbe(frame) { return await frame.evaluate(() => { const key = "__unideskOpenCodeEvents"; window[key] = Array.isArray(window[key]) ? window[key] : []; if (window.__unideskManualOpenCodeEventSourceStarted) { return { ok: true, reused: true, eventCount: window[key].length }; } window.__unideskManualOpenCodeEventSourceStarted = true; const eventUrl = new URL("/global/event", location.origin).toString(); const source = new EventSource(eventUrl); window.__unideskManualOpenCodeEventSource = source; const push = (type, event) => { const rawData = typeof event?.data === "string" ? event.data : ""; let parsedType = null; let status = null; if (rawData) { try { const parsed = JSON.parse(rawData); parsedType = typeof parsed?.type === "string" ? parsed.type : typeof parsed?.event === "string" ? parsed.event : typeof parsed?.name === "string" ? parsed.name : null; status = typeof parsed?.status === "string" ? parsed.status : typeof parsed?.part?.status === "string" ? parsed.part.status : null; } catch { parsedType = null; } } const label = parsedType || (status ? String(type) + ":" + status : String(type)); window[key].push({ at: Date.now(), source: eventUrl, type: String(type), parsedType, status, label, dataBytes: rawData.length, manualProbe: true }); if (window[key].length > 240) window[key].splice(0, window[key].length - 240); }; const eventTypes = ["message", "message.part.updated", "message.part.added", "message.updated", "step-finish", "step.finish", "session.idle", "session.status", "session.updated", "session.error", "error"]; for (const type of eventTypes) source.addEventListener(type, (event) => push(type, event)); source.addEventListener("open", () => push("open", { data: "" })); return { ok: true, reused: false, eventUrl, listenedTypes: eventTypes }; }); } async function clickButtonMatching(frame, predicate, label) { const buttons = frame.locator("button,[role='button']"); const count = Math.min(await buttons.count().catch(() => 0), 80); for (let index = 0; index < count; index += 1) { const button = buttons.nth(index); const visible = await button.isVisible().catch(() => false); if (!visible) continue; const text = normalizeButtonText(await button.textContent().catch(() => "")); if (!predicate(text)) continue; await button.click({ timeout: 5000 }); return { ok: true, label, index, text }; } return null; } function normalizeButtonText(value) { return String(value || "").replace(/\\s+/gu, " ").trim(); }`; const oldBlocked = "const blocked = /^(send|stop|cancel|thinking|loading|new session|settings)$/iu;"; const newBlocked = "const blocked = /^(send|stop|cancel|thinking|loading|new session|settings|build|shell|review|last turn changes|create git repository|all files|main branch|deepseek-v4-flash)$/iu;"; const withFlow = source.replace(oldFlow, newFlow); const withFrameFinder = withFlow.replace(oldFrameFinder, newFrameFinder); const patched = withFrameFinder.replace(oldBlocked, newBlocked); if (withFlow === source) throw new Error("internal opencode-smoke generated script patch did not apply: flow"); if (withFrameFinder === withFlow) throw new Error("internal opencode-smoke generated script patch did not apply: frame finder"); if (patched === withFrameFinder) throw new Error("internal opencode-smoke generated script patch did not apply: assistant text filter"); if (!patched.includes("ensureOpenCodeProjectOpen") || !patched.includes("startOpenCodeEventProbe")) throw new Error("internal opencode-smoke generated script patch is missing required helpers"); return patched; } function resolveWebProbeScreenshotUrl(spec: HwlabRuntimeLaneSpec, targetPath: string): string { const origin = nodeWebProbeDefaultUrl(spec); try { return new URL(targetPath || "/", origin).toString(); } catch { throw new Error(`web-probe screenshot --path cannot be resolved against ${origin}: ${targetPath}`); } } function parseWebProbeScreenshotViewport(value: string): string { if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(value)) throw new Error(`web-probe screenshot --viewport must look like 1440x900, got ${value}`); const [widthRaw, heightRaw] = value.split("x"); const width = Number(widthRaw); const height = Number(heightRaw); if (width < 240 || width > 7680 || height < 240 || height > 4320) throw new Error(`web-probe screenshot --viewport out of range: ${value}`); return value; } function parseWebProbeScreenshotName(value: string): string { if (!/^[A-Za-z0-9._-]{1,120}$/u.test(value)) throw new Error("web-probe screenshot --name must contain only letters, numbers, dot, underscore, or dash, max 120 chars"); return value.endsWith(".png") ? value : `${value}.png`; } function parseWebProbeScreenshotWaitUntil(value: string): "load" | "domcontentloaded" | "networkidle" | "commit" { if (value === "load" || value === "domcontentloaded" || value === "networkidle" || value === "commit") return value; throw new Error(`web-probe screenshot --wait-until must be load, domcontentloaded, networkidle, or commit; got ${value}`); }