// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-probe module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:6032-7426 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { BootstrapAdminPasswordMaterial, BootstrapAdminSecretMaterial, NodeRuntimeGitMirrorEgressProxySpec, NodeRuntimeGitMirrorGithubTransportSpec, NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult, NodeWebProbeOptions, RuntimeSecretSpec } from "./entry"; import { isCommandSuccess, nodeRuntimeGitopsRoot, runNodeK3sArgs, runNodeK3sScript } from "./cleanup"; import { nodeRuntimeCicdWaitWarning } from "./git-mirror"; import { parseNodeScopedDelegatedOptions } from "./plan"; import { nodeRuntimePipelineFailureSummary, nodeRuntimePipelineRunDiagnostics } from "./render"; import { compactRuntimeCommand, runNodeHostScript } from "./runtime-common"; import { assertLane, assertNodeId, keyValueLinesFromText, numericField, optionValue, optionalStringValue, positiveIntegerOption, positiveIntegerValue, record, requiredOption, shellQuote, statusText, stringValue, stripOptions } from "./utils"; import { discoverWebObserveIndexEntry, readWebObserveIndexEntry } from "./web-observe-render"; import { assertKnownOptions, nodeWebProbeAutoCommandTimeoutSeconds, nodeWebProbeDefaultUrl, normalizeNodeWebProbeObserveArgs, parseNodeWebProbeObserveOptions, parseNodeWebProbeSentinelOptions, parseWebProbeBrowserProxyMode } from "./web-probe-observe"; import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes"; import { resolveEgressProxySourceRef } from "../egress-proxy-sources"; export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record { const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http); const gitMirror = nodeRuntimeGitMirrorTarget(spec); const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec); const renderGitMirror = { ...gitMirror, egressProxy: gitMirror.egressProxy.mode === "direct" ? { mode: "direct", required: false, } : { ...gitMirror.egressProxy, mode: "node-global", required: true, }, }; return { nodeId: spec.nodeId, lane: spec.lane, sourceBranch: spec.sourceBranch, gitopsBranch: spec.gitopsBranch, gitopsRoot: nodeRuntimeGitopsRoot(spec), runtimePath: spec.runtimePath, runtimeRenderDir: spec.runtimeRenderDir, runtimeNamespace: spec.runtimeNamespace, catalogPath: spec.catalogPath, tektonDir: spec.tektonDir, argoApplicationFile: spec.argoApplicationFile, argoRepoUrl: spec.argoRepoUrl, gitUrl: spec.gitUrl, gitReadUrl: spec.gitReadUrl, gitWriteUrl: spec.gitWriteUrl, toolsImage: gitMirror.toolsImage, toolsImagePullPolicy: gitMirror.toolsImagePullPolicy, gitMirror: renderGitMirror, networkProfileId: spec.networkProfileId, downloadProfileId: spec.downloadProfileId, gitSshProxyHost: gitSshProxy?.host, gitSshProxyPort: gitSshProxy?.port, proxyHttp: spec.networkProfile.proxy.http, proxyHttps: spec.networkProfile.proxy.https, proxyAll: spec.networkProfile.proxy.all, noProxy: spec.networkProfile.proxy.noProxy.join(","), dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http, dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https, dockerProxyAll: spec.networkProfile.dockerBuildProxy.all, dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","), dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy, npmRegistry: spec.downloadProfile.npm.registry, npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000, npmRetries: spec.downloadProfile.npm.retries, stepEnv: spec.stepEnv, observability: spec.observability, runtimeStore: spec.runtimeStore, runtimeImageRewrites: spec.runtimeImageRewrites, registryPrefix: spec.registryPrefix, buildkitSidecarImage: spec.buildkit?.sidecarImage, publicWebUrl: spec.publicWebUrl, publicApiUrl: spec.publicApiUrl, publicExposure: spec.publicExposure === null ? undefined : { enabled: spec.publicExposure.enabled, mode: spec.publicExposure.mode, publicBaseUrl: spec.publicExposure.publicBaseUrl, hostname: spec.publicExposure.hostname, expectedA: spec.publicExposure.expectedA, serverAddr: spec.publicExposure.serverAddr, serverPort: spec.publicExposure.serverPort, secretName: spec.publicExposure.secretName, secretKey: spec.publicExposure.secretKey, tokenKey: spec.publicExposure.tokenKey, webProxy: spec.publicExposure.webProxy, apiProxy: spec.publicExposure.apiProxy, }, externalPostgres: activeExternalPostgres === undefined ? undefined : { enabled: true, serviceName: activeExternalPostgres.serviceName, endpointAddress: activeExternalPostgres.endpointAddress, port: activeExternalPostgres.port, runtimeAccess: activeExternalPostgres.runtimeAccess ?? null, sslmode: activeExternalPostgres.sslmode, }, }; } export function httpProxyEndpoint(value: string): { host: string; port: number } | null { let parsed: URL; try { parsed = new URL(value); } catch { return null; } if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10); if (!Number.isInteger(port) || port <= 0) return null; return { host: parsed.hostname, port }; } export function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] { return [ `${renderDir}/devops-infra/git-mirror.yaml`, `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`, `${renderDir}/${spec.tektonDir}/rbac.yaml`, `${renderDir}/${spec.tektonDir}/pipeline.yaml`, `${renderDir}/argocd/project.yaml`, `${renderDir}/argocd/${spec.argoApplicationFile}`, ]; } export function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { const files = nodeRuntimeControlPlaneFiles(spec, renderDir); const args = [ "kubectl", "apply", ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), ...files.flatMap((file) => ["-f", file]), ]; return runNodeK3sArgs(spec, args, timeoutSeconds); } export function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir) .map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`) .join(""); const args = [ "kubectl", "apply", ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), "-f", "-", ]; return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest); } export function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult { const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; const script = [ "set -eu", `render_dir=${shellQuote(renderDir)}`, `prefix=${shellQuote(prefix)}`, "case \"$render_dir\" in", " \"$prefix\"*) rm -rf \"$render_dir\" ;;", " *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;", "esac", ].join("\n"); return runNodeHostScript(spec, script, 60); } export function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult { const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`; if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) { return { command: ["rm", "-rf", ""], cwd: repoRoot, exitCode: 44, stdout: "", stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`, signal: null, timedOut: false, }; } return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 }); } export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record { return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, namespace: "hwlab-ci", labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": spec.lane, "hwlab.pikastech.local/source-commit": sourceCommit, "hwlab.pikastech.local/trigger": "unidesk-node-cli", }, annotations: { "hwlab.pikastech.local/node": spec.nodeId, "hwlab.pikastech.local/source-branch": spec.sourceBranch, "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch, "hwlab.pikastech.local/runtime-path": spec.runtimePath, "hwlab.pikastech.local/network-profile": spec.networkProfileId, "hwlab.pikastech.local/download-profile": spec.downloadProfileId, }, }, spec: { pipelineRef: { name: spec.pipeline }, taskRunTemplate: { serviceAccountName: spec.serviceAccountName, podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, }, }, params: [ { name: "git-url", value: spec.gitUrl }, { name: "git-read-url", value: spec.gitReadUrl }, { name: "git-write-url", value: spec.gitWriteUrl }, { name: "source-branch", value: spec.sourceBranch }, { name: "gitops-branch", value: spec.gitopsBranch }, { name: "lane", value: spec.lane }, { name: "catalog-path", value: spec.catalogPath }, { name: "image-tag-mode", value: "full" }, { name: "runtime-path", value: spec.runtimePath }, { name: "revision", value: sourceCommit }, { name: "registry-prefix", value: spec.registryPrefix }, { name: "services", value: spec.serviceIds.join(",") }, { name: "base-image", value: spec.baseImage }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, ], }, }; } export function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult { const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64"); const script = [ "set -eu", `manifest_b64=${shellQuote(manifestB64)}`, `pipeline_run=${shellQuote(pipelineRun)}`, `rerun=${rerun ? "true" : "false"}`, "manifest_path=\"/tmp/$pipeline_run.json\"", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", "existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", "if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then", " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "elif [ \"$existing_status\" = False ]; then", " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "fi", "if kubectl create -f \"$manifest_path\"; then", " :", "else", " code=$?", " if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then", " printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2", " else", " exit \"$code\"", " fi", "fi", "kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'", ].join("\n"); return runNodeK3sScript(spec, script, timeoutSeconds); } export function waitForNodeRuntimePipelineRunTerminal( spec: HwlabRuntimeLaneSpec, pipelineRun: string, timeoutSeconds: number, options: { opportunisticPostFlush?: () => Record | null; opportunisticPostSync?: () => Record | null } = {}, ): Record { const severeWarningThresholdMs = 120_000; const startedAt = Date.now(); const deadline = startedAt + timeoutSeconds * 1000; let polls = 0; let last: Record = { exists: false, name: pipelineRun }; let lastOpportunisticPostFlushAt = 0; let severeWarningEmitted = false; let opportunisticPostSyncAttempted = false; const opportunisticPostFlushes: Record[] = []; const opportunisticPostSyncs: Record[] = []; printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds }); while (Date.now() <= deadline) { polls += 1; last = getNodeRuntimePipelineRun(spec, pipelineRun); const status = typeof last.status === "string" ? last.status : null; const reason = typeof last.reason === "string" ? last.reason : null; const elapsedMs = Date.now() - startedAt; printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs }); if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") { severeWarningEmitted = true; printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", warning: "pipelinerun-wait-severe-timeout", severity: "warning", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs, thresholdMs: severeWarningThresholdMs, message: "PipelineRun has been non-terminal for more than 120s; this is a severe timeout and requires further investigation of Tekton taskRuns/pods instead of treating it as normal wait time.", next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, }, }); } if (status === "True" || status === "False") { const ok = status === "True"; const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics); printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); return { ok, status: ok ? "succeeded" : "failed", pipelineRun: last, diagnostics, failureSummary, polls, elapsedMs: Date.now() - startedAt, opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined, opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined, degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed", }; } if ( options.opportunisticPostSync !== undefined && !opportunisticPostSyncAttempted && nodeRuntimePipelineRunSourceCloneCompleted(spec, pipelineRun) ) { opportunisticPostSyncAttempted = true; const sync = options.opportunisticPostSync(); if (sync !== null) opportunisticPostSyncs.push(sync); } if (options.opportunisticPostFlush !== undefined && Date.now() - lastOpportunisticPostFlushAt >= 15_000) { lastOpportunisticPostFlushAt = Date.now(); const flush = options.opportunisticPostFlush(); if (flush !== null) opportunisticPostFlushes.push(flush); last = getNodeRuntimePipelineRun(spec, pipelineRun); const refreshedStatus = typeof last.status === "string" ? last.status : null; const refreshedReason = typeof last.reason === "string" ? last.reason : null; const refreshedElapsedMs = Date.now() - startedAt; if (refreshedStatus === "True" || refreshedStatus === "False") { const ok = refreshedStatus === "True"; const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics); printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: refreshedStatus, reason: refreshedReason, polls, elapsedMs: refreshedElapsedMs, observedAfterPostFlush: true }); return { ok, status: ok ? "succeeded" : "failed", pipelineRun: last, diagnostics, failureSummary, polls, elapsedMs: refreshedElapsedMs, opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined, opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined, observedAfterPostFlush: true, degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed", }; } } sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now()))); } const elapsedMs = Date.now() - startedAt; const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs }); printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds }); return { ok: true, status: "pending", pipelineRun: last, polls, elapsedMs, waitLimitSeconds: timeoutSeconds, warning, opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined, opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined, next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` }, }; } export function nodeRuntimePipelineRunSourceCloneCompleted(spec: HwlabRuntimeLaneSpec, pipelineRun: string): boolean { const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60); if (result.exitCode !== 0) return false; let parsed: unknown; try { parsed = JSON.parse(result.stdout); } catch { return false; } const items = Array.isArray(record(parsed).items) ? record(parsed).items as unknown[] : []; return items.some((item) => { const itemRecord = record(item); const metadata = record(itemRecord.metadata); const labels = record(metadata.labels); const name = typeof metadata.name === "string" ? metadata.name : ""; const pipelineTask = typeof labels["tekton.dev/pipelineTask"] === "string" ? labels["tekton.dev/pipelineTask"] : ""; const taskName = `${pipelineTask} ${name}`; if (!/(clone|checkout|local-git|source[-_ ]*worktree)/iu.test(taskName)) return false; const status = record(itemRecord.status); const conditions = Array.isArray(status.conditions) ? status.conditions as unknown[] : []; return conditions.some((condition) => { const conditionRecord = record(condition); return conditionRecord.type === "Succeeded" && conditionRecord.status === "True"; }); }); } export function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`); } export function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record { const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60); if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) }; const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u); return { exists: true, name: pipelineRun, status: status || null, reason: reason || null, message: message || null, createdAt: createdAt || null, result: compactRuntimeCommand(result), }; } export function sleepSync(ms: number): void { const buffer = new SharedArrayBuffer(4); Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms)); } export function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return null; const secretRoot = externalPostgresSecretSourceRoot(spec); const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey); const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey); const missing = [ ...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]), ...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]), ]; if (missing.length > 0) { if (dryRun) { return { ok: true, mode: "dry-run", mutation: false, skipped: true, reason: "external-postgres-secret-source-missing", missing, sourceRoot: secretRoot, valuesPrinted: false, next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, }; } return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, degradedReason: "external-postgres-secret-source-missing", missing, sourceRoot: secretRoot, valuesPrinted: false, next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, }; } const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds); if (!isCommandSuccess(setup)) { return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, phase: "namespace-authn-setup", setup: compactRuntimeCommand(setup), valuesPrinted: false, }; } const manifest = { apiVersion: "v1", kind: "List", items: [ { apiVersion: "v1", kind: "Secret", metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace }, type: "Opaque", stringData: { [pg.cloudApi.secretKey]: cloudApi.value }, }, { apiVersion: "v1", kind: "Secret", metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace }, type: "Opaque", stringData: { [pg.openfga.secretKey]: openfga.value }, }, ], }; const applyScript = [ "set -eu", [ "kubectl", "apply", "--server-side", "--force-conflicts", "--field-manager=unidesk-hwlab-node-external-postgres-secret", ...(dryRun ? ["--dry-run=server"] : []), "-f", "-", ].map(shellQuote).join(" "), ].join("\n"); const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest)); return { ok: isCommandSuccess(apply), mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: !dryRun && isCommandSuccess(apply), namespace: spec.runtimeNamespace, sourceRoot: secretRoot, secrets: [ { name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey }, { name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null }, ], setup: compactRuntimeCommand(setup), apply: compactRuntimeCommand(apply), valuesPrinted: false, }; } export function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { const pg = spec.runtimeStore?.postgres; if (pg?.mode !== "local-k3s") return null; const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`; const sourceRef = pg.adminPasswordSourceRef ?? ""; const sourceKey = pg.adminPasswordSourceKey ?? ""; if (sourceRef.length === 0 || sourceKey.length === 0) { return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, namespace: spec.runtimeNamespace, secretName, secretKey: "POSTGRES_PASSWORD", degradedReason: "local-postgres-secret-source-not-configured", valuesPrinted: false, next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` }, }; } const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun }); if (material.ok !== true) { return { ok: false, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: false, namespace: spec.runtimeNamespace, secretName, secretKey: "POSTGRES_PASSWORD", source: material, degradedReason: material.error ?? "local-postgres-secret-source-missing", valuesPrinted: false, next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=` }, }; } const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? ""); const fields = keyValueLinesFromText(statusText(result)); const afterBytes = numericField(fields.afterPasswordBytes) ?? 0; const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0; return { ok, mode: dryRun ? "dry-run" : "confirmed-secret-sync", mutation: !dryRun && fields.mutation === "true", namespace: spec.runtimeNamespace, secretName, secretKey: "POSTGRES_PASSWORD", source: { ok: material.ok, sourceRef, sourceKey, sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath), generated: material.generated, fingerprint: material.fingerprint, valueBytes: material.value?.length ?? 0, valuesPrinted: false, }, before: { namespaceExists: fields.beforeNamespaceExists === "yes", secretExists: fields.beforeSecretExists === "yes", passwordBytes: numericField(fields.beforePasswordBytes), fingerprint: fields.beforePasswordFingerprint || null, }, after: { namespaceExists: fields.afterNamespaceExists === "yes", secretExists: fields.afterSecretExists === "yes", passwordBytes: numericField(fields.afterPasswordBytes), fingerprint: fields.afterPasswordFingerprint || null, }, action: fields.action || null, applyExitCode: numericField(fields.applyExitCode), namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode), degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed", result: compactRuntimeCommand(result), valuesPrinted: false, }; } export function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): { ok: boolean; sourceRef: string; sourceKey: string; sourcePath: string | null; checkedPaths: string[]; value: string | null; fingerprint: string | null; generated: boolean; error?: string; } { const checkedPaths = localSecretSourcePaths(input.sourceRef); const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null; const sourcePath = existingPath ?? checkedPaths[0] ?? null; if (sourcePath === null) { return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" }; } if (existingPath === null && input.dryRun) { return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" }; } if (existingPath === null) { const value = randomBytes(32).toString("base64url"); mkdirSync(dirname(sourcePath), { recursive: true }); writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 }); return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true }; } const text = readFileSync(existingPath, "utf8"); const values = parseEnvFile(text); const existingValue = values[input.sourceKey]; if (existingValue !== undefined && existingValue.length > 0) { return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false }; } if (input.dryRun) { return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" }; } const value = randomBytes(32).toString("base64url"); const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n"; writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 }); return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true }; } export function localSecretSourcePaths(sourceRef: string): string[] { const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); const paths = index >= 0 ? [ join(repoRoot.slice(0, index), ".state", "secrets", sourceRef), join(repoRoot, ".state", "secrets", sourceRef), ] : [join(repoRoot, ".state", "secrets", sourceRef)]; return [...new Set(paths)]; } export function shortSecretFingerprint(value: string): string { return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`; } export function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string { return [ "set +e", `namespace=${shellQuote(spec.runtimeNamespace)}`, `secret=${shellQuote(secretName)}`, "secret_key=POSTGRES_PASSWORD", `source_ref=${shellQuote(sourceRef)}`, `source_key=${shellQuote(sourceKey)}`, `source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, "field_manager=unidesk-hwlab-node-local-postgres-secret", "password=$(cat)", "password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')", "exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }", "namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", "decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }", "before_namespace_exists=$(namespace_exists_flag)", "before_secret_exists=no", "before_b64=", "if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi", "before_password_bytes=$(decoded_length \"$before_b64\")", "before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")", "after_namespace_exists=$before_namespace_exists", "after_secret_exists=$before_secret_exists", "after_password_bytes=$before_password_bytes", "after_password_fingerprint=$before_password_fingerprint", "action=observed", "mutation=false", "namespace_create_exit=", "apply_exit=", "if [ \"$dry_run\" = true ]; then", " if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi", "elif [ \"$password_bytes\" -le 0 ]; then", " action=source-empty", " apply_exit=42", "elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then", " action=source-mismatch", " apply_exit=47", "else", " if [ \"$before_namespace_exists\" != yes ]; then", " kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err", " namespace_create_exit=$?", " else", " namespace_create_exit=0", " fi", " if [ \"$namespace_create_exit\" = 0 ]; then", " tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)", " password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')", " cat >\"$tmp\" </tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err", " apply_exit=$?", " rm -f \"$tmp\"", " if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi", " else", " action=namespace-create-failed", " apply_exit=$namespace_create_exit", " fi", "fi", "after_namespace_exists=$(namespace_exists_flag)", "if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi", "after_password_bytes=$(decoded_length \"$after_b64\")", "after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'key\\t%s\\n' \"$secret_key\"", "printf 'sourceRef\\t%s\\n' \"$source_ref\"", "printf 'sourceKey\\t%s\\n' \"$source_key\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", "printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"", "printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"", "printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", "printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"", "printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"", "printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "password=", "if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi", "if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi", ].join("\n"); } export function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record { const source = spec.baseImageSource ?? ""; const script = [ "set -eu", `target=${shellQuote(spec.baseImage)}`, `source=${shellQuote(source)}`, "repo_tag=${target#*/}", "repo=${repo_tag%:*}", "tag=${repo_tag##*:}", "if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", "registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", "registry_present=false", "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi", "target_present=false", "if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi", "source_present=false", "if [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi", "printf 'target\\t%s\\n' \"$target\"", "printf 'source\\t%s\\n' \"$source\"", "printf 'sourceConfigured\\t%s\\n' \"$([ -n \"$source\" ] && printf true || printf false)\"", "printf 'registryUrl\\t%s\\n' \"$registry_url\"", "printf 'registryTagPresent\\t%s\\n' \"$registry_present\"", "printf 'targetImagePresent\\t%s\\n' \"$target_present\"", "printf 'sourceImagePresent\\t%s\\n' \"$source_present\"", ].join("\n"); const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); const fields = keyValueLinesFromText(statusText(result)); const sourceConfigured = fields.sourceConfigured === "true"; const registryTagPresent = fields.registryTagPresent === "true"; return { ok: isCommandSuccess(result) && sourceConfigured && registryTagPresent, target: fields.target ?? spec.baseImage, source: fields.source || null, sourceConfigured, registryUrl: fields.registryUrl ?? null, registryTagPresent, targetImagePresent: fields.targetImagePresent === "true", sourceImagePresent: fields.sourceImagePresent === "true", result: compactRuntimeCommand(result), }; } export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { if (spec.baseImageSource === undefined) return null; const script = [ "set -eu", `target=${shellQuote(spec.baseImage)}`, `source=${shellQuote(spec.baseImageSource)}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, `pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`, "repo_tag=${target#*/}", "repo=${repo_tag%:*}", "tag=${repo_tag##*:}", "if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", "registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", "present=false", "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi", "action=none", "if [ \"$present\" = false ]; then", " action=seed", " if [ \"$dry_run\" = false ]; then", " source_present_before=true", " if ! docker image inspect \"$source\" >/dev/null 2>&1; then", " source_present_before=false", " attempt=1", " pulled_source=false", " while [ \"$attempt\" -le \"$pull_retries\" ]; do", " pull_attempts=$attempt", " if docker pull \"$source\" >/tmp/hwlab-node-base-image-pull.out 2>&1; then pulled_source=true; break; fi", " attempt=$((attempt + 1))", " sleep 2", " done", " if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-base-image-pull.out >&2 2>/dev/null || true; fi", " fi", " docker image inspect \"$source\" >/dev/null", " docker tag \"$source\" \"$target\"", " docker push \"$target\" >/tmp/hwlab-node-base-image-push.out", " fi", "fi", "after=false", "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi", "printf 'target\\t%s\\n' \"$target\"", "printf 'source\\t%s\\n' \"$source\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'presentBefore\\t%s\\n' \"$present\"", "printf 'presentAfter\\t%s\\n' \"$after\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'pullRetries\\t%s\\n' \"$pull_retries\"", "printf 'sourcePresentBefore\\t%s\\n' \"${source_present_before:-unknown}\"", "printf 'pulledSource\\t%s\\n' \"${pulled_source:-false}\"", "printf 'pullAttempts\\t%s\\n' \"${pull_attempts:-0}\"", "if [ \"$dry_run\" = true ] || [ \"$after\" = true ]; then exit 0; fi", "exit 1", ].join("\n"); const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); const fields = keyValueLinesFromText(statusText(result)); return { ok: isCommandSuccess(result), dryRun, target: fields.target ?? spec.baseImage, source: fields.source ?? spec.baseImageSource, presentBefore: fields.presentBefore === "true", presentAfter: fields.presentAfter === "true", action: fields.action ?? null, pullRetries: numericField(fields.pullRetries), sourcePresentBefore: fields.sourcePresentBefore ?? null, pulledSource: fields.pulledSource === "true", pullAttempts: numericField(fields.pullAttempts), result: compactRuntimeCommand(result), }; } export function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return "true"; const authnKey = pg.openfga.authnKey ?? "authn-preshared-key"; return [ "set -eu", `namespace=${shellQuote(spec.runtimeNamespace)}`, `secret=${shellQuote(pg.openfga.secretName)}`, `authn_key=${shellQuote(authnKey)}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, "namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"", "if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi", "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -", "authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')", "if [ \"${authn_present:-0}\" -gt 0 ]; then", " action=kept", "elif [ \"$dry_run\" = true ]; then", " action=would-generate-authn-key", "else", " if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi", " kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null", " authn_value=", " action=generated-authn-key", "fi", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", "printf 'authnKey\\t%s\\n' \"$authn_key\"", "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'valuesPrinted\\tfalse\\n'", ].join("\n"); } export function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string { const configRef = spec.externalPostgres?.configRef; if (configRef === undefined) return join(repoRoot, ".state", "secrets"); const configPath = rootPath(configRef); const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown; if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`); const secrets = (parsed as Record).secrets; if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`); const root = (secrets as Record).root; if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`); return root.startsWith("/") ? root : rootPath(root); } export function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } { const sourcePath = join(secretRoot, sourceRef); if (!existsSync(sourcePath)) return { ok: false, sourcePath }; const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const value = values[key]; if (value === undefined || value.length === 0) return { ok: false, sourcePath }; return { ok: true, value, sourcePath }; } export function secretSourcePaths(sourceRef: string): string[] { const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef)); return [...new Set(paths)]; } export function displayRepoPath(path: string): string { const normalizedRoot = repoRoot.replace(/\/+$/u, ""); if (path === normalizedRoot) return "."; if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1); const marker = "/.worktree/"; const index = normalizedRoot.indexOf(marker); if (index >= 0) { const mainRoot = normalizedRoot.slice(0, index); if (path === mainRoot) return "."; if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1); } return path; } export function hwlabPasswordHash(password: string): string { const salt = randomBytes(16).toString("hex"); return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`; } export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial { const sourceRef = spec.bootstrapAdminPasswordSourceRef; const sourceKey = spec.bootstrapAdminPasswordSourceKey; if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "bootstrap-admin-yaml-source-missing" }; } const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) { return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "secret-source-missing" }; } const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const password = values[sourceKey]; if (password === undefined || password.length === 0) { return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, passwordHash: null, error: "secret-key-missing" }; } return { ok: true, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`, passwordHash: hwlabPasswordHash(password), error: null, }; } export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial { const sourceRef = spec.bootstrapAdminPasswordSourceRef; const sourceKey = spec.bootstrapAdminPasswordSourceKey; if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, password: null, error: "bootstrap-admin-yaml-source-missing" }; } const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) { return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" }; } const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const password = values[sourceKey]; if (password === undefined || password.length === 0) { return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" }; } return { ok: true, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`, password, error: null, }; } export function parseEnvFile(text: string): Record { const values: Record = {}; for (const rawLine of text.split(/\r?\n/u)) { const line = rawLine.trim(); if (line.length === 0 || line.startsWith("#")) continue; const eq = line.indexOf("="); if (eq <= 0) continue; const key = line.slice(0, eq).trim(); const rawValue = line.slice(eq + 1).trim(); values[key] = rawValue.startsWith("'") && rawValue.endsWith("'") ? rawValue.slice(1, -1).replace(/'"'"'/gu, "'") : rawValue.startsWith("\"") && rawValue.endsWith("\"") ? rawValue.slice(1, -1) : rawValue; } return values; } export function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean { if (!/postgres|pgsql|pgdata/iu.test(name)) return false; const externalServiceName = spec.externalPostgres?.serviceName; if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false; return true; } export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return { required: false, ready: true }; if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; const runtimeAccess = pg.runtimeAccess ?? { endpointAddress: pg.endpointAddress, port: pg.port }; const endpointIsIpv4 = /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/u.test(runtimeAccess.endpointAddress); const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "jsonpath={.spec.type}{\"\\n\"}{.spec.externalName}{\"\\n\"}{.spec.ports[0].port}{\"\\n\"}"], 60); const [serviceType = "", externalName = "", servicePort = ""] = service.stdout.split(/\r?\n/u); if (!endpointIsIpv4) { const ready = service.exitCode === 0 && serviceType === "ExternalName" && externalName === runtimeAccess.endpointAddress && Number(servicePort) === runtimeAccess.port; return { required: true, ready, serviceName: pg.serviceName, serviceType: serviceType || null, externalName: externalName || null, expectedExternalName: runtimeAccess.endpointAddress, port: numericField(servicePort), expectedPort: runtimeAccess.port, providerEndpointAddress: pg.endpointAddress, providerPort: pg.port, runtimeAccess: pg.runtimeAccess ?? null, service: compactRuntimeCommand(service), }; } const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60); const [addressType = "", address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u); return { required: true, ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port, serviceName: pg.serviceName, serviceType: serviceType || null, addressType: addressType || null, endpointAddress: address || null, expectedEndpointAddress: runtimeAccess.endpointAddress, port: numericField(port), expectedPort: runtimeAccess.port, providerEndpointAddress: pg.endpointAddress, providerPort: pg.port, runtimeAccess: pg.runtimeAccess ?? null, service: compactRuntimeCommand(service), endpointSlice: compactRuntimeCommand(endpointSlice), }; } export function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { const pg = hwlabRuntimeActiveExternalPostgres(spec); if (pg === undefined) return { required: false, ready: true }; if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey); const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey); const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key"); return { required: true, ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present, valuesPrinted: false, cloudApi, openfga: { datastoreUri: openfgaUri, authnPresharedKey: openfgaAuthn, }, }; } export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec { const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown); const nodes = record(parsed.nodes); const node = record(nodes[spec.nodeId]); const targets = Array.isArray(parsed.targets) ? parsed.targets : []; const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane); if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); const gitMirror = record(target.gitMirror); const gitMirrorEgressProxy = record(gitMirror.egressProxy); const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode"); if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`); const nodeEgressProxy = gitMirrorEgressMode === "direct" ? { mode: "direct" as const, required: false as const } : nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`); if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`); const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport"); const source = record(target.source); const gitops = record(target.gitops); const tekton = record(target.tekton); const toolsImage = record(tekton.toolsImage); const toolsImagePullPolicy = optionalStringValue(toolsImage.imagePullPolicy, "tekton.toolsImage.imagePullPolicy") ?? "IfNotPresent"; if (toolsImagePullPolicy !== "Always" && toolsImagePullPolicy !== "IfNotPresent" && toolsImagePullPolicy !== "Never") throw new Error("tekton.toolsImage.imagePullPolicy must be Always, IfNotPresent, or Never"); return { id: stringValue(target.id, "target.id"), node: stringValue(target.node, "target.node"), lane: stringValue(target.lane, "target.lane"), namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"), serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"), serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"), cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"), cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"), cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"), servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"), secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"), syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"), syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"), flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"), toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"), toolsImagePullPolicy, sourceRepository: stringValue(source.repository, "source.repository"), sourceBranch: stringValue(source.branch, "source.branch"), gitopsBranch: stringValue(gitops.branch, "gitops.branch"), egressProxy: nodeEgressProxy, githubTransport, }; } export function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record, path: string): NodeRuntimeGitMirrorGithubTransportSpec { const mode = stringValue(raw.mode, `${path}.mode`); if (mode === "ssh") return { mode }; if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`); return { mode, username: stringValue(raw.username, `${path}.username`), tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`), tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`), tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`), tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`), }; } export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record, path: string): NodeRuntimeGitMirrorEgressProxySpec { const mode = stringValue(raw.mode, `${path}.mode`); if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`); const port = Number(raw.port); if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`); const sourceConfigRef = optionalStringValue(raw.sourceConfigRef, `${path}.sourceConfigRef`) ?? null; const source = sourceConfigRef === null ? null : resolveEgressProxySourceRef(sourceConfigRef, `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.${path}.sourceConfigRef`); const sourceType = source?.sourceType ?? stringValue(raw.sourceType, `${path}.sourceType`); if (sourceType !== "subscription-url" && sourceType !== "master-shadowsocks") throw new Error(`${path}.sourceType must be subscription-url or master-shadowsocks`); if (raw.sourceType !== undefined && raw.sourceType !== sourceType) throw new Error(`${path}.sourceType must match sourceConfigRef value ${sourceType}`); const sourceRef = source?.sourceRef ?? stringValue(raw.sourceRef, `${path}.sourceRef`); const sourceKey = source?.sourceKey ?? stringValue(raw.sourceKey, `${path}.sourceKey`); if (source !== null) { if (raw.sourceRef !== undefined && raw.sourceRef !== source.sourceRef) throw new Error(`${path}.sourceRef must match sourceConfigRef value ${source.sourceRef}`); if (raw.sourceKey !== undefined && raw.sourceKey !== source.sourceKey) throw new Error(`${path}.sourceKey must match sourceConfigRef value ${source.sourceKey}`); } const preferredOutbound = source?.preferredOutbound ?? (raw.preferredOutbound === undefined ? null : stringValue(raw.preferredOutbound, `${path}.preferredOutbound`)); if (preferredOutbound !== null && preferredOutbound !== "vless-reality" && preferredOutbound !== "hysteria2") throw new Error(`${path}.preferredOutbound must be vless-reality or hysteria2`); if (source !== null && source.preferredOutbound !== null && raw.preferredOutbound !== undefined && raw.preferredOutbound !== source.preferredOutbound) { throw new Error(`${path}.preferredOutbound must match sourceConfigRef value ${source.preferredOutbound}`); } const noProxyRaw = raw.noProxy; if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`); const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)); return { mode, clientName: stringValue(raw.clientName, `${path}.clientName`), namespace: stringValue(raw.namespace, `${path}.namespace`), serviceName: stringValue(raw.serviceName, `${path}.serviceName`), port, sourceConfigRef, sourceRef, sourceKey, sourceType, preferredOutbound, sourceFingerprint: source?.fingerprint ?? null, noProxy, }; } export function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record { const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60); return { secret, key, present: result.exitCode === 0 && result.stdout.trim().length > 0, valueBytesBase64: result.stdout.trim().length, exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500), valuesPrinted: false, }; } export function startNodeDelegatedJob(options: ReturnType): Record { const commandArgs = [ "hwlab", "nodes", options.domain, ...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]), "--node", options.node, "--lane", options.lane, "--confirm", "--timeout-seconds", String(options.timeoutSeconds), "--wait", ]; const command = ["bun", "scripts/cli.ts", ...commandArgs]; const job = startJob( `hwlab_nodes_${options.lane}_${options.domain}_${options.action}`, command, `Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`, ); return { ok: true, command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`, node: options.node, lane: options.lane, mode: "async-job", reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking", job, statusCommand: `bun scripts/cli.ts job status ${job.id}`, waitCommand: command.join(" "), }; } export function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType): Record { const rewritten = rewriteDelegatedNodeValue(value, scoped); const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record : { value: rewritten }; return { ...result, node: scoped.node, commandFamily: "hwlab nodes", }; } export function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType): unknown { if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped); if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped)); if (typeof value !== "object" || value === null) return value; return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)])); } export function rewriteDelegatedNodeString(value: string, scoped: ReturnType): string { const replaceCommand = (text: string, domain: DelegatedNodeDomain) => { const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); return text .replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`) .replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`); }; return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror"); } export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions { const [actionRaw] = args; if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "observe" && actionRaw !== "sentinel") throw new Error("web-probe usage: run|script|observe|sentinel --node NODE --lane vNN [--url URL]"); if (actionRaw === "sentinel") return parseNodeWebProbeSentinelOptions(args.slice(1)); if (actionRaw === "observe") { const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1)); const explicitNode = optionValue(args, "--node"); const explicitLane = optionValue(args, "--lane"); const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null; const indexed = requestedId === null ? null : readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null); const node = explicitNode ?? indexed?.node; const lane = explicitLane ?? indexed?.lane; if (node === undefined || lane === undefined) { if (requestedId !== null) { throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`); } throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id"); } assertNodeId(node); assertLane(lane); if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`); const spec = hwlabRuntimeLaneSpecForNode(lane, node); return parseNodeWebProbeObserveOptions(normalized.args, node, lane, spec, normalized.id, indexed); } const node = requiredOption(args, "--node"); assertNodeId(node); const lane = requiredOption(args, "--lane"); assertLane(lane); if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`); const spec = hwlabRuntimeLaneSpecForNode(lane, node); if (actionRaw === "script") { assertKnownOptions(args.slice(1), new Set([ "--node", "--lane", "--url", "--timeout-ms", "--viewport", "--browser-proxy-mode", "--script-file", "--command-timeout-seconds", ]), new Set([])); const scriptFile = optionValue(args, "--script-file"); if (scriptFile === undefined && process.stdin.isTTY) { throw new Error("web-probe script requires a stdin heredoc or --script-file "); } const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8"); if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script"); const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000); const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600); return { action: "script", node, lane, url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec), timeoutMs, viewport: optionValue(args, "--viewport") ?? "1440x900", browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode), commandTimeoutSeconds, scriptText, scriptSource: { kind: scriptFile === undefined ? "stdin" : "file", path: scriptFile ?? null, byteCount: Buffer.byteLength(scriptText), sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`, }, }; } assertKnownOptions(args.slice(1), new Set([ "--node", "--lane", "--url", "--timeout-ms", "--wait-after-submit-ms", "--wait-messages-ms", "--wait-agent-terminal-ms", "--trace-sample-count", "--trace-sample-interval-ms", "--message", "--conversation-id", "--command-timeout-seconds", ]), new Set([ "--fresh-session", "--no-cancel-running", ])); const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000); const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000); const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000); const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000); const message = optionValue(args, "--message") ?? null; const hasMessage = message !== null; const traceSampleCountDefault = hasMessage ? 2 : 0; const traceSampleIntervalDefault = hasMessage ? 1500 : 0; const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200); const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000); const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({ timeoutMs, waitAfterSubmitMs, waitMessagesMs, waitAgentTerminalMs, traceSampleCount, traceSampleIntervalMs, freshSession: args.includes("--fresh-session"), hasMessage, }); const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds"); const commandTimeoutUserProvided = commandTimeoutRaw !== undefined; const commandTimeoutSeconds = commandTimeoutUserProvided ? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds) : commandTimeoutAutoSeconds; return { action: "run", node, lane, url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec), timeoutMs, waitAfterSubmitMs, waitMessagesMs, waitAgentTerminalMs, traceSampleCount, traceSampleIntervalMs, message, conversationId: optionValue(args, "--conversation-id") ?? null, freshSession: args.includes("--fresh-session"), cancelRunning: !args.includes("--no-cancel-running"), commandTimeoutSeconds, commandTimeoutAutoSeconds, commandTimeoutUserProvided, }; }