Codex
2351 lines
133 KiB
TypeScript
2351 lines
133 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-probe module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:6032-7426 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneCiGitWorkspaceSecret, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { BootstrapAdminPasswordMaterial, BootstrapAdminSecretMaterial, NodeRuntimeGitMirrorEgressProxySpec, NodeRuntimeGitMirrorGithubTransportSpec, NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult, NodeWebProbeOptions, RuntimeSecretSpec } from "./entry";
|
|
import { isCommandSuccess, nodeRuntimeGitopsRoot, runNodeK3sArgs, runNodeK3sScript } from "./cleanup";
|
|
import { nodeRuntimeCicdWaitWarning } from "./git-mirror";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { nodeRuntimePipelineFailureSummary, nodeRuntimePipelineRunDiagnostics } from "./render";
|
|
import { compactRuntimeCommand, runNodeHostScript, runNodeHostScriptAsync } from "./runtime-common";
|
|
import { assertLane, assertNodeId, keyValueLinesFromText, numericField, optionValue, optionalStringValue, positiveIntegerOption, positiveIntegerValue, record, requiredOption, shellQuote, statusText, stringValue, stripOptions } from "./utils";
|
|
import { discoverWebObserveIndexEntry, readWebObserveIndexEntry } from "./web-observe-render";
|
|
import { assertKnownOptions, nodeWebProbeAutoCommandTimeoutSeconds, nodeWebProbeDefaultUrl, normalizeNodeWebProbeObserveArgs, parseNodeWebProbeObserveOptions, parseNodeWebProbeSentinelOptions, parseWebProbeBrowserProxyMode } from "./web-probe-observe";
|
|
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
|
|
import { resolveEgressProxySourceRef } from "../egress-proxy-sources";
|
|
|
|
export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
|
|
const gitMirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
|
|
const renderGitMirror = {
|
|
...gitMirror,
|
|
egressProxy: gitMirror.egressProxy.mode === "direct" ? {
|
|
mode: "direct",
|
|
required: false,
|
|
} : gitMirror.egressProxy.mode === "host-route" ? {
|
|
...gitMirror.egressProxy,
|
|
mode: "host-route",
|
|
required: true,
|
|
} : {
|
|
...gitMirror.egressProxy,
|
|
mode: "node-global",
|
|
required: true,
|
|
},
|
|
};
|
|
const deployYamlGitMirror = {
|
|
...renderGitMirror,
|
|
egressProxy: renderGitMirror.egressProxy.mode !== "host-route" ? renderGitMirror.egressProxy : {
|
|
mode: "node-global",
|
|
required: true,
|
|
clientName: renderGitMirror.egressProxy.clientName,
|
|
namespace: "platform-infra",
|
|
serviceName: renderGitMirror.egressProxy.clientName,
|
|
port: httpProxyEndpoint(renderGitMirror.egressProxy.proxyUrl)?.port ?? 10808,
|
|
proxyUrl: renderGitMirror.egressProxy.proxyUrl,
|
|
noProxy: renderGitMirror.egressProxy.noProxy,
|
|
},
|
|
};
|
|
return {
|
|
nodeId: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceBranch: spec.sourceBranch,
|
|
gitopsBranch: spec.gitopsBranch,
|
|
gitopsRoot: nodeRuntimeGitopsRoot(spec),
|
|
runtimePath: spec.runtimePath,
|
|
runtimeRenderDir: spec.runtimeRenderDir,
|
|
runtimeNamespace: spec.runtimeNamespace,
|
|
catalogPath: spec.catalogPath,
|
|
tektonDir: spec.tektonDir,
|
|
pipelineName: spec.pipeline,
|
|
argoApplicationFile: spec.argoApplicationFile,
|
|
argoRepoUrl: spec.argoRepoUrl,
|
|
gitUrl: spec.gitUrl,
|
|
gitReadUrl: spec.gitReadUrl,
|
|
gitWriteUrl: spec.gitWriteUrl,
|
|
toolsImage: gitMirror.toolsImage,
|
|
toolsImagePullPolicy: gitMirror.toolsImagePullPolicy,
|
|
gitMirror: renderGitMirror,
|
|
deployYamlGitMirror,
|
|
networkProfileId: spec.networkProfileId,
|
|
downloadProfileId: spec.downloadProfileId,
|
|
gitSshProxyHost: gitSshProxy?.host,
|
|
gitSshProxyPort: gitSshProxy?.port,
|
|
proxyHttp: spec.networkProfile.proxy.http,
|
|
proxyHttps: spec.networkProfile.proxy.https,
|
|
proxyAll: spec.networkProfile.proxy.all,
|
|
noProxy: spec.networkProfile.proxy.noProxy.join(","),
|
|
dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http,
|
|
dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https,
|
|
dockerProxyAll: spec.networkProfile.dockerBuildProxy.all,
|
|
dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
|
|
dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy,
|
|
npmRegistry: spec.downloadProfile.npm.registry,
|
|
npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000,
|
|
npmRetries: spec.downloadProfile.npm.retries,
|
|
stepEnv: spec.stepEnv,
|
|
observability: spec.observability,
|
|
runtimeStore: spec.runtimeStore,
|
|
codeAgentRuntime: spec.codeAgentRuntime === undefined ? undefined : {
|
|
enabled: spec.codeAgentRuntime.enabled,
|
|
adapter: spec.codeAgentRuntime.adapter,
|
|
managerUrl: spec.codeAgentRuntime.managerUrl,
|
|
apiKeySecretName: spec.codeAgentRuntime.apiKeySecretName,
|
|
apiKeySecretKey: spec.codeAgentRuntime.apiKeySecretKey,
|
|
runnerNamespace: spec.codeAgentRuntime.runnerNamespace,
|
|
secretNamespace: spec.codeAgentRuntime.secretNamespace,
|
|
repoUrlFrom: spec.codeAgentRuntime.repoUrlFrom,
|
|
repoUrl: spec.gitReadUrl,
|
|
providerIdFrom: spec.codeAgentRuntime.providerIdFrom,
|
|
providerId: spec.nodeId,
|
|
defaultProviderProfile: spec.codeAgentRuntime.defaultProviderProfile,
|
|
codexStdioSupervisor: spec.codeAgentRuntime.codexStdioSupervisor,
|
|
kafkaShadowProducer: spec.codeAgentRuntime.kafkaShadowProducer ?? null,
|
|
valuesPrinted: false,
|
|
},
|
|
runtimeImageRewrites: spec.runtimeImageRewrites,
|
|
runtimeImageBuilds: spec.runtimeImageBuilds,
|
|
registryPrefix: spec.registryPrefix,
|
|
buildkitSidecarImage: spec.buildkit?.sidecarImage,
|
|
publicWebUrl: spec.publicWebUrl,
|
|
publicApiUrl: spec.publicApiUrl,
|
|
publicExposure: spec.publicExposure === null ? undefined : {
|
|
enabled: spec.publicExposure.enabled,
|
|
mode: spec.publicExposure.mode,
|
|
publicBaseUrl: spec.publicExposure.publicBaseUrl,
|
|
hostname: spec.publicExposure.hostname,
|
|
expectedA: spec.publicExposure.expectedA,
|
|
serverAddr: spec.publicExposure.serverAddr,
|
|
serverPort: spec.publicExposure.serverPort,
|
|
secretName: spec.publicExposure.secretName,
|
|
secretKey: spec.publicExposure.secretKey,
|
|
tokenKey: spec.publicExposure.tokenKey,
|
|
webProxy: spec.publicExposure.webProxy,
|
|
apiProxy: spec.publicExposure.apiProxy,
|
|
extraProxies: spec.publicExposure.extraProxies,
|
|
},
|
|
externalPostgres: activeExternalPostgres === undefined ? undefined : {
|
|
enabled: true,
|
|
serviceName: activeExternalPostgres.serviceName,
|
|
endpointAddress: activeExternalPostgres.endpointAddress,
|
|
port: activeExternalPostgres.port,
|
|
runtimeAccess: activeExternalPostgres.runtimeAccess ?? null,
|
|
sslmode: activeExternalPostgres.sslmode,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function httpProxyEndpoint(value: string): { host: string; port: number } | null {
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(value);
|
|
} catch {
|
|
return null;
|
|
}
|
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10);
|
|
if (!Number.isInteger(port) || port <= 0) return null;
|
|
return { host: parsed.hostname, port };
|
|
}
|
|
|
|
export function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
|
|
return [
|
|
`${renderDir}/devops-infra/git-mirror.yaml`,
|
|
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
|
|
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
|
|
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,
|
|
`${renderDir}/argocd/project.yaml`,
|
|
`${renderDir}/argocd/${spec.argoApplicationFile}`,
|
|
];
|
|
}
|
|
|
|
export function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
|
const files = nodeRuntimeControlPlaneFiles(spec, renderDir);
|
|
const args = [
|
|
"kubectl",
|
|
"apply",
|
|
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
|
...files.flatMap((file) => ["-f", file]),
|
|
];
|
|
return runNodeK3sArgs(spec, args, timeoutSeconds);
|
|
}
|
|
|
|
export function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
|
const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir)
|
|
.map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`)
|
|
.join("");
|
|
const args = [
|
|
"kubectl",
|
|
"apply",
|
|
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
|
"-f",
|
|
"-",
|
|
];
|
|
return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest);
|
|
}
|
|
|
|
export function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult {
|
|
const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
|
const script = [
|
|
"set -eu",
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`prefix=${shellQuote(prefix)}`,
|
|
"case \"$render_dir\" in",
|
|
" \"$prefix\"*) rm -rf \"$render_dir\" ;;",
|
|
" *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;",
|
|
"esac",
|
|
].join("\n");
|
|
return runNodeHostScript(spec, script, 60);
|
|
}
|
|
|
|
export function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult {
|
|
const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
|
const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`;
|
|
if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) {
|
|
return {
|
|
command: ["rm", "-rf", "<refused>"],
|
|
cwd: repoRoot,
|
|
exitCode: 44,
|
|
stdout: "",
|
|
stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`,
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 });
|
|
}
|
|
|
|
export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
|
|
const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane);
|
|
return {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "PipelineRun",
|
|
metadata: {
|
|
name: pipelineRun,
|
|
namespace: gitWorkspaceSecret.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/gitops-target": spec.lane,
|
|
"hwlab.pikastech.local/source-commit": sourceCommit,
|
|
"hwlab.pikastech.local/trigger": "unidesk-node-cli",
|
|
},
|
|
annotations: {
|
|
"hwlab.pikastech.local/node": spec.nodeId,
|
|
"hwlab.pikastech.local/source-branch": spec.sourceBranch,
|
|
"hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
|
|
"hwlab.pikastech.local/runtime-path": spec.runtimePath,
|
|
"hwlab.pikastech.local/network-profile": spec.networkProfileId,
|
|
"hwlab.pikastech.local/download-profile": spec.downloadProfileId,
|
|
},
|
|
},
|
|
spec: {
|
|
pipelineRef: { name: spec.pipeline },
|
|
taskRunTemplate: {
|
|
serviceAccountName: spec.serviceAccountName,
|
|
podTemplate: {
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
securityContext: { fsGroup: 1000 },
|
|
},
|
|
},
|
|
params: [
|
|
{ name: "git-url", value: spec.gitUrl },
|
|
{ name: "git-read-url", value: spec.gitReadUrl },
|
|
{ name: "git-write-url", value: spec.gitWriteUrl },
|
|
{ name: "source-branch", value: spec.sourceBranch },
|
|
{ name: "gitops-branch", value: spec.gitopsBranch },
|
|
{ name: "lane", value: spec.lane },
|
|
{ name: "catalog-path", value: spec.catalogPath },
|
|
{ name: "image-tag-mode", value: "full" },
|
|
{ name: "runtime-path", value: spec.runtimePath },
|
|
{ name: "source-commit", value: sourceCommit },
|
|
{ name: "revision", value: sourceCommit },
|
|
{ name: "registry-prefix", value: spec.registryPrefix },
|
|
{ name: "services", value: spec.serviceIds.join(",") },
|
|
{ name: "base-image", value: spec.baseImage },
|
|
],
|
|
workspaces: [
|
|
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
|
{ name: "git-ssh", secret: { secretName: gitWorkspaceSecret.name } },
|
|
],
|
|
},
|
|
};
|
|
}
|
|
|
|
export function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult {
|
|
const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane);
|
|
const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
`pipeline_run=${shellQuote(pipelineRun)}`,
|
|
`ci_namespace=${shellQuote(gitWorkspaceSecret.namespace)}`,
|
|
`git_workspace_secret=${shellQuote(gitWorkspaceSecret.name)}`,
|
|
`rerun=${rerun ? "true" : "false"}`,
|
|
"manifest_path=\"/tmp/$pipeline_run.json\"",
|
|
"if ! kubectl -n \"$ci_namespace\" get secret \"$git_workspace_secret\" >/dev/null 2>&1; then",
|
|
" printf 'missing Tekton git workspace Secret %s/%s; run: bun scripts/cli.ts hwlab nodes control-plane infra apply --node %s --lane %s --confirm\\n' \"$ci_namespace\" \"$git_workspace_secret\" " + shellQuote(spec.nodeId) + " " + shellQuote(spec.lane) + " >&2",
|
|
" exit 44",
|
|
"fi",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
"existing_status=$(kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
|
|
"if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then",
|
|
" kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
"elif [ \"$existing_status\" = False ]; then",
|
|
" kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
"fi",
|
|
"if kubectl create -f \"$manifest_path\"; then",
|
|
" :",
|
|
"else",
|
|
" code=$?",
|
|
" if kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" >/dev/null 2>&1; then",
|
|
" printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2",
|
|
" else",
|
|
" exit \"$code\"",
|
|
" fi",
|
|
"fi",
|
|
"kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'",
|
|
].join("\n");
|
|
return runNodeK3sScript(spec, script, timeoutSeconds);
|
|
}
|
|
|
|
export function waitForNodeRuntimePipelineRunTerminal(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
pipelineRun: string,
|
|
timeoutSeconds: number,
|
|
options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
|
|
): Record<string, unknown> {
|
|
const severeWarningThresholdMs = 120_000;
|
|
const startedAt = Date.now();
|
|
const deadline = startedAt + timeoutSeconds * 1000;
|
|
let polls = 0;
|
|
let last: Record<string, unknown> = { exists: false, name: pipelineRun };
|
|
let lastOpportunisticPostFlushAt = 0;
|
|
let severeWarningEmitted = false;
|
|
let opportunisticPostSyncAttempted = false;
|
|
const opportunisticPostFlushes: Record<string, unknown>[] = [];
|
|
const opportunisticPostSyncs: Record<string, unknown>[] = [];
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds });
|
|
while (Date.now() <= deadline) {
|
|
polls += 1;
|
|
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const status = typeof last.status === "string" ? last.status : null;
|
|
const reason = typeof last.reason === "string" ? last.reason : null;
|
|
const elapsedMs = Date.now() - startedAt;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs });
|
|
if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") {
|
|
severeWarningEmitted = true;
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: "pipelinerun-wait",
|
|
status: "warning",
|
|
warning: "pipelinerun-wait-severe-timeout",
|
|
severity: "warning",
|
|
pipelineRun,
|
|
pipelineStatus: status,
|
|
reason,
|
|
polls,
|
|
elapsedMs,
|
|
thresholdMs: severeWarningThresholdMs,
|
|
message: "PipelineRun has been non-terminal for more than 120s; this is a severe timeout and requires further investigation of Tekton taskRuns/pods instead of treating it as normal wait time.",
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
},
|
|
});
|
|
}
|
|
if (status === "True" || status === "False") {
|
|
const ok = status === "True";
|
|
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
|
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
|
|
return {
|
|
ok,
|
|
status: ok ? "succeeded" : "failed",
|
|
pipelineRun: last,
|
|
diagnostics,
|
|
failureSummary,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
|
};
|
|
}
|
|
if (
|
|
options.opportunisticPostSync !== undefined
|
|
&& !opportunisticPostSyncAttempted
|
|
&& nodeRuntimePipelineRunSourceCloneCompleted(spec, pipelineRun)
|
|
) {
|
|
opportunisticPostSyncAttempted = true;
|
|
const sync = options.opportunisticPostSync();
|
|
if (sync !== null) opportunisticPostSyncs.push(sync);
|
|
}
|
|
if (options.opportunisticPostFlush !== undefined && Date.now() - lastOpportunisticPostFlushAt >= 15_000) {
|
|
lastOpportunisticPostFlushAt = Date.now();
|
|
const flush = options.opportunisticPostFlush();
|
|
if (flush !== null) opportunisticPostFlushes.push(flush);
|
|
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const refreshedStatus = typeof last.status === "string" ? last.status : null;
|
|
const refreshedReason = typeof last.reason === "string" ? last.reason : null;
|
|
const refreshedElapsedMs = Date.now() - startedAt;
|
|
if (refreshedStatus === "True" || refreshedStatus === "False") {
|
|
const ok = refreshedStatus === "True";
|
|
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
|
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: refreshedStatus, reason: refreshedReason, polls, elapsedMs: refreshedElapsedMs, observedAfterPostFlush: true });
|
|
return {
|
|
ok,
|
|
status: ok ? "succeeded" : "failed",
|
|
pipelineRun: last,
|
|
diagnostics,
|
|
failureSummary,
|
|
polls,
|
|
elapsedMs: refreshedElapsedMs,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
observedAfterPostFlush: true,
|
|
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
|
};
|
|
}
|
|
}
|
|
sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
|
|
}
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
|
|
return {
|
|
ok: true,
|
|
status: "pending",
|
|
pipelineRun: last,
|
|
polls,
|
|
elapsedMs,
|
|
waitLimitSeconds: timeoutSeconds,
|
|
warning,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimePipelineRunSourceCloneCompleted(spec: HwlabRuntimeLaneSpec, pipelineRun: string): boolean {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
|
if (result.exitCode !== 0) return false;
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(result.stdout);
|
|
} catch {
|
|
return false;
|
|
}
|
|
const items = Array.isArray(record(parsed).items) ? record(parsed).items as unknown[] : [];
|
|
return items.some((item) => {
|
|
const itemRecord = record(item);
|
|
const metadata = record(itemRecord.metadata);
|
|
const labels = record(metadata.labels);
|
|
const name = typeof metadata.name === "string" ? metadata.name : "";
|
|
const pipelineTask = typeof labels["tekton.dev/pipelineTask"] === "string" ? labels["tekton.dev/pipelineTask"] : "";
|
|
const taskName = `${pipelineTask} ${name}`;
|
|
if (!/(clone|checkout|local-git|source[-_ ]*worktree)/iu.test(taskName)) return false;
|
|
const status = record(itemRecord.status);
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions as unknown[] : [];
|
|
return conditions.some((condition) => {
|
|
const conditionRecord = record(condition);
|
|
return conditionRecord.type === "Succeeded" && conditionRecord.status === "True";
|
|
});
|
|
});
|
|
}
|
|
|
|
export function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
|
process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`);
|
|
}
|
|
|
|
export function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60);
|
|
if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) };
|
|
const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u);
|
|
return {
|
|
exists: true,
|
|
name: pipelineRun,
|
|
status: status || null,
|
|
reason: reason || null,
|
|
message: message || null,
|
|
createdAt: createdAt || null,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function sleepSync(ms: number): void {
|
|
const buffer = new SharedArrayBuffer(4);
|
|
Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms));
|
|
}
|
|
|
|
export function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return null;
|
|
const secretRoot = externalPostgresSecretSourceRoot(spec);
|
|
const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey);
|
|
const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey);
|
|
const missing = [
|
|
...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]),
|
|
...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]),
|
|
];
|
|
if (missing.length > 0) {
|
|
if (dryRun) {
|
|
return {
|
|
ok: true,
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
skipped: true,
|
|
reason: "external-postgres-secret-source-missing",
|
|
missing,
|
|
sourceRoot: secretRoot,
|
|
valuesPrinted: false,
|
|
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
degradedReason: "external-postgres-secret-source-missing",
|
|
missing,
|
|
sourceRoot: secretRoot,
|
|
valuesPrinted: false,
|
|
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
|
};
|
|
}
|
|
const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds);
|
|
if (!isCommandSuccess(setup)) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
phase: "namespace-authn-setup",
|
|
setup: compactRuntimeCommand(setup),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const manifest = {
|
|
apiVersion: "v1",
|
|
kind: "List",
|
|
items: [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace },
|
|
type: "Opaque",
|
|
stringData: { [pg.cloudApi.secretKey]: cloudApi.value },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace },
|
|
type: "Opaque",
|
|
stringData: { [pg.openfga.secretKey]: openfga.value },
|
|
},
|
|
],
|
|
};
|
|
const applyScript = [
|
|
"set -eu",
|
|
[
|
|
"kubectl",
|
|
"apply",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
"--field-manager=unidesk-hwlab-node-external-postgres-secret",
|
|
...(dryRun ? ["--dry-run=server"] : []),
|
|
"-f",
|
|
"-",
|
|
].map(shellQuote).join(" "),
|
|
].join("\n");
|
|
const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest));
|
|
return {
|
|
ok: isCommandSuccess(apply),
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: !dryRun && isCommandSuccess(apply),
|
|
namespace: spec.runtimeNamespace,
|
|
sourceRoot: secretRoot,
|
|
secrets: [
|
|
{ name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey },
|
|
{ name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null },
|
|
],
|
|
setup: compactRuntimeCommand(setup),
|
|
apply: compactRuntimeCommand(apply),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
const pg = spec.runtimeStore?.postgres;
|
|
if (pg?.mode !== "local-k3s") return null;
|
|
const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`;
|
|
const sourceRef = pg.adminPasswordSourceRef ?? "";
|
|
const sourceKey = pg.adminPasswordSourceKey ?? "";
|
|
if (sourceRef.length === 0 || sourceKey.length === 0) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
degradedReason: "local-postgres-secret-source-not-configured",
|
|
valuesPrinted: false,
|
|
next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` },
|
|
};
|
|
}
|
|
const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun });
|
|
if (material.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
source: material,
|
|
degradedReason: material.error ?? "local-postgres-secret-source-missing",
|
|
valuesPrinted: false,
|
|
next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=<redacted>` },
|
|
};
|
|
}
|
|
const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? "");
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const afterBytes = numericField(fields.afterPasswordBytes) ?? 0;
|
|
const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0;
|
|
return {
|
|
ok,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: !dryRun && fields.mutation === "true",
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
source: {
|
|
ok: material.ok,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath),
|
|
generated: material.generated,
|
|
fingerprint: material.fingerprint,
|
|
valueBytes: material.value?.length ?? 0,
|
|
valuesPrinted: false,
|
|
},
|
|
before: {
|
|
namespaceExists: fields.beforeNamespaceExists === "yes",
|
|
secretExists: fields.beforeSecretExists === "yes",
|
|
passwordBytes: numericField(fields.beforePasswordBytes),
|
|
fingerprint: fields.beforePasswordFingerprint || null,
|
|
},
|
|
after: {
|
|
namespaceExists: fields.afterNamespaceExists === "yes",
|
|
secretExists: fields.afterSecretExists === "yes",
|
|
passwordBytes: numericField(fields.afterPasswordBytes),
|
|
fingerprint: fields.afterPasswordFingerprint || null,
|
|
},
|
|
action: fields.action || null,
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode),
|
|
degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed",
|
|
result: compactRuntimeCommand(result),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): {
|
|
ok: boolean;
|
|
sourceRef: string;
|
|
sourceKey: string;
|
|
sourcePath: string | null;
|
|
checkedPaths: string[];
|
|
value: string | null;
|
|
fingerprint: string | null;
|
|
generated: boolean;
|
|
error?: string;
|
|
} {
|
|
const checkedPaths = localSecretSourcePaths(input.sourceRef);
|
|
const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null;
|
|
const sourcePath = existingPath ?? checkedPaths[0] ?? null;
|
|
if (sourcePath === null) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" };
|
|
}
|
|
if (existingPath === null && input.dryRun) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" };
|
|
}
|
|
if (existingPath === null) {
|
|
const value = randomBytes(32).toString("base64url");
|
|
mkdirSync(dirname(sourcePath), { recursive: true });
|
|
writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
|
}
|
|
const text = readFileSync(existingPath, "utf8");
|
|
const values = parseEnvFile(text);
|
|
const existingValue = values[input.sourceKey];
|
|
if (existingValue !== undefined && existingValue.length > 0) {
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false };
|
|
}
|
|
if (input.dryRun) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" };
|
|
}
|
|
const value = randomBytes(32).toString("base64url");
|
|
const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n";
|
|
writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
|
}
|
|
|
|
export function localSecretSourcePaths(sourceRef: string): string[] {
|
|
if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef);
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
const paths = index >= 0
|
|
? [
|
|
join(repoRoot.slice(0, index), ".state", "secrets", sourceRef),
|
|
join(repoRoot, ".state", "secrets", sourceRef),
|
|
]
|
|
: [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
export function shortSecretFingerprint(value: string): string {
|
|
return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`;
|
|
}
|
|
|
|
export function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(secretName)}`,
|
|
"secret_key=POSTGRES_PASSWORD",
|
|
`source_ref=${shellQuote(sourceRef)}`,
|
|
`source_key=${shellQuote(sourceKey)}`,
|
|
`source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
"field_manager=unidesk-hwlab-node-local-postgres-secret",
|
|
"password=$(cat)",
|
|
"password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')",
|
|
"exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }",
|
|
"before_namespace_exists=$(namespace_exists_flag)",
|
|
"before_secret_exists=no",
|
|
"before_b64=",
|
|
"if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi",
|
|
"before_password_bytes=$(decoded_length \"$before_b64\")",
|
|
"before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")",
|
|
"after_namespace_exists=$before_namespace_exists",
|
|
"after_secret_exists=$before_secret_exists",
|
|
"after_password_bytes=$before_password_bytes",
|
|
"after_password_fingerprint=$before_password_fingerprint",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"namespace_create_exit=",
|
|
"apply_exit=",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi",
|
|
"elif [ \"$password_bytes\" -le 0 ]; then",
|
|
" action=source-empty",
|
|
" apply_exit=42",
|
|
"elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then",
|
|
" action=source-mismatch",
|
|
" apply_exit=47",
|
|
"else",
|
|
" if [ \"$before_namespace_exists\" != yes ]; then",
|
|
" kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err",
|
|
" namespace_create_exit=$?",
|
|
" else",
|
|
" namespace_create_exit=0",
|
|
" fi",
|
|
" if [ \"$namespace_create_exit\" = 0 ]; then",
|
|
" tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)",
|
|
" password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')",
|
|
" cat >\"$tmp\" <<EOF_SECRET",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $secret",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" app.kubernetes.io/managed-by: unidesk",
|
|
" annotations:",
|
|
" hwlab.pikastech.local/secret-source-ref: $source_ref",
|
|
" hwlab.pikastech.local/secret-source-key: $source_key",
|
|
" hwlab.pikastech.local/secret-source-fingerprint: $source_fingerprint",
|
|
"type: Opaque",
|
|
"data:",
|
|
" $secret_key: $password_b64",
|
|
"EOF_SECRET",
|
|
" kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err",
|
|
" apply_exit=$?",
|
|
" rm -f \"$tmp\"",
|
|
" if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
|
|
" else",
|
|
" action=namespace-create-failed",
|
|
" apply_exit=$namespace_create_exit",
|
|
" fi",
|
|
"fi",
|
|
"after_namespace_exists=$(namespace_exists_flag)",
|
|
"if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi",
|
|
"after_password_bytes=$(decoded_length \"$after_b64\")",
|
|
"after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'key\\t%s\\n' \"$secret_key\"",
|
|
"printf 'sourceRef\\t%s\\n' \"$source_ref\"",
|
|
"printf 'sourceKey\\t%s\\n' \"$source_key\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
|
"printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"",
|
|
"printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"",
|
|
"printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
|
"printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"",
|
|
"printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"",
|
|
"printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"password=",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
interface NodeRuntimeImageDependency {
|
|
id: string;
|
|
target: string;
|
|
source: string;
|
|
mode: "pull" | "build-moonbridge" | "build-opencode-git";
|
|
sourceRepo?: string;
|
|
sourceRef?: string;
|
|
builderImage?: string;
|
|
goProxy?: string;
|
|
dockerNetworkMode?: "default" | "host";
|
|
}
|
|
|
|
function nodeRuntimeImageDependencies(spec: HwlabRuntimeLaneSpec): NodeRuntimeImageDependency[] {
|
|
const images: NodeRuntimeImageDependency[] = [
|
|
{ id: "base", target: spec.baseImage, source: spec.baseImageSource ?? "", mode: "pull" },
|
|
];
|
|
if (spec.buildkit !== undefined) {
|
|
images.push({ id: "buildkit", target: spec.buildkit.sidecarImage, source: spec.buildkit.sourceImage, mode: "pull" });
|
|
}
|
|
spec.runtimeImageRewrites.forEach((rewrite, index) => {
|
|
images.push({ id: `rewrite-${index + 1}`, target: rewrite.target, source: rewrite.source, mode: "pull" });
|
|
});
|
|
spec.runtimeImageBuilds.forEach((build) => {
|
|
if (build.kind === "moonbridge") {
|
|
images.push({
|
|
id: build.id,
|
|
target: build.target,
|
|
source: `${build.sourceRepo}#${build.sourceRef}`,
|
|
mode: "build-moonbridge",
|
|
sourceRepo: build.sourceRepo,
|
|
sourceRef: build.sourceRef,
|
|
builderImage: build.builderImage,
|
|
goProxy: build.goProxy,
|
|
dockerNetworkMode: build.dockerNetworkMode,
|
|
});
|
|
return;
|
|
}
|
|
images.push({
|
|
id: build.id,
|
|
target: build.target,
|
|
source: build.sourceImage ?? "",
|
|
mode: "build-opencode-git",
|
|
dockerNetworkMode: build.dockerNetworkMode,
|
|
});
|
|
});
|
|
return images;
|
|
}
|
|
|
|
function nodeRuntimeImageCalls(spec: HwlabRuntimeLaneSpec, functionName: string): string {
|
|
return nodeRuntimeImageDependencies(spec)
|
|
.map((image) => [
|
|
functionName,
|
|
image.id,
|
|
image.target,
|
|
image.source,
|
|
image.mode,
|
|
image.sourceRepo ?? "",
|
|
image.sourceRef ?? "",
|
|
image.builderImage ?? "",
|
|
image.goProxy ?? "",
|
|
image.dockerNetworkMode ?? "",
|
|
].map(shellQuote).join(" "))
|
|
.join("\n");
|
|
}
|
|
|
|
function parseNodeRuntimeImageRows(text: string): Record<string, unknown>[] {
|
|
return text.split(/\r?\n/u).flatMap((line) => {
|
|
if (!line.startsWith("imageJson\t")) return [];
|
|
try {
|
|
return [JSON.parse(line.slice("imageJson\t".length)) as Record<string, unknown>];
|
|
} catch {
|
|
return [];
|
|
}
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
|
|
const script = [
|
|
"set -eu",
|
|
"check_image() {",
|
|
" id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"",
|
|
" repo_tag=${target#*/}",
|
|
" repo=${repo_tag%:*}",
|
|
" tag=${repo_tag##*:}",
|
|
" if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
|
" registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
|
" registry_present=false",
|
|
" if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi",
|
|
" target_present=false",
|
|
" if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi",
|
|
" source_present=false",
|
|
" if [ \"$mode\" = build-moonbridge ]; then source_present=true; elif [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi",
|
|
" source_configured=false",
|
|
" if [ -n \"$source\" ]; then source_configured=true; fi",
|
|
" python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$source_configured\" \"$registry_url\" \"$registry_present\" \"$target_present\" \"$source_present\" <<'PY'",
|
|
"import json, sys",
|
|
"keys=['id','target','source','mode','sourceConfigured','registryUrl','registryTagPresent','targetImagePresent','sourceImagePresent']",
|
|
"data=dict(zip(keys, sys.argv[1:]))",
|
|
"for key in ['sourceConfigured','registryTagPresent','targetImagePresent','sourceImagePresent']:",
|
|
" data[key] = data[key] == 'true'",
|
|
"print('imageJson\\t' + json.dumps(data, separators=(',', ':')))",
|
|
"PY",
|
|
"}",
|
|
nodeRuntimeImageCalls(spec, "check_image"),
|
|
].join("\n");
|
|
const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
|
const imageRows = parseNodeRuntimeImageRows(statusText(result));
|
|
const base = imageRows.find((image) => image.id === "base") ?? {};
|
|
return {
|
|
ok: isCommandSuccess(result) && imageRows.length === nodeRuntimeImageDependencies(spec).length && imageRows.every((image) => image.sourceConfigured === true && image.registryTagPresent === true),
|
|
target: base.target ?? spec.baseImage,
|
|
source: base.source || null,
|
|
sourceConfigured: base.sourceConfigured === true,
|
|
registryUrl: base.registryUrl ?? null,
|
|
registryTagPresent: base.registryTagPresent === true,
|
|
targetImagePresent: base.targetImagePresent === true,
|
|
sourceImagePresent: base.sourceImagePresent === true,
|
|
images: imageRows,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
if (spec.baseImageSource === undefined) return null;
|
|
const script = [
|
|
"set -eu",
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
`pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`,
|
|
`build_http_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.http)}`,
|
|
`build_https_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.https)}`,
|
|
`build_all_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.all)}`,
|
|
`build_no_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.noProxy.join(","))}`,
|
|
"resolve_host_proxy_url() {",
|
|
" raw_url=\"$1\"",
|
|
" if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi",
|
|
" info=$(python3 - \"$raw_url\" <<'PY'",
|
|
"from urllib.parse import urlparse",
|
|
"import sys",
|
|
"url = sys.argv[1]",
|
|
"parsed = urlparse(url)",
|
|
"host = parsed.hostname or ''",
|
|
"port = parsed.port",
|
|
"scheme = parsed.scheme or 'http'",
|
|
"parts = host.split('.')",
|
|
"if len(parts) >= 4 and parts[2] == 'svc':",
|
|
" print('\\t'.join(['service', scheme, parts[0], parts[1], str(port or 80)]))",
|
|
"else:",
|
|
" print('\\t'.join(['literal', url]))",
|
|
"PY",
|
|
" )",
|
|
" kind=$(printf '%s' \"$info\" | cut -f1)",
|
|
" service_cluster_ip() {",
|
|
" service_namespace=\"$1\"; service_name=\"$2\"",
|
|
" cluster_ip=\"\"",
|
|
" if command -v kubectl >/dev/null 2>&1; then",
|
|
" cluster_ip=$(kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)",
|
|
" fi",
|
|
" if { [ -z \"$cluster_ip\" ] || [ \"$cluster_ip\" = None ]; } && command -v k3s >/dev/null 2>&1; then",
|
|
" cluster_ip=$(k3s kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)",
|
|
" fi",
|
|
" printf '%s\\n' \"$cluster_ip\"",
|
|
" }",
|
|
" if [ \"$kind\" = service ]; then",
|
|
" scheme=$(printf '%s' \"$info\" | cut -f2)",
|
|
" service_name=$(printf '%s' \"$info\" | cut -f3)",
|
|
" service_namespace=$(printf '%s' \"$info\" | cut -f4)",
|
|
" service_port=$(printf '%s' \"$info\" | cut -f5)",
|
|
" cluster_ip=$(service_cluster_ip \"$service_namespace\" \"$service_name\")",
|
|
" if [ -n \"$cluster_ip\" ] && [ \"$cluster_ip\" != None ]; then printf '%s://%s:%s\\n' \"$scheme\" \"$cluster_ip\" \"$service_port\"; else printf '%s\\n' \"$raw_url\"; fi",
|
|
" else",
|
|
" printf '%s\\n' \"$raw_url\"",
|
|
" fi",
|
|
"}",
|
|
"build_http_proxy=$(resolve_host_proxy_url \"$build_http_proxy\")",
|
|
"build_https_proxy=$(resolve_host_proxy_url \"$build_https_proxy\")",
|
|
"build_all_proxy=$(resolve_host_proxy_url \"$build_all_proxy\")",
|
|
"build_container_proxy_url() {",
|
|
" raw_url=\"$1\"",
|
|
" if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi",
|
|
" python3 - \"$raw_url\" <<'PY'",
|
|
"from urllib.parse import urlparse, urlunparse",
|
|
"import sys",
|
|
"url = sys.argv[1]",
|
|
"parsed = urlparse(url)",
|
|
"host = parsed.hostname or ''",
|
|
"if host in {'127.0.0.1', 'localhost'}:",
|
|
" netloc = 'host.docker.internal'",
|
|
" if parsed.port is not None:",
|
|
" netloc = f'{netloc}:{parsed.port}'",
|
|
" print(urlunparse(parsed._replace(netloc=netloc)))",
|
|
"else:",
|
|
" print(url)",
|
|
"PY",
|
|
"}",
|
|
"build_container_http_proxy=$(build_container_proxy_url \"$build_http_proxy\")",
|
|
"build_container_https_proxy=$(build_container_proxy_url \"$build_https_proxy\")",
|
|
"build_container_all_proxy=$(build_container_proxy_url \"$build_all_proxy\")",
|
|
"docker_build_add_host_args=\"\"",
|
|
"case \"$build_container_http_proxy $build_container_https_proxy $build_container_all_proxy\" in",
|
|
" *host.docker.internal*) docker_build_add_host_args=\"--add-host host.docker.internal:host-gateway\" ;;",
|
|
"esac",
|
|
"failed=false",
|
|
"ensure_image() {",
|
|
" id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"; source_repo=\"$5\"; source_ref=\"$6\"; builder_image=\"$7\"; go_proxy=\"$8\"; docker_network_mode=\"$9\"",
|
|
" repo_tag=${target#*/}",
|
|
" repo=${repo_tag%:*}",
|
|
" tag=${repo_tag##*:}",
|
|
" if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
|
" registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
|
" present=false",
|
|
" if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi",
|
|
" action=none",
|
|
" source_present_before=unknown",
|
|
" pulled_source=false",
|
|
" pull_attempts=0",
|
|
" if [ \"$present\" = false ]; then",
|
|
" action=seed",
|
|
" if [ \"$dry_run\" = false ]; then",
|
|
" if [ \"$mode\" = build-moonbridge ]; then",
|
|
" action=build",
|
|
" source_present_before=build-source",
|
|
" effective_build_container_http_proxy=\"$build_container_http_proxy\"",
|
|
" effective_build_container_https_proxy=\"$build_container_https_proxy\"",
|
|
" effective_build_container_all_proxy=\"$build_container_all_proxy\"",
|
|
" effective_docker_build_add_host_args=\"$docker_build_add_host_args\"",
|
|
" if [ \"$docker_network_mode\" = host ]; then",
|
|
" effective_build_container_http_proxy=\"$build_http_proxy\"",
|
|
" effective_build_container_https_proxy=\"$build_https_proxy\"",
|
|
" effective_build_container_all_proxy=\"$build_all_proxy\"",
|
|
" effective_docker_build_add_host_args=\"\"",
|
|
" fi",
|
|
" tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)",
|
|
" dockerfile=\"$tmpdir/Dockerfile\"",
|
|
" cat > \"$dockerfile\" <<'DOCKERFILE'",
|
|
"ARG BUILDER_IMAGE",
|
|
"FROM ${BUILDER_IMAGE} AS builder",
|
|
"ARG MOONBRIDGE_REPO",
|
|
"ARG MOONBRIDGE_REF",
|
|
"ARG GOPROXY_VALUE",
|
|
"ARG HTTP_PROXY",
|
|
"ARG HTTPS_PROXY",
|
|
"ARG ALL_PROXY",
|
|
"ARG NO_PROXY",
|
|
"ENV GOPROXY=${GOPROXY_VALUE}",
|
|
"ENV HTTP_PROXY=${HTTP_PROXY}",
|
|
"ENV HTTPS_PROXY=${HTTPS_PROXY}",
|
|
"ENV ALL_PROXY=${ALL_PROXY}",
|
|
"ENV NO_PROXY=${NO_PROXY}",
|
|
"ENV http_proxy=${HTTP_PROXY}",
|
|
"ENV https_proxy=${HTTPS_PROXY}",
|
|
"ENV all_proxy=${ALL_PROXY}",
|
|
"ENV no_proxy=${NO_PROXY}",
|
|
"WORKDIR /src",
|
|
"RUN git clone --filter=blob:none \"${MOONBRIDGE_REPO}\" moon-bridge",
|
|
"WORKDIR /src/moon-bridge",
|
|
"RUN git checkout \"${MOONBRIDGE_REF}\" \\",
|
|
" && go mod download \\",
|
|
" && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags=\"-s -w\" -o /out/moonbridge ./cmd/moonbridge",
|
|
"FROM scratch",
|
|
"WORKDIR /app",
|
|
"COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt",
|
|
"COPY --from=builder /out/moonbridge /app/moonbridge",
|
|
"EXPOSE 4001",
|
|
"USER 65532:65532",
|
|
"ENTRYPOINT [\"/app/moonbridge\"]",
|
|
"DOCKERFILE",
|
|
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then",
|
|
" docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }",
|
|
" else",
|
|
" cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true",
|
|
" failed=true",
|
|
" fi",
|
|
" rm -rf \"$tmpdir\"",
|
|
" elif [ \"$mode\" = build-opencode-git ]; then",
|
|
" action=build",
|
|
" source_present_before=false",
|
|
" if [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present_before=true; fi",
|
|
" effective_build_container_http_proxy=\"$build_container_http_proxy\"",
|
|
" effective_build_container_https_proxy=\"$build_container_https_proxy\"",
|
|
" effective_build_container_all_proxy=\"$build_container_all_proxy\"",
|
|
" effective_docker_build_add_host_args=\"$docker_build_add_host_args\"",
|
|
" if [ \"$docker_network_mode\" = host ]; then",
|
|
" effective_build_container_http_proxy=\"$build_http_proxy\"",
|
|
" effective_build_container_https_proxy=\"$build_https_proxy\"",
|
|
" effective_build_container_all_proxy=\"$build_all_proxy\"",
|
|
" effective_docker_build_add_host_args=\"\"",
|
|
" fi",
|
|
" tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)",
|
|
" dockerfile=\"$tmpdir/Dockerfile\"",
|
|
" cat > \"$dockerfile\" <<'DOCKERFILE'",
|
|
"ARG SOURCE_IMAGE",
|
|
"FROM ${SOURCE_IMAGE}",
|
|
"ARG HTTP_PROXY",
|
|
"ARG HTTPS_PROXY",
|
|
"ARG ALL_PROXY",
|
|
"ARG NO_PROXY",
|
|
"ENV HTTP_PROXY=${HTTP_PROXY}",
|
|
"ENV HTTPS_PROXY=${HTTPS_PROXY}",
|
|
"ENV ALL_PROXY=${ALL_PROXY}",
|
|
"ENV NO_PROXY=${NO_PROXY}",
|
|
"ENV http_proxy=${HTTP_PROXY}",
|
|
"ENV https_proxy=${HTTPS_PROXY}",
|
|
"ENV all_proxy=${ALL_PROXY}",
|
|
"ENV no_proxy=${NO_PROXY}",
|
|
"RUN apk add --no-cache git",
|
|
"DOCKERFILE",
|
|
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg SOURCE_IMAGE=\"$source\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then",
|
|
" docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }",
|
|
" else",
|
|
" cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true",
|
|
" failed=true",
|
|
" fi",
|
|
" rm -rf \"$tmpdir\"",
|
|
" else",
|
|
" source_present_before=true",
|
|
" if ! docker image inspect \"$source\" >/dev/null 2>&1; then",
|
|
" source_present_before=false",
|
|
" attempt=1",
|
|
" while [ \"$attempt\" -le \"$pull_retries\" ]; do",
|
|
" pull_attempts=$attempt",
|
|
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker pull \"$source\" >/tmp/hwlab-node-runtime-image-$id-pull.out 2>&1; then pulled_source=true; break; fi",
|
|
" attempt=$((attempt + 1))",
|
|
" sleep 2",
|
|
" done",
|
|
" if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-runtime-image-$id-pull.out >&2 2>/dev/null || true; failed=true; fi",
|
|
" fi",
|
|
" if [ \"$failed\" != true ]; then",
|
|
" if docker image inspect \"$source\" >/dev/null 2>&1; then",
|
|
" docker tag \"$source\" \"$target\"",
|
|
" docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || failed=true",
|
|
" else",
|
|
" failed=true",
|
|
" fi",
|
|
" fi",
|
|
" fi",
|
|
" fi",
|
|
" fi",
|
|
" after=false",
|
|
" if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi",
|
|
" python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$dry_run\" \"$present\" \"$after\" \"$action\" \"$pull_retries\" \"$source_present_before\" \"$pulled_source\" \"$pull_attempts\" <<'PY'",
|
|
"import json, sys",
|
|
"keys=['id','target','source','mode','dryRun','presentBefore','presentAfter','action','pullRetries','sourcePresentBefore','pulledSource','pullAttempts']",
|
|
"data=dict(zip(keys, sys.argv[1:]))",
|
|
"for key in ['dryRun','presentBefore','presentAfter','pulledSource']:",
|
|
" data[key] = data[key] == 'true'",
|
|
"for key in ['pullRetries','pullAttempts']:",
|
|
" data[key] = int(data[key]) if str(data[key]).isdigit() else None",
|
|
"print('imageJson\\t' + json.dumps(data, separators=(',', ':')))",
|
|
"PY",
|
|
" if [ \"$dry_run\" = false ] && [ \"$after\" != true ]; then failed=true; fi",
|
|
"}",
|
|
nodeRuntimeImageCalls(spec, "ensure_image"),
|
|
"if [ \"$failed\" = true ]; then exit 1; fi",
|
|
].join("\n");
|
|
const hasBuild = nodeRuntimeImageDependencies(spec).some((image) => image.mode === "build-moonbridge" || image.mode === "build-opencode-git");
|
|
const result = hasBuild && !dryRun
|
|
? runNodeHostScriptAsync(spec, script, Math.min(timeoutSeconds, 600), "runtime-image-build")
|
|
: runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
|
const imageRows = parseNodeRuntimeImageRows(statusText(result));
|
|
const dependencyCount = nodeRuntimeImageDependencies(spec).length;
|
|
const imageRowsComplete = imageRows.length === dependencyCount
|
|
&& imageRows.every((image) => image.presentAfter === true || image.registryTagPresent === true);
|
|
const base = imageRows.find((image) => image.id === "base") ?? {};
|
|
return {
|
|
ok: isCommandSuccess(result) || imageRowsComplete,
|
|
dryRun,
|
|
target: base.target ?? spec.baseImage,
|
|
source: base.source ?? spec.baseImageSource,
|
|
presentBefore: base.presentBefore === true,
|
|
presentAfter: base.presentAfter === true,
|
|
action: base.action ?? null,
|
|
pullRetries: numericField(typeof base.pullRetries === "number" ? String(base.pullRetries) : undefined),
|
|
sourcePresentBefore: typeof base.sourcePresentBefore === "string" ? base.sourcePresentBefore : null,
|
|
pulledSource: base.pulledSource === true,
|
|
pullAttempts: numericField(typeof base.pullAttempts === "number" ? String(base.pullAttempts) : undefined),
|
|
images: imageRows,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return "true";
|
|
const authnKey = pg.openfga.authnKey ?? "authn-preshared-key";
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(pg.openfga.secretName)}`,
|
|
`authn_key=${shellQuote(authnKey)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
"namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"",
|
|
"if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi",
|
|
"kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -",
|
|
"authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')",
|
|
"if [ \"${authn_present:-0}\" -gt 0 ]; then",
|
|
" action=kept",
|
|
"elif [ \"$dry_run\" = true ]; then",
|
|
" action=would-generate-authn-key",
|
|
"else",
|
|
" if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi",
|
|
" kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null",
|
|
" authn_value=",
|
|
" action=generated-authn-key",
|
|
"fi",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'authnKey\\t%s\\n' \"$authn_key\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'valuesPrinted\\tfalse\\n'",
|
|
].join("\n");
|
|
}
|
|
|
|
export function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string {
|
|
const configRef = spec.externalPostgres?.configRef;
|
|
if (configRef === undefined) return join(repoRoot, ".state", "secrets");
|
|
const configPath = rootPath(configRef);
|
|
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown;
|
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`);
|
|
const secrets = (parsed as Record<string, unknown>).secrets;
|
|
if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`);
|
|
const root = (secrets as Record<string, unknown>).root;
|
|
if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`);
|
|
return root.startsWith("/") ? root : rootPath(root);
|
|
}
|
|
|
|
export function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } {
|
|
const sourcePath = join(secretRoot, sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, sourcePath };
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const value = values[key];
|
|
if (value === undefined || value.length === 0) return { ok: false, sourcePath };
|
|
return { ok: true, value, sourcePath };
|
|
}
|
|
|
|
export function secretSourcePaths(sourceRef: string): string[] {
|
|
if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef);
|
|
const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef));
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
function ownerFileSourcePaths(sourceRef: string): string[] {
|
|
if (sourceRef.includes("..") || sourceRef.includes("\0")) return [];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot];
|
|
return [...new Set(roots.map((root) => join(root, sourceRef)))];
|
|
}
|
|
|
|
export function displayRepoPath(path: string): string {
|
|
const normalizedRoot = repoRoot.replace(/\/+$/u, "");
|
|
if (path === normalizedRoot) return ".";
|
|
if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1);
|
|
const marker = "/.worktree/";
|
|
const index = normalizedRoot.indexOf(marker);
|
|
if (index >= 0) {
|
|
const mainRoot = normalizedRoot.slice(0, index);
|
|
if (path === mainRoot) return ".";
|
|
if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1);
|
|
}
|
|
return path;
|
|
}
|
|
|
|
export function hwlabPasswordHash(password: string): string {
|
|
const salt = randomBytes(16).toString("hex");
|
|
return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`;
|
|
}
|
|
|
|
export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial {
|
|
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
|
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
|
const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null;
|
|
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
|
return bootstrapAdminSecretMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing");
|
|
}
|
|
const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine);
|
|
if (!password.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error);
|
|
const username = readBootstrapAdminUsername(spec);
|
|
if (!username.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error);
|
|
return {
|
|
ok: true,
|
|
username: username.value,
|
|
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
|
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
|
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
|
usernameFingerprint: shortSecretFingerprint(username.value),
|
|
sourceRef,
|
|
sourceKey,
|
|
sourceLine,
|
|
sourcePath: password.sourcePath,
|
|
sourcePresent: true,
|
|
sourceFingerprint: shortSecretFingerprint(password.value),
|
|
passwordHash: hwlabPasswordHash(password.value),
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial {
|
|
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
|
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
|
const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null;
|
|
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
|
return bootstrapAdminPasswordMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing");
|
|
}
|
|
const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine);
|
|
if (!password.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error);
|
|
const username = readBootstrapAdminUsername(spec);
|
|
if (!username.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error);
|
|
return {
|
|
ok: true,
|
|
username: username.value,
|
|
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
|
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
|
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
|
usernameFingerprint: shortSecretFingerprint(username.value),
|
|
sourceRef,
|
|
sourceKey,
|
|
sourceLine,
|
|
sourcePath: password.sourcePath,
|
|
sourcePresent: true,
|
|
sourceFingerprint: shortSecretFingerprint(password.value),
|
|
password: password.value,
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
function readBootstrapAdminUsername(spec: RuntimeSecretSpec): { ok: true; value: string } | { ok: false; error: string } {
|
|
const ref = spec.bootstrapAdminUsernameSourceRef;
|
|
if (ref === undefined) return { ok: true, value: spec.bootstrapAdminUsername };
|
|
const key = spec.bootstrapAdminUsernameSourceKey ?? "username";
|
|
const material = readSecretSourceScalar(ref, key, spec.bootstrapAdminUsernameSourceLine ?? null);
|
|
if (!material.ok) return { ok: false, error: `username-${material.error}` };
|
|
return { ok: true, value: material.value };
|
|
}
|
|
|
|
function readSecretSourceScalar(sourceRef: string, sourceKey: string, sourceLine: number | null): { ok: true; value: string; sourcePath: string; sourcePresent: true } | { ok: false; sourcePath: string; sourcePresent: boolean; error: string } {
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, sourcePath, sourcePresent: false, error: "secret-source-missing" };
|
|
const text = readFileSync(sourcePath, "utf8");
|
|
if (sourceLine !== null) {
|
|
const line = text.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? "";
|
|
if (line.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-line-missing" };
|
|
return { ok: true, value: line, sourcePath, sourcePresent: true };
|
|
}
|
|
const values = parseEnvFile(text);
|
|
const runtimeValue = process.env[sourceKey];
|
|
const value = values[sourceKey] ?? runtimeValue;
|
|
if (value === undefined || value.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-key-missing" };
|
|
return { ok: true, value, sourcePath, sourcePresent: true };
|
|
}
|
|
|
|
function bootstrapAdminSecretMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminSecretMaterial {
|
|
return {
|
|
ok: false,
|
|
username: null,
|
|
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
|
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
|
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
|
usernameFingerprint: null,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourceLine,
|
|
sourcePath,
|
|
sourcePresent: sourcePresent === true,
|
|
sourceFingerprint: null,
|
|
passwordHash: null,
|
|
error,
|
|
};
|
|
}
|
|
|
|
function bootstrapAdminPasswordMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminPasswordMaterial {
|
|
return {
|
|
ok: false,
|
|
username: null,
|
|
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
|
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
|
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
|
usernameFingerprint: null,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourceLine,
|
|
sourcePath,
|
|
sourcePresent: sourcePresent === true,
|
|
sourceFingerprint: null,
|
|
password: null,
|
|
error,
|
|
};
|
|
}
|
|
|
|
export function parseEnvFile(text: string): Record<string, string> {
|
|
const values: Record<string, string> = {};
|
|
for (const rawLine of text.split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (line.length === 0 || line.startsWith("#")) continue;
|
|
const eq = line.indexOf("=");
|
|
if (eq <= 0) continue;
|
|
const key = line.slice(0, eq).trim();
|
|
const rawValue = line.slice(eq + 1).trim();
|
|
values[key] = rawValue.startsWith("'") && rawValue.endsWith("'")
|
|
? rawValue.slice(1, -1).replace(/'"'"'/gu, "'")
|
|
: rawValue.startsWith("\"") && rawValue.endsWith("\"")
|
|
? rawValue.slice(1, -1)
|
|
: rawValue;
|
|
}
|
|
return values;
|
|
}
|
|
|
|
export function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean {
|
|
if (!/postgres|pgsql|pgdata/iu.test(name)) return false;
|
|
const externalServiceName = spec.externalPostgres?.serviceName;
|
|
if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false;
|
|
return true;
|
|
}
|
|
|
|
export function nodeRuntimeCodeAgentRuntimeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
|
const runtime = spec.codeAgentRuntime;
|
|
if (runtime === undefined || runtime.enabled === false) return { required: false, ready: true, enabled: runtime?.enabled ?? false };
|
|
if (!namespaceExists) return { required: true, ready: false, enabled: true, degradedReason: "runtime-namespace-missing" };
|
|
const managerTarget = nodeRuntimeCodeAgentManagerTarget(runtime.managerUrl, runtime.runnerNamespace);
|
|
const runnerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.runnerNamespace, "-o", "name"], 60);
|
|
const secretNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.secretNamespace, "-o", "name"], 60);
|
|
const managerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", managerTarget.namespace, "-o", "name"], 60);
|
|
const managerService = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "service", managerTarget.serviceName, "-o", "name"], 60);
|
|
const managerDeployment = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "deployment", managerTarget.serviceName, "-o", "jsonpath={.status.readyReplicas}{\"\\n\"}{.spec.replicas}{\"\\n\"}"], 60);
|
|
const [managerReadyReplicasRaw = "", managerDesiredReplicasRaw = ""] = managerDeployment.stdout.split(/\r?\n/u);
|
|
const managerReadyReplicas = numericField(managerReadyReplicasRaw);
|
|
const managerDesiredReplicas = numericField(managerDesiredReplicasRaw);
|
|
const managerDeploymentReady = managerDeployment.exitCode === 0 && (managerReadyReplicas ?? 0) >= Math.max(1, managerDesiredReplicas ?? 1);
|
|
const apiKey = secretKeyStatus(spec, runtime.apiKeySecretName, runtime.apiKeySecretKey);
|
|
const cloudApiEnv = nodeRuntimeCodeAgentCloudApiEnvStatus(spec, runtime);
|
|
const ready = runnerNamespace.exitCode === 0
|
|
&& secretNamespace.exitCode === 0
|
|
&& managerNamespace.exitCode === 0
|
|
&& managerService.exitCode === 0
|
|
&& managerDeploymentReady
|
|
&& apiKey.present === true
|
|
&& cloudApiEnv.ready === true;
|
|
const degradedReason = ready
|
|
? undefined
|
|
: runnerNamespace.exitCode !== 0
|
|
? "agentrun-runner-namespace-missing"
|
|
: secretNamespace.exitCode !== 0
|
|
? "agentrun-secret-namespace-missing"
|
|
: managerNamespace.exitCode !== 0
|
|
? "agentrun-manager-namespace-missing"
|
|
: managerService.exitCode !== 0
|
|
? "agentrun-manager-service-missing"
|
|
: !managerDeploymentReady
|
|
? "agentrun-manager-deployment-not-ready"
|
|
: apiKey.present !== true
|
|
? "agentrun-api-key-secret-missing"
|
|
: "hwlab-cloud-api-code-agent-env-mismatch";
|
|
return {
|
|
required: true,
|
|
enabled: true,
|
|
ready,
|
|
degradedReason,
|
|
adapter: runtime.adapter,
|
|
managerUrl: runtime.managerUrl,
|
|
manager: {
|
|
namespace: managerTarget.namespace,
|
|
serviceName: managerTarget.serviceName,
|
|
serviceExists: managerService.exitCode === 0,
|
|
deploymentReady: managerDeploymentReady,
|
|
readyReplicas: managerReadyReplicas,
|
|
desiredReplicas: managerDesiredReplicas,
|
|
namespaceResult: compactRuntimeCommand(managerNamespace),
|
|
serviceResult: compactRuntimeCommand(managerService),
|
|
deploymentResult: compactRuntimeCommand(managerDeployment),
|
|
},
|
|
runnerNamespace: {
|
|
name: runtime.runnerNamespace,
|
|
exists: runnerNamespace.exitCode === 0,
|
|
result: compactRuntimeCommand(runnerNamespace),
|
|
},
|
|
secretNamespace: {
|
|
name: runtime.secretNamespace,
|
|
exists: secretNamespace.exitCode === 0,
|
|
result: compactRuntimeCommand(secretNamespace),
|
|
},
|
|
apiKey,
|
|
cloudApiEnv,
|
|
repoUrlFrom: runtime.repoUrlFrom,
|
|
providerIdFrom: runtime.providerIdFrom,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeCodeAgentManagerTarget(managerUrl: string, fallbackNamespace: string): { namespace: string; serviceName: string } {
|
|
let parsed: URL | null = null;
|
|
try {
|
|
parsed = new URL(managerUrl);
|
|
} catch {
|
|
parsed = null;
|
|
}
|
|
const host = parsed?.hostname ?? "";
|
|
const parts = host.split(".").filter(Boolean);
|
|
if (parts.length >= 3 && parts[2] === "svc") return { serviceName: parts[0], namespace: parts[1] };
|
|
return { serviceName: parts[0] || "agentrun-mgr", namespace: parts[1] || fallbackNamespace };
|
|
}
|
|
|
|
function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable<HwlabRuntimeLaneSpec["codeAgentRuntime"]>): Record<string, unknown> {
|
|
const deployment = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deployment", "hwlab-cloud-api", "-o", "json"], 60);
|
|
if (deployment.exitCode !== 0) {
|
|
return {
|
|
ready: false,
|
|
deploymentExists: false,
|
|
mismatches: [{ name: "hwlab-cloud-api", reason: "deployment-missing" }],
|
|
result: compactCodeAgentDeploymentProbeResult(deployment),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(deployment.stdout) as unknown;
|
|
} catch (error) {
|
|
return {
|
|
ready: false,
|
|
deploymentExists: true,
|
|
mismatches: [{ name: "hwlab-cloud-api", reason: "deployment-json-invalid", message: error instanceof Error ? error.message : String(error) }],
|
|
result: compactCodeAgentDeploymentProbeResult(deployment),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const deploy = record(parsed);
|
|
const specRecord = record(record(record(deploy.spec).template).spec);
|
|
const containers = Array.isArray(specRecord.containers) ? specRecord.containers.map(record) : [];
|
|
const container = containers.find((item) => item.name === "hwlab-cloud-api") ?? containers[0];
|
|
const env = Array.isArray(container?.env) ? container.env.map(record) : [];
|
|
const envByName = new Map(env.map((item) => [String(item.name || ""), item]));
|
|
const mismatches: Array<Record<string, unknown>> = [];
|
|
const expectValue = (name: string, expected: string): void => {
|
|
const item = envByName.get(name);
|
|
const value = typeof item?.value === "string" ? item.value : null;
|
|
if (value !== expected) mismatches.push({ name, expected, value, kind: "value" });
|
|
};
|
|
const expectSecret = (name: string, expectedSecret: string, expectedKey: string): void => {
|
|
const item = envByName.get(name);
|
|
const secretRef = record(record(item?.valueFrom).secretKeyRef);
|
|
if (secretRef.name !== expectedSecret || secretRef.key !== expectedKey) {
|
|
mismatches.push({ name, expectedSecret, expectedKey, secret: secretRef.name ?? null, key: secretRef.key ?? null, kind: "secretKeyRef" });
|
|
}
|
|
};
|
|
expectValue("HWLAB_CODE_AGENT_ADAPTER", runtime.adapter);
|
|
expectValue("AGENTRUN_MGR_URL", runtime.managerUrl);
|
|
expectSecret("AGENTRUN_API_KEY", runtime.apiKeySecretName, runtime.apiKeySecretKey);
|
|
expectValue("HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE", runtime.runnerNamespace);
|
|
expectValue("HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE", runtime.secretNamespace);
|
|
expectValue("HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", spec.gitReadUrl);
|
|
expectValue("HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", spec.nodeId);
|
|
expectValue("HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE", runtime.defaultProviderProfile);
|
|
expectValue("HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", runtime.codexStdioSupervisor);
|
|
const kafkaShadowProducer = runtime.kafkaShadowProducer;
|
|
if (kafkaShadowProducer !== undefined) {
|
|
expectValue("HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED", String(kafkaShadowProducer.enabled));
|
|
expectValue("HWLAB_KAFKA_SHADOW_CONSUME_ENABLED", String(kafkaShadowProducer.consumeEnabled));
|
|
expectValue("HWLAB_KAFKA_BOOTSTRAP_SERVERS", kafkaShadowProducer.bootstrapServers);
|
|
expectValue("HWLAB_KAFKA_COMMAND_TOPIC", kafkaShadowProducer.commandTopic);
|
|
expectValue("HWLAB_KAFKA_CLIENT_ID", kafkaShadowProducer.clientId);
|
|
}
|
|
return {
|
|
ready: mismatches.length === 0,
|
|
deploymentExists: true,
|
|
deploymentName: "hwlab-cloud-api",
|
|
containerName: container?.name ?? null,
|
|
checkedEnvCount: kafkaShadowProducer === undefined ? 9 : 14,
|
|
mismatches,
|
|
result: compactCodeAgentDeploymentProbeResult(deployment),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactCodeAgentDeploymentProbeResult(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
stderrTail: result.stderr.slice(-600),
|
|
timedOut: result.timedOut === true,
|
|
};
|
|
}
|
|
|
|
export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return { required: false, ready: true };
|
|
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
|
const runtimeAccess = pg.runtimeAccess ?? { endpointAddress: pg.endpointAddress, port: pg.port };
|
|
const endpointIsIpv4 = /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/u.test(runtimeAccess.endpointAddress);
|
|
const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "jsonpath={.spec.type}{\"\\n\"}{.spec.externalName}{\"\\n\"}{.spec.ports[0].port}{\"\\n\"}"], 60);
|
|
const [serviceType = "", externalName = "", servicePort = ""] = service.stdout.split(/\r?\n/u);
|
|
if (!endpointIsIpv4) {
|
|
const ready = service.exitCode === 0 && serviceType === "ExternalName" && externalName === runtimeAccess.endpointAddress && Number(servicePort) === runtimeAccess.port;
|
|
return {
|
|
required: true,
|
|
ready,
|
|
serviceName: pg.serviceName,
|
|
serviceType: serviceType || null,
|
|
externalName: externalName || null,
|
|
expectedExternalName: runtimeAccess.endpointAddress,
|
|
port: numericField(servicePort),
|
|
expectedPort: runtimeAccess.port,
|
|
providerEndpointAddress: pg.endpointAddress,
|
|
providerPort: pg.port,
|
|
runtimeAccess: pg.runtimeAccess ?? null,
|
|
service: compactRuntimeCommand(service),
|
|
};
|
|
}
|
|
const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
|
|
const [addressType = "", address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u);
|
|
return {
|
|
required: true,
|
|
ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port,
|
|
serviceName: pg.serviceName,
|
|
serviceType: serviceType || null,
|
|
addressType: addressType || null,
|
|
endpointAddress: address || null,
|
|
expectedEndpointAddress: runtimeAccess.endpointAddress,
|
|
port: numericField(port),
|
|
expectedPort: runtimeAccess.port,
|
|
providerEndpointAddress: pg.endpointAddress,
|
|
providerPort: pg.port,
|
|
runtimeAccess: pg.runtimeAccess ?? null,
|
|
service: compactRuntimeCommand(service),
|
|
endpointSlice: compactRuntimeCommand(endpointSlice),
|
|
};
|
|
}
|
|
|
|
export function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return { required: false, ready: true };
|
|
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
|
const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey);
|
|
const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey);
|
|
const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key");
|
|
return {
|
|
required: true,
|
|
ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present,
|
|
valuesPrinted: false,
|
|
cloudApi,
|
|
openfga: {
|
|
datastoreUri: openfgaUri,
|
|
authnPresharedKey: openfgaAuthn,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec {
|
|
const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
|
|
const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
|
|
const nodes = record(parsed.nodes);
|
|
const node = record(nodes[spec.nodeId]);
|
|
const targets = Array.isArray(parsed.targets) ? parsed.targets : [];
|
|
const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane);
|
|
if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
|
|
const gitMirror = record(target.gitMirror);
|
|
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
|
|
const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode");
|
|
if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "host-route" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global, host-route, or direct for node=${spec.nodeId} lane=${spec.lane}`);
|
|
const nodeEgressProxy = gitMirrorEgressMode === "direct"
|
|
? { mode: "direct" as const, required: false as const }
|
|
: nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
|
|
if ((gitMirrorEgressMode === "node-global" || gitMirrorEgressMode === "host-route") && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
|
|
if (gitMirrorEgressMode === "node-global" && nodeEgressProxy.mode !== "k8s-service-cluster-ip") throw new Error(`gitMirror.egressProxy.mode=node-global requires nodes.${spec.nodeId}.egressProxy.mode=k8s-service-cluster-ip`);
|
|
if (gitMirrorEgressMode === "host-route" && nodeEgressProxy.mode !== "host-route") throw new Error(`gitMirror.egressProxy.mode=host-route requires nodes.${spec.nodeId}.egressProxy.mode=host-route`);
|
|
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
|
|
const source = record(target.source);
|
|
const gitops = record(target.gitops);
|
|
const tekton = record(target.tekton);
|
|
const toolsImage = record(tekton.toolsImage);
|
|
const toolsImagePullPolicy = optionalStringValue(toolsImage.imagePullPolicy, "tekton.toolsImage.imagePullPolicy") ?? "IfNotPresent";
|
|
if (toolsImagePullPolicy !== "Always" && toolsImagePullPolicy !== "IfNotPresent" && toolsImagePullPolicy !== "Never") throw new Error("tekton.toolsImage.imagePullPolicy must be Always, IfNotPresent, or Never");
|
|
return {
|
|
id: stringValue(target.id, "target.id"),
|
|
node: stringValue(target.node, "target.node"),
|
|
lane: stringValue(target.lane, "target.lane"),
|
|
namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"),
|
|
serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
|
|
serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
|
|
cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
|
|
cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"),
|
|
cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
|
|
servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"),
|
|
secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
|
|
syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"),
|
|
syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
|
|
flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"),
|
|
toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"),
|
|
toolsImagePullPolicy,
|
|
sourceRepository: stringValue(source.repository, "source.repository"),
|
|
sourceBranch: stringValue(source.branch, "source.branch"),
|
|
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
|
|
egressProxy: nodeEgressProxy,
|
|
githubTransport,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorGithubTransportSpec {
|
|
const mode = stringValue(raw.mode, `${path}.mode`);
|
|
if (mode === "ssh") {
|
|
const knownHostsSecretKey = optionalStringValue(raw.knownHostsSecretKey, `${path}.knownHostsSecretKey`);
|
|
const knownHostsSourceRef = optionalStringValue(raw.knownHostsSourceRef, `${path}.knownHostsSourceRef`);
|
|
const knownHostsSourceKey = optionalStringValue(raw.knownHostsSourceKey, `${path}.knownHostsSourceKey`);
|
|
const knownHostsSourceEncoding = raw.knownHostsSourceEncoding === undefined ? null : gitMirrorSecretSourceEncoding(raw.knownHostsSourceEncoding, `${path}.knownHostsSourceEncoding`);
|
|
const knownHostsValues = [knownHostsSecretKey, knownHostsSourceRef, knownHostsSourceKey, knownHostsSourceEncoding];
|
|
if (knownHostsValues.some((value) => value !== null) && knownHostsValues.some((value) => value === null)) {
|
|
throw new Error(`${path}.knownHostsSecretKey/sourceRef/sourceKey/sourceEncoding must be declared together`);
|
|
}
|
|
return {
|
|
mode,
|
|
privateKeySecretKey: stringValue(raw.privateKeySecretKey, `${path}.privateKeySecretKey`),
|
|
privateKeySourceRef: stringValue(raw.privateKeySourceRef, `${path}.privateKeySourceRef`),
|
|
privateKeySourceKey: stringValue(raw.privateKeySourceKey, `${path}.privateKeySourceKey`),
|
|
privateKeySourceEncoding: gitMirrorSecretSourceEncoding(raw.privateKeySourceEncoding, `${path}.privateKeySourceEncoding`),
|
|
knownHostsSecretKey,
|
|
knownHostsSourceRef,
|
|
knownHostsSourceKey,
|
|
knownHostsSourceEncoding,
|
|
};
|
|
}
|
|
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
|
|
return {
|
|
mode,
|
|
username: stringValue(raw.username, `${path}.username`),
|
|
tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`),
|
|
tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`),
|
|
tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`),
|
|
tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`),
|
|
};
|
|
}
|
|
|
|
function gitMirrorSecretSourceEncoding(raw: unknown, path: string): "plain" | "base64" {
|
|
const value = stringValue(raw, path);
|
|
if (value !== "plain" && value !== "base64") throw new Error(`${path} must be plain or base64`);
|
|
return value;
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorEgressProxySpec {
|
|
const mode = stringValue(raw.mode, `${path}.mode`);
|
|
if (mode === "host-route") {
|
|
const noProxyRaw = raw.noProxy;
|
|
if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
|
|
return {
|
|
mode,
|
|
clientName: stringValue(raw.clientName, `${path}.clientName`),
|
|
hostProxyConfigRef: stringValue(raw.hostProxyConfigRef, `${path}.hostProxyConfigRef`),
|
|
proxyEnvPath: stringValue(raw.proxyEnvPath, `${path}.proxyEnvPath`),
|
|
proxyUrl: stringValue(raw.proxyUrl, `${path}.proxyUrl`),
|
|
noProxy: noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)),
|
|
};
|
|
}
|
|
if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip or host-route`);
|
|
const port = Number(raw.port);
|
|
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`);
|
|
const sourceConfigRef = optionalStringValue(raw.sourceConfigRef, `${path}.sourceConfigRef`) ?? null;
|
|
const source = sourceConfigRef === null ? null : resolveEgressProxySourceRef(sourceConfigRef, `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.${path}.sourceConfigRef`);
|
|
const sourceType = source?.sourceType ?? stringValue(raw.sourceType, `${path}.sourceType`);
|
|
if (sourceType !== "subscription-url" && sourceType !== "master-shadowsocks") throw new Error(`${path}.sourceType must be subscription-url or master-shadowsocks`);
|
|
if (raw.sourceType !== undefined && raw.sourceType !== sourceType) throw new Error(`${path}.sourceType must match sourceConfigRef value ${sourceType}`);
|
|
const sourceRef = source?.sourceRef ?? stringValue(raw.sourceRef, `${path}.sourceRef`);
|
|
const sourceKey = source?.sourceKey ?? stringValue(raw.sourceKey, `${path}.sourceKey`);
|
|
if (source !== null) {
|
|
if (raw.sourceRef !== undefined && raw.sourceRef !== source.sourceRef) throw new Error(`${path}.sourceRef must match sourceConfigRef value ${source.sourceRef}`);
|
|
if (raw.sourceKey !== undefined && raw.sourceKey !== source.sourceKey) throw new Error(`${path}.sourceKey must match sourceConfigRef value ${source.sourceKey}`);
|
|
}
|
|
const preferredOutbound = source?.preferredOutbound ?? (raw.preferredOutbound === undefined ? null : stringValue(raw.preferredOutbound, `${path}.preferredOutbound`));
|
|
if (preferredOutbound !== null && preferredOutbound !== "vless-reality" && preferredOutbound !== "hysteria2") throw new Error(`${path}.preferredOutbound must be vless-reality or hysteria2`);
|
|
if (source !== null && source.preferredOutbound !== null && raw.preferredOutbound !== undefined && raw.preferredOutbound !== source.preferredOutbound) {
|
|
throw new Error(`${path}.preferredOutbound must match sourceConfigRef value ${source.preferredOutbound}`);
|
|
}
|
|
const noProxyRaw = raw.noProxy;
|
|
if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
|
|
const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`));
|
|
return {
|
|
mode,
|
|
clientName: stringValue(raw.clientName, `${path}.clientName`),
|
|
namespace: stringValue(raw.namespace, `${path}.namespace`),
|
|
serviceName: stringValue(raw.serviceName, `${path}.serviceName`),
|
|
port,
|
|
sourceConfigRef,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourceType,
|
|
preferredOutbound,
|
|
sourceFingerprint: source?.fingerprint ?? null,
|
|
noProxy,
|
|
};
|
|
}
|
|
|
|
export function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record<string, unknown> {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60);
|
|
return {
|
|
secret,
|
|
key,
|
|
present: result.exitCode === 0 && result.stdout.trim().length > 0,
|
|
valueBytesBase64: result.stdout.trim().length,
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const commandArgs = [
|
|
"hwlab",
|
|
"nodes",
|
|
options.domain,
|
|
...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
|
|
"--node",
|
|
options.node,
|
|
"--lane",
|
|
options.lane,
|
|
"--confirm",
|
|
"--timeout-seconds",
|
|
String(options.timeoutSeconds),
|
|
"--wait",
|
|
];
|
|
const command = ["bun", "scripts/cli.ts", ...commandArgs];
|
|
const job = startJob(
|
|
`hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
|
|
command,
|
|
`Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
|
|
);
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: "async-job",
|
|
reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
|
|
job,
|
|
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
|
|
waitCommand: command.join(" "),
|
|
};
|
|
}
|
|
|
|
export function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const rewritten = rewriteDelegatedNodeValue(value, scoped);
|
|
const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
|
|
return {
|
|
...result,
|
|
node: scoped.node,
|
|
commandFamily: "hwlab nodes",
|
|
};
|
|
}
|
|
|
|
export function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
|
|
if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
|
|
if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
|
|
if (typeof value !== "object" || value === null) return value;
|
|
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
|
|
}
|
|
|
|
export function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
|
|
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
return text
|
|
.replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
|
|
.replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
|
|
};
|
|
return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
|
|
}
|
|
|
|
export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
|
const [actionRaw] = args;
|
|
if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "screenshot" && actionRaw !== "observe" && actionRaw !== "sentinel" && actionRaw !== "opencode-smoke") throw new Error("web-probe usage: run|script|screenshot|observe|sentinel|opencode-smoke --node NODE --lane vNN [--url URL]");
|
|
if (actionRaw === "sentinel") return parseNodeWebProbeSentinelOptions(args.slice(1));
|
|
if (actionRaw === "observe") {
|
|
const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1));
|
|
const explicitNode = optionValue(args, "--node");
|
|
const explicitLane = optionValue(args, "--lane");
|
|
const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null;
|
|
const indexed = requestedId === null
|
|
? null
|
|
: readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null);
|
|
const node = explicitNode ?? indexed?.node;
|
|
const lane = explicitLane ?? indexed?.lane;
|
|
if (node === undefined || lane === undefined) {
|
|
if (requestedId !== null) {
|
|
throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`);
|
|
}
|
|
throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id");
|
|
}
|
|
assertNodeId(node);
|
|
assertLane(lane);
|
|
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
return parseNodeWebProbeObserveOptions(normalized.args, node, lane, spec, normalized.id, indexed);
|
|
}
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const lane = requiredOption(args, "--lane");
|
|
assertLane(lane);
|
|
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
if (actionRaw === "screenshot") {
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--path",
|
|
"--viewport",
|
|
"--local-dir",
|
|
"--name",
|
|
"--timeout-ms",
|
|
"--wait-until",
|
|
"--selector",
|
|
"--wait-timeout-ms",
|
|
"--command-timeout-seconds",
|
|
]), new Set([
|
|
"--full-page",
|
|
"--no-full-page",
|
|
"--keep-remote",
|
|
]));
|
|
const url = optionValue(args, "--url");
|
|
const targetPath = optionValue(args, "--path") ?? null;
|
|
if (url !== undefined && targetPath !== null) throw new Error("web-probe screenshot accepts --url or --path, not both");
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000);
|
|
const waitTimeoutMs = positiveIntegerOption(args, "--wait-timeout-ms", Math.max(90000, timeoutMs + 30000), 600000);
|
|
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.ceil(waitTimeoutMs / 1000) + 45, 900);
|
|
return {
|
|
action: "screenshot",
|
|
node,
|
|
lane,
|
|
url: url ?? resolveWebProbeScreenshotUrl(spec, targetPath ?? "/"),
|
|
path: targetPath,
|
|
viewport: parseWebProbeScreenshotViewport(optionValue(args, "--viewport") ?? "1440x900"),
|
|
localDir: optionValue(args, "--local-dir") ?? "/tmp",
|
|
name: parseWebProbeScreenshotName(optionValue(args, "--name") ?? `web-probe-${node.toLowerCase()}-${lane}.png`),
|
|
timeoutMs,
|
|
waitUntil: parseWebProbeScreenshotWaitUntil(optionValue(args, "--wait-until") ?? "networkidle"),
|
|
fullPage: !args.includes("--no-full-page"),
|
|
selector: optionValue(args, "--selector") ?? null,
|
|
keepRemote: args.includes("--keep-remote"),
|
|
waitTimeoutMs,
|
|
commandTimeoutSeconds,
|
|
};
|
|
}
|
|
if (actionRaw === "script") {
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--timeout-ms",
|
|
"--viewport",
|
|
"--browser-proxy-mode",
|
|
"--script-file",
|
|
"--command-timeout-seconds",
|
|
]), new Set([]));
|
|
const scriptFile = optionValue(args, "--script-file");
|
|
if (scriptFile === undefined && process.stdin.isTTY) {
|
|
throw new Error("web-probe script requires a stdin heredoc or --script-file <path>");
|
|
}
|
|
const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8");
|
|
if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script");
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
|
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600);
|
|
return {
|
|
action: "script",
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
|
|
timeoutMs,
|
|
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
|
browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode),
|
|
commandTimeoutSeconds,
|
|
scriptText,
|
|
scriptSource: {
|
|
kind: scriptFile === undefined ? "stdin" : "file",
|
|
path: scriptFile ?? null,
|
|
byteCount: Buffer.byteLength(scriptText),
|
|
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
|
},
|
|
};
|
|
}
|
|
if (actionRaw === "opencode-smoke") {
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--path",
|
|
"--message",
|
|
"--expect-text",
|
|
"--timeout-ms",
|
|
"--response-timeout-ms",
|
|
"--viewport",
|
|
"--browser-proxy-mode",
|
|
"--command-timeout-seconds",
|
|
]), new Set([]));
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 40000, 300000);
|
|
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600);
|
|
const requestedResponseTimeoutMs = positiveIntegerOption(args, "--response-timeout-ms", Math.min(30000, timeoutMs), timeoutMs);
|
|
const responseTimeoutMs = Math.min(requestedResponseTimeoutMs, Math.max(5000, (commandTimeoutSeconds * 1000) - 20000));
|
|
const path = optionValue(args, "--path") ?? "/opencode";
|
|
if (!path.startsWith("/") || path.includes("\0") || path.length > 200) throw new Error("web-probe opencode-smoke --path must be an absolute path up to 200 chars");
|
|
const message = optionValue(args, "--message") ?? "hi";
|
|
if (message.includes("\0") || message.length > 500) throw new Error("web-probe opencode-smoke --message must be 0-500 non-NUL chars");
|
|
const expectText = optionValue(args, "--expect-text") ?? null;
|
|
if (expectText !== null && (expectText.includes("\0") || expectText.length > 500)) throw new Error("web-probe opencode-smoke --expect-text must be 0-500 non-NUL chars");
|
|
const scriptText = nodeWebProbeOpencodeSmokeScript({
|
|
path,
|
|
message,
|
|
expectText,
|
|
responseTimeoutMs,
|
|
});
|
|
const commandLabel = `web-probe opencode-smoke --node ${node} --lane ${lane}`;
|
|
return {
|
|
action: "script",
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
|
|
timeoutMs,
|
|
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
|
browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode),
|
|
commandTimeoutSeconds,
|
|
scriptText,
|
|
commandLabel,
|
|
suppressAdHocWarning: true,
|
|
generatedHints: [
|
|
"OpenCode smoke is a repo-owned typed web-probe command; it uses the same managed auth, remote browser, report recovery and artifact contract as web-probe script.",
|
|
"Passing means the browser DOM reached assistant text and the OpenCode EventSource produced update, finish and idle evidence.",
|
|
],
|
|
generatedPreferredCommands: {
|
|
withExpectedText: `${commandLabel} --expect-text <assistant-substring>`,
|
|
},
|
|
scriptSource: {
|
|
kind: "generated",
|
|
path: "builtin:opencode-smoke",
|
|
byteCount: Buffer.byteLength(scriptText),
|
|
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
|
},
|
|
};
|
|
}
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--timeout-ms",
|
|
"--wait-after-submit-ms",
|
|
"--wait-messages-ms",
|
|
"--wait-agent-terminal-ms",
|
|
"--trace-sample-count",
|
|
"--trace-sample-interval-ms",
|
|
"--message",
|
|
"--conversation-id",
|
|
"--command-timeout-seconds",
|
|
]), new Set([
|
|
"--fresh-session",
|
|
"--no-cancel-running",
|
|
]));
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
|
const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000);
|
|
const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000);
|
|
const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000);
|
|
const message = optionValue(args, "--message") ?? null;
|
|
const hasMessage = message !== null;
|
|
const traceSampleCountDefault = hasMessage ? 2 : 0;
|
|
const traceSampleIntervalDefault = hasMessage ? 1500 : 0;
|
|
const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200);
|
|
const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000);
|
|
const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({
|
|
timeoutMs,
|
|
waitAfterSubmitMs,
|
|
waitMessagesMs,
|
|
waitAgentTerminalMs,
|
|
traceSampleCount,
|
|
traceSampleIntervalMs,
|
|
freshSession: args.includes("--fresh-session"),
|
|
hasMessage,
|
|
});
|
|
const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds");
|
|
const commandTimeoutUserProvided = commandTimeoutRaw !== undefined;
|
|
const commandTimeoutSeconds = commandTimeoutUserProvided
|
|
? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds)
|
|
: commandTimeoutAutoSeconds;
|
|
return {
|
|
action: "run",
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
|
|
timeoutMs,
|
|
waitAfterSubmitMs,
|
|
waitMessagesMs,
|
|
waitAgentTerminalMs,
|
|
traceSampleCount,
|
|
traceSampleIntervalMs,
|
|
message,
|
|
conversationId: optionValue(args, "--conversation-id") ?? null,
|
|
freshSession: args.includes("--fresh-session"),
|
|
cancelRunning: !args.includes("--no-cancel-running"),
|
|
commandTimeoutSeconds,
|
|
commandTimeoutAutoSeconds,
|
|
commandTimeoutUserProvided,
|
|
};
|
|
}
|
|
|
|
function nodeWebProbeOpencodeSmokeScript(input: { path: string; message: string; expectText: string | null; responseTimeoutMs: number }): string {
|
|
const config = {
|
|
path: input.path,
|
|
message: input.message,
|
|
expectText: input.expectText,
|
|
responseTimeoutMs: input.responseTimeoutMs,
|
|
};
|
|
return patchOpenCodeSmokeScript(`const config = ${JSON.stringify(config)};\n\nexport default async function opencodeSmoke({ page, baseUrl, recordStep, wait, screenshot, jsonArtifact }) {\n await page.addInitScript(() => {\n const key = \"__unideskOpenCodeEvents\";\n window[key] = Array.isArray(window[key]) ? window[key] : [];\n if (window.__unideskOpenCodeEventSourceWrapped) return;\n window.__unideskOpenCodeEventSourceWrapped = true;\n const OriginalEventSource = window.EventSource;\n if (!OriginalEventSource) return;\n const push = (source, type, event) => {\n let parsedType = null;\n let parsedStatus = null;\n let dataBytes = 0;\n const rawData = typeof event?.data === \"string\" ? event.data : \"\";\n dataBytes = rawData.length;\n if (rawData) {\n try {\n const parsed = JSON.parse(rawData);\n parsedType = typeof parsed?.type === \"string\" ? parsed.type : typeof parsed?.event === \"string\" ? parsed.event : typeof parsed?.name === \"string\" ? parsed.name : null;\n parsedStatus = typeof parsed?.status === \"string\" ? parsed.status : typeof parsed?.part?.status === \"string\" ? parsed.part.status : null;\n } catch {\n parsedType = null;\n }\n }\n const label = parsedType || (parsedStatus ? type + \":\" + parsedStatus : type);\n window[key].push({ at: Date.now(), source, type, parsedType, status: parsedStatus, label, dataBytes });\n if (window[key].length > 240) window[key].splice(0, window[key].length - 240);\n };\n const originalAddEventListener = OriginalEventSource.prototype.addEventListener;\n OriginalEventSource.prototype.addEventListener = function wrappedAddEventListener(type, listener, options) {\n if (typeof listener !== \"function\") return originalAddEventListener.call(this, type, listener, options);\n const wrapped = function wrappedOpenCodeEvent(event) {\n push(String(this?.url || \"eventsource\"), String(type || \"message\"), event);\n return listener.call(this, event);\n };\n return originalAddEventListener.call(this, type, wrapped, options);\n };\n function WrappedEventSource(url, options) {\n const instance = new OriginalEventSource(url, options);\n originalAddEventListener.call(instance, \"message\", (event) => push(String(url || \"eventsource\"), \"message\", event));\n return instance;\n }\n WrappedEventSource.prototype = OriginalEventSource.prototype;\n WrappedEventSource.CONNECTING = OriginalEventSource.CONNECTING;\n WrappedEventSource.OPEN = OriginalEventSource.OPEN;\n WrappedEventSource.CLOSED = OriginalEventSource.CLOSED;\n window.EventSource = WrappedEventSource;\n });\n\n const targetUrl = new URL(config.path, baseUrl).toString();\n await page.goto(targetUrl, { waitUntil: \"domcontentloaded\", timeout: Math.min(Math.max(config.responseTimeoutMs, 1000), 120000) });\n recordStep(\"opencode-navigate\", { ok: true, targetPath: config.path, finalUrl: page.url() });\n\n const frame = await findOpenCodeFrame(page, config.responseTimeoutMs);\n const before = await collectDomState(frame, [], config);\n const beforeLines = before.lines;\n recordStep(\"opencode-frame\", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit });\n\n const prompt = await fillOpenCodePrompt(frame, config.message);\n recordStep(\"opencode-prompt-filled\", { ok: true, selector: prompt.selector, textBytes: config.message.length });\n\n const submit = frame.locator(\"[data-action='prompt-submit']\").first();\n await submit.waitFor({ state: \"visible\", timeout: Math.min(config.responseTimeoutMs, 10000) });\n await frame.waitForFunction(() => {\n const button = document.querySelector(\"[data-action='prompt-submit']\");\n return Boolean(button && !button.disabled && button.getAttribute(\"aria-disabled\") !== \"true\");\n }, null, { timeout: 5000 }).catch(() => null);\n await submit.click({ timeout: 10000 });\n recordStep(\"opencode-submit\", { ok: true, selector: \"[data-action='prompt-submit']\" });\n\n const final = await waitForOpenCodeFinal(page, frame, beforeLines, config);\n await jsonArtifact(\"opencode-smoke-events.json\", final.events);\n const finalScreenshot = await screenshot(\"opencode-smoke-final.png\").catch((error) => ({ error: error instanceof Error ? error.message : String(error) }));\n const ok = final.conditions.assistantText && final.conditions.notThinking && final.conditions.updateEvent && final.conditions.finishEvent && final.conditions.idleEvent && final.conditions.expectedText;\n recordStep(\"opencode-final\", { ok, conditions: final.conditions, assistantTextPreview: final.assistantTextPreview, eventTypes: final.events.eventTypes });\n return {\n ok,\n status: ok ? \"pass\" : \"blocked\",\n failedCondition: ok ? null : failedCondition(final.conditions),\n frameUrl: final.url,\n finalUrl: page.url(),\n assistantTextPreview: final.assistantTextPreview,\n bodyPreview: final.bodyPreview,\n eventTypes: final.events.eventTypes,\n eventCount: final.events.eventCount,\n conditions: final.conditions,\n screenshot: finalScreenshot,\n issueEvidence: {\n frameUrl: final.url,\n assistantTextPreview: final.assistantTextPreview,\n eventTypes: final.events.eventTypes,\n conditions: final.conditions,\n screenshotSha256: typeof finalScreenshot?.sha256 === \"string\" ? finalScreenshot.sha256 : null,\n valuesRedacted: true,\n },\n valuesRedacted: true,\n };\n}\n\nasync function findOpenCodeFrame(page, timeoutMs) {\n const deadline = Date.now() + timeoutMs;\n let best = null;\n while (Date.now() < deadline) {\n for (const frame of page.frames()) {\n const summary = await frameSummary(frame);\n const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0);\n if (best === null || score > best.score) best = { frame, summary, score };\n if (summary.hasSubmit && summary.textboxCount > 0) return frame;\n }\n await waitFor(500);\n }\n throw new Error(\"OpenCode frame with prompt submit button was not found; best=\" + JSON.stringify(best?.summary || null));\n}\n\nasync function frameSummary(frame) {\n return await frame.evaluate(() => ({\n url: location.href,\n title: document.title,\n hasSubmit: Boolean(document.querySelector(\"[data-action='prompt-submit']\")),\n textboxCount: document.querySelectorAll(\"[contenteditable='true'], textarea, [role='textbox'], input:not([type]), input[type='text']\").length,\n bodyPreview: (document.body?.innerText || \"\").replace(/\\s+/gu, \" \").trim().slice(0, 240),\n })).catch((error) => ({ url: frame.url(), title: \"\", hasSubmit: false, textboxCount: 0, bodyPreview: String(error).slice(0, 160) }));\n}\n\nasync function fillOpenCodePrompt(frame, message) {\n const selectors = [\n \"[contenteditable='true']\",\n \"textarea\",\n \"[role='textbox']\",\n \"input:not([type])\",\n \"input[type='text']\",\n ];\n let lastError = null;\n for (const selector of selectors) {\n const locator = frame.locator(selector).last();\n const count = await locator.count().catch(() => 0);\n if (count < 1) continue;\n try {\n await locator.click({ timeout: 5000 });\n await locator.fill(message, { timeout: 5000 });\n return { selector };\n } catch (error) {\n lastError = error;\n try {\n await locator.evaluate((element, text) => {\n if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {\n element.value = text;\n element.dispatchEvent(new InputEvent(\"input\", { bubbles: true, data: text, inputType: \"insertText\" }));\n element.dispatchEvent(new Event(\"change\", { bubbles: true }));\n return;\n }\n element.textContent = text;\n element.dispatchEvent(new InputEvent(\"input\", { bubbles: true, data: text, inputType: \"insertText\" }));\n }, message);\n return { selector, fallback: \"dom-input-event\" };\n } catch (fallbackError) {\n lastError = fallbackError;\n }\n }\n }\n throw new Error(\"OpenCode prompt input was not fillable: \" + (lastError instanceof Error ? lastError.message : String(lastError || \"not found\")));\n}\n\nasync function waitForOpenCodeFinal(page, frame, beforeLines, config) {\n const deadline = Date.now() + config.responseTimeoutMs;\n let latest = await collectState(page, frame, beforeLines, config);\n while (Date.now() < deadline) {\n latest = await collectState(page, frame, beforeLines, config);\n if (latest.conditions.assistantText && latest.conditions.notThinking && latest.conditions.updateEvent && latest.conditions.finishEvent && latest.conditions.idleEvent && latest.conditions.expectedText) return latest;\n await waitFor(750);\n }\n return latest;\n}\n\nasync function collectState(page, frame, beforeLines, config) {\n const dom = await collectDomState(frame, beforeLines, config);\n const events = await collectEventSummary(page);\n const labels = events.eventTypes.join(\"\\n\");\n const conditions = {\n assistantText: Boolean(dom.assistantTextPreview),\n notThinking: dom.thinkingVisible !== true,\n updateEvent: /message[._-]?part[._-]?updated|part[._-]?updated|delta|updated/iu.test(labels),\n finishEvent: /step[._-]?finish|finish|completed|done/iu.test(labels),\n idleEvent: /session[._-]?idle|session[._-]?status[:/._-]?idle|\\bidle\\b/iu.test(labels),\n expectedText: config.expectText === null || dom.expectedTextPresent === true,\n };\n return { ...dom, events, conditions };\n}\n\nasync function collectDomState(frame, beforeLines, config) {\n return await frame.evaluate(({ beforeLines, message, expectText }) => {\n const normalize = (value) => String(value || \"\").replace(/\\r/g, \"\").replace(/[ \\t]+/g, \" \").trim();\n const bodyText = normalize(document.body?.innerText || \"\");\n const lines = bodyText.split(\"\\n\").map((line) => normalize(line)).filter(Boolean).slice(-400);\n const beforeSet = new Set(Array.isArray(beforeLines) ? beforeLines : []);\n const prompt = normalize(message).toLowerCase();\n const blocked = /^(send|stop|cancel|thinking|loading|new session|settings)$/iu;\n const selectorTexts = Array.from(document.querySelectorAll(\"[data-role='assistant'], [data-message-role='assistant'], [data-testid*='assistant' i], [aria-label*='assistant' i], [class*='assistant' i], main article, main [role='article'], main [data-message-id]\")).map((element) => normalize(element.textContent || \"\")).filter((text) => text.length > 0 && text.toLowerCase() !== prompt && !blocked.test(text)).slice(-20);\n const newLines = lines.filter((line) => !beforeSet.has(line) && line.toLowerCase() !== prompt && !blocked.test(line) && !/^you\\b/iu.test(line)).slice(-40);\n const assistantText = selectorTexts.find((text) => text.length > 0) || newLines.find((line) => line.length > 0) || null;\n return {\n url: location.href,\n title: document.title,\n hasSubmit: Boolean(document.querySelector(\"[data-action='prompt-submit']\")),\n lines,\n lineCount: lines.length,\n newLineCount: newLines.length,\n assistantCandidateCount: selectorTexts.length,\n assistantTextPreview: assistantText === null ? null : assistantText.slice(0, 500),\n thinkingVisible: /(^|\\n|\\b)Thinking(\\b|\\n)/u.test(bodyText),\n expectedTextPresent: expectText === null ? true : bodyText.includes(expectText),\n bodyPreview: bodyText.slice(-1200),\n };\n }, { beforeLines, message: config.message, expectText: config.expectText });\n}\n\nasync function collectEventSummary(page) {\n const frameEvents = [];\n for (const frame of page.frames()) {\n const events = await frame.evaluate(() => Array.isArray(window.__unideskOpenCodeEvents) ? window.__unideskOpenCodeEvents.slice(-120) : []).catch(() => []);\n if (events.length > 0) frameEvents.push({ frameUrl: frame.url(), events });\n }\n const labels = [];\n for (const item of frameEvents) {\n for (const event of item.events) {\n const raw = [event.label, event.parsedType, event.status, event.type].filter(Boolean).join(\":\");\n if (raw && !labels.includes(raw)) labels.push(raw);\n }\n }\n return {\n eventCount: frameEvents.reduce((sum, item) => sum + item.events.length, 0),\n eventTypes: labels.slice(0, 40),\n frames: frameEvents.map((item) => ({ frameUrl: item.frameUrl, count: item.events.length, last: item.events.slice(-5) })),\n valuesRedacted: true,\n };\n}\n\nfunction failedCondition(conditions) {\n return Object.entries(conditions).filter(([, ok]) => ok !== true).map(([key]) => key).join(\",\") || \"unknown\";\n}\n\nfunction waitFor(ms) {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n`);
|
|
}
|
|
|
|
function patchOpenCodeSmokeScript(source: string): string {
|
|
const oldFlow = ` const frame = await findOpenCodeFrame(page, config.responseTimeoutMs);
|
|
const before = await collectDomState(frame, [], config);
|
|
const beforeLines = before.lines;
|
|
recordStep("opencode-frame", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit });
|
|
|
|
const prompt = await fillOpenCodePrompt(frame, config.message);`;
|
|
const newFlow = ` const frame = await findOpenCodeFrame(page, config.responseTimeoutMs);
|
|
const project = await ensureOpenCodeProjectOpen(frame, config.responseTimeoutMs);
|
|
recordStep("opencode-project", project);
|
|
const composer = await waitForOpenCodeComposer(frame, config.responseTimeoutMs);
|
|
recordStep("opencode-composer", composer);
|
|
const eventProbe = await startOpenCodeEventProbe(frame);
|
|
recordStep("opencode-event-probe", eventProbe);
|
|
const before = await collectDomState(frame, [], config);
|
|
const beforeLines = before.lines;
|
|
recordStep("opencode-frame", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit });
|
|
|
|
const prompt = await fillOpenCodePrompt(frame, config.message);`;
|
|
const oldFrameFinder = `async function findOpenCodeFrame(page, timeoutMs) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let best = null;
|
|
while (Date.now() < deadline) {
|
|
for (const frame of page.frames()) {
|
|
const summary = await frameSummary(frame);
|
|
const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0);
|
|
if (best === null || score > best.score) best = { frame, summary, score };
|
|
if (summary.hasSubmit && summary.textboxCount > 0) return frame;
|
|
}
|
|
await waitFor(500);
|
|
}
|
|
throw new Error("OpenCode frame with prompt submit button was not found; best=" + JSON.stringify(best?.summary || null));
|
|
}`;
|
|
const newFrameFinder = `async function findOpenCodeFrame(page, timeoutMs) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let best = null;
|
|
while (Date.now() < deadline) {
|
|
for (const frame of page.frames()) {
|
|
const summary = await frameSummary(frame);
|
|
const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0) + (isOpenCodeFrame(frame, summary) ? 50 : 0);
|
|
if (best === null || score > best.score) best = { frame, summary, score };
|
|
if (summary.hasSubmit && summary.textboxCount > 0) return frame;
|
|
if (isOpenCodeFrame(frame, summary)) return frame;
|
|
}
|
|
await waitFor(500);
|
|
}
|
|
throw new Error("OpenCode frame was not found; best=" + JSON.stringify(best?.summary || null));
|
|
}
|
|
|
|
function isOpenCodeFrame(frame, summary) {
|
|
const rawUrl = String(summary?.url || frame.url() || "");
|
|
let hostname = "";
|
|
try {
|
|
hostname = new URL(rawUrl).hostname;
|
|
} catch {
|
|
hostname = "";
|
|
}
|
|
const hasParent = typeof frame.parentFrame === "function" && frame.parentFrame() !== null;
|
|
if (hasParent && (/opencode/iu.test(rawUrl) || String(summary?.title || "") === "OpenCode")) return true;
|
|
return /^opencode[-.]/iu.test(hostname);
|
|
}
|
|
|
|
async function ensureOpenCodeProjectOpen(frame, timeoutMs) {
|
|
const deadline = Date.now() + Math.min(timeoutMs, 20000);
|
|
let clickedProject = null;
|
|
let dismissed = [];
|
|
let lastSummary = null;
|
|
while (Date.now() < deadline) {
|
|
lastSummary = await frameSummary(frame);
|
|
if (lastSummary.hasSubmit && lastSummary.textboxCount > 0) {
|
|
return { ok: true, alreadyOpen: true, frameUrl: lastSummary.url, dismissed, clickedProject };
|
|
}
|
|
const dismissedButton = await clickButtonMatching(frame, (text) => /^(not yet|skip|maybe later)$/iu.test(text), "dismiss-provider");
|
|
if (dismissedButton) {
|
|
dismissed.push(dismissedButton.text);
|
|
await waitFor(500);
|
|
continue;
|
|
}
|
|
if (/No projects open|Recent projects|Open a project/iu.test(String(lastSummary.bodyPreview || ""))) {
|
|
const clicked = await clickButtonMatching(frame, (text) => text === "/" || /^\\/\\s*\\d/iu.test(text) || /^\\/.*ago$/iu.test(text), "recent-project");
|
|
if (clicked) {
|
|
clickedProject = clicked.text;
|
|
await waitFor(1500);
|
|
return { ok: true, alreadyOpen: false, frameUrl: lastSummary.url, dismissed, clickedProject };
|
|
}
|
|
}
|
|
await waitFor(500);
|
|
}
|
|
return { ok: false, alreadyOpen: false, frameUrl: lastSummary?.url ?? frame.url(), dismissed, clickedProject, bodyPreview: lastSummary?.bodyPreview ?? null };
|
|
}
|
|
|
|
async function waitForOpenCodeComposer(frame, timeoutMs) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastSummary = null;
|
|
while (Date.now() < deadline) {
|
|
lastSummary = await frameSummary(frame);
|
|
if (lastSummary.hasSubmit && lastSummary.textboxCount > 0) {
|
|
return { ok: true, frameUrl: lastSummary.url, textboxCount: lastSummary.textboxCount, hasSubmit: true };
|
|
}
|
|
await waitFor(750);
|
|
}
|
|
throw new Error("OpenCode composer was not ready; last=" + JSON.stringify(lastSummary));
|
|
}
|
|
|
|
async function startOpenCodeEventProbe(frame) {
|
|
return await frame.evaluate(() => {
|
|
const key = "__unideskOpenCodeEvents";
|
|
window[key] = Array.isArray(window[key]) ? window[key] : [];
|
|
if (window.__unideskManualOpenCodeEventSourceStarted) {
|
|
return { ok: true, reused: true, eventCount: window[key].length };
|
|
}
|
|
window.__unideskManualOpenCodeEventSourceStarted = true;
|
|
const eventUrl = new URL("/global/event", location.origin).toString();
|
|
const source = new EventSource(eventUrl);
|
|
window.__unideskManualOpenCodeEventSource = source;
|
|
const push = (type, event) => {
|
|
const rawData = typeof event?.data === "string" ? event.data : "";
|
|
let parsedType = null;
|
|
let status = null;
|
|
if (rawData) {
|
|
try {
|
|
const parsed = JSON.parse(rawData);
|
|
parsedType = typeof parsed?.type === "string" ? parsed.type : typeof parsed?.event === "string" ? parsed.event : typeof parsed?.name === "string" ? parsed.name : null;
|
|
status = typeof parsed?.status === "string" ? parsed.status : typeof parsed?.part?.status === "string" ? parsed.part.status : null;
|
|
} catch {
|
|
parsedType = null;
|
|
}
|
|
}
|
|
const label = parsedType || (status ? String(type) + ":" + status : String(type));
|
|
window[key].push({ at: Date.now(), source: eventUrl, type: String(type), parsedType, status, label, dataBytes: rawData.length, manualProbe: true });
|
|
if (window[key].length > 240) window[key].splice(0, window[key].length - 240);
|
|
};
|
|
const eventTypes = ["message", "message.part.updated", "message.part.added", "message.updated", "step-finish", "step.finish", "session.idle", "session.status", "session.updated", "session.error", "error"];
|
|
for (const type of eventTypes) source.addEventListener(type, (event) => push(type, event));
|
|
source.addEventListener("open", () => push("open", { data: "" }));
|
|
return { ok: true, reused: false, eventUrl, listenedTypes: eventTypes };
|
|
});
|
|
}
|
|
|
|
async function clickButtonMatching(frame, predicate, label) {
|
|
const buttons = frame.locator("button,[role='button']");
|
|
const count = Math.min(await buttons.count().catch(() => 0), 80);
|
|
for (let index = 0; index < count; index += 1) {
|
|
const button = buttons.nth(index);
|
|
const visible = await button.isVisible().catch(() => false);
|
|
if (!visible) continue;
|
|
const text = normalizeButtonText(await button.textContent().catch(() => ""));
|
|
if (!predicate(text)) continue;
|
|
await button.click({ timeout: 5000 });
|
|
return { ok: true, label, index, text };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function normalizeButtonText(value) {
|
|
return String(value || "").replace(/\\s+/gu, " ").trim();
|
|
}`;
|
|
const oldBlocked = "const blocked = /^(send|stop|cancel|thinking|loading|new session|settings)$/iu;";
|
|
const newBlocked = "const blocked = /^(send|stop|cancel|thinking|loading|new session|settings|build|shell|review|last turn changes|create git repository|all files|main branch|deepseek-v4-flash)$/iu;";
|
|
const withFlow = source.replace(oldFlow, newFlow);
|
|
const withFrameFinder = withFlow.replace(oldFrameFinder, newFrameFinder);
|
|
const patched = withFrameFinder.replace(oldBlocked, newBlocked);
|
|
if (withFlow === source) throw new Error("internal opencode-smoke generated script patch did not apply: flow");
|
|
if (withFrameFinder === withFlow) throw new Error("internal opencode-smoke generated script patch did not apply: frame finder");
|
|
if (patched === withFrameFinder) throw new Error("internal opencode-smoke generated script patch did not apply: assistant text filter");
|
|
if (!patched.includes("ensureOpenCodeProjectOpen") || !patched.includes("startOpenCodeEventProbe")) throw new Error("internal opencode-smoke generated script patch is missing required helpers");
|
|
return patched;
|
|
}
|
|
|
|
function resolveWebProbeScreenshotUrl(spec: HwlabRuntimeLaneSpec, targetPath: string): string {
|
|
const origin = nodeWebProbeDefaultUrl(spec);
|
|
try {
|
|
return new URL(targetPath || "/", origin).toString();
|
|
} catch {
|
|
throw new Error(`web-probe screenshot --path cannot be resolved against ${origin}: ${targetPath}`);
|
|
}
|
|
}
|
|
|
|
function parseWebProbeScreenshotViewport(value: string): string {
|
|
if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(value)) throw new Error(`web-probe screenshot --viewport must look like 1440x900, got ${value}`);
|
|
const [widthRaw, heightRaw] = value.split("x");
|
|
const width = Number(widthRaw);
|
|
const height = Number(heightRaw);
|
|
if (width < 240 || width > 7680 || height < 240 || height > 4320) throw new Error(`web-probe screenshot --viewport out of range: ${value}`);
|
|
return value;
|
|
}
|
|
|
|
function parseWebProbeScreenshotName(value: string): string {
|
|
if (!/^[A-Za-z0-9._-]{1,120}$/u.test(value)) throw new Error("web-probe screenshot --name must contain only letters, numbers, dot, underscore, or dash, max 120 chars");
|
|
return value.endsWith(".png") ? value : `${value}.png`;
|
|
}
|
|
|
|
function parseWebProbeScreenshotWaitUntil(value: string): "load" | "domcontentloaded" | "networkidle" | "commit" {
|
|
if (value === "load" || value === "domcontentloaded" || value === "networkidle" || value === "commit") return value;
|
|
throw new Error(`web-probe screenshot --wait-until must be load, domcontentloaded, networkidle, or commit; got ${value}`);
|
|
}
|