Lyon
1426 lines
71 KiB
TypeScript
1426 lines
71 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-probe module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:6032-7426 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { BootstrapAdminPasswordMaterial, BootstrapAdminSecretMaterial, NodeRuntimeGitMirrorEgressProxySpec, NodeRuntimeGitMirrorGithubTransportSpec, NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult, NodeWebProbeOptions, RuntimeSecretSpec } from "./entry";
|
|
import { isCommandSuccess, nodeRuntimeGitopsRoot, runNodeK3sArgs, runNodeK3sScript } from "./cleanup";
|
|
import { nodeRuntimeCicdWaitWarning } from "./git-mirror";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { nodeRuntimePipelineFailureSummary, nodeRuntimePipelineRunDiagnostics } from "./render";
|
|
import { compactRuntimeCommand, runNodeHostScript } from "./runtime-common";
|
|
import { assertLane, assertNodeId, keyValueLinesFromText, numericField, optionValue, optionalStringValue, positiveIntegerOption, positiveIntegerValue, record, requiredOption, shellQuote, statusText, stringValue, stripOptions } from "./utils";
|
|
import { discoverWebObserveIndexEntry, readWebObserveIndexEntry } from "./web-observe-render";
|
|
import { assertKnownOptions, nodeWebProbeAutoCommandTimeoutSeconds, nodeWebProbeDefaultUrl, normalizeNodeWebProbeObserveArgs, parseNodeWebProbeObserveOptions, parseNodeWebProbeSentinelOptions, parseWebProbeBrowserProxyMode } from "./web-probe-observe";
|
|
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
|
|
|
|
export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
|
|
const gitMirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
|
|
const renderGitMirror = {
|
|
...gitMirror,
|
|
egressProxy: gitMirror.egressProxy.mode === "direct" ? {
|
|
mode: "direct",
|
|
required: false,
|
|
} : {
|
|
...gitMirror.egressProxy,
|
|
mode: "node-global",
|
|
required: true,
|
|
},
|
|
};
|
|
return {
|
|
nodeId: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceBranch: spec.sourceBranch,
|
|
gitopsBranch: spec.gitopsBranch,
|
|
gitopsRoot: nodeRuntimeGitopsRoot(spec),
|
|
runtimePath: spec.runtimePath,
|
|
runtimeRenderDir: spec.runtimeRenderDir,
|
|
runtimeNamespace: spec.runtimeNamespace,
|
|
catalogPath: spec.catalogPath,
|
|
tektonDir: spec.tektonDir,
|
|
argoApplicationFile: spec.argoApplicationFile,
|
|
argoRepoUrl: spec.argoRepoUrl,
|
|
gitUrl: spec.gitUrl,
|
|
gitReadUrl: spec.gitReadUrl,
|
|
gitWriteUrl: spec.gitWriteUrl,
|
|
toolsImage: gitMirror.toolsImage,
|
|
toolsImagePullPolicy: gitMirror.toolsImagePullPolicy,
|
|
gitMirror: renderGitMirror,
|
|
networkProfileId: spec.networkProfileId,
|
|
downloadProfileId: spec.downloadProfileId,
|
|
gitSshProxyHost: gitSshProxy?.host,
|
|
gitSshProxyPort: gitSshProxy?.port,
|
|
proxyHttp: spec.networkProfile.proxy.http,
|
|
proxyHttps: spec.networkProfile.proxy.https,
|
|
proxyAll: spec.networkProfile.proxy.all,
|
|
noProxy: spec.networkProfile.proxy.noProxy.join(","),
|
|
dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http,
|
|
dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https,
|
|
dockerProxyAll: spec.networkProfile.dockerBuildProxy.all,
|
|
dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
|
|
dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy,
|
|
npmRegistry: spec.downloadProfile.npm.registry,
|
|
npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000,
|
|
npmRetries: spec.downloadProfile.npm.retries,
|
|
stepEnv: spec.stepEnv,
|
|
observability: spec.observability,
|
|
runtimeStore: spec.runtimeStore,
|
|
runtimeImageRewrites: spec.runtimeImageRewrites,
|
|
registryPrefix: spec.registryPrefix,
|
|
buildkitSidecarImage: spec.buildkit?.sidecarImage,
|
|
publicWebUrl: spec.publicWebUrl,
|
|
publicApiUrl: spec.publicApiUrl,
|
|
publicExposure: spec.publicExposure === null ? undefined : {
|
|
enabled: spec.publicExposure.enabled,
|
|
mode: spec.publicExposure.mode,
|
|
publicBaseUrl: spec.publicExposure.publicBaseUrl,
|
|
hostname: spec.publicExposure.hostname,
|
|
expectedA: spec.publicExposure.expectedA,
|
|
serverAddr: spec.publicExposure.serverAddr,
|
|
serverPort: spec.publicExposure.serverPort,
|
|
secretName: spec.publicExposure.secretName,
|
|
secretKey: spec.publicExposure.secretKey,
|
|
tokenKey: spec.publicExposure.tokenKey,
|
|
webProxy: spec.publicExposure.webProxy,
|
|
apiProxy: spec.publicExposure.apiProxy,
|
|
},
|
|
externalPostgres: activeExternalPostgres === undefined ? undefined : {
|
|
enabled: true,
|
|
serviceName: activeExternalPostgres.serviceName,
|
|
endpointAddress: activeExternalPostgres.endpointAddress,
|
|
port: activeExternalPostgres.port,
|
|
runtimeAccess: activeExternalPostgres.runtimeAccess ?? null,
|
|
sslmode: activeExternalPostgres.sslmode,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function httpProxyEndpoint(value: string): { host: string; port: number } | null {
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(value);
|
|
} catch {
|
|
return null;
|
|
}
|
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10);
|
|
if (!Number.isInteger(port) || port <= 0) return null;
|
|
return { host: parsed.hostname, port };
|
|
}
|
|
|
|
export function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
|
|
return [
|
|
`${renderDir}/devops-infra/git-mirror.yaml`,
|
|
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
|
|
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
|
|
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,
|
|
`${renderDir}/argocd/project.yaml`,
|
|
`${renderDir}/argocd/${spec.argoApplicationFile}`,
|
|
];
|
|
}
|
|
|
|
export function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
|
const files = nodeRuntimeControlPlaneFiles(spec, renderDir);
|
|
const args = [
|
|
"kubectl",
|
|
"apply",
|
|
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
|
...files.flatMap((file) => ["-f", file]),
|
|
];
|
|
return runNodeK3sArgs(spec, args, timeoutSeconds);
|
|
}
|
|
|
|
export function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
|
const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir)
|
|
.map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`)
|
|
.join("");
|
|
const args = [
|
|
"kubectl",
|
|
"apply",
|
|
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
|
"-f",
|
|
"-",
|
|
];
|
|
return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest);
|
|
}
|
|
|
|
export function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult {
|
|
const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
|
const script = [
|
|
"set -eu",
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`prefix=${shellQuote(prefix)}`,
|
|
"case \"$render_dir\" in",
|
|
" \"$prefix\"*) rm -rf \"$render_dir\" ;;",
|
|
" *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;",
|
|
"esac",
|
|
].join("\n");
|
|
return runNodeHostScript(spec, script, 60);
|
|
}
|
|
|
|
export function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult {
|
|
const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
|
const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`;
|
|
if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) {
|
|
return {
|
|
command: ["rm", "-rf", "<refused>"],
|
|
cwd: repoRoot,
|
|
exitCode: 44,
|
|
stdout: "",
|
|
stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`,
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 });
|
|
}
|
|
|
|
export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
|
|
return {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "PipelineRun",
|
|
metadata: {
|
|
name: pipelineRun,
|
|
namespace: "hwlab-ci",
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/gitops-target": spec.lane,
|
|
"hwlab.pikastech.local/source-commit": sourceCommit,
|
|
"hwlab.pikastech.local/trigger": "unidesk-node-cli",
|
|
},
|
|
annotations: {
|
|
"hwlab.pikastech.local/node": spec.nodeId,
|
|
"hwlab.pikastech.local/source-branch": spec.sourceBranch,
|
|
"hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
|
|
"hwlab.pikastech.local/runtime-path": spec.runtimePath,
|
|
"hwlab.pikastech.local/network-profile": spec.networkProfileId,
|
|
"hwlab.pikastech.local/download-profile": spec.downloadProfileId,
|
|
},
|
|
},
|
|
spec: {
|
|
pipelineRef: { name: spec.pipeline },
|
|
taskRunTemplate: {
|
|
serviceAccountName: spec.serviceAccountName,
|
|
podTemplate: {
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
securityContext: { fsGroup: 1000 },
|
|
},
|
|
},
|
|
params: [
|
|
{ name: "git-url", value: spec.gitUrl },
|
|
{ name: "git-read-url", value: spec.gitReadUrl },
|
|
{ name: "git-write-url", value: spec.gitWriteUrl },
|
|
{ name: "source-branch", value: spec.sourceBranch },
|
|
{ name: "gitops-branch", value: spec.gitopsBranch },
|
|
{ name: "lane", value: spec.lane },
|
|
{ name: "catalog-path", value: spec.catalogPath },
|
|
{ name: "image-tag-mode", value: "full" },
|
|
{ name: "runtime-path", value: spec.runtimePath },
|
|
{ name: "revision", value: sourceCommit },
|
|
{ name: "registry-prefix", value: spec.registryPrefix },
|
|
{ name: "services", value: spec.serviceIds.join(",") },
|
|
{ name: "base-image", value: spec.baseImage },
|
|
],
|
|
workspaces: [
|
|
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
|
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
|
|
],
|
|
},
|
|
};
|
|
}
|
|
|
|
export function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult {
|
|
const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
`pipeline_run=${shellQuote(pipelineRun)}`,
|
|
`rerun=${rerun ? "true" : "false"}`,
|
|
"manifest_path=\"/tmp/$pipeline_run.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
"existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
|
|
"if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then",
|
|
" kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
"elif [ \"$existing_status\" = False ]; then",
|
|
" kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
"fi",
|
|
"if kubectl create -f \"$manifest_path\"; then",
|
|
" :",
|
|
"else",
|
|
" code=$?",
|
|
" if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then",
|
|
" printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2",
|
|
" else",
|
|
" exit \"$code\"",
|
|
" fi",
|
|
"fi",
|
|
"kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'",
|
|
].join("\n");
|
|
return runNodeK3sScript(spec, script, timeoutSeconds);
|
|
}
|
|
|
|
export function waitForNodeRuntimePipelineRunTerminal(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
pipelineRun: string,
|
|
timeoutSeconds: number,
|
|
options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
|
|
): Record<string, unknown> {
|
|
const severeWarningThresholdMs = 120_000;
|
|
const startedAt = Date.now();
|
|
const deadline = startedAt + timeoutSeconds * 1000;
|
|
let polls = 0;
|
|
let last: Record<string, unknown> = { exists: false, name: pipelineRun };
|
|
let lastOpportunisticPostFlushAt = 0;
|
|
let severeWarningEmitted = false;
|
|
let opportunisticPostSyncAttempted = false;
|
|
const opportunisticPostFlushes: Record<string, unknown>[] = [];
|
|
const opportunisticPostSyncs: Record<string, unknown>[] = [];
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds });
|
|
while (Date.now() <= deadline) {
|
|
polls += 1;
|
|
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const status = typeof last.status === "string" ? last.status : null;
|
|
const reason = typeof last.reason === "string" ? last.reason : null;
|
|
const elapsedMs = Date.now() - startedAt;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs });
|
|
if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") {
|
|
severeWarningEmitted = true;
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: "pipelinerun-wait",
|
|
status: "warning",
|
|
warning: "pipelinerun-wait-severe-timeout",
|
|
severity: "warning",
|
|
pipelineRun,
|
|
pipelineStatus: status,
|
|
reason,
|
|
polls,
|
|
elapsedMs,
|
|
thresholdMs: severeWarningThresholdMs,
|
|
message: "PipelineRun has been non-terminal for more than 120s; this is a severe timeout and requires further investigation of Tekton taskRuns/pods instead of treating it as normal wait time.",
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
},
|
|
});
|
|
}
|
|
if (status === "True" || status === "False") {
|
|
const ok = status === "True";
|
|
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
|
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
|
|
return {
|
|
ok,
|
|
status: ok ? "succeeded" : "failed",
|
|
pipelineRun: last,
|
|
diagnostics,
|
|
failureSummary,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
|
};
|
|
}
|
|
if (
|
|
options.opportunisticPostSync !== undefined
|
|
&& !opportunisticPostSyncAttempted
|
|
&& nodeRuntimePipelineRunSourceCloneCompleted(spec, pipelineRun)
|
|
) {
|
|
opportunisticPostSyncAttempted = true;
|
|
const sync = options.opportunisticPostSync();
|
|
if (sync !== null) opportunisticPostSyncs.push(sync);
|
|
}
|
|
if (options.opportunisticPostFlush !== undefined && Date.now() - lastOpportunisticPostFlushAt >= 15_000) {
|
|
lastOpportunisticPostFlushAt = Date.now();
|
|
const flush = options.opportunisticPostFlush();
|
|
if (flush !== null) opportunisticPostFlushes.push(flush);
|
|
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const refreshedStatus = typeof last.status === "string" ? last.status : null;
|
|
const refreshedReason = typeof last.reason === "string" ? last.reason : null;
|
|
const refreshedElapsedMs = Date.now() - startedAt;
|
|
if (refreshedStatus === "True" || refreshedStatus === "False") {
|
|
const ok = refreshedStatus === "True";
|
|
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
|
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: refreshedStatus, reason: refreshedReason, polls, elapsedMs: refreshedElapsedMs, observedAfterPostFlush: true });
|
|
return {
|
|
ok,
|
|
status: ok ? "succeeded" : "failed",
|
|
pipelineRun: last,
|
|
diagnostics,
|
|
failureSummary,
|
|
polls,
|
|
elapsedMs: refreshedElapsedMs,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
observedAfterPostFlush: true,
|
|
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
|
};
|
|
}
|
|
}
|
|
sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
|
|
}
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
|
|
return {
|
|
ok: true,
|
|
status: "pending",
|
|
pipelineRun: last,
|
|
polls,
|
|
elapsedMs,
|
|
waitLimitSeconds: timeoutSeconds,
|
|
warning,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimePipelineRunSourceCloneCompleted(spec: HwlabRuntimeLaneSpec, pipelineRun: string): boolean {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
|
if (result.exitCode !== 0) return false;
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(result.stdout);
|
|
} catch {
|
|
return false;
|
|
}
|
|
const items = Array.isArray(record(parsed).items) ? record(parsed).items as unknown[] : [];
|
|
return items.some((item) => {
|
|
const itemRecord = record(item);
|
|
const metadata = record(itemRecord.metadata);
|
|
const labels = record(metadata.labels);
|
|
const name = typeof metadata.name === "string" ? metadata.name : "";
|
|
const pipelineTask = typeof labels["tekton.dev/pipelineTask"] === "string" ? labels["tekton.dev/pipelineTask"] : "";
|
|
const taskName = `${pipelineTask} ${name}`;
|
|
if (!/(clone|checkout|local-git|source[-_ ]*worktree)/iu.test(taskName)) return false;
|
|
const status = record(itemRecord.status);
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions as unknown[] : [];
|
|
return conditions.some((condition) => {
|
|
const conditionRecord = record(condition);
|
|
return conditionRecord.type === "Succeeded" && conditionRecord.status === "True";
|
|
});
|
|
});
|
|
}
|
|
|
|
export function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
|
process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`);
|
|
}
|
|
|
|
export function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60);
|
|
if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) };
|
|
const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u);
|
|
return {
|
|
exists: true,
|
|
name: pipelineRun,
|
|
status: status || null,
|
|
reason: reason || null,
|
|
message: message || null,
|
|
createdAt: createdAt || null,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function sleepSync(ms: number): void {
|
|
const buffer = new SharedArrayBuffer(4);
|
|
Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms));
|
|
}
|
|
|
|
export function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return null;
|
|
const secretRoot = externalPostgresSecretSourceRoot(spec);
|
|
const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey);
|
|
const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey);
|
|
const missing = [
|
|
...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]),
|
|
...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]),
|
|
];
|
|
if (missing.length > 0) {
|
|
if (dryRun) {
|
|
return {
|
|
ok: true,
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
skipped: true,
|
|
reason: "external-postgres-secret-source-missing",
|
|
missing,
|
|
sourceRoot: secretRoot,
|
|
valuesPrinted: false,
|
|
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
degradedReason: "external-postgres-secret-source-missing",
|
|
missing,
|
|
sourceRoot: secretRoot,
|
|
valuesPrinted: false,
|
|
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
|
};
|
|
}
|
|
const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds);
|
|
if (!isCommandSuccess(setup)) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
phase: "namespace-authn-setup",
|
|
setup: compactRuntimeCommand(setup),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const manifest = {
|
|
apiVersion: "v1",
|
|
kind: "List",
|
|
items: [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace },
|
|
type: "Opaque",
|
|
stringData: { [pg.cloudApi.secretKey]: cloudApi.value },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace },
|
|
type: "Opaque",
|
|
stringData: { [pg.openfga.secretKey]: openfga.value },
|
|
},
|
|
],
|
|
};
|
|
const applyScript = [
|
|
"set -eu",
|
|
[
|
|
"kubectl",
|
|
"apply",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
"--field-manager=unidesk-hwlab-node-external-postgres-secret",
|
|
...(dryRun ? ["--dry-run=server"] : []),
|
|
"-f",
|
|
"-",
|
|
].map(shellQuote).join(" "),
|
|
].join("\n");
|
|
const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest));
|
|
return {
|
|
ok: isCommandSuccess(apply),
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: !dryRun && isCommandSuccess(apply),
|
|
namespace: spec.runtimeNamespace,
|
|
sourceRoot: secretRoot,
|
|
secrets: [
|
|
{ name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey },
|
|
{ name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null },
|
|
],
|
|
setup: compactRuntimeCommand(setup),
|
|
apply: compactRuntimeCommand(apply),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
const pg = spec.runtimeStore?.postgres;
|
|
if (pg?.mode !== "local-k3s") return null;
|
|
const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`;
|
|
const sourceRef = pg.adminPasswordSourceRef ?? "";
|
|
const sourceKey = pg.adminPasswordSourceKey ?? "";
|
|
if (sourceRef.length === 0 || sourceKey.length === 0) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
degradedReason: "local-postgres-secret-source-not-configured",
|
|
valuesPrinted: false,
|
|
next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` },
|
|
};
|
|
}
|
|
const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun });
|
|
if (material.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
source: material,
|
|
degradedReason: material.error ?? "local-postgres-secret-source-missing",
|
|
valuesPrinted: false,
|
|
next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=<redacted>` },
|
|
};
|
|
}
|
|
const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? "");
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const afterBytes = numericField(fields.afterPasswordBytes) ?? 0;
|
|
const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0;
|
|
return {
|
|
ok,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: !dryRun && fields.mutation === "true",
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
source: {
|
|
ok: material.ok,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath),
|
|
generated: material.generated,
|
|
fingerprint: material.fingerprint,
|
|
valueBytes: material.value?.length ?? 0,
|
|
valuesPrinted: false,
|
|
},
|
|
before: {
|
|
namespaceExists: fields.beforeNamespaceExists === "yes",
|
|
secretExists: fields.beforeSecretExists === "yes",
|
|
passwordBytes: numericField(fields.beforePasswordBytes),
|
|
fingerprint: fields.beforePasswordFingerprint || null,
|
|
},
|
|
after: {
|
|
namespaceExists: fields.afterNamespaceExists === "yes",
|
|
secretExists: fields.afterSecretExists === "yes",
|
|
passwordBytes: numericField(fields.afterPasswordBytes),
|
|
fingerprint: fields.afterPasswordFingerprint || null,
|
|
},
|
|
action: fields.action || null,
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode),
|
|
degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed",
|
|
result: compactRuntimeCommand(result),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): {
|
|
ok: boolean;
|
|
sourceRef: string;
|
|
sourceKey: string;
|
|
sourcePath: string | null;
|
|
checkedPaths: string[];
|
|
value: string | null;
|
|
fingerprint: string | null;
|
|
generated: boolean;
|
|
error?: string;
|
|
} {
|
|
const checkedPaths = localSecretSourcePaths(input.sourceRef);
|
|
const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null;
|
|
const sourcePath = existingPath ?? checkedPaths[0] ?? null;
|
|
if (sourcePath === null) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" };
|
|
}
|
|
if (existingPath === null && input.dryRun) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" };
|
|
}
|
|
if (existingPath === null) {
|
|
const value = randomBytes(32).toString("base64url");
|
|
mkdirSync(dirname(sourcePath), { recursive: true });
|
|
writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
|
}
|
|
const text = readFileSync(existingPath, "utf8");
|
|
const values = parseEnvFile(text);
|
|
const existingValue = values[input.sourceKey];
|
|
if (existingValue !== undefined && existingValue.length > 0) {
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false };
|
|
}
|
|
if (input.dryRun) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" };
|
|
}
|
|
const value = randomBytes(32).toString("base64url");
|
|
const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n";
|
|
writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
|
}
|
|
|
|
export function localSecretSourcePaths(sourceRef: string): string[] {
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
const paths = index >= 0
|
|
? [
|
|
join(repoRoot.slice(0, index), ".state", "secrets", sourceRef),
|
|
join(repoRoot, ".state", "secrets", sourceRef),
|
|
]
|
|
: [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
export function shortSecretFingerprint(value: string): string {
|
|
return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`;
|
|
}
|
|
|
|
export function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(secretName)}`,
|
|
"secret_key=POSTGRES_PASSWORD",
|
|
`source_ref=${shellQuote(sourceRef)}`,
|
|
`source_key=${shellQuote(sourceKey)}`,
|
|
`source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
"field_manager=unidesk-hwlab-node-local-postgres-secret",
|
|
"password=$(cat)",
|
|
"password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')",
|
|
"exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }",
|
|
"before_namespace_exists=$(namespace_exists_flag)",
|
|
"before_secret_exists=no",
|
|
"before_b64=",
|
|
"if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi",
|
|
"before_password_bytes=$(decoded_length \"$before_b64\")",
|
|
"before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")",
|
|
"after_namespace_exists=$before_namespace_exists",
|
|
"after_secret_exists=$before_secret_exists",
|
|
"after_password_bytes=$before_password_bytes",
|
|
"after_password_fingerprint=$before_password_fingerprint",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"namespace_create_exit=",
|
|
"apply_exit=",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi",
|
|
"elif [ \"$password_bytes\" -le 0 ]; then",
|
|
" action=source-empty",
|
|
" apply_exit=42",
|
|
"elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then",
|
|
" action=source-mismatch",
|
|
" apply_exit=47",
|
|
"else",
|
|
" if [ \"$before_namespace_exists\" != yes ]; then",
|
|
" kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err",
|
|
" namespace_create_exit=$?",
|
|
" else",
|
|
" namespace_create_exit=0",
|
|
" fi",
|
|
" if [ \"$namespace_create_exit\" = 0 ]; then",
|
|
" tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)",
|
|
" password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')",
|
|
" cat >\"$tmp\" <<EOF_SECRET",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $secret",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" app.kubernetes.io/managed-by: unidesk",
|
|
" annotations:",
|
|
" hwlab.pikastech.local/secret-source-ref: $source_ref",
|
|
" hwlab.pikastech.local/secret-source-key: $source_key",
|
|
" hwlab.pikastech.local/secret-source-fingerprint: $source_fingerprint",
|
|
"type: Opaque",
|
|
"data:",
|
|
" $secret_key: $password_b64",
|
|
"EOF_SECRET",
|
|
" kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err",
|
|
" apply_exit=$?",
|
|
" rm -f \"$tmp\"",
|
|
" if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
|
|
" else",
|
|
" action=namespace-create-failed",
|
|
" apply_exit=$namespace_create_exit",
|
|
" fi",
|
|
"fi",
|
|
"after_namespace_exists=$(namespace_exists_flag)",
|
|
"if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi",
|
|
"after_password_bytes=$(decoded_length \"$after_b64\")",
|
|
"after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'key\\t%s\\n' \"$secret_key\"",
|
|
"printf 'sourceRef\\t%s\\n' \"$source_ref\"",
|
|
"printf 'sourceKey\\t%s\\n' \"$source_key\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
|
"printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"",
|
|
"printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"",
|
|
"printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
|
"printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"",
|
|
"printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"",
|
|
"printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"password=",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
|
|
const source = spec.baseImageSource ?? "";
|
|
const script = [
|
|
"set -eu",
|
|
`target=${shellQuote(spec.baseImage)}`,
|
|
`source=${shellQuote(source)}`,
|
|
"repo_tag=${target#*/}",
|
|
"repo=${repo_tag%:*}",
|
|
"tag=${repo_tag##*:}",
|
|
"if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
|
"registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
|
"registry_present=false",
|
|
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi",
|
|
"target_present=false",
|
|
"if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi",
|
|
"source_present=false",
|
|
"if [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi",
|
|
"printf 'target\\t%s\\n' \"$target\"",
|
|
"printf 'source\\t%s\\n' \"$source\"",
|
|
"printf 'sourceConfigured\\t%s\\n' \"$([ -n \"$source\" ] && printf true || printf false)\"",
|
|
"printf 'registryUrl\\t%s\\n' \"$registry_url\"",
|
|
"printf 'registryTagPresent\\t%s\\n' \"$registry_present\"",
|
|
"printf 'targetImagePresent\\t%s\\n' \"$target_present\"",
|
|
"printf 'sourceImagePresent\\t%s\\n' \"$source_present\"",
|
|
].join("\n");
|
|
const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const sourceConfigured = fields.sourceConfigured === "true";
|
|
const registryTagPresent = fields.registryTagPresent === "true";
|
|
return {
|
|
ok: isCommandSuccess(result) && sourceConfigured && registryTagPresent,
|
|
target: fields.target ?? spec.baseImage,
|
|
source: fields.source || null,
|
|
sourceConfigured,
|
|
registryUrl: fields.registryUrl ?? null,
|
|
registryTagPresent,
|
|
targetImagePresent: fields.targetImagePresent === "true",
|
|
sourceImagePresent: fields.sourceImagePresent === "true",
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
if (spec.baseImageSource === undefined) return null;
|
|
const script = [
|
|
"set -eu",
|
|
`target=${shellQuote(spec.baseImage)}`,
|
|
`source=${shellQuote(spec.baseImageSource)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
`pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`,
|
|
"repo_tag=${target#*/}",
|
|
"repo=${repo_tag%:*}",
|
|
"tag=${repo_tag##*:}",
|
|
"if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
|
"registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
|
"present=false",
|
|
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi",
|
|
"action=none",
|
|
"if [ \"$present\" = false ]; then",
|
|
" action=seed",
|
|
" if [ \"$dry_run\" = false ]; then",
|
|
" source_present_before=true",
|
|
" if ! docker image inspect \"$source\" >/dev/null 2>&1; then",
|
|
" source_present_before=false",
|
|
" attempt=1",
|
|
" pulled_source=false",
|
|
" while [ \"$attempt\" -le \"$pull_retries\" ]; do",
|
|
" pull_attempts=$attempt",
|
|
" if docker pull \"$source\" >/tmp/hwlab-node-base-image-pull.out 2>&1; then pulled_source=true; break; fi",
|
|
" attempt=$((attempt + 1))",
|
|
" sleep 2",
|
|
" done",
|
|
" if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-base-image-pull.out >&2 2>/dev/null || true; fi",
|
|
" fi",
|
|
" docker image inspect \"$source\" >/dev/null",
|
|
" docker tag \"$source\" \"$target\"",
|
|
" docker push \"$target\" >/tmp/hwlab-node-base-image-push.out",
|
|
" fi",
|
|
"fi",
|
|
"after=false",
|
|
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi",
|
|
"printf 'target\\t%s\\n' \"$target\"",
|
|
"printf 'source\\t%s\\n' \"$source\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'presentBefore\\t%s\\n' \"$present\"",
|
|
"printf 'presentAfter\\t%s\\n' \"$after\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'pullRetries\\t%s\\n' \"$pull_retries\"",
|
|
"printf 'sourcePresentBefore\\t%s\\n' \"${source_present_before:-unknown}\"",
|
|
"printf 'pulledSource\\t%s\\n' \"${pulled_source:-false}\"",
|
|
"printf 'pullAttempts\\t%s\\n' \"${pull_attempts:-0}\"",
|
|
"if [ \"$dry_run\" = true ] || [ \"$after\" = true ]; then exit 0; fi",
|
|
"exit 1",
|
|
].join("\n");
|
|
const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
return {
|
|
ok: isCommandSuccess(result),
|
|
dryRun,
|
|
target: fields.target ?? spec.baseImage,
|
|
source: fields.source ?? spec.baseImageSource,
|
|
presentBefore: fields.presentBefore === "true",
|
|
presentAfter: fields.presentAfter === "true",
|
|
action: fields.action ?? null,
|
|
pullRetries: numericField(fields.pullRetries),
|
|
sourcePresentBefore: fields.sourcePresentBefore ?? null,
|
|
pulledSource: fields.pulledSource === "true",
|
|
pullAttempts: numericField(fields.pullAttempts),
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return "true";
|
|
const authnKey = pg.openfga.authnKey ?? "authn-preshared-key";
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(pg.openfga.secretName)}`,
|
|
`authn_key=${shellQuote(authnKey)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
"namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"",
|
|
"if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi",
|
|
"kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -",
|
|
"authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')",
|
|
"if [ \"${authn_present:-0}\" -gt 0 ]; then",
|
|
" action=kept",
|
|
"elif [ \"$dry_run\" = true ]; then",
|
|
" action=would-generate-authn-key",
|
|
"else",
|
|
" if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi",
|
|
" kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null",
|
|
" authn_value=",
|
|
" action=generated-authn-key",
|
|
"fi",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'authnKey\\t%s\\n' \"$authn_key\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'valuesPrinted\\tfalse\\n'",
|
|
].join("\n");
|
|
}
|
|
|
|
export function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string {
|
|
const configRef = spec.externalPostgres?.configRef;
|
|
if (configRef === undefined) return join(repoRoot, ".state", "secrets");
|
|
const configPath = rootPath(configRef);
|
|
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown;
|
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`);
|
|
const secrets = (parsed as Record<string, unknown>).secrets;
|
|
if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`);
|
|
const root = (secrets as Record<string, unknown>).root;
|
|
if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`);
|
|
return root.startsWith("/") ? root : rootPath(root);
|
|
}
|
|
|
|
export function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } {
|
|
const sourcePath = join(secretRoot, sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, sourcePath };
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const value = values[key];
|
|
if (value === undefined || value.length === 0) return { ok: false, sourcePath };
|
|
return { ok: true, value, sourcePath };
|
|
}
|
|
|
|
export function secretSourcePaths(sourceRef: string): string[] {
|
|
const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef));
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
export function displayRepoPath(path: string): string {
|
|
const normalizedRoot = repoRoot.replace(/\/+$/u, "");
|
|
if (path === normalizedRoot) return ".";
|
|
if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1);
|
|
const marker = "/.worktree/";
|
|
const index = normalizedRoot.indexOf(marker);
|
|
if (index >= 0) {
|
|
const mainRoot = normalizedRoot.slice(0, index);
|
|
if (path === mainRoot) return ".";
|
|
if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1);
|
|
}
|
|
return path;
|
|
}
|
|
|
|
export function hwlabPasswordHash(password: string): string {
|
|
const salt = randomBytes(16).toString("hex");
|
|
return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`;
|
|
}
|
|
|
|
export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial {
|
|
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
|
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
|
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
|
return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "bootstrap-admin-yaml-source-missing" };
|
|
}
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "secret-source-missing" };
|
|
}
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const password = values[sourceKey];
|
|
if (password === undefined || password.length === 0) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, passwordHash: null, error: "secret-key-missing" };
|
|
}
|
|
return {
|
|
ok: true,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath,
|
|
sourcePresent: true,
|
|
sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
|
|
passwordHash: hwlabPasswordHash(password),
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial {
|
|
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
|
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
|
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
|
return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, password: null, error: "bootstrap-admin-yaml-source-missing" };
|
|
}
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" };
|
|
}
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const password = values[sourceKey];
|
|
if (password === undefined || password.length === 0) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" };
|
|
}
|
|
return {
|
|
ok: true,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath,
|
|
sourcePresent: true,
|
|
sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
|
|
password,
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
export function parseEnvFile(text: string): Record<string, string> {
|
|
const values: Record<string, string> = {};
|
|
for (const rawLine of text.split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (line.length === 0 || line.startsWith("#")) continue;
|
|
const eq = line.indexOf("=");
|
|
if (eq <= 0) continue;
|
|
const key = line.slice(0, eq).trim();
|
|
const rawValue = line.slice(eq + 1).trim();
|
|
values[key] = rawValue.startsWith("'") && rawValue.endsWith("'")
|
|
? rawValue.slice(1, -1).replace(/'"'"'/gu, "'")
|
|
: rawValue.startsWith("\"") && rawValue.endsWith("\"")
|
|
? rawValue.slice(1, -1)
|
|
: rawValue;
|
|
}
|
|
return values;
|
|
}
|
|
|
|
export function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean {
|
|
if (!/postgres|pgsql|pgdata/iu.test(name)) return false;
|
|
const externalServiceName = spec.externalPostgres?.serviceName;
|
|
if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false;
|
|
return true;
|
|
}
|
|
|
|
export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return { required: false, ready: true };
|
|
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
|
const runtimeAccess = pg.runtimeAccess ?? { endpointAddress: pg.endpointAddress, port: pg.port };
|
|
const endpointIsIpv4 = /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/u.test(runtimeAccess.endpointAddress);
|
|
const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "jsonpath={.spec.type}{\"\\n\"}{.spec.externalName}{\"\\n\"}{.spec.ports[0].port}{\"\\n\"}"], 60);
|
|
const [serviceType = "", externalName = "", servicePort = ""] = service.stdout.split(/\r?\n/u);
|
|
if (!endpointIsIpv4) {
|
|
const ready = service.exitCode === 0 && serviceType === "ExternalName" && externalName === runtimeAccess.endpointAddress && Number(servicePort) === runtimeAccess.port;
|
|
return {
|
|
required: true,
|
|
ready,
|
|
serviceName: pg.serviceName,
|
|
serviceType: serviceType || null,
|
|
externalName: externalName || null,
|
|
expectedExternalName: runtimeAccess.endpointAddress,
|
|
port: numericField(servicePort),
|
|
expectedPort: runtimeAccess.port,
|
|
providerEndpointAddress: pg.endpointAddress,
|
|
providerPort: pg.port,
|
|
runtimeAccess: pg.runtimeAccess ?? null,
|
|
service: compactRuntimeCommand(service),
|
|
};
|
|
}
|
|
const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
|
|
const [addressType = "", address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u);
|
|
return {
|
|
required: true,
|
|
ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port,
|
|
serviceName: pg.serviceName,
|
|
serviceType: serviceType || null,
|
|
addressType: addressType || null,
|
|
endpointAddress: address || null,
|
|
expectedEndpointAddress: runtimeAccess.endpointAddress,
|
|
port: numericField(port),
|
|
expectedPort: runtimeAccess.port,
|
|
providerEndpointAddress: pg.endpointAddress,
|
|
providerPort: pg.port,
|
|
runtimeAccess: pg.runtimeAccess ?? null,
|
|
service: compactRuntimeCommand(service),
|
|
endpointSlice: compactRuntimeCommand(endpointSlice),
|
|
};
|
|
}
|
|
|
|
export function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
|
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
|
if (pg === undefined) return { required: false, ready: true };
|
|
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
|
const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey);
|
|
const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey);
|
|
const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key");
|
|
return {
|
|
required: true,
|
|
ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present,
|
|
valuesPrinted: false,
|
|
cloudApi,
|
|
openfga: {
|
|
datastoreUri: openfgaUri,
|
|
authnPresharedKey: openfgaAuthn,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec {
|
|
const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
|
|
const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
|
|
const nodes = record(parsed.nodes);
|
|
const node = record(nodes[spec.nodeId]);
|
|
const targets = Array.isArray(parsed.targets) ? parsed.targets : [];
|
|
const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane);
|
|
if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
|
|
const gitMirror = record(target.gitMirror);
|
|
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
|
|
const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode");
|
|
if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`);
|
|
const nodeEgressProxy = gitMirrorEgressMode === "direct"
|
|
? { mode: "direct" as const, required: false as const }
|
|
: nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
|
|
if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
|
|
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
|
|
const source = record(target.source);
|
|
const gitops = record(target.gitops);
|
|
const tekton = record(target.tekton);
|
|
const toolsImage = record(tekton.toolsImage);
|
|
const toolsImagePullPolicy = optionalStringValue(toolsImage.imagePullPolicy, "tekton.toolsImage.imagePullPolicy") ?? "IfNotPresent";
|
|
if (toolsImagePullPolicy !== "Always" && toolsImagePullPolicy !== "IfNotPresent" && toolsImagePullPolicy !== "Never") throw new Error("tekton.toolsImage.imagePullPolicy must be Always, IfNotPresent, or Never");
|
|
return {
|
|
id: stringValue(target.id, "target.id"),
|
|
node: stringValue(target.node, "target.node"),
|
|
lane: stringValue(target.lane, "target.lane"),
|
|
namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"),
|
|
serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
|
|
serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
|
|
cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
|
|
cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"),
|
|
cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
|
|
servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"),
|
|
secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
|
|
syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"),
|
|
syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
|
|
flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"),
|
|
toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"),
|
|
toolsImagePullPolicy,
|
|
sourceRepository: stringValue(source.repository, "source.repository"),
|
|
sourceBranch: stringValue(source.branch, "source.branch"),
|
|
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
|
|
egressProxy: nodeEgressProxy,
|
|
githubTransport,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorGithubTransportSpec {
|
|
const mode = stringValue(raw.mode, `${path}.mode`);
|
|
if (mode === "ssh") return { mode };
|
|
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
|
|
return {
|
|
mode,
|
|
username: stringValue(raw.username, `${path}.username`),
|
|
tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`),
|
|
tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`),
|
|
tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`),
|
|
tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`),
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorEgressProxySpec {
|
|
const mode = stringValue(raw.mode, `${path}.mode`);
|
|
if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`);
|
|
const port = Number(raw.port);
|
|
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`);
|
|
const sourceType = stringValue(raw.sourceType, `${path}.sourceType`);
|
|
if (sourceType !== "subscription-url") throw new Error(`${path}.sourceType must be subscription-url`);
|
|
const noProxyRaw = raw.noProxy;
|
|
if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
|
|
const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`));
|
|
return {
|
|
mode,
|
|
clientName: stringValue(raw.clientName, `${path}.clientName`),
|
|
namespace: stringValue(raw.namespace, `${path}.namespace`),
|
|
serviceName: stringValue(raw.serviceName, `${path}.serviceName`),
|
|
port,
|
|
sourceRef: stringValue(raw.sourceRef, `${path}.sourceRef`),
|
|
sourceKey: stringValue(raw.sourceKey, `${path}.sourceKey`),
|
|
sourceType,
|
|
noProxy,
|
|
};
|
|
}
|
|
|
|
export function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record<string, unknown> {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60);
|
|
return {
|
|
secret,
|
|
key,
|
|
present: result.exitCode === 0 && result.stdout.trim().length > 0,
|
|
valueBytesBase64: result.stdout.trim().length,
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const commandArgs = [
|
|
"hwlab",
|
|
"nodes",
|
|
options.domain,
|
|
...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
|
|
"--node",
|
|
options.node,
|
|
"--lane",
|
|
options.lane,
|
|
"--confirm",
|
|
"--timeout-seconds",
|
|
String(options.timeoutSeconds),
|
|
"--wait",
|
|
];
|
|
const command = ["bun", "scripts/cli.ts", ...commandArgs];
|
|
const job = startJob(
|
|
`hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
|
|
command,
|
|
`Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
|
|
);
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: "async-job",
|
|
reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
|
|
job,
|
|
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
|
|
waitCommand: command.join(" "),
|
|
};
|
|
}
|
|
|
|
export function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const rewritten = rewriteDelegatedNodeValue(value, scoped);
|
|
const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
|
|
return {
|
|
...result,
|
|
node: scoped.node,
|
|
commandFamily: "hwlab nodes",
|
|
};
|
|
}
|
|
|
|
export function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
|
|
if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
|
|
if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
|
|
if (typeof value !== "object" || value === null) return value;
|
|
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
|
|
}
|
|
|
|
export function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
|
|
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
return text
|
|
.replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
|
|
.replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
|
|
};
|
|
return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
|
|
}
|
|
|
|
export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
|
const [actionRaw] = args;
|
|
if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "observe" && actionRaw !== "sentinel") throw new Error("web-probe usage: run|script|observe|sentinel --node NODE --lane vNN [--url URL]");
|
|
if (actionRaw === "sentinel") return parseNodeWebProbeSentinelOptions(args.slice(1));
|
|
if (actionRaw === "observe") {
|
|
const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1));
|
|
const explicitNode = optionValue(args, "--node");
|
|
const explicitLane = optionValue(args, "--lane");
|
|
const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null;
|
|
const indexed = requestedId === null
|
|
? null
|
|
: readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null);
|
|
const node = explicitNode ?? indexed?.node;
|
|
const lane = explicitLane ?? indexed?.lane;
|
|
if (node === undefined || lane === undefined) {
|
|
if (requestedId !== null) {
|
|
throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`);
|
|
}
|
|
throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id");
|
|
}
|
|
assertNodeId(node);
|
|
assertLane(lane);
|
|
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
return parseNodeWebProbeObserveOptions(normalized.args, node, lane, spec, normalized.id, indexed);
|
|
}
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const lane = requiredOption(args, "--lane");
|
|
assertLane(lane);
|
|
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
if (actionRaw === "script") {
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--timeout-ms",
|
|
"--viewport",
|
|
"--browser-proxy-mode",
|
|
"--script-file",
|
|
"--command-timeout-seconds",
|
|
]), new Set([]));
|
|
const scriptFile = optionValue(args, "--script-file");
|
|
if (scriptFile === undefined && process.stdin.isTTY) {
|
|
throw new Error("web-probe script requires a stdin heredoc or --script-file <path>");
|
|
}
|
|
const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8");
|
|
if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script");
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
|
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600);
|
|
return {
|
|
action: "script",
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
|
|
timeoutMs,
|
|
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
|
browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode),
|
|
commandTimeoutSeconds,
|
|
scriptText,
|
|
scriptSource: {
|
|
kind: scriptFile === undefined ? "stdin" : "file",
|
|
path: scriptFile ?? null,
|
|
byteCount: Buffer.byteLength(scriptText),
|
|
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
|
},
|
|
};
|
|
}
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--timeout-ms",
|
|
"--wait-after-submit-ms",
|
|
"--wait-messages-ms",
|
|
"--wait-agent-terminal-ms",
|
|
"--trace-sample-count",
|
|
"--trace-sample-interval-ms",
|
|
"--message",
|
|
"--conversation-id",
|
|
"--command-timeout-seconds",
|
|
]), new Set([
|
|
"--fresh-session",
|
|
"--no-cancel-running",
|
|
]));
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
|
const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000);
|
|
const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000);
|
|
const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000);
|
|
const message = optionValue(args, "--message") ?? null;
|
|
const hasMessage = message !== null;
|
|
const traceSampleCountDefault = hasMessage ? 2 : 0;
|
|
const traceSampleIntervalDefault = hasMessage ? 1500 : 0;
|
|
const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200);
|
|
const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000);
|
|
const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({
|
|
timeoutMs,
|
|
waitAfterSubmitMs,
|
|
waitMessagesMs,
|
|
waitAgentTerminalMs,
|
|
traceSampleCount,
|
|
traceSampleIntervalMs,
|
|
freshSession: args.includes("--fresh-session"),
|
|
hasMessage,
|
|
});
|
|
const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds");
|
|
const commandTimeoutUserProvided = commandTimeoutRaw !== undefined;
|
|
const commandTimeoutSeconds = commandTimeoutUserProvided
|
|
? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds)
|
|
: commandTimeoutAutoSeconds;
|
|
return {
|
|
action: "run",
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
|
|
timeoutMs,
|
|
waitAfterSubmitMs,
|
|
waitMessagesMs,
|
|
waitAgentTerminalMs,
|
|
traceSampleCount,
|
|
traceSampleIntervalMs,
|
|
message,
|
|
conversationId: optionValue(args, "--conversation-id") ?? null,
|
|
freshSession: args.includes("--fresh-session"),
|
|
cancelRunning: !args.includes("--no-cancel-running"),
|
|
commandTimeoutSeconds,
|
|
commandTimeoutAutoSeconds,
|
|
commandTimeoutUserProvided,
|
|
};
|
|
}
|