diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 749773cd..a50248ac 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -56,9 +56,13 @@ nodes: endpoint: 127.0.0.1:5000 egressProxy: mode: k8s-service-cluster-ip + clientName: d601-global-proxy namespace: platform-infra serviceName: sub2api-egress-proxy port: 10808 + sourceRef: platform-infra/master-vpn-subscription.env + sourceKey: MASTER_VPN_SUBSCRIPTION_URL + sourceType: subscription-url noProxy: - localhost - 127.0.0.1 @@ -109,6 +113,9 @@ targets: flushJobPrefix: git-mirror-hwlab-d601-v03-flush-manual readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git + egressProxy: + mode: node-global + required: true tekton: pipelineName: hwlab-d601-v03-ci-image-publish serviceAccountName: hwlab-d601-v03-tekton-runner diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index f83979c6..54c91cb9 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -40,12 +40,21 @@ interface ArgoOptions { interface ControlPlaneEgressProxySpec { mode: "k8s-service-cluster-ip"; + clientName: string; namespace: string; serviceName: string; port: number; + sourceRef: string; + sourceKey: string; + sourceType: "subscription-url"; noProxy: readonly string[]; } +interface ControlPlaneGitMirrorEgressProxySpec { + mode: "node-global"; + required: boolean; +} + interface ControlPlaneNodeSpec { id: string; route: string; @@ -99,6 +108,7 @@ interface ControlPlaneTargetSpec { flushJobPrefix: string; readUrl: string; writeUrl: string; + egressProxy: ControlPlaneGitMirrorEgressProxySpec | null; }; tekton: { pipelineName: string; @@ -816,15 +826,30 @@ function execStartPreField(raw: unknown, path: string): readonly (readonly strin function egressProxySpec(raw: Record, path: string): ControlPlaneEgressProxySpec { const mode = stringField(raw, "mode", path); if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`); + const sourceType = stringField(raw, "sourceType", path); + if (sourceType !== "subscription-url") throw new Error(`${path}.sourceType must be subscription-url`); return { mode, + clientName: stringField(raw, "clientName", path), namespace: stringField(raw, "namespace", path), serviceName: stringField(raw, "serviceName", path), port: positiveConfigIntegerField(raw, "port", path), + sourceRef: stringField(raw, "sourceRef", path), + sourceKey: stringField(raw, "sourceKey", path), + sourceType, noProxy: stringArrayField(raw, "noProxy", path), }; } +function gitMirrorEgressProxySpec(raw: Record, path: string): ControlPlaneGitMirrorEgressProxySpec { + const mode = stringField(raw, "mode", path); + if (mode !== "node-global") throw new Error(`${path}.mode must be node-global`); + return { + mode, + required: raw.required === undefined ? true : booleanField(raw, "required", path), + }; +} + function targetSpec(raw: Record, index: number): ControlPlaneTargetSpec { const path = `targets[${index}]`; const source = asRecord(raw.source, `${path}.source`); @@ -838,6 +863,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa const gitMirrorNamespace = stringField(gitMirror, "namespace", `${path}.gitMirror`); const serviceReadName = stringField(gitMirror, "serviceReadName", `${path}.gitMirror`); const serviceWriteName = stringField(gitMirror, "serviceWriteName", `${path}.gitMirror`); + const gitMirrorEgressProxy = gitMirror.egressProxy === undefined ? null : gitMirrorEgressProxySpec(asRecord(gitMirror.egressProxy, `${path}.gitMirror.egressProxy`), `${path}.gitMirror.egressProxy`); const sourceRepository = stringField(source, "repository", `${path}.source`); return { id: stringField(raw, "id", path), @@ -863,6 +889,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa flushJobPrefix: stringField(gitMirror, "flushJobPrefix", `${path}.gitMirror`), readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, + egressProxy: gitMirrorEgressProxy, }, tekton: { pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`), @@ -1175,8 +1202,13 @@ NODE } function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { - const proxyHost = node.egressProxy?.serviceName === undefined ? "127.0.0.1" : `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; - const proxyPort = node.egressProxy?.port ?? 10808; + if (target.gitMirror.egressProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`); + if (target.gitMirror.egressProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`); + if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub SSH no longer falls back to localhost`); + const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; + const proxyPort = node.egressProxy.port; + const proxyUrl = `http://${proxyHost}:${proxyPort}`; + const noProxy = node.egressProxy.noProxy.join(","); return [ "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", @@ -1212,6 +1244,14 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT "socket.on('close', () => process.exit(0));", "NODE_PROXY", "chmod 0700 /tmp/hwlab-github-proxy-connect.cjs", + `export HTTP_PROXY=${shQuote(proxyUrl)}`, + `export HTTPS_PROXY=${shQuote(proxyUrl)}`, + `export ALL_PROXY=${shQuote(proxyUrl)}`, + `export http_proxy=${shQuote(proxyUrl)}`, + `export https_proxy=${shQuote(proxyUrl)}`, + `export all_proxy=${shQuote(proxyUrl)}`, + `export NO_PROXY=${shQuote(noProxy)}`, + `export no_proxy=${shQuote(noProxy)}`, `export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p'`)}`, `repository=${shQuote(target.source.repository)}`, `source_branch=${shQuote(target.source.branch)}`, @@ -1411,6 +1451,7 @@ function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS servicePort: target.gitMirror.servicePort, deploymentReplicas: target.gitMirror.deploymentReplicas, syncConfigMap: target.gitMirror.syncConfigMapName, + egressProxy: target.gitMirror.egressProxy, statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"], }, pipeline: target.tekton.pipelineName, diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 56a307e9..aec3c82d 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -239,12 +239,26 @@ interface NodeRuntimeGitMirrorTargetSpec { cachePvcName: string; cacheHostPath: string | null; secretName: string; + syncConfigMapName: string; syncJobPrefix: string; flushJobPrefix: string; toolsImage: string; sourceRepository: string; sourceBranch: string; gitopsBranch: string; + egressProxy: NodeRuntimeGitMirrorEgressProxySpec; +} + +interface NodeRuntimeGitMirrorEgressProxySpec { + mode: "k8s-service-cluster-ip"; + clientName: string; + namespace: string; + serviceName: string; + port: number; + sourceRef: string; + sourceKey: string; + sourceType: "subscription-url"; + noProxy: string[]; } const MASTER_ADMIN_API_KEY_KEY = "api-key"; @@ -3228,13 +3242,14 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, volumes: [ { name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } }, - { name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } }, + { name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } }, ], containers: [{ name: action, image: mirror.toolsImage, imagePullPolicy: "IfNotPresent", command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"], + env: nodeRuntimeGitMirrorProxyEnv(mirror), volumeMounts: [ { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, @@ -3247,6 +3262,22 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, }; } +function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record[] { + const proxy = mirror.egressProxy; + const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`; + const noProxy = proxy.noProxy.join(","); + return [ + { name: "HTTP_PROXY", value: proxyUrl }, + { name: "HTTPS_PROXY", value: proxyUrl }, + { name: "ALL_PROXY", value: proxyUrl }, + { name: "http_proxy", value: proxyUrl }, + { name: "https_proxy", value: proxyUrl }, + { name: "all_proxy", value: proxyUrl }, + { name: "NO_PROXY", value: noProxy }, + { name: "no_proxy", value: noProxy }, + ]; +} + function nodeRuntimeControlPlaneStatus(scoped: ReturnType): Record { const spec = scoped.spec; const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit"); @@ -5831,10 +5862,16 @@ function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExist function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec { const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown); + const nodes = record(parsed.nodes); + const node = record(nodes[spec.nodeId]); + const nodeEgressProxy = nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`); const targets = Array.isArray(parsed.targets) ? parsed.targets : []; const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane); if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); const gitMirror = record(target.gitMirror); + const gitMirrorEgressProxy = record(gitMirror.egressProxy); + if (stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode") !== "node-global") throw new Error(`gitMirror.egressProxy.mode must be node-global for node=${spec.nodeId} lane=${spec.lane}`); + if (gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`); const source = record(target.source); const gitops = record(target.gitops); const tekton = record(target.tekton); @@ -5849,12 +5886,37 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"), cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"), secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"), + syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"), syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"), flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"), toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"), sourceRepository: stringValue(source.repository, "source.repository"), sourceBranch: stringValue(source.branch, "source.branch"), gitopsBranch: stringValue(gitops.branch, "gitops.branch"), + egressProxy: nodeEgressProxy, + }; +} + +function nodeRuntimeGitMirrorEgressProxySpec(raw: Record, path: string): NodeRuntimeGitMirrorEgressProxySpec { + const mode = stringValue(raw.mode, `${path}.mode`); + if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`); + const port = Number(raw.port); + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`); + const sourceType = stringValue(raw.sourceType, `${path}.sourceType`); + if (sourceType !== "subscription-url") throw new Error(`${path}.sourceType must be subscription-url`); + const noProxyRaw = raw.noProxy; + if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`); + const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)); + return { + mode, + clientName: stringValue(raw.clientName, `${path}.clientName`), + namespace: stringValue(raw.namespace, `${path}.namespace`), + serviceName: stringValue(raw.serviceName, `${path}.serviceName`), + port, + sourceRef: stringValue(raw.sourceRef, `${path}.sourceRef`), + sourceKey: stringValue(raw.sourceKey, `${path}.sourceKey`), + sourceType, + noProxy, }; }