fix: use https transport for hwlab git mirror (#580)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { rootPath } from "./config";
|
||||
import { runCommand, type CommandResult } from "./command";
|
||||
import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets";
|
||||
|
||||
export const HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH = "config/hwlab-node-control-plane.yaml";
|
||||
|
||||
@@ -55,6 +56,17 @@ interface ControlPlaneGitMirrorEgressProxySpec {
|
||||
required: boolean;
|
||||
}
|
||||
|
||||
type ControlPlaneGitMirrorGithubTransportSpec =
|
||||
| { mode: "ssh" }
|
||||
| {
|
||||
mode: "https";
|
||||
username: string;
|
||||
tokenSecretName: string;
|
||||
tokenSecretKey: string;
|
||||
tokenSourceRef: string;
|
||||
tokenSourceKey: string;
|
||||
};
|
||||
|
||||
interface ControlPlaneNodeSpec {
|
||||
id: string;
|
||||
route: string;
|
||||
@@ -109,6 +121,7 @@ interface ControlPlaneTargetSpec {
|
||||
readUrl: string;
|
||||
writeUrl: string;
|
||||
egressProxy: ControlPlaneGitMirrorEgressProxySpec | null;
|
||||
githubTransport: ControlPlaneGitMirrorGithubTransportSpec;
|
||||
};
|
||||
tekton: {
|
||||
pipelineName: string;
|
||||
@@ -850,6 +863,28 @@ function gitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): C
|
||||
};
|
||||
}
|
||||
|
||||
function gitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): ControlPlaneGitMirrorGithubTransportSpec {
|
||||
const mode = stringField(raw, "mode", path);
|
||||
if (mode === "ssh") return { mode };
|
||||
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
|
||||
const tokenSecretName = stringField(raw, "tokenSecretName", path);
|
||||
const tokenSecretKey = stringField(raw, "tokenSecretKey", path);
|
||||
const tokenSourceRef = stringField(raw, "tokenSourceRef", path);
|
||||
const tokenSourceKey = stringField(raw, "tokenSourceKey", path);
|
||||
validateKubernetesName(tokenSecretName, `${path}.tokenSecretName`);
|
||||
validateSecretKey(tokenSecretKey, `${path}.tokenSecretKey`);
|
||||
validateSourceRef(tokenSourceRef, `${path}.tokenSourceRef`);
|
||||
validateEnvKey(tokenSourceKey, `${path}.tokenSourceKey`);
|
||||
return {
|
||||
mode,
|
||||
username: stringField(raw, "username", path),
|
||||
tokenSecretName,
|
||||
tokenSecretKey,
|
||||
tokenSourceRef,
|
||||
tokenSourceKey,
|
||||
};
|
||||
}
|
||||
|
||||
function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTargetSpec {
|
||||
const path = `targets[${index}]`;
|
||||
const source = asRecord(raw.source, `${path}.source`);
|
||||
@@ -864,6 +899,7 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
|
||||
const serviceReadName = stringField(gitMirror, "serviceReadName", `${path}.gitMirror`);
|
||||
const serviceWriteName = stringField(gitMirror, "serviceWriteName", `${path}.gitMirror`);
|
||||
const gitMirrorEgressProxy = gitMirror.egressProxy === undefined ? null : gitMirrorEgressProxySpec(asRecord(gitMirror.egressProxy, `${path}.gitMirror.egressProxy`), `${path}.gitMirror.egressProxy`);
|
||||
const githubTransport = gitMirrorGithubTransportSpec(asRecord(gitMirror.githubTransport, `${path}.gitMirror.githubTransport`), `${path}.gitMirror.githubTransport`);
|
||||
const sourceRepository = stringField(source, "repository", `${path}.source`);
|
||||
return {
|
||||
id: stringField(raw, "id", path),
|
||||
@@ -890,6 +926,7 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
|
||||
readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`,
|
||||
writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`,
|
||||
egressProxy: gitMirrorEgressProxy,
|
||||
githubTransport,
|
||||
},
|
||||
tekton: {
|
||||
pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`),
|
||||
@@ -934,6 +971,8 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa
|
||||
},
|
||||
},
|
||||
);
|
||||
const githubTokenSecret = gitMirrorGithubTokenSecret(target, labels);
|
||||
if (githubTokenSecret !== null) manifests.push(githubTokenSecret);
|
||||
if (target.gitMirror.cacheHostPath === null) {
|
||||
manifests.push({
|
||||
apiVersion: "v1",
|
||||
@@ -975,6 +1014,43 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa
|
||||
return manifests;
|
||||
}
|
||||
|
||||
function gitMirrorGithubTokenSecret(target: ControlPlaneTargetSpec, labels: Record<string, string>): Record<string, unknown> | null {
|
||||
const transport = target.gitMirror.githubTransport;
|
||||
if (transport.mode !== "https") return null;
|
||||
const token = gitMirrorGithubHttpsToken(transport);
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "Secret",
|
||||
metadata: {
|
||||
name: transport.tokenSecretName,
|
||||
namespace: target.gitMirror.namespace,
|
||||
labels: { ...labels, "app.kubernetes.io/name": "git-mirror" },
|
||||
annotations: {
|
||||
"unidesk.ai/source-ref": transport.tokenSourceRef,
|
||||
"unidesk.ai/source-key": transport.tokenSourceKey,
|
||||
"unidesk.ai/target-key": transport.tokenSecretKey,
|
||||
"unidesk.ai/fingerprint": token.fingerprint,
|
||||
},
|
||||
},
|
||||
type: "Opaque",
|
||||
stringData: { [transport.tokenSecretKey]: token.value },
|
||||
};
|
||||
}
|
||||
|
||||
function gitMirrorGithubHttpsToken(transport: Extract<ControlPlaneGitMirrorGithubTransportSpec, { mode: "https" }>): { value: string; fingerprint: string } {
|
||||
const absolute = transport.tokenSourceRef.startsWith("/");
|
||||
const source = readEnvSourceFile({
|
||||
root: absolute ? "/" : rootPath("."),
|
||||
sourceRef: absolute ? transport.tokenSourceRef.slice(1) : transport.tokenSourceRef,
|
||||
missingMessage: () => `gitMirror.githubTransport token source ${transport.tokenSourceRef} is missing; create the YAML-declared sourceRef with ${transport.tokenSourceKey} before applying the control plane`,
|
||||
});
|
||||
const value = requiredEnvValue(source.values, transport.tokenSourceKey, transport.tokenSourceRef);
|
||||
return {
|
||||
value,
|
||||
fingerprint: fingerprintSecretValues({ [transport.tokenSourceKey]: value }, [transport.tokenSourceKey]),
|
||||
};
|
||||
}
|
||||
|
||||
function service(name: string, namespace: string, labels: Record<string, string>, port: number): Record<string, unknown> {
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
@@ -987,6 +1063,7 @@ function service(name: string, namespace: string, labels: Record<string, string>
|
||||
function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
||||
return sha256Short(JSON.stringify({
|
||||
repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }],
|
||||
githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport),
|
||||
server: gitMirrorServerJs(),
|
||||
status: gitMirrorStatusShell(),
|
||||
sync: gitMirrorSyncShell(node, target),
|
||||
@@ -1203,20 +1280,59 @@ NODE
|
||||
|
||||
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
||||
const gitMirrorProxy = target.gitMirror.egressProxy;
|
||||
if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`);
|
||||
const transport = target.gitMirror.githubTransport;
|
||||
if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub transport no longer falls back to localhost`);
|
||||
if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
|
||||
if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub SSH no longer falls back to localhost`);
|
||||
if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport no longer falls back to localhost`);
|
||||
const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
|
||||
const proxyPort = node.egressProxy.port;
|
||||
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
||||
const noProxy = node.egressProxy.noProxy.join(",");
|
||||
const proxySummary = `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`;
|
||||
const proxySummary = transport.mode === "https"
|
||||
? `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} source=yaml`
|
||||
: `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`;
|
||||
const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
|
||||
const common = [
|
||||
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
|
||||
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export ALL_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export http_proxy=${shQuote(proxyUrl)}`,
|
||||
`export https_proxy=${shQuote(proxyUrl)}`,
|
||||
`export all_proxy=${shQuote(proxyUrl)}`,
|
||||
`export NO_PROXY=${shQuote(noProxy)}`,
|
||||
`export no_proxy=${shQuote(noProxy)}`,
|
||||
`repository=${shQuote(target.source.repository)}`,
|
||||
`source_branch=${shQuote(target.source.branch)}`,
|
||||
`gitops_branch=${shQuote(target.gitops.branch)}`,
|
||||
"repo=\"/cache/${repository}.git\"",
|
||||
];
|
||||
if (transport.mode === "https") {
|
||||
return [
|
||||
...common,
|
||||
"if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi",
|
||||
"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'",
|
||||
"#!/bin/sh",
|
||||
"case \"$1\" in",
|
||||
" *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;",
|
||||
" *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;",
|
||||
" *) printf '\\n' ;;",
|
||||
"esac",
|
||||
"SH_ASKPASS",
|
||||
"chmod 0700 /tmp/hwlab-git-askpass.sh",
|
||||
`export GITHUB_USERNAME=${shQuote(transport.username)}`,
|
||||
"export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh",
|
||||
"export GIT_TERMINAL_PROMPT=0",
|
||||
"unset GIT_SSH",
|
||||
"unset GIT_SSH_COMMAND",
|
||||
"remote=\"https://github.com/${repository}.git\"",
|
||||
].join("\n");
|
||||
}
|
||||
return [
|
||||
"mkdir -p /root/.ssh",
|
||||
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
||||
"chmod 0400 /root/.ssh/id_rsa",
|
||||
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
|
||||
...common,
|
||||
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
|
||||
"#!/usr/bin/env node",
|
||||
"const net = require('node:net');",
|
||||
@@ -1273,20 +1389,8 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT
|
||||
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
|
||||
"SH_PROXY",
|
||||
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
|
||||
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export ALL_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export http_proxy=${shQuote(proxyUrl)}`,
|
||||
`export https_proxy=${shQuote(proxyUrl)}`,
|
||||
`export all_proxy=${shQuote(proxyUrl)}`,
|
||||
`export NO_PROXY=${shQuote(noProxy)}`,
|
||||
`export no_proxy=${shQuote(noProxy)}`,
|
||||
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
|
||||
"unset GIT_SSH_COMMAND",
|
||||
`repository=${shQuote(target.source.repository)}`,
|
||||
`source_branch=${shQuote(target.source.branch)}`,
|
||||
`gitops_branch=${shQuote(target.gitops.branch)}`,
|
||||
"repo=\"/cache/${repository}.git\"",
|
||||
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
||||
].join("\n");
|
||||
}
|
||||
@@ -1482,6 +1586,7 @@ function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS
|
||||
deploymentReplicas: target.gitMirror.deploymentReplicas,
|
||||
syncConfigMap: target.gitMirror.syncConfigMapName,
|
||||
egressProxy: target.gitMirror.egressProxy,
|
||||
githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport),
|
||||
statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"],
|
||||
},
|
||||
pipeline: target.tekton.pipelineName,
|
||||
@@ -1536,6 +1641,19 @@ function k3sDropInContent(spec: ControlPlaneK3sNodeSpec): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function gitMirrorGithubTransportSummary(transport: ControlPlaneGitMirrorGithubTransportSpec): Record<string, unknown> {
|
||||
if (transport.mode === "ssh") return { mode: "ssh", valuesPrinted: false };
|
||||
return {
|
||||
mode: "https",
|
||||
username: transport.username,
|
||||
tokenSecretName: transport.tokenSecretName,
|
||||
tokenSecretKey: transport.tokenSecretKey,
|
||||
tokenSourceRef: transport.tokenSourceRef,
|
||||
tokenSourceKey: transport.tokenSourceKey,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function systemdExecArg(value: string): string {
|
||||
if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value;
|
||||
return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`;
|
||||
@@ -2343,6 +2461,22 @@ function optionalStringField(obj: Record<string, unknown>, key: string, path: st
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateKubernetesName(value: string, path: string): void {
|
||||
if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} must be a Kubernetes resource name`);
|
||||
}
|
||||
|
||||
function validateSecretKey(value: string, path: string): void {
|
||||
if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a Kubernetes Secret key`);
|
||||
}
|
||||
|
||||
function validateEnvKey(value: string, path: string): void {
|
||||
if (!/^[A-Z0-9_]+$/u.test(value)) throw new Error(`${path} must be an env key`);
|
||||
}
|
||||
|
||||
function validateSourceRef(value: string, path: string): void {
|
||||
if (!/^[A-Za-z0-9_./-]+$/u.test(value) || value.includes("..")) throw new Error(`${path} has an unsupported sourceRef format`);
|
||||
}
|
||||
|
||||
function numberField(obj: Record<string, unknown>, key: string, path: string): number {
|
||||
const value = obj[key];
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`);
|
||||
|
||||
Reference in New Issue
Block a user