fix: use https transport for hwlab git mirror (#580)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
@@ -116,6 +116,13 @@ targets:
|
|||||||
egressProxy:
|
egressProxy:
|
||||||
mode: node-global
|
mode: node-global
|
||||||
required: true
|
required: true
|
||||||
|
githubTransport:
|
||||||
|
mode: https
|
||||||
|
username: x-access-token
|
||||||
|
tokenSecretName: git-mirror-github-token
|
||||||
|
tokenSecretKey: GITHUB_TOKEN
|
||||||
|
tokenSourceRef: /root/unidesk/.state/secrets/github/hwlab-git-mirror.env
|
||||||
|
tokenSourceKey: GH_TOKEN
|
||||||
tekton:
|
tekton:
|
||||||
pipelineName: hwlab-d601-v03-ci-image-publish
|
pipelineName: hwlab-d601-v03-ci-image-publish
|
||||||
serviceAccountName: hwlab-d601-v03-tekton-runner
|
serviceAccountName: hwlab-d601-v03-tekton-runner
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
|
|||||||
import { readFileSync } from "node:fs";
|
import { readFileSync } from "node:fs";
|
||||||
import { rootPath } from "./config";
|
import { rootPath } from "./config";
|
||||||
import { runCommand, type CommandResult } from "./command";
|
import { runCommand, type CommandResult } from "./command";
|
||||||
|
import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets";
|
||||||
|
|
||||||
export const HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH = "config/hwlab-node-control-plane.yaml";
|
export const HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH = "config/hwlab-node-control-plane.yaml";
|
||||||
|
|
||||||
@@ -55,6 +56,17 @@ interface ControlPlaneGitMirrorEgressProxySpec {
|
|||||||
required: boolean;
|
required: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ControlPlaneGitMirrorGithubTransportSpec =
|
||||||
|
| { mode: "ssh" }
|
||||||
|
| {
|
||||||
|
mode: "https";
|
||||||
|
username: string;
|
||||||
|
tokenSecretName: string;
|
||||||
|
tokenSecretKey: string;
|
||||||
|
tokenSourceRef: string;
|
||||||
|
tokenSourceKey: string;
|
||||||
|
};
|
||||||
|
|
||||||
interface ControlPlaneNodeSpec {
|
interface ControlPlaneNodeSpec {
|
||||||
id: string;
|
id: string;
|
||||||
route: string;
|
route: string;
|
||||||
@@ -109,6 +121,7 @@ interface ControlPlaneTargetSpec {
|
|||||||
readUrl: string;
|
readUrl: string;
|
||||||
writeUrl: string;
|
writeUrl: string;
|
||||||
egressProxy: ControlPlaneGitMirrorEgressProxySpec | null;
|
egressProxy: ControlPlaneGitMirrorEgressProxySpec | null;
|
||||||
|
githubTransport: ControlPlaneGitMirrorGithubTransportSpec;
|
||||||
};
|
};
|
||||||
tekton: {
|
tekton: {
|
||||||
pipelineName: string;
|
pipelineName: string;
|
||||||
@@ -850,6 +863,28 @@ function gitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): C
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function gitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): ControlPlaneGitMirrorGithubTransportSpec {
|
||||||
|
const mode = stringField(raw, "mode", path);
|
||||||
|
if (mode === "ssh") return { mode };
|
||||||
|
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
|
||||||
|
const tokenSecretName = stringField(raw, "tokenSecretName", path);
|
||||||
|
const tokenSecretKey = stringField(raw, "tokenSecretKey", path);
|
||||||
|
const tokenSourceRef = stringField(raw, "tokenSourceRef", path);
|
||||||
|
const tokenSourceKey = stringField(raw, "tokenSourceKey", path);
|
||||||
|
validateKubernetesName(tokenSecretName, `${path}.tokenSecretName`);
|
||||||
|
validateSecretKey(tokenSecretKey, `${path}.tokenSecretKey`);
|
||||||
|
validateSourceRef(tokenSourceRef, `${path}.tokenSourceRef`);
|
||||||
|
validateEnvKey(tokenSourceKey, `${path}.tokenSourceKey`);
|
||||||
|
return {
|
||||||
|
mode,
|
||||||
|
username: stringField(raw, "username", path),
|
||||||
|
tokenSecretName,
|
||||||
|
tokenSecretKey,
|
||||||
|
tokenSourceRef,
|
||||||
|
tokenSourceKey,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTargetSpec {
|
function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTargetSpec {
|
||||||
const path = `targets[${index}]`;
|
const path = `targets[${index}]`;
|
||||||
const source = asRecord(raw.source, `${path}.source`);
|
const source = asRecord(raw.source, `${path}.source`);
|
||||||
@@ -864,6 +899,7 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
|
|||||||
const serviceReadName = stringField(gitMirror, "serviceReadName", `${path}.gitMirror`);
|
const serviceReadName = stringField(gitMirror, "serviceReadName", `${path}.gitMirror`);
|
||||||
const serviceWriteName = stringField(gitMirror, "serviceWriteName", `${path}.gitMirror`);
|
const serviceWriteName = stringField(gitMirror, "serviceWriteName", `${path}.gitMirror`);
|
||||||
const gitMirrorEgressProxy = gitMirror.egressProxy === undefined ? null : gitMirrorEgressProxySpec(asRecord(gitMirror.egressProxy, `${path}.gitMirror.egressProxy`), `${path}.gitMirror.egressProxy`);
|
const gitMirrorEgressProxy = gitMirror.egressProxy === undefined ? null : gitMirrorEgressProxySpec(asRecord(gitMirror.egressProxy, `${path}.gitMirror.egressProxy`), `${path}.gitMirror.egressProxy`);
|
||||||
|
const githubTransport = gitMirrorGithubTransportSpec(asRecord(gitMirror.githubTransport, `${path}.gitMirror.githubTransport`), `${path}.gitMirror.githubTransport`);
|
||||||
const sourceRepository = stringField(source, "repository", `${path}.source`);
|
const sourceRepository = stringField(source, "repository", `${path}.source`);
|
||||||
return {
|
return {
|
||||||
id: stringField(raw, "id", path),
|
id: stringField(raw, "id", path),
|
||||||
@@ -890,6 +926,7 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
|
|||||||
readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`,
|
readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`,
|
||||||
writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`,
|
writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`,
|
||||||
egressProxy: gitMirrorEgressProxy,
|
egressProxy: gitMirrorEgressProxy,
|
||||||
|
githubTransport,
|
||||||
},
|
},
|
||||||
tekton: {
|
tekton: {
|
||||||
pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`),
|
pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`),
|
||||||
@@ -934,6 +971,8 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
const githubTokenSecret = gitMirrorGithubTokenSecret(target, labels);
|
||||||
|
if (githubTokenSecret !== null) manifests.push(githubTokenSecret);
|
||||||
if (target.gitMirror.cacheHostPath === null) {
|
if (target.gitMirror.cacheHostPath === null) {
|
||||||
manifests.push({
|
manifests.push({
|
||||||
apiVersion: "v1",
|
apiVersion: "v1",
|
||||||
@@ -975,6 +1014,43 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa
|
|||||||
return manifests;
|
return manifests;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function gitMirrorGithubTokenSecret(target: ControlPlaneTargetSpec, labels: Record<string, string>): Record<string, unknown> | null {
|
||||||
|
const transport = target.gitMirror.githubTransport;
|
||||||
|
if (transport.mode !== "https") return null;
|
||||||
|
const token = gitMirrorGithubHttpsToken(transport);
|
||||||
|
return {
|
||||||
|
apiVersion: "v1",
|
||||||
|
kind: "Secret",
|
||||||
|
metadata: {
|
||||||
|
name: transport.tokenSecretName,
|
||||||
|
namespace: target.gitMirror.namespace,
|
||||||
|
labels: { ...labels, "app.kubernetes.io/name": "git-mirror" },
|
||||||
|
annotations: {
|
||||||
|
"unidesk.ai/source-ref": transport.tokenSourceRef,
|
||||||
|
"unidesk.ai/source-key": transport.tokenSourceKey,
|
||||||
|
"unidesk.ai/target-key": transport.tokenSecretKey,
|
||||||
|
"unidesk.ai/fingerprint": token.fingerprint,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
type: "Opaque",
|
||||||
|
stringData: { [transport.tokenSecretKey]: token.value },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function gitMirrorGithubHttpsToken(transport: Extract<ControlPlaneGitMirrorGithubTransportSpec, { mode: "https" }>): { value: string; fingerprint: string } {
|
||||||
|
const absolute = transport.tokenSourceRef.startsWith("/");
|
||||||
|
const source = readEnvSourceFile({
|
||||||
|
root: absolute ? "/" : rootPath("."),
|
||||||
|
sourceRef: absolute ? transport.tokenSourceRef.slice(1) : transport.tokenSourceRef,
|
||||||
|
missingMessage: () => `gitMirror.githubTransport token source ${transport.tokenSourceRef} is missing; create the YAML-declared sourceRef with ${transport.tokenSourceKey} before applying the control plane`,
|
||||||
|
});
|
||||||
|
const value = requiredEnvValue(source.values, transport.tokenSourceKey, transport.tokenSourceRef);
|
||||||
|
return {
|
||||||
|
value,
|
||||||
|
fingerprint: fingerprintSecretValues({ [transport.tokenSourceKey]: value }, [transport.tokenSourceKey]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function service(name: string, namespace: string, labels: Record<string, string>, port: number): Record<string, unknown> {
|
function service(name: string, namespace: string, labels: Record<string, string>, port: number): Record<string, unknown> {
|
||||||
return {
|
return {
|
||||||
apiVersion: "v1",
|
apiVersion: "v1",
|
||||||
@@ -987,6 +1063,7 @@ function service(name: string, namespace: string, labels: Record<string, string>
|
|||||||
function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
||||||
return sha256Short(JSON.stringify({
|
return sha256Short(JSON.stringify({
|
||||||
repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }],
|
repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }],
|
||||||
|
githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport),
|
||||||
server: gitMirrorServerJs(),
|
server: gitMirrorServerJs(),
|
||||||
status: gitMirrorStatusShell(),
|
status: gitMirrorStatusShell(),
|
||||||
sync: gitMirrorSyncShell(node, target),
|
sync: gitMirrorSyncShell(node, target),
|
||||||
@@ -1203,20 +1280,59 @@ NODE
|
|||||||
|
|
||||||
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
||||||
const gitMirrorProxy = target.gitMirror.egressProxy;
|
const gitMirrorProxy = target.gitMirror.egressProxy;
|
||||||
if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`);
|
const transport = target.gitMirror.githubTransport;
|
||||||
|
if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub transport no longer falls back to localhost`);
|
||||||
if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
|
if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
|
||||||
if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub SSH no longer falls back to localhost`);
|
if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport no longer falls back to localhost`);
|
||||||
const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
|
const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
|
||||||
const proxyPort = node.egressProxy.port;
|
const proxyPort = node.egressProxy.port;
|
||||||
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
||||||
const noProxy = node.egressProxy.noProxy.join(",");
|
const noProxy = node.egressProxy.noProxy.join(",");
|
||||||
const proxySummary = `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`;
|
const proxySummary = transport.mode === "https"
|
||||||
|
? `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} source=yaml`
|
||||||
|
: `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`;
|
||||||
const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
|
const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
|
||||||
|
const common = [
|
||||||
|
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
|
||||||
|
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
|
||||||
|
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
|
||||||
|
`export ALL_PROXY=${shQuote(proxyUrl)}`,
|
||||||
|
`export http_proxy=${shQuote(proxyUrl)}`,
|
||||||
|
`export https_proxy=${shQuote(proxyUrl)}`,
|
||||||
|
`export all_proxy=${shQuote(proxyUrl)}`,
|
||||||
|
`export NO_PROXY=${shQuote(noProxy)}`,
|
||||||
|
`export no_proxy=${shQuote(noProxy)}`,
|
||||||
|
`repository=${shQuote(target.source.repository)}`,
|
||||||
|
`source_branch=${shQuote(target.source.branch)}`,
|
||||||
|
`gitops_branch=${shQuote(target.gitops.branch)}`,
|
||||||
|
"repo=\"/cache/${repository}.git\"",
|
||||||
|
];
|
||||||
|
if (transport.mode === "https") {
|
||||||
|
return [
|
||||||
|
...common,
|
||||||
|
"if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi",
|
||||||
|
"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'",
|
||||||
|
"#!/bin/sh",
|
||||||
|
"case \"$1\" in",
|
||||||
|
" *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;",
|
||||||
|
" *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;",
|
||||||
|
" *) printf '\\n' ;;",
|
||||||
|
"esac",
|
||||||
|
"SH_ASKPASS",
|
||||||
|
"chmod 0700 /tmp/hwlab-git-askpass.sh",
|
||||||
|
`export GITHUB_USERNAME=${shQuote(transport.username)}`,
|
||||||
|
"export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh",
|
||||||
|
"export GIT_TERMINAL_PROMPT=0",
|
||||||
|
"unset GIT_SSH",
|
||||||
|
"unset GIT_SSH_COMMAND",
|
||||||
|
"remote=\"https://github.com/${repository}.git\"",
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
return [
|
return [
|
||||||
"mkdir -p /root/.ssh",
|
"mkdir -p /root/.ssh",
|
||||||
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
||||||
"chmod 0400 /root/.ssh/id_rsa",
|
"chmod 0400 /root/.ssh/id_rsa",
|
||||||
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
|
...common,
|
||||||
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
|
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
|
||||||
"#!/usr/bin/env node",
|
"#!/usr/bin/env node",
|
||||||
"const net = require('node:net');",
|
"const net = require('node:net');",
|
||||||
@@ -1273,20 +1389,8 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT
|
|||||||
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
|
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
|
||||||
"SH_PROXY",
|
"SH_PROXY",
|
||||||
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
|
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
|
||||||
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
|
|
||||||
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
|
|
||||||
`export ALL_PROXY=${shQuote(proxyUrl)}`,
|
|
||||||
`export http_proxy=${shQuote(proxyUrl)}`,
|
|
||||||
`export https_proxy=${shQuote(proxyUrl)}`,
|
|
||||||
`export all_proxy=${shQuote(proxyUrl)}`,
|
|
||||||
`export NO_PROXY=${shQuote(noProxy)}`,
|
|
||||||
`export no_proxy=${shQuote(noProxy)}`,
|
|
||||||
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
|
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
|
||||||
"unset GIT_SSH_COMMAND",
|
"unset GIT_SSH_COMMAND",
|
||||||
`repository=${shQuote(target.source.repository)}`,
|
|
||||||
`source_branch=${shQuote(target.source.branch)}`,
|
|
||||||
`gitops_branch=${shQuote(target.gitops.branch)}`,
|
|
||||||
"repo=\"/cache/${repository}.git\"",
|
|
||||||
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
||||||
].join("\n");
|
].join("\n");
|
||||||
}
|
}
|
||||||
@@ -1482,6 +1586,7 @@ function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS
|
|||||||
deploymentReplicas: target.gitMirror.deploymentReplicas,
|
deploymentReplicas: target.gitMirror.deploymentReplicas,
|
||||||
syncConfigMap: target.gitMirror.syncConfigMapName,
|
syncConfigMap: target.gitMirror.syncConfigMapName,
|
||||||
egressProxy: target.gitMirror.egressProxy,
|
egressProxy: target.gitMirror.egressProxy,
|
||||||
|
githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport),
|
||||||
statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"],
|
statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"],
|
||||||
},
|
},
|
||||||
pipeline: target.tekton.pipelineName,
|
pipeline: target.tekton.pipelineName,
|
||||||
@@ -1536,6 +1641,19 @@ function k3sDropInContent(spec: ControlPlaneK3sNodeSpec): string {
|
|||||||
].join("\n");
|
].join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function gitMirrorGithubTransportSummary(transport: ControlPlaneGitMirrorGithubTransportSpec): Record<string, unknown> {
|
||||||
|
if (transport.mode === "ssh") return { mode: "ssh", valuesPrinted: false };
|
||||||
|
return {
|
||||||
|
mode: "https",
|
||||||
|
username: transport.username,
|
||||||
|
tokenSecretName: transport.tokenSecretName,
|
||||||
|
tokenSecretKey: transport.tokenSecretKey,
|
||||||
|
tokenSourceRef: transport.tokenSourceRef,
|
||||||
|
tokenSourceKey: transport.tokenSourceKey,
|
||||||
|
valuesPrinted: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function systemdExecArg(value: string): string {
|
function systemdExecArg(value: string): string {
|
||||||
if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value;
|
if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value;
|
||||||
return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`;
|
return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`;
|
||||||
@@ -2343,6 +2461,22 @@ function optionalStringField(obj: Record<string, unknown>, key: string, path: st
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validateKubernetesName(value: string, path: string): void {
|
||||||
|
if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} must be a Kubernetes resource name`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateSecretKey(value: string, path: string): void {
|
||||||
|
if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a Kubernetes Secret key`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateEnvKey(value: string, path: string): void {
|
||||||
|
if (!/^[A-Z0-9_]+$/u.test(value)) throw new Error(`${path} must be an env key`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateSourceRef(value: string, path: string): void {
|
||||||
|
if (!/^[A-Za-z0-9_./-]+$/u.test(value) || value.includes("..")) throw new Error(`${path} has an unsupported sourceRef format`);
|
||||||
|
}
|
||||||
|
|
||||||
function numberField(obj: Record<string, unknown>, key: string, path: string): number {
|
function numberField(obj: Record<string, unknown>, key: string, path: string): number {
|
||||||
const value = obj[key];
|
const value = obj[key];
|
||||||
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`);
|
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`);
|
||||||
|
|||||||
@@ -249,8 +249,20 @@ interface NodeRuntimeGitMirrorTargetSpec {
|
|||||||
sourceBranch: string;
|
sourceBranch: string;
|
||||||
gitopsBranch: string;
|
gitopsBranch: string;
|
||||||
egressProxy: NodeRuntimeGitMirrorEgressProxySpec;
|
egressProxy: NodeRuntimeGitMirrorEgressProxySpec;
|
||||||
|
githubTransport: NodeRuntimeGitMirrorGithubTransportSpec;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type NodeRuntimeGitMirrorGithubTransportSpec =
|
||||||
|
| { mode: "ssh" }
|
||||||
|
| {
|
||||||
|
mode: "https";
|
||||||
|
username: string;
|
||||||
|
tokenSecretName: string;
|
||||||
|
tokenSecretKey: string;
|
||||||
|
tokenSourceRef: string;
|
||||||
|
tokenSourceKey: string;
|
||||||
|
};
|
||||||
|
|
||||||
interface NodeRuntimeGitMirrorEgressProxySpec {
|
interface NodeRuntimeGitMirrorEgressProxySpec {
|
||||||
mode: "k8s-service-cluster-ip";
|
mode: "k8s-service-cluster-ip";
|
||||||
clientName: string;
|
clientName: string;
|
||||||
@@ -2954,6 +2966,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
|||||||
? undefined
|
? undefined
|
||||||
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
||||||
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
|
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
|
||||||
|
const stopped = retryExhausted || retryableFailure?.stopped === true;
|
||||||
return {
|
return {
|
||||||
ok: actionSucceeded && (status === undefined || status.ok === true),
|
ok: actionSucceeded && (status === undefined || status.ok === true),
|
||||||
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
||||||
@@ -2962,6 +2975,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
|||||||
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
|
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
|
||||||
mutation: !scoped.dryRun && actionSucceeded,
|
mutation: !scoped.dryRun && actionSucceeded,
|
||||||
namespace: mirror.namespace,
|
namespace: mirror.namespace,
|
||||||
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
||||||
jobName,
|
jobName,
|
||||||
manifest: scoped.dryRun ? manifest : undefined,
|
manifest: scoped.dryRun ? manifest : undefined,
|
||||||
result: compactRuntimeCommand(result),
|
result: compactRuntimeCommand(result),
|
||||||
@@ -2971,7 +2985,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
|||||||
maxAttempts: retryMaxAttempts,
|
maxAttempts: retryMaxAttempts,
|
||||||
attempts,
|
attempts,
|
||||||
exhausted: retryExhausted,
|
exhausted: retryExhausted,
|
||||||
stopped: retryExhausted,
|
stopped,
|
||||||
valuesPrinted: false,
|
valuesPrinted: false,
|
||||||
} : undefined,
|
} : undefined,
|
||||||
retryableFailure: retryableFailure ?? undefined,
|
retryableFailure: retryableFailure ?? undefined,
|
||||||
@@ -2998,6 +3012,25 @@ function nodeRuntimeGitMirrorRetryableFailure(
|
|||||||
): Record<string, unknown> | null {
|
): Record<string, unknown> | null {
|
||||||
const text = `${result.stdout}\n${result.stderr}`;
|
const text = `${result.stdout}\n${result.stderr}`;
|
||||||
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
|
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
|
||||||
|
const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text);
|
||||||
|
if (authFailure) {
|
||||||
|
return {
|
||||||
|
retryable: false,
|
||||||
|
retryAttempt: attempt,
|
||||||
|
retryMaxAttempts,
|
||||||
|
retryLabel: `${attempt}/${retryMaxAttempts}`,
|
||||||
|
retryExhausted: false,
|
||||||
|
stopped: true,
|
||||||
|
reason: `Git mirror GitHub ${mirror.githubTransport.mode} authentication failed. This is not retryable; fix the YAML-declared token source/Secret instead of consuming retry budget.`,
|
||||||
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
||||||
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
||||||
|
next: {
|
||||||
|
fixSecret: "apply the YAML-declared githubTransport token source through the node control-plane infra apply path",
|
||||||
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||||
|
},
|
||||||
|
valuesPrinted: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
const retryable = partialSuccess !== null
|
const retryable = partialSuccess !== null
|
||||||
|| proxyConnectFailure
|
|| proxyConnectFailure
|
||||||
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
||||||
@@ -3017,8 +3050,9 @@ function nodeRuntimeGitMirrorRetryableFailure(
|
|||||||
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
||||||
: proxyConnectFailure
|
: proxyConnectFailure
|
||||||
? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
|
? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
|
||||||
: "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.",
|
: `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`,
|
||||||
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
||||||
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
||||||
next: exhausted
|
next: exhausted
|
||||||
? {
|
? {
|
||||||
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||||
@@ -3187,6 +3221,7 @@ function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record
|
|||||||
partialSuccess: result.partialSuccess ?? null,
|
partialSuccess: result.partialSuccess ?? null,
|
||||||
fallback: result.fallback ?? null,
|
fallback: result.fallback ?? null,
|
||||||
retry: result.retry ?? null,
|
retry: result.retry ?? null,
|
||||||
|
githubTransport: result.githubTransport ?? null,
|
||||||
retryableFailure: result.retryableFailure ?? null,
|
retryableFailure: result.retryableFailure ?? null,
|
||||||
degradedReason: result.degradedReason ?? null,
|
degradedReason: result.degradedReason ?? null,
|
||||||
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
|
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
|
||||||
@@ -3214,6 +3249,18 @@ function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, act
|
|||||||
}
|
}
|
||||||
|
|
||||||
function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record<string, unknown> {
|
function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record<string, unknown> {
|
||||||
|
const volumes: Record<string, unknown>[] = [
|
||||||
|
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
|
||||||
|
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
|
||||||
|
];
|
||||||
|
const volumeMounts: Record<string, unknown>[] = [
|
||||||
|
{ name: "cache", mountPath: "/cache" },
|
||||||
|
{ name: "script", mountPath: "/script", readOnly: true },
|
||||||
|
];
|
||||||
|
if (mirror.githubTransport.mode === "ssh") {
|
||||||
|
volumes.splice(1, 0, { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } });
|
||||||
|
volumeMounts.splice(1, 0, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true });
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
apiVersion: "batch/v1",
|
apiVersion: "batch/v1",
|
||||||
kind: "Job",
|
kind: "Job",
|
||||||
@@ -3245,22 +3292,14 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec,
|
|||||||
},
|
},
|
||||||
spec: {
|
spec: {
|
||||||
restartPolicy: "Never",
|
restartPolicy: "Never",
|
||||||
volumes: [
|
volumes,
|
||||||
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
|
|
||||||
{ name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } },
|
|
||||||
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
|
|
||||||
],
|
|
||||||
containers: [{
|
containers: [{
|
||||||
name: action,
|
name: action,
|
||||||
image: mirror.toolsImage,
|
image: mirror.toolsImage,
|
||||||
imagePullPolicy: "IfNotPresent",
|
imagePullPolicy: "IfNotPresent",
|
||||||
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
|
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
|
||||||
env: nodeRuntimeGitMirrorProxyEnv(mirror),
|
env: [...nodeRuntimeGitMirrorProxyEnv(mirror), ...nodeRuntimeGitMirrorGithubTransportEnv(mirror)],
|
||||||
volumeMounts: [
|
volumeMounts,
|
||||||
{ name: "cache", mountPath: "/cache" },
|
|
||||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
||||||
{ name: "script", mountPath: "/script", readOnly: true },
|
|
||||||
],
|
|
||||||
}],
|
}],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -3268,7 +3307,7 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec,
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, string>[] {
|
function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
|
||||||
const proxy = mirror.egressProxy;
|
const proxy = mirror.egressProxy;
|
||||||
const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
|
const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
|
||||||
const noProxy = proxy.noProxy.join(",");
|
const noProxy = proxy.noProxy.join(",");
|
||||||
@@ -3284,6 +3323,28 @@ function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): R
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function nodeRuntimeGitMirrorGithubTransportEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
|
||||||
|
if (mirror.githubTransport.mode !== "https") return [];
|
||||||
|
return [{
|
||||||
|
name: "GITHUB_TOKEN",
|
||||||
|
valueFrom: { secretKeyRef: { name: mirror.githubTransport.tokenSecretName, key: mirror.githubTransport.tokenSecretKey } },
|
||||||
|
}];
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
|
||||||
|
const transport = mirror.githubTransport;
|
||||||
|
if (transport.mode === "ssh") return { mode: "ssh", secretName: mirror.secretName, valuesPrinted: false };
|
||||||
|
return {
|
||||||
|
mode: "https",
|
||||||
|
username: transport.username,
|
||||||
|
tokenSecretName: transport.tokenSecretName,
|
||||||
|
tokenSecretKey: transport.tokenSecretKey,
|
||||||
|
tokenSourceRef: transport.tokenSourceRef,
|
||||||
|
tokenSourceKey: transport.tokenSourceKey,
|
||||||
|
valuesPrinted: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||||
const spec = scoped.spec;
|
const spec = scoped.spec;
|
||||||
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
|
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
|
||||||
@@ -5891,6 +5952,7 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM
|
|||||||
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
|
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
|
||||||
if (stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode") !== "node-global") throw new Error(`gitMirror.egressProxy.mode must be node-global for node=${spec.nodeId} lane=${spec.lane}`);
|
if (stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode") !== "node-global") throw new Error(`gitMirror.egressProxy.mode must be node-global for node=${spec.nodeId} lane=${spec.lane}`);
|
||||||
if (gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
|
if (gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
|
||||||
|
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
|
||||||
const source = record(target.source);
|
const source = record(target.source);
|
||||||
const gitops = record(target.gitops);
|
const gitops = record(target.gitops);
|
||||||
const tekton = record(target.tekton);
|
const tekton = record(target.tekton);
|
||||||
@@ -5915,6 +5977,21 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM
|
|||||||
sourceBranch: stringValue(source.branch, "source.branch"),
|
sourceBranch: stringValue(source.branch, "source.branch"),
|
||||||
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
|
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
|
||||||
egressProxy: nodeEgressProxy,
|
egressProxy: nodeEgressProxy,
|
||||||
|
githubTransport,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorGithubTransportSpec {
|
||||||
|
const mode = stringValue(raw.mode, `${path}.mode`);
|
||||||
|
if (mode === "ssh") return { mode };
|
||||||
|
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
|
||||||
|
return {
|
||||||
|
mode,
|
||||||
|
username: stringValue(raw.username, `${path}.username`),
|
||||||
|
tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`),
|
||||||
|
tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`),
|
||||||
|
tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`),
|
||||||
|
tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user