mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use https transport for hwlab git mirror (#580)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-21 16:02:23 +08:00
committed by GitHub
parent d1c773e319
commit 476ced5ea3
3 changed files with 248 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+91 -14
View File
@@ -249,8 +249,20 @@ interface NodeRuntimeGitMirrorTargetSpec {
sourceBranch: string;
gitopsBranch: string;
egressProxy: NodeRuntimeGitMirrorEgressProxySpec;
githubTransport: NodeRuntimeGitMirrorGithubTransportSpec;
}
type NodeRuntimeGitMirrorGithubTransportSpec =
| { mode: "ssh" }
| {
mode: "https";
username: string;
tokenSecretName: string;
tokenSecretKey: string;
tokenSourceRef: string;
tokenSourceKey: string;
};
interface NodeRuntimeGitMirrorEgressProxySpec {
mode: "k8s-service-cluster-ip";
clientName: string;
@@ -2954,6 +2966,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
? undefined
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
const stopped = retryExhausted || retryableFailure?.stopped === true;
return {
ok: actionSucceeded && (status === undefined || status.ok === true),
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
@@ -2962,6 +2975,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
mutation: !scoped.dryRun && actionSucceeded,
namespace: mirror.namespace,
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
jobName,
manifest: scoped.dryRun ? manifest : undefined,
result: compactRuntimeCommand(result),
@@ -2971,7 +2985,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
maxAttempts: retryMaxAttempts,
attempts,
exhausted: retryExhausted,
stopped: retryExhausted,
stopped,
valuesPrinted: false,
} : undefined,
retryableFailure: retryableFailure ?? undefined,
@@ -2998,6 +3012,25 @@ function nodeRuntimeGitMirrorRetryableFailure(
): Record<string, unknown> | null {
const text = `${result.stdout}\n${result.stderr}`;
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text);
if (authFailure) {
return {
retryable: false,
retryAttempt: attempt,
retryMaxAttempts,
retryLabel: `${attempt}/${retryMaxAttempts}`,
retryExhausted: false,
stopped: true,
reason: `Git mirror GitHub ${mirror.githubTransport.mode} authentication failed. This is not retryable; fix the YAML-declared token source/Secret instead of consuming retry budget.`,
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
next: {
fixSecret: "apply the YAML-declared githubTransport token source through the node control-plane infra apply path",
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
},
valuesPrinted: false,
};
}
const retryable = partialSuccess !== null
|| proxyConnectFailure
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
@@ -3017,8 +3050,9 @@ function nodeRuntimeGitMirrorRetryableFailure(
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
: proxyConnectFailure
? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
: "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.",
: `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`,
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
next: exhausted
? {
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
@@ -3187,6 +3221,7 @@ function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record
partialSuccess: result.partialSuccess ?? null,
fallback: result.fallback ?? null,
retry: result.retry ?? null,
githubTransport: result.githubTransport ?? null,
retryableFailure: result.retryableFailure ?? null,
degradedReason: result.degradedReason ?? null,
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
@@ -3214,6 +3249,18 @@ function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, act
}
function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record<string, unknown> {
const volumes: Record<string, unknown>[] = [
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
];
const volumeMounts: Record<string, unknown>[] = [
{ name: "cache", mountPath: "/cache" },
{ name: "script", mountPath: "/script", readOnly: true },
];
if (mirror.githubTransport.mode === "ssh") {
volumes.splice(1, 0, { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } });
volumeMounts.splice(1, 0, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true });
}
return {
apiVersion: "batch/v1",
kind: "Job",
@@ -3245,22 +3292,14 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec,
},
spec: {
restartPolicy: "Never",
volumes: [
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
{ name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } },
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
],
volumes,
containers: [{
name: action,
image: mirror.toolsImage,
imagePullPolicy: "IfNotPresent",
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
env: nodeRuntimeGitMirrorProxyEnv(mirror),
volumeMounts: [
{ name: "cache", mountPath: "/cache" },
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
{ name: "script", mountPath: "/script", readOnly: true },
],
env: [...nodeRuntimeGitMirrorProxyEnv(mirror), ...nodeRuntimeGitMirrorGithubTransportEnv(mirror)],
volumeMounts,
}],
},
},
@@ -3268,7 +3307,7 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec,
};
}
function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, string>[] {
function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
const proxy = mirror.egressProxy;
const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
const noProxy = proxy.noProxy.join(",");
@@ -3284,6 +3323,28 @@ function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): R
];
}
function nodeRuntimeGitMirrorGithubTransportEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
if (mirror.githubTransport.mode !== "https") return [];
return [{
name: "GITHUB_TOKEN",
valueFrom: { secretKeyRef: { name: mirror.githubTransport.tokenSecretName, key: mirror.githubTransport.tokenSecretKey } },
}];
}
function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
const transport = mirror.githubTransport;
if (transport.mode === "ssh") return { mode: "ssh", secretName: mirror.secretName, valuesPrinted: false };
return {
mode: "https",
username: transport.username,
tokenSecretName: transport.tokenSecretName,
tokenSecretKey: transport.tokenSecretKey,
tokenSourceRef: transport.tokenSourceRef,
tokenSourceKey: transport.tokenSourceKey,
valuesPrinted: false,
};
}
function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
@@ -5891,6 +5952,7 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
if (stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode") !== "node-global") throw new Error(`gitMirror.egressProxy.mode must be node-global for node=${spec.nodeId} lane=${spec.lane}`);
if (gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
const source = record(target.source);
const gitops = record(target.gitops);
const tekton = record(target.tekton);
@@ -5915,6 +5977,21 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM
sourceBranch: stringValue(source.branch, "source.branch"),
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
egressProxy: nodeEgressProxy,
githubTransport,
};
}
function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorGithubTransportSpec {
const mode = stringValue(raw.mode, `${path}.mode`);
if (mode === "ssh") return { mode };
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
return {
mode,
username: stringValue(raw.username, `${path}.username`),
tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`),
tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`),
tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`),
tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`),
};
}