mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use k8s git mirror source snapshots

Browse Source
This commit is contained in:
Codex
2026-07-01 11:03:18 +00:00
parent b972ebd290
commit d687bdf668
20 changed files with 465 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-control-plane.ts
+55 -6
View File
@@ -1898,6 +1898,8 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
const ciNamespace = stringField(raw, "ciNamespace", path);
const runtimeNamespace = stringField(raw, "runtimeNamespace", path);
const argoNamespace = stringField(argo, "namespace", `${path}.argo`);
const sourceAuthority = controlPlaneSourceAuthoritySpec(asRecord(source.sourceAuthority, `${path}.source.sourceAuthority`), `${path}.source.sourceAuthority`);
const sourceSnapshot = controlPlaneSourceSnapshotSpec(asRecord(source.sourceSnapshot, `${path}.source.sourceSnapshot`), `${path}.source.sourceSnapshot`);
return {
id: stringField(raw, "id", path),
node,
@@ -1905,7 +1907,7 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
enabled: booleanField(raw, "enabled", path),
ciNamespace,
runtimeNamespace,
source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`) },
source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`), sourceAuthority, sourceSnapshot },
gitops: { branch: stringField(gitops, "branch", `${path}.gitops`), path: stringField(gitops, "path", `${path}.gitops`) },
gitMirror: {
namespace: gitMirrorNamespace,
@@ -1948,6 +1950,38 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
};
}
function controlPlaneSourceAuthoritySpec(raw: Record<string, unknown>, path: string): ControlPlaneTargetSpec["source"]["sourceAuthority"] {
const mode = stringField(raw, "mode", path);
const resolver = stringField(raw, "resolver", path);
if (mode !== "gitMirrorSnapshot") throw new Error(`${path}.mode must be gitMirrorSnapshot`);
if (resolver !== "k8s-git-mirror") throw new Error(`${path}.resolver must be k8s-git-mirror`);
return {
mode,
resolver,
allowHostGit: falseBooleanField(raw, "allowHostGit", path),
allowHostWorkspace: falseBooleanField(raw, "allowHostWorkspace", path),
allowGithubDirectInPipeline: falseBooleanField(raw, "allowGithubDirectInPipeline", path),
};
}
function controlPlaneSourceSnapshotSpec(raw: Record<string, unknown>, path: string): ControlPlaneTargetSpec["source"]["sourceSnapshot"] {
const stageRefPrefix = stringField(raw, "stageRefPrefix", path);
if (!stageRefPrefix.startsWith("refs/")) throw new Error(`${path}.stageRefPrefix must start with refs/`);
if (stageRefPrefix.includes("..") || /\s/u.test(stageRefPrefix)) throw new Error(`${path}.stageRefPrefix must not contain whitespace or ..`);
if (!stageRefPrefix.includes("{branch}")) throw new Error(`${path}.stageRefPrefix must include {branch}`);
const missingObjectPolicy = stringField(raw, "missingObjectPolicy", path);
const refreshPolicy = stringField(raw, "refreshPolicy", path);
if (missingObjectPolicy !== "fail-fast") throw new Error(`${path}.missingObjectPolicy must be fail-fast`);
if (refreshPolicy !== "sync-before-snapshot") throw new Error(`${path}.refreshPolicy must be sync-before-snapshot`);
return { stageRefPrefix, missingObjectPolicy, refreshPolicy };
}
function falseBooleanField(obj: Record<string, unknown>, key: string, path: string): false {
const value = booleanField(obj, key, path);
if (value !== false) throw new Error(`${path}.${key} must be false`);
return false;
}
function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record<string, unknown>[] {
const labels = {
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
@@ -1977,7 +2011,7 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa
kind: "ConfigMap",
metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } },
data: {
"repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2),
"repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, sourceStageRefPrefix: target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch), gitopsBranch: target.gitops.branch }], null, 2),
"server.js": gitMirrorServerJs(),
"status.sh": gitMirrorStatusShell(),
"sync.sh": gitMirrorSyncShell(_node, target),
@@ -2280,7 +2314,7 @@ function service(name: string, namespace: string, labels: Record<string, string>
function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
return sha256Short(JSON.stringify({
repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }],
repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, sourceStageRefPrefix: target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch), gitopsBranch: target.gitops.branch }],
ports: { read: target.gitMirror.readContainerPort, write: target.gitMirror.writeContainerPort },
githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport),
runtimeProxy: runtimeHostProxyConfig(node, gitMirrorRuntimeProxySpec(node, target)),
@@ -2467,6 +2501,9 @@ for (const spec of repositories) {
const repoPath = '/cache/' + spec.repository + '.git';
const localSource = rev(repoPath, 'refs/heads/' + spec.sourceBranch);
const githubSource = rev(repoPath, 'refs/mirror-stage/heads/' + spec.sourceBranch);
const sourceStageRefPrefix = spec.sourceStageRefPrefix || ('refs/unidesk/snapshots/hwlab-node-runtime/' + spec.sourceBranch);
const sourceStageRef = githubSource ? sourceStageRefPrefix.replace(/\/+$/, '') + '/' + githubSource : null;
const sourceSnapshot = sourceStageRef ? rev(repoPath, sourceStageRef) : null;
const localGitops = rev(repoPath, 'refs/heads/' + spec.gitopsBranch);
const githubGitops = rev(repoPath, 'refs/mirror-stage/heads/' + spec.gitopsBranch);
items[spec.key] = {
@@ -2474,10 +2511,13 @@ for (const spec of repositories) {
sourceBranch: spec.sourceBranch,
localSource,
githubSource,
sourceAuthority: 'git-mirror-snapshot',
sourceStageRef,
sourceSnapshot,
gitopsBranch: spec.gitopsBranch,
localGitops,
githubGitops,
sourceInSync: Boolean(localSource && githubSource && localSource === githubSource),
sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource),
gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops),
pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)),
};
@@ -2487,11 +2527,15 @@ const pendingFlush = Object.values(items).some((item) => Boolean(item.pendingFlu
console.log(JSON.stringify({
localSource: first.localSource || null,
githubSource: first.githubSource || null,
sourceAuthority: first.sourceAuthority || null,
sourceStageRef: first.sourceStageRef || null,
sourceSnapshot: first.sourceSnapshot || null,
localGitops: first.localGitops || null,
githubGitops: first.githubGitops || null,
refSources: {
localSource: 'refs/heads/' + (first.sourceBranch || ''),
githubSource: 'refs/mirror-stage/heads/' + (first.sourceBranch || ''),
sourceSnapshot: first.sourceStageRef || null,
localGitops: 'refs/heads/' + (first.gitopsBranch || ''),
githubGitops: 'refs/mirror-stage/heads/' + (first.gitopsBranch || ''),
githubFieldsAreMirrorStageCache: true
@@ -2557,6 +2601,7 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT
useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`,
`repository=${shQuote(target.source.repository)}`,
`source_branch=${shQuote(target.source.branch)}`,
`source_stage_ref_prefix=${shQuote(target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch))}`,
`gitops_branch=${shQuote(target.gitops.branch)}`,
"repo=\"/cache/${repository}.git\"",
].filter(Boolean);
@@ -2678,6 +2723,8 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg
"git --git-dir=\"$repo\" config http.receivepack true",
"timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"",
"source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")",
"source_stage_ref=\"${source_stage_ref_prefix%/}/$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"$source_stage_ref\" \"$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"",
"discarded_stale_gitops=false",
"if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then",
@@ -2694,7 +2741,7 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg
" fi",
"fi",
"git --git-dir=\"$repo\" update-server-info",
"export repository source_branch gitops_branch started_at discarded_stale_gitops",
"export repository source_branch source_stage_ref gitops_branch started_at discarded_stale_gitops",
"node <<'NODE' | tee /cache/HWLAB.last-sync.json",
"const { execFileSync } = require('node:child_process');",
"const repository = process.env.repository;",
@@ -2704,10 +2751,12 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg
"function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }",
"const localSource = rev(`refs/heads/${sourceBranch}`);",
"const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);",
"const sourceStageRef = process.env.source_stage_ref;",
"const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;",
"const localGitops = rev(`refs/heads/${gitopsBranch}`);",
"const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);",
"const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));",
"console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));",
"console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), sourceAuthority: 'git-mirror-snapshot', localSource, githubSource, sourceStageRef, sourceSnapshot, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));",
"NODE",
"cat /cache/HWLAB.last-sync.json",
"",