fix: make AgentRun git mirror cover Artificer bundles

This commit is contained in:
Codex
2026-06-11 08:47:04 +00:00
parent 0bef6e577a
commit 6d80bfec94
3 changed files with 170 additions and 48 deletions
+168 -46
View File
@@ -16,13 +16,18 @@ const argoNamespace = "argocd";
const argoApplication = "agentrun-g14-v01";
const gitopsBranch = "v0.1-gitops";
const gitMirrorNamespace = "devops-infra";
const gitMirrorReadUrl = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git";
const gitMirrorWriteUrl = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/agentrun.git";
const gitMirrorRepoPath = "/cache/pikasTech/agentrun.git";
const gitMirrorReadBaseUrl = "http://git-mirror-http.devops-infra.svc.cluster.local";
const gitMirrorWriteBaseUrl = "http://git-mirror-write.devops-infra.svc.cluster.local";
const gitMirrorReadUrl = `${gitMirrorReadBaseUrl}/pikasTech/agentrun.git`;
const gitMirrorWriteUrl = `${gitMirrorWriteBaseUrl}/pikasTech/agentrun.git`;
const gitMirrorSyncJobPrefix = "git-mirror-agentrun-sync-manual";
const gitMirrorFlushJobPrefix = "git-mirror-agentrun-flush-manual";
const githubSshUrl = "ssh://git@ssh.github.com:443/pikasTech/agentrun.git";
const mirrorToolsImage = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1";
const gitMirrorRepositories = [
{ key: "agentrun", repository: "pikasTech/agentrun", sourceBranch, gitopsBranch },
{ key: "unidesk", repository: "pikasTech/unidesk", sourceBranch: "master" },
{ key: "agent_skills", repository: "pikasTech/agent_skills", sourceBranch: "master" },
] as const;
export function agentRunHelp(): unknown {
return {
@@ -2614,7 +2619,7 @@ function gitMirrorJobManifest(action: "sync" | "flush", name: string): Record<st
},
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 300,
activeDeadlineSeconds: 600,
ttlSecondsAfterFinished: 3600,
template: {
metadata: {
@@ -2649,32 +2654,115 @@ function gitMirrorJobManifest(action: "sync" | "flush", name: string): Record<st
};
}
function gitMirrorSshSetupShellLines(): string[] {
return [
"mkdir -p /root/.ssh",
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
"chmod 0400 /root/.ssh/id_rsa",
"cat > /tmp/agentrun-github-proxy-connect.mjs <<'NODE_PROXY'",
"#!/usr/bin/env node",
"import net from \"node:net\";",
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
"const proxyPort = Number.parseInt(proxyPortRaw || \"\", 10);",
"const targetPort = Number.parseInt(targetPortRaw || \"\", 10);",
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);",
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
"let buffer = Buffer.alloc(0);",
"socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });",
"socket.on(\"connect\", () => socket.write(\"CONNECT \" + targetHost + \":\" + targetPort + \" HTTP/1.1\\r\\nHost: \" + targetHost + \":\" + targetPort + \"\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n\"));",
"socket.on(\"error\", () => process.exit(66));",
"function onData(chunk) {",
" buffer = Buffer.concat([buffer, chunk]);",
" const headerEnd = buffer.indexOf(\"\\r\\n\\r\\n\");",
" if (headerEnd === -1 && buffer.length < 8192) return;",
" const head = buffer.slice(0, headerEnd + 4).toString(\"latin1\");",
" const statusLine = head.split(\"\\r\\n\", 1)[0] || \"\";",
" const statusCode = Number.parseInt(statusLine.split(\" \")[1] || \"\", 10);",
" if (!statusLine.startsWith(\"HTTP/1.\") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }",
" socket.off(\"data\", onData);",
" socket.setTimeout(0);",
" const rest = buffer.slice(headerEnd + 4);",
" if (rest.length) process.stdout.write(rest);",
" process.stdin.pipe(socket);",
" socket.pipe(process.stdout);",
"}",
"socket.on(\"data\", onData);",
"socket.on(\"close\", () => process.exit(0));",
"NODE_PROXY",
"chmod 0700 /tmp/agentrun-github-proxy-connect.mjs",
"export GIT_SSH_COMMAND=\"ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/agentrun-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'\"",
];
}
function gitMirrorSyncShellScript(): string {
const repoSpecs = gitMirrorRepositories.map((repo) => [repo.key, repo.repository, repo.sourceBranch, repo.gitopsBranch ?? ""].join("|"));
return [
"set -eu",
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
"mkdir -p /cache/pikasTech",
"export GIT_SSH_COMMAND='ssh -i /git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/agentrun-known-hosts'",
`repo=${shQuote(gitMirrorRepoPath)}`,
`remote=${shQuote(githubSshUrl)}`,
"if [ ! -d \"$repo\" ]; then git clone --mirror \"$remote\" \"$repo\"; else git --git-dir=\"$repo\" remote set-url origin \"$remote\"; fi",
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
"git --git-dir=\"$repo\" config http.receivepack true",
"git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1:refs/heads/v0.1 +refs/heads/v0.1:refs/mirror-stage/heads/v0.1",
"if git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1-gitops:refs/mirror-stage/heads/v0.1-gitops; then",
" if ! git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' >/dev/null 2>&1; then",
" git --git-dir=\"$repo\" update-ref refs/heads/v0.1-gitops \"$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}')\"",
...gitMirrorSshSetupShellLines(),
"sync_repo() {",
" key=\"$1\"",
" repository=\"$2\"",
" source_branch=\"$3\"",
" gitops_branch=\"$4\"",
" repo=\"/cache/${repository}.git\"",
" remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
" mkdir -p \"$(dirname \"$repo\")\"",
" if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
" git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"",
" else",
" rm -rf \"$repo\"",
" git init --bare \"$repo\"",
" git --git-dir=\"$repo\" remote add origin \"$remote\"",
" fi",
"fi",
"local_v01=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1^{commit}')",
"github_v01=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1^{commit}')",
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
"json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }",
"cat > /cache/agentrun.last-sync.json <<EOF",
"{\"event\":\"git-mirror-sync\",\"repo\":\"pikasTech/agentrun\",\"status\":\"succeeded\",\"startedAt\":\"$started_at\",\"syncedAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"localV01\":\"$local_v01\",\"githubV01\":\"$github_v01\",\"localGitops\":$(json_ref \"$local_gitops\"),\"githubGitops\":$(json_ref \"$github_gitops\"),\"pendingFlush\":$pending}",
"EOF",
" git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
" git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
" git --git-dir=\"$repo\" config http.receivepack true",
" timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"",
" source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")",
" git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"",
" if [ -n \"$gitops_branch\" ]; then",
" if timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then",
" github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
" local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
" if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then",
" git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"",
" elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then",
" git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"",
" fi",
" fi",
" fi",
" git --git-dir=\"$repo\" update-server-info",
"}",
`for spec in ${repoSpecs.map(shQuote).join(" ")}; do`,
" IFS='|' read -r key repository source_branch gitops_branch <<EOF_SPEC",
"$spec",
"EOF_SPEC",
" sync_repo \"$key\" \"$repository\" \"$source_branch\" \"$gitops_branch\"",
"done",
`REPOSITORIES_JSON=${shQuote(JSON.stringify(gitMirrorRepositories))}`,
"export REPOSITORIES_JSON started_at",
"node <<'NODE' | tee /cache/agentrun.last-sync.json",
"const { execFileSync } = require('node:child_process');",
"const repositories = JSON.parse(process.env.REPOSITORIES_JSON || '[]');",
"function rev(repo, ref) {",
" try { execFileSync('test', ['-d', repo + '/objects']); } catch { return null; }",
" try { return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); }",
" catch { return null; }",
"}",
"const items = {};",
"for (const spec of repositories) {",
" const repoPath = `/cache/${spec.repository}.git`;",
" const localSource = rev(repoPath, `refs/heads/${spec.sourceBranch}`);",
" const githubSource = rev(repoPath, `refs/mirror-stage/heads/${spec.sourceBranch}`);",
" const localGitops = spec.gitopsBranch ? rev(repoPath, `refs/heads/${spec.gitopsBranch}`) : null;",
" const githubGitops = spec.gitopsBranch ? rev(repoPath, `refs/mirror-stage/heads/${spec.gitopsBranch}`) : null;",
" items[spec.key] = { repository: spec.repository, sourceBranch: spec.sourceBranch, localSource, githubSource, gitopsBranch: spec.gitopsBranch || null, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: spec.gitopsBranch ? Boolean(localGitops && githubGitops && localGitops === githubGitops) : null, pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)) };",
"}",
"const agentrun = items.agentrun || {};",
"console.log(JSON.stringify({ event: 'git-mirror-sync', repo: 'pikasTech/agentrun', status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localV01: agentrun.localSource || null, githubV01: agentrun.githubSource || null, localGitops: agentrun.localGitops || null, githubGitops: agentrun.githubGitops || null, pendingFlush: Boolean(agentrun.pendingFlush), repositories: items }));",
"NODE",
"cat /cache/agentrun.last-sync.json",
].join("\n");
}
@@ -2683,9 +2771,9 @@ function gitMirrorFlushShellScript(): string {
return [
"set -eu",
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
"export GIT_SSH_COMMAND='ssh -i /git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/agentrun-known-hosts'",
`repo=${shQuote(gitMirrorRepoPath)}`,
`remote=${shQuote(githubSshUrl)}`,
...gitMirrorSshSetupShellLines(),
"repo=/cache/pikasTech/agentrun.git",
"remote=ssh://git@ssh.github.com:443/pikasTech/agentrun.git",
"test -d \"$repo\"",
"git --git-dir=\"$repo\" remote set-url origin \"$remote\"",
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
@@ -2707,23 +2795,47 @@ function gitMirrorCacheProbeScript(): string {
return [
"printf 'lastSync='; cat /cache/agentrun.last-sync.json 2>/dev/null || true; printf '\\n'",
"printf 'lastFlush='; cat /cache/agentrun.last-flush.json 2>/dev/null || true; printf '\\n'",
`repo=${shQuote(gitMirrorRepoPath)}`,
"rev() { git --git-dir=\"$repo\" rev-parse --verify \"$1^{commit}\" 2>/dev/null || true; }",
"local_v01=$(rev refs/heads/v0.1)",
"github_v01=$(rev refs/mirror-stage/heads/v0.1)",
"local_gitops=$(rev refs/heads/v0.1-gitops)",
"github_gitops=$(rev refs/mirror-stage/heads/v0.1-gitops)",
"exact=false",
"exact_commit=\"\"",
"if [ -n \"$local_v01\" ]; then",
" tmp=$(mktemp -d)",
" if git init -q \"$tmp\" && git -C \"$tmp\" remote add origin http://127.0.0.1:8080/pikasTech/agentrun.git && git -C \"$tmp\" fetch --depth=1 origin \"$local_v01\" >/tmp/agentrun-exact-fetch.log 2>&1; then",
" exact_commit=$(git -C \"$tmp\" rev-parse FETCH_HEAD 2>/dev/null || true)",
" if [ \"$exact_commit\" = \"$local_v01\" ]; then exact=true; fi",
" fi",
" rm -rf \"$tmp\"",
"fi",
"node -e 'const [localV01,githubV01,localGitops,githubGitops,exact,exactCommit]=process.argv.slice(1); const n=v=>v&&v.length>0?v:null; const pending=Boolean(n(localGitops)&&(!n(githubGitops)||localGitops!==githubGitops)); console.log(\"refs=\"+JSON.stringify({refs:{localV01:n(localV01),githubV01:n(githubV01),localGitops:n(localGitops),githubGitops:n(githubGitops)},pendingFlush:pending,exactFetch:{localV01:exact===\"true\",commit:n(exactCommit)}}));' \"$local_v01\" \"$github_v01\" \"$local_gitops\" \"$github_gitops\" \"$exact\" \"$exact_commit\"",
`REPOSITORIES_JSON=${shQuote(JSON.stringify(gitMirrorRepositories))}`,
"export REPOSITORIES_JSON",
"node <<'NODE'",
"const { execFileSync } = require('node:child_process');",
"const { mkdtempSync, rmSync } = require('node:fs');",
"const { tmpdir } = require('node:os');",
"const { join } = require('node:path');",
"const repositories = JSON.parse(process.env.REPOSITORIES_JSON || '[]');",
"function rev(repo, ref) {",
" try { execFileSync('test', ['-d', repo + '/objects']); } catch { return null; }",
" try { return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); }",
" catch { return null; }",
"}",
"function exactFetch(repository, commit) {",
" if (!commit) return { ok: false, commit: null };",
" const dir = mkdtempSync(join(tmpdir(), 'agentrun-mirror-probe-'));",
" try {",
" execFileSync('git', ['init', '-q', dir]);",
" execFileSync('git', ['-C', dir, 'remote', 'add', 'origin', `http://127.0.0.1:8080/${repository}.git`]);",
" execFileSync('git', ['-C', dir, 'fetch', '--depth=1', 'origin', commit], { stdio: 'ignore' });",
" const fetched = execFileSync('git', ['-C', dir, 'rev-parse', 'FETCH_HEAD'], { encoding: 'utf8' }).trim();",
" return { ok: fetched === commit, commit: fetched || null };",
" } catch {",
" return { ok: false, commit: null };",
" } finally {",
" rmSync(dir, { recursive: true, force: true });",
" }",
"}",
"const items = {};",
"for (const spec of repositories) {",
" const repoPath = `/cache/${spec.repository}.git`;",
" const localSource = rev(repoPath, `refs/heads/${spec.sourceBranch}`);",
" const githubSource = rev(repoPath, `refs/mirror-stage/heads/${spec.sourceBranch}`);",
" const localGitops = spec.gitopsBranch ? rev(repoPath, `refs/heads/${spec.gitopsBranch}`) : null;",
" const githubGitops = spec.gitopsBranch ? rev(repoPath, `refs/mirror-stage/heads/${spec.gitopsBranch}`) : null;",
" const exact = exactFetch(spec.repository, localSource);",
" items[spec.key] = { repository: spec.repository, sourceBranch: spec.sourceBranch, localSource, githubSource, gitopsBranch: spec.gitopsBranch || null, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: spec.gitopsBranch ? Boolean(localGitops && githubGitops && localGitops === githubGitops) : null, pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)), exactFetch: exact };",
"}",
"const agentrun = items.agentrun || {};",
"console.log('refs=' + JSON.stringify({ refs: { localV01: agentrun.localSource || null, githubV01: agentrun.githubSource || null, localGitops: agentrun.localGitops || null, githubGitops: agentrun.githubGitops || null }, pendingFlush: Boolean(agentrun.pendingFlush), exactFetch: { localV01: Boolean(agentrun.exactFetch && agentrun.exactFetch.ok), commit: agentrun.exactFetch ? agentrun.exactFetch.commit : null }, repositories: items }));",
"NODE",
].join("\n");
}
@@ -2743,9 +2855,10 @@ function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
localGitops,
githubGitops,
pendingFlush,
repositories: record(refs).repositories ?? record(lastSync).repositories ?? null,
sourceInSync: Boolean(localV01 && githubV01 && localV01 === githubV01),
gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops),
githubInSync: Boolean(localV01 && githubV01 && localV01 === githubV01 && (!localGitops || localGitops === githubGitops)),
githubInSync: Boolean(localV01 && githubV01 && localV01 === githubV01 && (!localGitops || localGitops === githubGitops)) && gitMirrorBundleRepositoriesReady(refs),
flushNeeded: pendingFlush === true,
flushCommand: pendingFlush === true ? "bun scripts/cli.ts agentrun git-mirror flush --confirm" : null,
exactFetch: record(refs).exactFetch ?? null,
@@ -2754,6 +2867,15 @@ function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
};
}
function gitMirrorBundleRepositoriesReady(refs: unknown): boolean {
const repositories = record(record(refs).repositories);
const expected = gitMirrorRepositories.filter((repo) => repo.key !== "agentrun").map((repo) => repo.key);
return expected.every((key) => {
const item = record(repositories[key]);
return item.sourceInSync === true && record(item.exactFetch).ok === true;
});
}
function gitMirrorSyncRequirement(sourceCommit: string, rawStatus: string): { required: boolean; reason: string; localV01: string | null } {
const summary = gitMirrorStatusSummary(rawStatus);
const localV01 = stringOrNull(summary.localV01);