feat: add agentrun git mirror controls
This commit is contained in:
Codex
+500
-8
@@ -1,5 +1,6 @@
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
||||
import { startJob } from "./jobs";
|
||||
|
||||
const g14SourceRoute = "G14:/root/agentrun-v01";
|
||||
const g14K3sRoute = "G14:k3s";
|
||||
@@ -10,10 +11,18 @@ const pipelineName = "agentrun-v01-ci-image-publish";
|
||||
const argoNamespace = "argocd";
|
||||
const argoApplication = "agentrun-g14-v01";
|
||||
const gitopsBranch = "v0.1-gitops";
|
||||
const gitMirrorNamespace = "devops-infra";
|
||||
const gitMirrorReadUrl = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git";
|
||||
const gitMirrorWriteUrl = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/agentrun.git";
|
||||
const gitMirrorRepoPath = "/cache/pikasTech/agentrun.git";
|
||||
const gitMirrorSyncJobPrefix = "git-mirror-agentrun-sync-manual";
|
||||
const gitMirrorFlushJobPrefix = "git-mirror-agentrun-flush-manual";
|
||||
const githubSshUrl = "ssh://git@ssh.github.com:443/pikasTech/agentrun.git";
|
||||
const mirrorToolsImage = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1";
|
||||
|
||||
export function agentRunHelp(): unknown {
|
||||
return {
|
||||
command: "agentrun v01 control-plane status|trigger-current|refresh",
|
||||
command: "agentrun v01 control-plane status|trigger-current|refresh | git-mirror status|sync|flush",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts agentrun v01 control-plane status",
|
||||
@@ -21,17 +30,30 @@ export function agentRunHelp(): unknown {
|
||||
"bun scripts/cli.ts agentrun v01 control-plane trigger-current --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane refresh --dry-run",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane refresh --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 git-mirror status",
|
||||
"bun scripts/cli.ts agentrun v01 git-mirror sync --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 git-mirror flush --confirm",
|
||||
],
|
||||
description: "Operate AgentRun v0.1 Tekton/Argo control plane through G14 routes; trigger-current is short-return and status is read-only.",
|
||||
description: "Operate AgentRun v0.1 Tekton/Argo control plane and devops-infra git mirror through G14 routes; status is read-only and trigger-current pre-syncs mirror refs before creating the PipelineRun.",
|
||||
};
|
||||
}
|
||||
|
||||
export async function runAgentRunCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
||||
const [lane, group, action] = args;
|
||||
if (lane !== "v01" || group !== "control-plane") return unsupported(args);
|
||||
if (action === "status") return await status(config);
|
||||
if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(args.slice(3)));
|
||||
if (action === "refresh") return await refresh(config, parseConfirmOptions(args.slice(3)));
|
||||
if (lane !== "v01") return unsupported(args);
|
||||
if (group === "control-plane") {
|
||||
if (action === "status") return await status(config);
|
||||
if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(args.slice(3)));
|
||||
if (action === "refresh") return await refresh(config, parseConfirmOptions(args.slice(3)));
|
||||
}
|
||||
if (group === "git-mirror") {
|
||||
if (action === "status") return await gitMirrorStatus(config);
|
||||
if (action === "sync" || action === "flush") {
|
||||
const options = parseGitMirrorOptions(args.slice(3));
|
||||
if (options.confirm && !options.wait) return startAsyncAgentRunJob(`agentrun_v01_git_mirror_${action}`, ["bun", "scripts/cli.ts", "agentrun", "v01", "git-mirror", action, "--confirm", "--wait", "--timeout-seconds", String(options.timeoutSeconds)], `Run AgentRun v0.1 git mirror ${action} on G14`);
|
||||
return await runGitMirrorJob(config, action, options);
|
||||
}
|
||||
}
|
||||
return unsupported(args);
|
||||
}
|
||||
|
||||
@@ -45,6 +67,11 @@ interface ConfirmOptions {
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface GitMirrorOptions extends ConfirmOptions {
|
||||
timeoutSeconds: number;
|
||||
wait: boolean;
|
||||
}
|
||||
|
||||
function parseTriggerOptions(args: string[]): TriggerOptions {
|
||||
return parseConfirmOptions(args);
|
||||
}
|
||||
@@ -57,6 +84,14 @@ function parseConfirmOptions(args: string[]): ConfirmOptions {
|
||||
};
|
||||
}
|
||||
|
||||
function parseGitMirrorOptions(args: string[]): GitMirrorOptions {
|
||||
const base = parseConfirmOptions(args);
|
||||
const timeoutIndex = args.indexOf("--timeout-seconds");
|
||||
const timeoutSeconds = timeoutIndex >= 0 ? Number(args[timeoutIndex + 1]) : 300;
|
||||
if (!Number.isFinite(timeoutSeconds) || timeoutSeconds < 30) throw new Error("--timeout-seconds must be a number >= 30");
|
||||
return { ...base, timeoutSeconds, wait: args.includes("--wait") };
|
||||
}
|
||||
|
||||
async function status(config: UniDeskConfig): Promise<Record<string, unknown>> {
|
||||
const source = await capture(config, g14SourceRoute, ["script", "--", [
|
||||
"cd /root/agentrun-v01",
|
||||
@@ -76,8 +111,10 @@ async function status(config: UniDeskConfig): Promise<Record<string, unknown>> {
|
||||
const pipelineRun = sourceCommit ? pipelineRunName(sourceCommit) : null;
|
||||
const k3s = await capture(config, g14K3sRoute, ["script", "--", statusScript(pipelineRun)]);
|
||||
const argo = parseArgoStatus(k3s.stdout);
|
||||
const mirror = await gitMirrorStatus(config);
|
||||
const ciSummary = labeledJson(k3s.stdout, "ciSummary");
|
||||
return {
|
||||
ok: source.exitCode === 0 && k3s.exitCode === 0,
|
||||
ok: source.exitCode === 0 && k3s.exitCode === 0 && mirror.ok === true,
|
||||
command: "agentrun v01 control-plane status",
|
||||
lane: "v0.1",
|
||||
sourceCommit,
|
||||
@@ -88,6 +125,13 @@ async function status(config: UniDeskConfig): Promise<Record<string, unknown>> {
|
||||
expectedPipelineRun: pipelineRun,
|
||||
source: compactCapture(source),
|
||||
runtime: compactCapture(k3s),
|
||||
ciSummary,
|
||||
gitMirror: {
|
||||
ok: mirror.ok,
|
||||
readUrl: mirror.readUrl,
|
||||
writeUrl: mirror.writeUrl,
|
||||
summary: mirror.summary,
|
||||
},
|
||||
runtimeAlignment: {
|
||||
localHeadMatchesOrigin: Boolean(localSourceCommit && originSourceCommit && localSourceCommit === originSourceCommit),
|
||||
argoRevision: argo.revision,
|
||||
@@ -127,22 +171,55 @@ async function triggerCurrent(config: UniDeskConfig, options: TriggerOptions): P
|
||||
namespace: ciNamespace,
|
||||
pipeline: pipelineName,
|
||||
runtimeNamespace,
|
||||
gitMirror: {
|
||||
readUrl: gitMirrorReadUrl,
|
||||
writeUrl: gitMirrorWriteUrl,
|
||||
},
|
||||
};
|
||||
const mirrorBefore = await gitMirrorStatus(config);
|
||||
const mirrorRequirement = gitMirrorSyncRequirement(sourceCommit, String(mirrorBefore.raw ?? ""));
|
||||
if (options.dryRun || !options.confirm) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "agentrun v01 control-plane trigger-current",
|
||||
dryRun: true,
|
||||
plan,
|
||||
gitMirrorPreSync: {
|
||||
required: mirrorRequirement.required,
|
||||
reason: mirrorRequirement.reason,
|
||||
before: mirrorBefore.summary,
|
||||
},
|
||||
next: { confirm: "bun scripts/cli.ts agentrun v01 control-plane trigger-current --confirm" },
|
||||
};
|
||||
}
|
||||
let gitMirrorPreSync: Record<string, unknown> = {
|
||||
required: mirrorRequirement.required,
|
||||
reason: mirrorRequirement.reason,
|
||||
before: mirrorBefore.summary,
|
||||
};
|
||||
if (mirrorRequirement.required) {
|
||||
const synced = await runGitMirrorJob(config, "sync", { confirm: true, dryRun: false, timeoutSeconds: 300, wait: true });
|
||||
const after = await gitMirrorStatus(config);
|
||||
const afterRequirement = gitMirrorSyncRequirement(sourceCommit, String(after.raw ?? ""));
|
||||
gitMirrorPreSync = { ...gitMirrorPreSync, sync: synced, after: after.summary, ok: synced.ok === true && afterRequirement.required === false };
|
||||
if (synced.ok !== true || afterRequirement.required !== false) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "agentrun v01 control-plane trigger-current",
|
||||
dryRun: false,
|
||||
degradedReason: "git-mirror-local-v01-not-current-after-sync",
|
||||
plan,
|
||||
gitMirrorPreSync,
|
||||
};
|
||||
}
|
||||
}
|
||||
const created = await capture(config, g14K3sRoute, ["script", "--", triggerScript(sourceCommit, pipelineRun)]);
|
||||
return {
|
||||
ok: created.exitCode === 0,
|
||||
command: "agentrun v01 control-plane trigger-current",
|
||||
dryRun: false,
|
||||
plan,
|
||||
gitMirrorPreSync,
|
||||
created: compactCapture(created),
|
||||
next: {
|
||||
status: "bun scripts/cli.ts agentrun v01 control-plane status",
|
||||
@@ -204,6 +281,31 @@ function statusScript(pipelineRun: string | null): string {
|
||||
: "true",
|
||||
"printf 'recentPipelineRuns\\n'",
|
||||
`kubectl -n ${ciNamespace} get pipelinerun --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,STATUS:.status.conditions[0].status,REASON:.status.conditions[0].reason,CREATED:.metadata.creationTimestamp' --no-headers 2>/dev/null | tail -n 5 || true`,
|
||||
"printf 'ciSummary='",
|
||||
pr.length > 0
|
||||
? [
|
||||
`if kubectl -n ${ciNamespace} get taskrun -l tekton.dev/pipelineRun=${shQuote(pr)} -o json >/tmp/agentrun-v01-taskruns.json 2>/dev/null; then`,
|
||||
"node <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const data = JSON.parse(fs.readFileSync('/tmp/agentrun-v01-taskruns.json', 'utf8'));",
|
||||
"const items = Array.isArray(data.items) ? data.items : [];",
|
||||
"function task(item) { return item?.metadata?.labels?.['tekton.dev/pipelineTask'] || item?.metadata?.name || null; }",
|
||||
"function result(item, name) { return (item?.status?.results || item?.status?.taskResults || []).find((entry) => entry.name === name)?.value ?? null; }",
|
||||
"function seconds(start, end) { const s = Date.parse(start || ''); const e = Date.parse(end || ''); return Number.isFinite(s) && Number.isFinite(e) ? Math.round((e - s) / 1000) : null; }",
|
||||
"const taskRuns = items.map((item) => ({ name: item?.metadata?.name || null, task: task(item), status: item?.status?.conditions?.[0]?.status || null, reason: item?.status?.conditions?.[0]?.reason || null, seconds: seconds(item?.status?.startTime, item?.status?.completionTime) }));",
|
||||
"const plan = items.find((item) => task(item) === 'plan-artifacts');",
|
||||
"const image = items.find((item) => task(item) === 'image-publish');",
|
||||
"const imageSeconds = taskRuns.find((item) => item.task === 'image-publish')?.seconds ?? null;",
|
||||
"console.log(JSON.stringify({",
|
||||
" planArtifacts: { summary: plan ? result(plan, 'summary') : null, envIdentity: plan ? result(plan, 'env-identity') : null, buildCount: plan ? Number(result(plan, 'build-count') || 0) : null, reuseCount: plan ? Number(result(plan, 'reuse-count') || 0) : null },",
|
||||
" envImage: { status: image ? result(image, 'status') : null, envIdentity: image ? result(image, 'env-identity') : null, image: image ? result(image, 'image') : null, digest: image ? result(image, 'digest') : null, repositoryDigest: image ? result(image, 'repository-digest') : null },",
|
||||
" taskRuns,",
|
||||
" performance: { imagePublishSeconds: imageSeconds, buildWarning: typeof imageSeconds === 'number' && imageSeconds > 120, criticalWarning: typeof imageSeconds === 'number' && imageSeconds > 180 }",
|
||||
"}));",
|
||||
"NODE",
|
||||
"else printf '{}\\n'; fi",
|
||||
].join("\n")
|
||||
: "printf '{}\\n'",
|
||||
"printf 'argo\\n'",
|
||||
`kubectl -n ${argoNamespace} get application ${argoApplication} -o 'jsonpath={.status.sync.revision}{"\\t"}{.status.sync.status}{"\\t"}{.status.health.status}{"\\n"}' 2>/dev/null || true`,
|
||||
"printf 'workloads\\n'",
|
||||
@@ -266,6 +368,10 @@ function triggerScript(sourceCommit: string, pipelineRun: string): string {
|
||||
" params:",
|
||||
" - name: revision",
|
||||
` value: ${sourceCommit}`,
|
||||
" - name: git-read-url",
|
||||
` value: ${gitMirrorReadUrl}`,
|
||||
" - name: git-write-url",
|
||||
` value: ${gitMirrorWriteUrl}`,
|
||||
" workspaces:",
|
||||
" - name: source",
|
||||
" volumeClaimTemplate:",
|
||||
@@ -282,6 +388,365 @@ function triggerScript(sourceCommit: string, pipelineRun: string): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function gitMirrorStatus(config: UniDeskConfig): Promise<Record<string, unknown>> {
|
||||
const script = [
|
||||
"set +e",
|
||||
"printf 'resources\\n'",
|
||||
`kubectl -n ${gitMirrorNamespace} get deploy,svc,pvc,cm -l app.kubernetes.io/name=git-mirror -o name 2>/dev/null || true`,
|
||||
"printf 'jobs\\n'",
|
||||
`kubectl -n ${gitMirrorNamespace} get job -l app.kubernetes.io/name=git-mirror --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime' --no-headers 2>/dev/null | tail -n 8 || true`,
|
||||
"printf 'cache\\n'",
|
||||
`kubectl exec -n ${gitMirrorNamespace} deploy/git-mirror-http -- sh -lc ${shQuote(gitMirrorCacheProbeScript())}`,
|
||||
].join("\n");
|
||||
const result = await capture(config, g14K3sRoute, ["script", "--", script]);
|
||||
const raw = result.stdout;
|
||||
const summary = gitMirrorStatusSummary(raw);
|
||||
return {
|
||||
ok: result.exitCode === 0 && summary.localV01 !== null,
|
||||
command: "agentrun v01 git-mirror status",
|
||||
namespace: gitMirrorNamespace,
|
||||
readUrl: gitMirrorReadUrl,
|
||||
writeUrl: gitMirrorWriteUrl,
|
||||
raw,
|
||||
summary,
|
||||
probe: compactCapture(result),
|
||||
next: {
|
||||
sync: "bun scripts/cli.ts agentrun v01 git-mirror sync --confirm",
|
||||
flush: summary.pendingFlush === true ? "bun scripts/cli.ts agentrun v01 git-mirror flush --confirm" : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", options: GitMirrorOptions): Promise<Record<string, unknown>> {
|
||||
const jobName = `${action === "sync" ? gitMirrorSyncJobPrefix : gitMirrorFlushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63);
|
||||
const manifest = gitMirrorJobManifest(action, jobName);
|
||||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||||
const command = `agentrun v01 git-mirror ${action}`;
|
||||
if (options.dryRun || !options.confirm) {
|
||||
return {
|
||||
ok: true,
|
||||
command,
|
||||
dryRun: true,
|
||||
namespace: gitMirrorNamespace,
|
||||
jobName,
|
||||
manifest,
|
||||
next: { confirm: `bun scripts/cli.ts ${command} --confirm` },
|
||||
};
|
||||
}
|
||||
const created = await createGitMirrorJob(config, jobName, manifestB64);
|
||||
if (created.exitCode !== 0 || !options.wait) {
|
||||
const status = await gitMirrorStatus(config);
|
||||
return {
|
||||
ok: created.exitCode === 0,
|
||||
command,
|
||||
dryRun: false,
|
||||
mode: options.wait ? "create-failed" : "submitted",
|
||||
namespace: gitMirrorNamespace,
|
||||
jobName,
|
||||
result: compactCapture(created),
|
||||
status,
|
||||
next: {
|
||||
status: "bun scripts/cli.ts agentrun v01 git-mirror status",
|
||||
wait: `bun scripts/cli.ts agentrun v01 git-mirror ${action} --confirm --wait`,
|
||||
},
|
||||
};
|
||||
}
|
||||
const wait = await waitForGitMirrorJob(config, action, jobName, options.timeoutSeconds);
|
||||
const status = await gitMirrorStatus(config);
|
||||
return {
|
||||
ok: created.exitCode === 0 && wait.ok === true,
|
||||
command,
|
||||
dryRun: false,
|
||||
mode: "waited",
|
||||
namespace: gitMirrorNamespace,
|
||||
jobName,
|
||||
result: compactCapture(created),
|
||||
wait,
|
||||
status,
|
||||
next: {
|
||||
status: "bun scripts/cli.ts agentrun v01 git-mirror status",
|
||||
flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? "bun scripts/cli.ts agentrun v01 git-mirror flush --confirm" : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function createGitMirrorJob(config: UniDeskConfig, jobName: string, manifestB64: string): Promise<SshCaptureResult> {
|
||||
const script = [
|
||||
"set -eu",
|
||||
`job=${shQuote(jobName)}`,
|
||||
`manifest_b64=${shQuote(manifestB64)}`,
|
||||
"manifest_path=\"/tmp/$job.json\"",
|
||||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||||
`kubectl delete job -n ${gitMirrorNamespace} "$job" --ignore-not-found=true >/dev/null`,
|
||||
"kubectl create -f \"$manifest_path\"",
|
||||
`kubectl get job -n ${gitMirrorNamespace} "$job" -o 'jsonpath=created={.metadata.name}{\"\\n\"}'`,
|
||||
].join("\n");
|
||||
return await capture(config, g14K3sRoute, ["script", "--", script]);
|
||||
}
|
||||
|
||||
async function waitForGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", jobName: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
||||
const startedAtMs = Date.now();
|
||||
let lastProbe: SshCaptureResult | null = null;
|
||||
let polls = 0;
|
||||
while (Date.now() - startedAtMs <= timeoutSeconds * 1000) {
|
||||
polls += 1;
|
||||
lastProbe = await capture(config, g14K3sRoute, ["script", "--", gitMirrorJobProbeScript(jobName)]);
|
||||
const summary = gitMirrorJobProbeSummary(lastProbe.stdout);
|
||||
process.stderr.write(`${JSON.stringify({
|
||||
event: `agentrun.git-mirror.${action}.progress`,
|
||||
at: new Date().toISOString(),
|
||||
stage: "k8s-job",
|
||||
status: summary.succeeded ? "succeeded" : summary.failed ? "failed" : "running",
|
||||
jobName,
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
})}\n`);
|
||||
if (summary.succeeded) {
|
||||
return {
|
||||
ok: true,
|
||||
action,
|
||||
jobName,
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
summary,
|
||||
probe: compactCapture(lastProbe),
|
||||
};
|
||||
}
|
||||
if (summary.failed) {
|
||||
return {
|
||||
ok: false,
|
||||
action,
|
||||
jobName,
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
degradedReason: "git-mirror-job-failed",
|
||||
summary,
|
||||
probe: compactCapture(lastProbe),
|
||||
};
|
||||
}
|
||||
await sleep(5_000);
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
action,
|
||||
jobName,
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAtMs,
|
||||
degradedReason: "git-mirror-job-timeout",
|
||||
probe: lastProbe ? compactCapture(lastProbe) : null,
|
||||
};
|
||||
}
|
||||
|
||||
function gitMirrorJobProbeScript(jobName: string): string {
|
||||
return [
|
||||
"set +e",
|
||||
`job=${shQuote(jobName)}`,
|
||||
"printf 'jobStatus='",
|
||||
`kubectl get job -n ${gitMirrorNamespace} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed} active={.status.active} start={.status.startTime} completion={.status.completionTime}' 2>/dev/null || true`,
|
||||
"printf '\\n'",
|
||||
"printf 'pods\\n'",
|
||||
`kubectl get pod -n ${gitMirrorNamespace} -l job-name="$job" -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,START:.status.startTime' --no-headers 2>/dev/null || true`,
|
||||
"printf 'logs\\n'",
|
||||
`kubectl logs -n ${gitMirrorNamespace} "job/$job" --tail=120 2>/dev/null || true`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function gitMirrorJobProbeSummary(stdout: string): Record<string, unknown> {
|
||||
const line = matchLine(stdout, "jobStatus=") ?? "";
|
||||
const succeeded = /(?:^|\s)succeeded=1(?:\s|$)/u.test(line);
|
||||
const failedRaw = firstMatch(line, /(?:^|\s)failed=([0-9]+)/u);
|
||||
const failed = failedRaw !== null && Number(failedRaw) > 0;
|
||||
const activeRaw = firstMatch(line, /(?:^|\s)active=([0-9]+)/u);
|
||||
return {
|
||||
statusLine: line,
|
||||
succeeded,
|
||||
failed,
|
||||
active: activeRaw === null ? null : Number(activeRaw),
|
||||
};
|
||||
}
|
||||
|
||||
function gitMirrorJobManifest(action: "sync" | "flush", name: string): Record<string, unknown> {
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name,
|
||||
namespace: gitMirrorNamespace,
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "git-mirror",
|
||||
"app.kubernetes.io/part-of": "devops-infra",
|
||||
"app.kubernetes.io/component": action === "sync" ? "sync-controller" : "flush-controller",
|
||||
"agentrun.pikastech.local/trigger": "manual-cli",
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
activeDeadlineSeconds: 300,
|
||||
ttlSecondsAfterFinished: 3600,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "git-mirror",
|
||||
"app.kubernetes.io/part-of": "devops-infra",
|
||||
"app.kubernetes.io/component": action === "sync" ? "sync-controller" : "flush-controller",
|
||||
"agentrun.pikastech.local/trigger": "manual-cli",
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
restartPolicy: "Never",
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
volumes: [
|
||||
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
|
||||
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
|
||||
],
|
||||
containers: [{
|
||||
name: action,
|
||||
image: mirrorToolsImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["/bin/sh", "-ec", action === "sync" ? gitMirrorSyncShellScript() : gitMirrorFlushShellScript()],
|
||||
volumeMounts: [
|
||||
{ name: "cache", mountPath: "/cache" },
|
||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||||
],
|
||||
}],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function gitMirrorSyncShellScript(): string {
|
||||
return [
|
||||
"set -eu",
|
||||
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"mkdir -p /cache/pikasTech",
|
||||
"export GIT_SSH_COMMAND='ssh -i /git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/agentrun-known-hosts'",
|
||||
`repo=${shQuote(gitMirrorRepoPath)}`,
|
||||
`remote=${shQuote(githubSshUrl)}`,
|
||||
"if [ ! -d \"$repo\" ]; then git clone --mirror \"$remote\" \"$repo\"; else git --git-dir=\"$repo\" remote set-url origin \"$remote\"; fi",
|
||||
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
|
||||
"git --git-dir=\"$repo\" config http.receivepack true",
|
||||
"git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1:refs/heads/v0.1 +refs/heads/v0.1:refs/mirror-stage/heads/v0.1",
|
||||
"if git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1-gitops:refs/mirror-stage/heads/v0.1-gitops; then",
|
||||
" if ! git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' >/dev/null 2>&1; then",
|
||||
" git --git-dir=\"$repo\" update-ref refs/heads/v0.1-gitops \"$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}')\"",
|
||||
" fi",
|
||||
"fi",
|
||||
"local_v01=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1^{commit}')",
|
||||
"github_v01=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1^{commit}')",
|
||||
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
||||
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
||||
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
|
||||
"json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }",
|
||||
"cat > /cache/agentrun.last-sync.json <<EOF",
|
||||
"{\"event\":\"git-mirror-sync\",\"repo\":\"pikasTech/agentrun\",\"status\":\"succeeded\",\"startedAt\":\"$started_at\",\"syncedAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"localV01\":\"$local_v01\",\"githubV01\":\"$github_v01\",\"localGitops\":$(json_ref \"$local_gitops\"),\"githubGitops\":$(json_ref \"$github_gitops\"),\"pendingFlush\":$pending}",
|
||||
"EOF",
|
||||
"cat /cache/agentrun.last-sync.json",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function gitMirrorFlushShellScript(): string {
|
||||
return [
|
||||
"set -eu",
|
||||
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"export GIT_SSH_COMMAND='ssh -i /git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/agentrun-known-hosts'",
|
||||
`repo=${shQuote(gitMirrorRepoPath)}`,
|
||||
`remote=${shQuote(githubSshUrl)}`,
|
||||
"test -d \"$repo\"",
|
||||
"git --git-dir=\"$repo\" remote set-url origin \"$remote\"",
|
||||
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
||||
"if [ -n \"$local_gitops\" ]; then",
|
||||
" git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin refs/heads/v0.1-gitops:refs/heads/v0.1-gitops",
|
||||
" git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1-gitops:refs/mirror-stage/heads/v0.1-gitops",
|
||||
"fi",
|
||||
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
||||
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
|
||||
"json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }",
|
||||
"cat > /cache/agentrun.last-flush.json <<EOF",
|
||||
"{\"event\":\"git-mirror-flush\",\"repo\":\"pikasTech/agentrun\",\"status\":\"succeeded\",\"startedAt\":\"$started_at\",\"flushedAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"localGitops\":$(json_ref \"$local_gitops\"),\"githubGitops\":$(json_ref \"$github_gitops\"),\"pendingFlush\":$pending}",
|
||||
"EOF",
|
||||
"cat /cache/agentrun.last-flush.json",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function gitMirrorCacheProbeScript(): string {
|
||||
return [
|
||||
"printf 'lastSync='; cat /cache/agentrun.last-sync.json 2>/dev/null || true; printf '\\n'",
|
||||
"printf 'lastFlush='; cat /cache/agentrun.last-flush.json 2>/dev/null || true; printf '\\n'",
|
||||
`repo=${shQuote(gitMirrorRepoPath)}`,
|
||||
"rev() { git --git-dir=\"$repo\" rev-parse --verify \"$1^{commit}\" 2>/dev/null || true; }",
|
||||
"local_v01=$(rev refs/heads/v0.1)",
|
||||
"github_v01=$(rev refs/mirror-stage/heads/v0.1)",
|
||||
"local_gitops=$(rev refs/heads/v0.1-gitops)",
|
||||
"github_gitops=$(rev refs/mirror-stage/heads/v0.1-gitops)",
|
||||
"exact=false",
|
||||
"exact_commit=\"\"",
|
||||
"if [ -n \"$local_v01\" ]; then",
|
||||
" tmp=$(mktemp -d)",
|
||||
" if git init -q \"$tmp\" && git -C \"$tmp\" remote add origin http://127.0.0.1:8080/pikasTech/agentrun.git && git -C \"$tmp\" fetch --depth=1 origin \"$local_v01\" >/tmp/agentrun-exact-fetch.log 2>&1; then",
|
||||
" exact_commit=$(git -C \"$tmp\" rev-parse FETCH_HEAD 2>/dev/null || true)",
|
||||
" if [ \"$exact_commit\" = \"$local_v01\" ]; then exact=true; fi",
|
||||
" fi",
|
||||
" rm -rf \"$tmp\"",
|
||||
"fi",
|
||||
"node -e 'const [localV01,githubV01,localGitops,githubGitops,exact,exactCommit]=process.argv.slice(1); const n=v=>v&&v.length>0?v:null; const pending=Boolean(n(localGitops)&&(!n(githubGitops)||localGitops!==githubGitops)); console.log(\"refs=\"+JSON.stringify({refs:{localV01:n(localV01),githubV01:n(githubV01),localGitops:n(localGitops),githubGitops:n(githubGitops)},pendingFlush:pending,exactFetch:{localV01:exact===\"true\",commit:n(exactCommit)}}));' \"$local_v01\" \"$github_v01\" \"$local_gitops\" \"$github_gitops\" \"$exact\" \"$exact_commit\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
|
||||
const refs = labeledJson(raw, "refs");
|
||||
const lastSync = labeledJson(raw, "lastSync");
|
||||
const lastFlush = labeledJson(raw, "lastFlush");
|
||||
const refsRecord = record(record(refs).refs);
|
||||
const localV01 = stringOrNull(refsRecord.localV01);
|
||||
const githubV01 = stringOrNull(refsRecord.githubV01);
|
||||
const localGitops = stringOrNull(refsRecord.localGitops);
|
||||
const githubGitops = stringOrNull(refsRecord.githubGitops);
|
||||
const pendingFlush = typeof record(refs).pendingFlush === "boolean" ? Boolean(record(refs).pendingFlush) : null;
|
||||
return {
|
||||
localV01,
|
||||
githubV01,
|
||||
localGitops,
|
||||
githubGitops,
|
||||
pendingFlush,
|
||||
sourceInSync: Boolean(localV01 && githubV01 && localV01 === githubV01),
|
||||
gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops),
|
||||
githubInSync: Boolean(localV01 && githubV01 && localV01 === githubV01 && (!localGitops || localGitops === githubGitops)),
|
||||
flushNeeded: pendingFlush === true,
|
||||
flushCommand: pendingFlush === true ? "bun scripts/cli.ts agentrun v01 git-mirror flush --confirm" : null,
|
||||
exactFetch: record(refs).exactFetch ?? null,
|
||||
lastSync,
|
||||
lastFlush,
|
||||
};
|
||||
}
|
||||
|
||||
function gitMirrorSyncRequirement(sourceCommit: string, rawStatus: string): { required: boolean; reason: string; localV01: string | null } {
|
||||
const summary = gitMirrorStatusSummary(rawStatus);
|
||||
const localV01 = stringOrNull(summary.localV01);
|
||||
return {
|
||||
required: localV01 !== sourceCommit,
|
||||
reason: localV01 === sourceCommit ? "local-v01-current" : localV01 === null ? "local-v01-unresolved" : "local-v01-stale",
|
||||
localV01,
|
||||
};
|
||||
}
|
||||
|
||||
function startAsyncAgentRunJob(name: string, command: string[], note: string): Record<string, unknown> {
|
||||
const job = startJob(name, command, note);
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
||||
return {
|
||||
ok: true,
|
||||
mode: "async-job",
|
||||
job,
|
||||
statusCommand,
|
||||
tailCommand: `tail -f ${job.stdoutFile}`,
|
||||
next: {
|
||||
status: statusCommand,
|
||||
tail: `tail -f ${job.stdoutFile}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function capture(config: UniDeskConfig, target: string, args: string[]): Promise<SshCaptureResult> {
|
||||
return await runSshCommandCapture(config, target, args);
|
||||
}
|
||||
@@ -303,6 +768,29 @@ function matchLine(text: string, prefix: string): string | null {
|
||||
return line ? line.slice(prefix.length).trim() || null : null;
|
||||
}
|
||||
|
||||
function firstMatch(text: string, pattern: RegExp): string | null {
|
||||
return pattern.exec(text)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function labeledJson(text: string, label: string): Record<string, unknown> {
|
||||
const prefix = `${label}=`;
|
||||
const line = text.split(/\r?\n/u).find((item) => item.startsWith(prefix));
|
||||
if (!line) return {};
|
||||
try {
|
||||
return record(JSON.parse(line.slice(prefix.length)));
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function stringOrNull(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function parseArgoStatus(text: string): { revision: string | null; syncStatus: string | null; healthStatus: string | null } {
|
||||
const lines = text.split(/\r?\n/u);
|
||||
const index = lines.findIndex((line) => line.trim() === "argo");
|
||||
@@ -323,6 +811,10 @@ function tail(text: string, maxChars: number): string {
|
||||
return text.length > maxChars ? text.slice(-maxChars) : text;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function shQuote(value: string): string {
|
||||
return `'${value.replace(/'/gu, "'\\''")}'`;
|
||||
}
|
||||
@@ -332,6 +824,6 @@ function unsupported(args: string[]): Record<string, unknown> {
|
||||
ok: false,
|
||||
command: `agentrun ${args.join(" ")}`.trim(),
|
||||
degradedReason: "unsupported-command",
|
||||
message: "supported commands: agentrun v01 control-plane status|trigger-current|refresh",
|
||||
message: "supported commands: agentrun v01 control-plane status|trigger-current|refresh; agentrun v01 git-mirror status|sync|flush",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2652,9 +2652,9 @@ function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown>
|
||||
"kubectl create -f \"$manifest_path\"",
|
||||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||||
"while :; do",
|
||||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='{.status.succeeded} {.status.failed}' 2>/dev/null || true)`,
|
||||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{print $1}')",
|
||||
" failed=$(printf '%s\\n' \"$status\" | awk '{print $2}')",
|
||||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||||
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||||
@@ -2702,9 +2702,9 @@ function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown
|
||||
"kubectl create -f \"$manifest_path\"",
|
||||
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
||||
"while :; do",
|
||||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='{.status.succeeded} {.status.failed}' 2>/dev/null || true)`,
|
||||
" succeeded=$(printf '%s\n' \"$status\" | awk '{print $1}')",
|
||||
" failed=$(printf '%s\n' \"$status\" | awk '{print $2}')",
|
||||
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
||||
" succeeded=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||||
" failed=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
||||
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
||||
|
||||
+6
-4
@@ -25,7 +25,7 @@ export interface JobRecord {
|
||||
}
|
||||
|
||||
export interface JobProgressSummary {
|
||||
kind: "hwlab-v02-trigger" | "hwlab-git-mirror" | "generic";
|
||||
kind: "hwlab-v02-trigger" | "git-mirror" | "generic";
|
||||
stage: string | null;
|
||||
stageStatus: string | null;
|
||||
sourceCommit: string | null;
|
||||
@@ -196,7 +196,7 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
|
||||
|
||||
function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
|
||||
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
|
||||
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush";
|
||||
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
|
||||
if (!knownWorkflow && !gitMirrorWorkflow && tails === undefined) return genericJobProgress(job);
|
||||
const nowMs = Date.now();
|
||||
const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
|
||||
@@ -290,7 +290,7 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder
|
||||
.filter(([key]) => key.startsWith(`gitMirror.${action}.`) || key === "sshRuntimeMaxMs")
|
||||
.map(([key, value]) => `${key}=${value}ms`);
|
||||
return {
|
||||
kind: "hwlab-git-mirror",
|
||||
kind: "git-mirror",
|
||||
stage: action,
|
||||
stageStatus: status ?? (job.status === "running" ? "running" : null),
|
||||
sourceCommit: null,
|
||||
@@ -315,7 +315,9 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder
|
||||
].filter(Boolean).join(" "),
|
||||
nextCommand: job.status === "running"
|
||||
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
||||
: "bun scripts/cli.ts hwlab g14 git-mirror status",
|
||||
: job.name.startsWith("agentrun_")
|
||||
? "bun scripts/cli.ts agentrun v01 git-mirror status"
|
||||
: "bun scripts/cli.ts hwlab g14 git-mirror status",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user