Codex
953 lines
40 KiB
TypeScript
953 lines
40 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. git-mirror module for scripts/src/hwlab-g14.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-g14.ts:5846-6777 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult, G14ControlPlaneOptions, G14GitMirrorOptions } from "./types";
|
|
import { tailText } from "./cleanup";
|
|
import { statusText } from "./pr-monitor";
|
|
import { compactCommandResult, g14K3s, isCommandSuccess, nested, printProgressEvent, record, shellQuote, shortSha, stringOrNull } from "./remote";
|
|
import { cleanupRuntimeLaneRenderDir, runRuntimeLaneRenderToTemp } from "./render";
|
|
import { resolveRuntimeLaneHead } from "./source";
|
|
import { GIT_MIRROR_LEGACY_CRONJOB, GIT_MIRROR_MANIFEST_FIELD_MANAGER, GIT_MIRROR_NAMESPACE, GIT_MIRROR_SYNC_JOB_PREFIX, V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS, V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, V02_GIT_MIRROR_PRESYNC_POLL_MS, V02_GIT_READ_URL, V02_GIT_WRITE_URL } from "./types";
|
|
import { parseShellSections, shellSectionOk, timestampMs } from "./v02-status";
|
|
|
|
export function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult {
|
|
return g14K3s([
|
|
"kubectl",
|
|
"delete",
|
|
"cronjob",
|
|
"-n",
|
|
GIT_MIRROR_NAMESPACE,
|
|
GIT_MIRROR_LEGACY_CRONJOB,
|
|
"--ignore-not-found=true",
|
|
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
|
], 60_000);
|
|
}
|
|
|
|
export function applyGitMirrorManifestFile(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
|
return g14K3s([
|
|
"kubectl",
|
|
"apply",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
`--field-manager=${GIT_MIRROR_MANIFEST_FIELD_MANAGER}`,
|
|
...(dryRun ? ["--dry-run=server"] : []),
|
|
"-f",
|
|
`${renderDir}/devops-infra/git-mirror.yaml`,
|
|
], timeoutSeconds * 1000);
|
|
}
|
|
|
|
export function gitMirrorSyncJobName(): string {
|
|
return `${GIT_MIRROR_SYNC_JOB_PREFIX}-${Date.now().toString(36)}`.slice(0, 63);
|
|
}
|
|
|
|
export function gitMirrorFlushJobName(): string {
|
|
return `git-mirror-hwlab-flush-manual-${Date.now().toString(36)}`.slice(0, 63);
|
|
}
|
|
|
|
export function gitMirrorSyncJobManifest(name: string): Record<string, unknown> {
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name,
|
|
namespace: GIT_MIRROR_NAMESPACE,
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "devops-infra",
|
|
"app.kubernetes.io/component": "sync-controller",
|
|
"hwlab.pikastech.local/trigger": "manual-cli",
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 300,
|
|
ttlSecondsAfterFinished: 3600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "devops-infra",
|
|
"app.kubernetes.io/component": "sync-controller",
|
|
"hwlab.pikastech.local/trigger": "manual-cli",
|
|
},
|
|
},
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
volumes: [
|
|
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
|
|
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
|
|
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } },
|
|
],
|
|
containers: [{
|
|
name: "sync",
|
|
image: "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1",
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/script/sync.sh"],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
{ name: "script", mountPath: "/script", readOnly: true },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
export function gitMirrorFlushJobManifest(name: string): Record<string, unknown> {
|
|
const manifest = gitMirrorSyncJobManifest(name);
|
|
record(record(manifest.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
|
const template = record(record(manifest.spec).template);
|
|
record(record(template.metadata).labels)["app.kubernetes.io/component"] = "flush-controller";
|
|
const podSpec = record(template.spec);
|
|
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
|
|
const first = record(containers[0]);
|
|
first.name = "flush";
|
|
first.command = ["/script/flush.sh"];
|
|
podSpec.containers = [first];
|
|
return manifest;
|
|
}
|
|
|
|
export function parseGitMirrorStatusRefs(raw: string): { refs: Record<string, string | null>; pendingFlush: boolean | null } {
|
|
const line = raw.split(/\r?\n/u).find((item) => item.startsWith("refs="));
|
|
if (line === undefined) return { refs: {}, pendingFlush: null };
|
|
try {
|
|
const parsed = record(JSON.parse(line.slice("refs=".length)) as unknown);
|
|
const refs = record(parsed.refs);
|
|
const parsedRefs: Record<string, string | null> = {};
|
|
for (const [key, value] of Object.entries(refs)) parsedRefs[key] = stringOrNull(value);
|
|
return {
|
|
refs: {
|
|
...parsedRefs,
|
|
localV02: stringOrNull(refs.localV02),
|
|
githubV02: stringOrNull(refs.githubV02),
|
|
localG14: stringOrNull(refs.localG14),
|
|
githubG14: stringOrNull(refs.githubG14),
|
|
localGitops: stringOrNull(refs.localGitops),
|
|
githubGitops: stringOrNull(refs.githubGitops),
|
|
localV03: stringOrNull(refs.localV03),
|
|
githubV03: stringOrNull(refs.githubV03),
|
|
localV03Gitops: stringOrNull(refs.localV03Gitops),
|
|
githubV03Gitops: stringOrNull(refs.githubV03Gitops),
|
|
},
|
|
pendingFlush: typeof parsed.pendingFlush === "boolean" ? parsed.pendingFlush : null,
|
|
};
|
|
} catch {
|
|
return { refs: {}, pendingFlush: null };
|
|
}
|
|
}
|
|
|
|
export function parseLabeledJsonLine(raw: string, label: string): Record<string, unknown> {
|
|
const line = raw.split(/\r?\n/u).find((item) => item.startsWith(`${label}=`));
|
|
if (line === undefined) return {};
|
|
const value = line.slice(label.length + 1).trim();
|
|
if (value.length === 0) return {};
|
|
try {
|
|
return record(JSON.parse(value) as unknown);
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
export function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
|
|
const refs = parseGitMirrorStatusRefs(raw);
|
|
const lastSync = parseLabeledJsonLine(raw, "lastSync");
|
|
const lastWrite = parseLabeledJsonLine(raw, "lastWrite");
|
|
const lastFlush = parseLabeledJsonLine(raw, "lastFlush");
|
|
const localV02 = refs.refs.localV02 ?? null;
|
|
const githubV02 = refs.refs.githubV02 ?? null;
|
|
const localGitops = refs.refs.localGitops ?? null;
|
|
const githubGitops = refs.refs.githubGitops ?? null;
|
|
const localV03 = refs.refs.localV03 ?? null;
|
|
const githubV03 = refs.refs.githubV03 ?? null;
|
|
const localV03Gitops = refs.refs.localV03Gitops ?? null;
|
|
const githubV03Gitops = refs.refs.githubV03Gitops ?? null;
|
|
const sourceInSync = Boolean(localV02 && githubV02 && localV02 === githubV02);
|
|
const gitopsInSync = Boolean(localGitops && githubGitops && localGitops === githubGitops);
|
|
const runtimeLanes = Object.fromEntries(hwlabRuntimeLaneIds().map((lane) => {
|
|
const normalized = lane.slice(0, 1).toUpperCase() + lane.slice(1).toLowerCase();
|
|
const localSource = refs.refs[`local${normalized}`] ?? null;
|
|
const githubSource = refs.refs[`github${normalized}`] ?? null;
|
|
const localLaneGitops = refs.refs[`local${normalized}Gitops`] ?? null;
|
|
const githubLaneGitops = refs.refs[`github${normalized}Gitops`] ?? null;
|
|
return [lane, {
|
|
localSource,
|
|
githubSource,
|
|
localGitops: localLaneGitops,
|
|
githubGitops: githubLaneGitops,
|
|
sourceInSync: Boolean(localSource && githubSource && localSource === githubSource),
|
|
gitopsInSync: Boolean(localLaneGitops && githubLaneGitops && localLaneGitops === githubLaneGitops),
|
|
}];
|
|
}));
|
|
const allRuntimeLanesInSync = Object.values(runtimeLanes).every((item) => {
|
|
const laneSummary = record(item);
|
|
return laneSummary.sourceInSync === true && laneSummary.gitopsInSync === true;
|
|
});
|
|
return {
|
|
localV02,
|
|
githubV02,
|
|
localV03,
|
|
githubV03,
|
|
localG14: refs.refs.localG14 ?? null,
|
|
githubG14: refs.refs.githubG14 ?? null,
|
|
localGitops,
|
|
githubGitops,
|
|
localV03Gitops,
|
|
githubV03Gitops,
|
|
pendingFlush: refs.pendingFlush,
|
|
lastSync: {
|
|
status: stringOrNull(lastSync.status) ?? null,
|
|
at: stringOrNull(lastSync.publishedAt) ?? stringOrNull(lastSync.syncedAt) ?? null,
|
|
pendingFlush: typeof lastSync.pendingFlush === "boolean" ? lastSync.pendingFlush : null,
|
|
},
|
|
lastWrite: {
|
|
status: stringOrNull(lastWrite.status) ?? null,
|
|
at: stringOrNull(lastWrite.writtenAt) ?? null,
|
|
pendingFlush: typeof lastWrite.pendingFlush === "boolean" ? lastWrite.pendingFlush : null,
|
|
},
|
|
lastFlush: {
|
|
status: stringOrNull(lastFlush.status) ?? null,
|
|
at: stringOrNull(lastFlush.flushedAt) ?? null,
|
|
pendingFlush: typeof lastFlush.pendingFlush === "boolean" ? lastFlush.pendingFlush : null,
|
|
},
|
|
flushNeeded: refs.pendingFlush === true,
|
|
flushCommand: refs.pendingFlush === true ? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" : null,
|
|
sourceInSync,
|
|
gitopsInSync,
|
|
v03SourceInSync: Boolean(localV03 && githubV03 && localV03 === githubV03),
|
|
v03GitopsInSync: Boolean(localV03Gitops && githubV03Gitops && localV03Gitops === githubV03Gitops),
|
|
runtimeLanes,
|
|
allRuntimeLanesInSync,
|
|
githubInSync: sourceInSync && gitopsInSync,
|
|
};
|
|
}
|
|
|
|
export function gitMirrorV02SyncRequirement(sourceCommit: string, rawStatus: string): Record<string, unknown> {
|
|
const parsed = parseGitMirrorStatusRefs(rawStatus);
|
|
const localV02 = parsed.refs.localV02 ?? null;
|
|
return {
|
|
required: localV02 !== sourceCommit,
|
|
sourceCommit,
|
|
localV02,
|
|
pendingFlush: parsed.pendingFlush,
|
|
refs: parsed.refs,
|
|
reason: localV02 === sourceCommit ? "local-v02-current" : localV02 === null ? "local-v02-unresolved" : "local-v02-stale",
|
|
};
|
|
}
|
|
|
|
export function runtimeLaneGitMirrorSourceInSyncForTest(lane: string, sourceCommit: string, summary: Record<string, unknown>): boolean {
|
|
const localKey = lane === "v02" ? "localV02" : lane === "v03" ? "localV03" : null;
|
|
return localKey !== null && typeof summary[localKey] === "string" && summary[localKey] === sourceCommit;
|
|
}
|
|
|
|
export function gitMirrorStatusCacheRaw(status: Record<string, unknown>): string {
|
|
return String(nested(status, ["cache", "raw"]) ?? "");
|
|
}
|
|
|
|
export function runtimeLaneGitMirrorSourceInSync(spec: HwlabRuntimeLaneSpec, sourceCommit: string, status: Record<string, unknown>): boolean {
|
|
return runtimeLaneGitMirrorSourceInSyncForTest(spec.lane, sourceCommit, record(nested(status, ["cache", "summary"])));
|
|
}
|
|
|
|
export function compactGitMirrorStatus(status: Record<string, unknown>, sourceCommit: string): Record<string, unknown> {
|
|
const requirement = gitMirrorV02SyncRequirement(sourceCommit, gitMirrorStatusCacheRaw(status));
|
|
return {
|
|
ok: status.ok === true,
|
|
readUrl: status.readUrl ?? V02_GIT_READ_URL,
|
|
writeUrl: status.writeUrl ?? V02_GIT_WRITE_URL,
|
|
elapsedMs: status.elapsedMs ?? null,
|
|
legacyCronJobExists: nested(status, ["legacyCronJob", "exists"]) === true,
|
|
required: requirement.required,
|
|
sourceCommit,
|
|
localV02: requirement.localV02 ?? null,
|
|
githubV02: requirement.refs.githubV02 ?? null,
|
|
pendingFlush: requirement.pendingFlush ?? null,
|
|
sourceInSync: Boolean(requirement.refs.localV02 && requirement.refs.githubV02 && requirement.refs.localV02 === requirement.refs.githubV02),
|
|
reason: requirement.reason,
|
|
cacheOk: nested(status, ["cache", "ok"]) === true,
|
|
cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000),
|
|
resourcesOk: nested(status, ["resources", "ok"]) === true,
|
|
};
|
|
}
|
|
|
|
export function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
ok: sync.ok === true,
|
|
command: sync.command ?? "hwlab g14 git-mirror sync",
|
|
mode: sync.mode ?? null,
|
|
skipped: sync.skipped === true,
|
|
reason: sync.reason ?? null,
|
|
namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE,
|
|
jobName: sync.jobName ?? null,
|
|
elapsedMs: sync.elapsedMs ?? null,
|
|
exitCode: nested(sync, ["result", "exitCode"]) ?? null,
|
|
stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600),
|
|
stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200),
|
|
stdoutBytes: nested(sync, ["result", "stdoutBytes"]) ?? null,
|
|
stderrBytes: nested(sync, ["result", "stderrBytes"]) ?? null,
|
|
};
|
|
}
|
|
|
|
export interface V02GitMirrorPreSyncMarker {
|
|
ok: boolean;
|
|
sourceCommit: string;
|
|
syncedAt: string;
|
|
localV02?: string | null;
|
|
githubV02?: string | null;
|
|
reason?: string | null;
|
|
}
|
|
|
|
export function v02GitMirrorPreSyncStateDir(): string {
|
|
return rootPath(".state", "hwlab-g14", "v02-git-mirror-presync");
|
|
}
|
|
|
|
export function v02GitMirrorPreSyncMarkerPath(sourceCommit: string): string {
|
|
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.json`);
|
|
}
|
|
|
|
export function v02GitMirrorPreSyncLockDir(sourceCommit: string): string {
|
|
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.lock`);
|
|
}
|
|
|
|
export function v02GitMirrorPreSyncWaitMs(timeoutSeconds: number): number {
|
|
const requestedMs = Math.max(0, Math.floor(timeoutSeconds * 1000));
|
|
if (requestedMs === 0) return V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS;
|
|
return Math.min(V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, requestedMs);
|
|
}
|
|
|
|
export function v02ReusableGitMirrorPreSyncMarker(marker: unknown, sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
|
|
const candidate = record(marker) as V02GitMirrorPreSyncMarker;
|
|
const syncedAtMs = timestampMs(candidate.syncedAt);
|
|
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit) return null;
|
|
if (syncedAtMs === null || nowMs - syncedAtMs > V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS) return null;
|
|
if (syncedAtMs < minSyncedAtMs) return null;
|
|
return candidate;
|
|
}
|
|
|
|
export function readV02GitMirrorPreSyncMarker(sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
|
|
const path = v02GitMirrorPreSyncMarkerPath(sourceCommit);
|
|
if (!existsSync(path)) return null;
|
|
try {
|
|
return v02ReusableGitMirrorPreSyncMarker(JSON.parse(readFileSync(path, "utf8")) as unknown, sourceCommit, nowMs, minSyncedAtMs);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function writeV02GitMirrorPreSyncMarker(sourceCommit: string, summary: Record<string, unknown>): V02GitMirrorPreSyncMarker {
|
|
const marker = {
|
|
ok: true,
|
|
sourceCommit,
|
|
syncedAt: new Date().toISOString(),
|
|
localV02: stringOrNull(summary.localV02),
|
|
githubV02: stringOrNull(summary.githubV02),
|
|
reason: stringOrNull(summary.reason),
|
|
};
|
|
const dir = v02GitMirrorPreSyncStateDir();
|
|
mkdirSync(dir, { recursive: true });
|
|
writeFileSync(v02GitMirrorPreSyncMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
|
return marker;
|
|
}
|
|
|
|
export function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number, minSyncedAtMs = 0): { acquired: boolean; lockDir: string; waitedMs: number; marker?: V02GitMirrorPreSyncMarker } {
|
|
const stateDir = v02GitMirrorPreSyncStateDir();
|
|
mkdirSync(stateDir, { recursive: true });
|
|
const lockDir = v02GitMirrorPreSyncLockDir(sourceCommit);
|
|
const startedAtMs = Date.now();
|
|
for (;;) {
|
|
const marker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), minSyncedAtMs);
|
|
if (marker !== null) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs, marker };
|
|
try {
|
|
mkdirSync(lockDir);
|
|
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
|
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
} catch {
|
|
const ageMs = (() => {
|
|
try {
|
|
return Date.now() - statSync(lockDir).mtimeMs;
|
|
} catch {
|
|
return 0;
|
|
}
|
|
})();
|
|
if (ageMs > V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS) {
|
|
try {
|
|
rmSync(lockDir, { recursive: true, force: true });
|
|
continue;
|
|
} catch {
|
|
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
}
|
|
}
|
|
if (Date.now() - startedAtMs >= waitMs) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
|
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
}
|
|
}
|
|
}
|
|
|
|
export function releaseV02GitMirrorPreSyncLock(lockDir: string): void {
|
|
try {
|
|
rmSync(lockDir, { recursive: true, force: true });
|
|
} catch {
|
|
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
|
}
|
|
}
|
|
|
|
export function waitForV02GitMirrorPreSync(sourceCommit: string, waitMs: number): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const observations: Record<string, unknown>[] = [];
|
|
let attempt = 0;
|
|
for (;;) {
|
|
attempt += 1;
|
|
const statusStartMs = Date.now();
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync-wait",
|
|
status: "polling",
|
|
sourceCommit,
|
|
attempt,
|
|
elapsedMs: statusStartMs - startedAtMs,
|
|
});
|
|
const status = runGitMirrorStatus();
|
|
const summary = compactGitMirrorStatus(status, sourceCommit);
|
|
observations.push({
|
|
attempt,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
statusElapsedMs: Date.now() - statusStartMs,
|
|
ok: summary.ok,
|
|
required: summary.required,
|
|
localV02: summary.localV02,
|
|
githubV02: summary.githubV02,
|
|
pendingFlush: summary.pendingFlush,
|
|
reason: summary.reason,
|
|
});
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync-wait",
|
|
status: summary.required === false ? "succeeded" : "waiting",
|
|
sourceCommit,
|
|
attempt,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
statusElapsedMs: Date.now() - statusStartMs,
|
|
required: summary.required,
|
|
localV02: summary.localV02,
|
|
githubV02: summary.githubV02,
|
|
pendingFlush: summary.pendingFlush,
|
|
reason: summary.reason,
|
|
});
|
|
if (summary.ok === true && summary.required === false) {
|
|
return {
|
|
ok: true,
|
|
sourceCommit,
|
|
attempts: attempt,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
final: summary,
|
|
observations: observations.slice(-6),
|
|
};
|
|
}
|
|
if (Date.now() - startedAtMs >= waitMs) {
|
|
return {
|
|
ok: false,
|
|
sourceCommit,
|
|
attempts: attempt,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
final: summary,
|
|
observations: observations.slice(-6),
|
|
degradedReason: "git-mirror-local-v02-not-current-after-wait",
|
|
};
|
|
}
|
|
const remainingMs = waitMs - (Date.now() - startedAtMs);
|
|
const sleepMs = Math.max(500, Math.min(V02_GIT_MIRROR_PRESYNC_POLL_MS, remainingMs));
|
|
const wait = runCommand(["sleep", String(Math.ceil(sleepMs / 1000))], repoRoot, { timeoutMs: sleepMs + 2_000 });
|
|
if (wait.exitCode !== 0 && wait.timedOut) {
|
|
return {
|
|
ok: false,
|
|
sourceCommit,
|
|
attempts: attempt,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
final: summary,
|
|
observations: observations.slice(-6),
|
|
degradedReason: "git-mirror-pre-sync-wait-sleep-timeout",
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
export function gitMirrorCacheProbeScript(): string {
|
|
const runtimeLanesJson = JSON.stringify(hwlabRuntimeLaneIds().map((lane) => {
|
|
const spec = hwlabRuntimeLaneSpec(lane);
|
|
return { lane, sourceBranch: spec.sourceBranch, gitopsBranch: spec.gitopsBranch };
|
|
}));
|
|
return [
|
|
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
|
|
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
|
|
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
|
|
"printf 'refs='",
|
|
"node - <<'NODE' 2>/dev/null || true",
|
|
"const { execFileSync } = require('node:child_process');",
|
|
"const repo = '/cache/pikasTech/HWLAB.git';",
|
|
"function rev(ref) {",
|
|
" try {",
|
|
" return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim();",
|
|
" } catch {",
|
|
" return null;",
|
|
" }",
|
|
"}",
|
|
`const runtimeLanes = ${runtimeLanesJson};`,
|
|
"const refs = {};",
|
|
"for (const spec of runtimeLanes) {",
|
|
" const key = spec.lane.toUpperCase();",
|
|
" const normalized = key.slice(0, 1) + key.slice(1).toLowerCase();",
|
|
" const localSource = rev('refs/heads/' + spec.sourceBranch);",
|
|
" const githubSource = rev('refs/mirror-stage/heads/' + spec.sourceBranch);",
|
|
" const localGitops = rev('refs/heads/' + spec.gitopsBranch);",
|
|
" const githubGitops = rev('refs/mirror-stage/heads/' + spec.gitopsBranch);",
|
|
" refs['local' + normalized] = localSource;",
|
|
" refs['github' + normalized] = githubSource;",
|
|
" refs['local' + normalized + 'Gitops'] = localGitops;",
|
|
" refs['github' + normalized + 'Gitops'] = githubGitops;",
|
|
" if (spec.lane === 'v02') {",
|
|
" refs.localV02 = localSource;",
|
|
" refs.githubV02 = githubSource;",
|
|
" refs.localGitops = localGitops;",
|
|
" refs.githubGitops = githubGitops;",
|
|
" }",
|
|
"}",
|
|
"refs.localG14 = rev('refs/heads/G14');",
|
|
"refs.githubG14 = rev('refs/mirror-stage/heads/G14');",
|
|
"const pendingFlush = runtimeLanes.some((spec) => {",
|
|
" const localGitops = rev('refs/heads/' + spec.gitopsBranch);",
|
|
" const githubGitops = rev('refs/mirror-stage/heads/' + spec.gitopsBranch);",
|
|
" return Boolean(localGitops && githubGitops && localGitops !== githubGitops);",
|
|
"});",
|
|
"console.log(JSON.stringify({ refs, pendingFlush }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
|
|
const waitMs = v02GitMirrorPreSyncWaitMs(options.timeoutSeconds);
|
|
const statusStartMs = Date.now();
|
|
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
|
|
const before = runGitMirrorStatus();
|
|
const beforeSummary = compactGitMirrorStatus(before, sourceCommit);
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync-status",
|
|
status: beforeSummary.ok === true ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
durationMs: Date.now() - statusStartMs,
|
|
required: beforeSummary.required,
|
|
localV02: beforeSummary.localV02,
|
|
pendingFlush: beforeSummary.pendingFlush,
|
|
reason: beforeSummary.reason,
|
|
});
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
mode: "dry-run",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
action: beforeSummary.required === true ? "would-sync-before-trigger" : "already-current",
|
|
next: beforeSummary.required === true ? { syncAndTrigger: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" } : undefined,
|
|
};
|
|
}
|
|
if (beforeSummary.required !== true) {
|
|
return {
|
|
ok: true,
|
|
mode: "already-current",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
};
|
|
}
|
|
const existingMarker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
|
if (existingMarker !== null) {
|
|
return {
|
|
ok: true,
|
|
mode: "reused-recent-git-mirror-presync-marker",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
marker: existingMarker,
|
|
waitMs,
|
|
};
|
|
}
|
|
const lock = acquireV02GitMirrorPreSyncLock(sourceCommit, waitMs, statusStartMs);
|
|
if (!lock.acquired) {
|
|
const markerAfterLockTimeout = lock.marker ?? readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
|
if (markerAfterLockTimeout !== null) {
|
|
return {
|
|
ok: true,
|
|
mode: "waited-for-recent-git-mirror-presync-marker",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
marker: markerAfterLockTimeout,
|
|
waitedMs: lock.waitedMs,
|
|
waitMs,
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
mode: "local-git-mirror-presync-lock",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
lockDir: lock.lockDir,
|
|
waitedMs: lock.waitedMs,
|
|
waitMs,
|
|
degradedReason: "git-mirror-pre-sync-lock-timeout",
|
|
};
|
|
}
|
|
try {
|
|
const markerAfterWait = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
|
|
if (markerAfterWait !== null) {
|
|
return {
|
|
ok: true,
|
|
mode: "waited-for-recent-git-mirror-presync-marker",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
marker: markerAfterWait,
|
|
waitedMs: lock.waitedMs,
|
|
waitMs,
|
|
};
|
|
}
|
|
const recheckStartMs = Date.now();
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync-recheck",
|
|
status: "started",
|
|
sourceCommit,
|
|
waitedMs: lock.waitedMs,
|
|
});
|
|
const recheck = runGitMirrorStatus();
|
|
const recheckSummary = compactGitMirrorStatus(recheck, sourceCommit);
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync-recheck",
|
|
status: recheckSummary.ok === true ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
durationMs: Date.now() - recheckStartMs,
|
|
required: recheckSummary.required,
|
|
localV02: recheckSummary.localV02,
|
|
githubV02: recheckSummary.githubV02,
|
|
pendingFlush: recheckSummary.pendingFlush,
|
|
reason: recheckSummary.reason,
|
|
});
|
|
if (recheckSummary.ok === true && recheckSummary.required === false) {
|
|
const marker = writeV02GitMirrorPreSyncMarker(sourceCommit, recheckSummary);
|
|
return {
|
|
ok: true,
|
|
mode: "became-current-after-lock-wait",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
recheck: recheckSummary,
|
|
marker,
|
|
waitedMs: lock.waitedMs,
|
|
waitMs,
|
|
};
|
|
}
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync-sync",
|
|
status: "started",
|
|
sourceCommit,
|
|
reason: recheckSummary.reason,
|
|
localV02: recheckSummary.localV02,
|
|
pendingFlush: recheckSummary.pendingFlush,
|
|
});
|
|
const sync = runGitMirrorSync({
|
|
action: "sync",
|
|
lane: "v02",
|
|
confirm: true,
|
|
dryRun: false,
|
|
wait: true,
|
|
timeoutSeconds: options.timeoutSeconds,
|
|
});
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync-sync",
|
|
status: sync.ok === true ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
durationMs: sync.elapsedMs ?? null,
|
|
jobName: sync.jobName ?? null,
|
|
});
|
|
const syncStatus = record(sync.status);
|
|
const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
|
|
const afterSummary = compactGitMirrorStatus(after, sourceCommit);
|
|
const wait = sync.ok === true && afterSummary.required === true ? waitForV02GitMirrorPreSync(sourceCommit, waitMs) : null;
|
|
const waitFinal = wait === null ? null : record(record(wait).final);
|
|
const finalSummary = wait !== null && wait.ok === true ? waitFinal : afterSummary;
|
|
const ok = sync.ok === true && finalSummary.required === false;
|
|
const marker = ok ? writeV02GitMirrorPreSyncMarker(sourceCommit, finalSummary) : null;
|
|
return {
|
|
ok,
|
|
mode: "auto-sync-before-trigger",
|
|
sourceCommit,
|
|
before: beforeSummary,
|
|
recheck: recheckSummary,
|
|
sync: compactGitMirrorSync(sync),
|
|
after: afterSummary,
|
|
wait,
|
|
final: finalSummary,
|
|
marker,
|
|
waitedMs: lock.waitedMs,
|
|
waitMs,
|
|
degradedReason: ok ? undefined : stringOrNull(record(wait).degradedReason) ?? "git-mirror-local-v02-not-current-after-sync",
|
|
};
|
|
} finally {
|
|
releaseV02GitMirrorPreSyncLock(lock.lockDir);
|
|
}
|
|
}
|
|
|
|
export function runGitMirrorStatus(lane: HwlabRuntimeLane = "v02"): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const spec = hwlabRuntimeLaneSpec(lane);
|
|
const script = [
|
|
"set +e",
|
|
"section() {",
|
|
" name=\"$1\"",
|
|
" shift",
|
|
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
|
" \"$@\"",
|
|
" code=$?",
|
|
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
|
"}",
|
|
[
|
|
"section resources kubectl get deploy,svc,pvc,cm",
|
|
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
|
"-l app.kubernetes.io/name=git-mirror",
|
|
"-o name",
|
|
].join(" "),
|
|
[
|
|
"section legacyCronJob kubectl get cronjob",
|
|
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
|
shellQuote(GIT_MIRROR_LEGACY_CRONJOB),
|
|
"--ignore-not-found",
|
|
"-o name",
|
|
].join(" "),
|
|
[
|
|
`section jobs kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
|
"-l app.kubernetes.io/name=git-mirror",
|
|
"--sort-by=.metadata.creationTimestamp",
|
|
"-o custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime",
|
|
"--no-headers",
|
|
].join(" "),
|
|
[
|
|
"section cache kubectl exec",
|
|
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
|
|
"deploy/git-mirror-http",
|
|
"-- sh -lc",
|
|
shellQuote(gitMirrorCacheProbeScript()),
|
|
].join(" "),
|
|
].join("\n");
|
|
const bundle = g14K3s(["sh", "--", script], 60_000);
|
|
const sections = parseShellSections(statusText(bundle));
|
|
const resources = sections.resources;
|
|
const legacyCronJob = sections.legacyCronJob;
|
|
const jobs = sections.jobs;
|
|
const cache = sections.cache;
|
|
const bundleStderr = bundle.stderr.trim().slice(0, 2000);
|
|
return {
|
|
ok: isCommandSuccess(bundle) && shellSectionOk(resources),
|
|
command: `hwlab g14 git-mirror status --lane ${lane}`,
|
|
lane,
|
|
namespace: GIT_MIRROR_NAMESPACE,
|
|
readUrl: V02_GIT_READ_URL,
|
|
writeUrl: V02_GIT_WRITE_URL,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
resources: {
|
|
ok: shellSectionOk(resources),
|
|
names: String(resources?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
|
stderr: shellSectionOk(resources) ? "" : bundleStderr,
|
|
},
|
|
legacyCronJob: {
|
|
ok: shellSectionOk(legacyCronJob),
|
|
exists: String(legacyCronJob?.stdout ?? "").trim().length > 0,
|
|
name: String(legacyCronJob?.stdout ?? "").trim() || null,
|
|
cleanupCommand: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm",
|
|
},
|
|
jobs: {
|
|
ok: shellSectionOk(jobs),
|
|
raw: String(jobs?.stdout ?? "").trim(),
|
|
stderr: shellSectionOk(jobs) ? "" : bundleStderr,
|
|
},
|
|
cache: {
|
|
ok: shellSectionOk(cache),
|
|
summary: gitMirrorStatusSummary(String(cache?.stdout ?? "").trim()),
|
|
raw: String(cache?.stdout ?? "").trim(),
|
|
stderr: shellSectionOk(cache) ? "" : bundleStderr,
|
|
},
|
|
next: {
|
|
apply: `bun scripts/cli.ts hwlab g14 git-mirror apply --lane ${spec.lane} --confirm`,
|
|
sync: `bun scripts/cli.ts hwlab g14 git-mirror sync --lane ${spec.lane} --confirm`,
|
|
flush: `bun scripts/cli.ts hwlab g14 git-mirror flush --lane ${spec.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
|
|
const spec = hwlabRuntimeLaneSpec(options.lane);
|
|
const head = resolveRuntimeLaneHead(spec);
|
|
const sourceCommit = head.sourceCommit;
|
|
if (sourceCommit === null) {
|
|
return { ok: false, command: `hwlab g14 git-mirror apply --lane ${spec.lane}`, lane: spec.lane, degradedReason: `${spec.lane}-head-unresolved`, sourceRepo: spec.cicdRepo, workspace: spec.workspace, headProbe: compactCommandResult(head.result) };
|
|
}
|
|
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
|
|
if (!isCommandSuccess(render.result)) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 git-mirror apply --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
phase: "source-render",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
};
|
|
}
|
|
const apply = applyGitMirrorManifestFile(render.renderDir, options.dryRun, options.timeoutSeconds);
|
|
const cleanupLegacyCronJob = isCommandSuccess(apply)
|
|
? deleteLegacyGitMirrorCronJob(options.dryRun)
|
|
: {
|
|
ok: false,
|
|
command: [],
|
|
exitCode: null,
|
|
stdout: "",
|
|
stderr: "skipped because apply failed",
|
|
parsed: null,
|
|
};
|
|
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
|
|
return {
|
|
ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob),
|
|
command: `hwlab g14 git-mirror apply --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
|
sourceCommit,
|
|
manifest: `${render.renderDir}/devops-infra/git-mirror.yaml`,
|
|
render: compactCommandResult(render.result),
|
|
apply: compactCommandResult(apply),
|
|
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
|
|
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
|
status: runGitMirrorStatus(spec.lane),
|
|
next: options.dryRun
|
|
? { apply: `bun scripts/cli.ts hwlab g14 git-mirror apply --lane ${spec.lane} --confirm` }
|
|
: { sync: `bun scripts/cli.ts hwlab g14 git-mirror sync --lane ${spec.lane} --confirm` },
|
|
};
|
|
}
|
|
|
|
export function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown> {
|
|
const startedAtMs = Date.now();
|
|
const spec = hwlabRuntimeLaneSpec(options.lane);
|
|
const jobName = gitMirrorSyncJobName();
|
|
const manifest = gitMirrorSyncJobManifest(jobName);
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
`job=${shellQuote(jobName)}`,
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
"manifest_path=\"/tmp/$job.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
options.dryRun
|
|
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
|
: [
|
|
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
|
"kubectl create -f \"$manifest_path\"",
|
|
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
|
"while :; do",
|
|
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
|
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
|
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
|
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
|
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
|
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
|
" exit 44",
|
|
" fi",
|
|
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
|
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
|
" exit 45",
|
|
" fi",
|
|
" sleep 2",
|
|
"done",
|
|
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
|
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(`cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/${spec.sourceBranch} refs/mirror-stage/heads/${spec.sourceBranch} refs/heads/${spec.gitopsBranch} refs/mirror-stage/heads/${spec.gitopsBranch} 2>/dev/null || true`)}`,
|
|
].join("\n"),
|
|
].join("\n");
|
|
const result = g14K3s(["sh", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
|
const syncCommand = spec.lane === "v02"
|
|
? "bun scripts/cli.ts hwlab g14 git-mirror sync --lane v02 --confirm"
|
|
: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`;
|
|
const triggerCommand = spec.lane === "v02"
|
|
? "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm"
|
|
: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`;
|
|
return {
|
|
ok: isCommandSuccess(result),
|
|
command: spec.lane === "v02" ? "hwlab g14 git-mirror sync" : "hwlab nodes git-mirror sync",
|
|
mode: options.dryRun ? "dry-run" : "confirmed-sync",
|
|
namespace: GIT_MIRROR_NAMESPACE,
|
|
jobName,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
manifest: options.dryRun ? manifest : undefined,
|
|
result: compactCommandResult(result),
|
|
status: options.dryRun ? undefined : runGitMirrorStatus(spec.lane),
|
|
next: options.dryRun ? { sync: syncCommand } : { triggerCurrent: triggerCommand },
|
|
};
|
|
}
|
|
|
|
export function runGitMirrorFlush(options: G14GitMirrorOptions): Record<string, unknown> {
|
|
const jobName = gitMirrorFlushJobName();
|
|
const manifest = gitMirrorFlushJobManifest(jobName);
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
`job=${shellQuote(jobName)}`,
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
"manifest_path=\"/tmp/$job.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
options.dryRun
|
|
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
|
: [
|
|
`kubectl delete job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" --ignore-not-found=true >/dev/null`,
|
|
"kubectl create -f \"$manifest_path\"",
|
|
`deadline=$(( $(date +%s) + ${options.timeoutSeconds} ))`,
|
|
"while :; do",
|
|
` status=$(kubectl get job -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)`,
|
|
" succeeded=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
|
" failed=$(printf '%s\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
|
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
|
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then",
|
|
` kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
|
" exit 54",
|
|
" fi",
|
|
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then",
|
|
` kubectl get job,pod -n ${shellQuote(GIT_MIRROR_NAMESPACE)} -l job-name="$job" -o wide || true`,
|
|
" exit 55",
|
|
" fi",
|
|
" sleep 2",
|
|
"done",
|
|
`kubectl logs -n ${shellQuote(GIT_MIRROR_NAMESPACE)} "job/$job" --tail=200 || true`,
|
|
`kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(`cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf "\\n"; git --git-dir=/cache/pikasTech/HWLAB.git rev-parse refs/heads/${options.lane === "v02" ? "v0.2-gitops" : hwlabRuntimeLaneSpec(options.lane).gitopsBranch} refs/mirror-stage/heads/${options.lane === "v02" ? "v0.2-gitops" : hwlabRuntimeLaneSpec(options.lane).gitopsBranch} 2>/dev/null || true`)}`,
|
|
].join("\n"),
|
|
].join("\n");
|
|
const result = g14K3s(["sh", "--", script], options.timeoutSeconds * 1000 + 30_000);
|
|
return {
|
|
ok: isCommandSuccess(result),
|
|
command: "hwlab g14 git-mirror flush",
|
|
mode: options.dryRun ? "dry-run" : "confirmed-flush",
|
|
namespace: GIT_MIRROR_NAMESPACE,
|
|
jobName,
|
|
manifest: options.dryRun ? manifest : undefined,
|
|
result: compactCommandResult(result),
|
|
status: options.dryRun ? undefined : runGitMirrorStatus(options.lane),
|
|
next: options.dryRun
|
|
? { flush: `bun scripts/cli.ts hwlab g14 git-mirror flush --lane ${options.lane} --confirm` }
|
|
: { status: `bun scripts/cli.ts hwlab g14 git-mirror status --lane ${options.lane}` },
|
|
};
|
|
}
|
|
|
|
export function runG14GitMirror(options: G14GitMirrorOptions): Record<string, unknown> {
|
|
if (options.action === "status") return runGitMirrorStatus(options.lane);
|
|
if (options.action === "apply") return runGitMirrorApply(options);
|
|
if (options.action === "flush") return runGitMirrorFlush(options);
|
|
return runGitMirrorSync(options);
|
|
}
|