fix: support D601 v03 git mirror sync (#327)

Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-13 15:22:44 +08:00
committed by GitHub
parent 7ba28c8f40
commit 4e7c10d132
4 changed files with 643 additions and 27 deletions
+319 -11
View File
@@ -5,7 +5,7 @@ import { repoRoot, rootPath, type Config } from "./config";
import { runCommand, type CommandResult } from "./command";
import { startJob } from "./jobs";
import { runHwlabG14Command } from "./hwlab-g14";
import { hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete";
@@ -79,6 +79,24 @@ interface RuntimeSecretSpec {
fieldManager: string;
}
interface NodeRuntimeGitMirrorTargetSpec {
id: string;
node: string;
lane: string;
namespace: string;
serviceReadName: string;
serviceWriteName: string;
cachePvcName: string;
cacheHostPath: string | null;
secretName: string;
syncJobPrefix: string;
flushJobPrefix: string;
toolsImage: string;
sourceRepository: string;
sourceBranch: string;
gitopsBranch: string;
}
const MASTER_ADMIN_API_KEY_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env";
const MASTER_ADMIN_API_KEY_KEY = "api-key";
const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash";
@@ -179,16 +197,12 @@ async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomai
return nodeRuntimeUnsupportedAction(scoped);
}
if (domain === "git-mirror" && scoped.node !== defaultSpec.nodeId) {
return {
ok: false,
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mutation: false,
degradedReason: "node-scoped-git-mirror-action-not-enabled",
message: "D601 runtime GitOps is declared in UniDesk YAML and triggered by node-scoped control-plane commands; this command intentionally does not delegate D601 git-mirror actions to G14.",
expected: nodeRuntimeExpected(scoped.spec),
};
if (scoped.action === "status") return nodeRuntimeGitMirrorStatus(scoped);
if (scoped.action === "sync" || scoped.action === "flush") {
if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped);
return nodeRuntimeGitMirrorRun(scoped);
}
return nodeRuntimeUnsupportedAction(scoped);
}
if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") {
return runNodeEndpointBridge(scoped);
@@ -955,6 +969,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
const before = getNodeRuntimePipelineRun(spec, pipelineRun);
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null });
if (scoped.dryRun) {
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
return {
ok: true,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
@@ -965,6 +980,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
pipelineRun,
rerun: scoped.rerun,
before,
gitMirror,
manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
@@ -985,6 +1001,24 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` },
};
}
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun });
const gitMirror = nodeRuntimeEnsureGitMirrorSourceCurrent(scoped, sourceCommit);
if (gitMirror.ok !== true) {
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "failed", sourceCommit, pipelineRun, reason: gitMirror.degradedReason ?? null });
return {
ok: false,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "git-mirror-pre-sync",
sourceCommit,
pipelineRun,
before,
gitMirror,
degradedReason: "node-runtime-git-mirror-pre-sync-failed",
};
}
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "succeeded", sourceCommit, pipelineRun });
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun });
const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false });
if (refresh.ok !== true) {
@@ -1019,6 +1053,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
pipelineRun,
rerun: scoped.rerun,
before,
gitMirror,
refresh,
create: compactRuntimeCommand(create),
after,
@@ -1028,6 +1063,217 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
};
}
function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const mirror = nodeRuntimeGitMirrorTarget(spec);
const script = [
"set +e",
`namespace=${shellQuote(mirror.namespace)}`,
`read_deploy=${shellQuote(mirror.serviceReadName)}`,
`write_deploy=${shellQuote(mirror.serviceWriteName)}`,
`read_svc=${shellQuote(mirror.serviceReadName)}`,
`write_svc=${shellQuote(mirror.serviceWriteName)}`,
`cache_pvc=${shellQuote(mirror.cachePvcName)}`,
`cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`,
"deploy_ready() { desired=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n \"$desired\" ] && [ \"$desired\" -gt 0 ] 2>/dev/null && [ \"${ready:-0}\" = \"$desired\" ] && printf true || printf false; }",
"exists_res() { kubectl -n \"$1\" get \"$2\" \"$3\" >/dev/null 2>&1 && printf true || printf false; }",
"endpoint_ready() { endpoints=$(kubectl -n \"$1\" get endpoints \"$2\" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n \"$endpoints\" ] && printf true || printf false; }",
"summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc '/etc/git-mirror/status.sh' 2>/tmp/hwlab-node-gitmirror-status.err || true)",
"if [ -z \"$summary_json\" ]; then summary_json='{}'; fi",
"read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")",
"write_deployment_ready=$(deploy_ready \"$namespace\" \"$write_deploy\")",
"read_service_exists=$(exists_res \"$namespace\" service \"$read_svc\")",
"write_service_exists=$(exists_res \"$namespace\" service \"$write_svc\")",
"read_endpoints_ready=$(endpoint_ready \"$namespace\" \"$read_svc\")",
"write_endpoints_ready=$(endpoint_ready \"$namespace\" \"$write_svc\")",
"cache_pvc_exists=$(exists_res \"$namespace\" pvc \"$cache_pvc\")",
"cache_host_path_exists=false",
"if [ -n \"$cache_host_path\" ] && kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_exists=true; fi",
"SUMMARY_JSON=\"$summary_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" node <<'NODE'",
"const summary = (() => { try { return JSON.parse(process.env.SUMMARY_JSON || '{}'); } catch { return {}; } })();",
"const env = process.env;",
"const ok = env.read_deployment_ready === 'true' && env.write_deployment_ready === 'true' && env.read_service_exists === 'true' && env.write_service_exists === 'true' && env.read_endpoints_ready === 'true' && env.write_endpoints_ready === 'true' && (env.cache_pvc_exists === 'true' || env.cache_host_path_exists === 'true') && summary.localSource;",
"console.log(JSON.stringify({",
" ok: Boolean(ok),",
" resources: {",
" readDeploymentReady: env.read_deployment_ready === 'true',",
" writeDeploymentReady: env.write_deployment_ready === 'true',",
" readServiceExists: env.read_service_exists === 'true',",
" writeServiceExists: env.write_service_exists === 'true',",
" readEndpointsReady: env.read_endpoints_ready === 'true',",
" writeEndpointsReady: env.write_endpoints_ready === 'true',",
" cachePvcExists: env.cache_pvc_exists === 'true',",
" cacheHostPathConfigured: Boolean(env.cache_host_path),",
" cacheHostPathExists: env.cache_host_path_exists === 'true'",
" },",
" summary,",
" valuesPrinted: false",
"}));",
"NODE",
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const parsed = parseJsonObject(statusText(result));
const summary = record(parsed.summary);
const resources = record(parsed.resources);
const ok = result.exitCode === 0 && parsed.ok === true;
return {
ok,
command: `hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "status",
mutation: false,
namespace: mirror.namespace,
readUrl: spec.gitReadUrl,
writeUrl: spec.gitWriteUrl,
sourceBranch: mirror.sourceBranch,
gitopsBranch: mirror.gitopsBranch,
resources,
summary,
result: compactRuntimeCommand(result),
degradedReason: ok ? undefined : "node-runtime-git-mirror-not-ready",
valuesPrinted: false,
next: {
sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm`,
flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm`,
},
};
}
function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped);
if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`);
const spec = scoped.spec;
const mirror = nodeRuntimeGitMirrorTarget(spec);
const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName);
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
const script = [
"set -eu",
`namespace=${shellQuote(mirror.namespace)}`,
`job=${shellQuote(jobName)}`,
`manifest_b64=${shellQuote(manifestB64)}`,
"manifest_path=\"/tmp/$job.json\"",
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
scoped.dryRun
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
: [
"kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null",
"kubectl create -f \"$manifest_path\"",
`deadline=$(( $(date +%s) + ${scoped.timeoutSeconds} ))`,
"while :; do",
" status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)",
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi",
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi",
" sleep 2",
"done",
"kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
].join("\n"),
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const status = scoped.dryRun || !isCommandSuccess(result) ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
return {
ok: isCommandSuccess(result) && (status === undefined || status.ok === true),
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
mutation: !scoped.dryRun && isCommandSuccess(result),
namespace: mirror.namespace,
jobName,
manifest: scoped.dryRun ? manifest : undefined,
result: compactRuntimeCommand(result),
status,
degradedReason: isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
next: scoped.dryRun
? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` }
: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, sourceCommit: string): Record<string, unknown> {
const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
const beforeSummary = record(before.summary);
if (before.ok === true && beforeSummary.localSource === sourceCommit) {
return { ok: true, mode: "already-current", sourceCommit, before };
}
const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true });
const after = record(sync.status);
const afterSummary = record(after.summary);
const ok = sync.ok === true && afterSummary.localSource === sourceCommit;
return {
ok,
mode: "synced-before-trigger",
sourceCommit,
before,
sync,
after: sync.status ?? null,
degradedReason: ok ? undefined : "node-runtime-git-mirror-local-source-not-current-after-sync",
};
}
function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string {
const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix;
return `${prefix}-${Date.now().toString(36)}`.slice(0, 63);
}
function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record<string, unknown> {
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name: jobName,
namespace: mirror.namespace,
labels: {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
"app.kubernetes.io/component": `${action}-controller`,
"hwlab.pikastech.local/node": mirror.node,
"hwlab.pikastech.local/lane": mirror.lane,
"hwlab.pikastech.local/trigger": "manual-cli",
},
},
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 600,
ttlSecondsAfterFinished: 3600,
template: {
metadata: {
labels: {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
"app.kubernetes.io/component": `${action}-controller`,
"hwlab.pikastech.local/node": mirror.node,
"hwlab.pikastech.local/lane": mirror.lane,
},
},
spec: {
restartPolicy: "Never",
volumes: [
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
{ name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } },
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } },
],
containers: [{
name: action,
image: mirror.toolsImage,
imagePullPolicy: "IfNotPresent",
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
volumeMounts: [
{ name: "cache", mountPath: "/cache" },
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
{ name: "script", mountPath: "/script", readOnly: true },
],
}],
},
},
},
};
}
function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
@@ -2798,6 +3044,36 @@ function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExist
};
}
function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec {
const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
const targets = Array.isArray(parsed.targets) ? parsed.targets : [];
const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane);
if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
const gitMirror = record(target.gitMirror);
const source = record(target.source);
const gitops = record(target.gitops);
const tekton = record(target.tekton);
const toolsImage = record(tekton.toolsImage);
return {
id: stringValue(target.id, "target.id"),
node: stringValue(target.node, "target.node"),
lane: stringValue(target.lane, "target.lane"),
namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"),
serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"),
toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"),
sourceRepository: stringValue(source.repository, "source.repository"),
sourceBranch: stringValue(source.branch, "source.branch"),
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
};
}
function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record<string, unknown> {
const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60);
return {
@@ -5385,6 +5661,38 @@ function assertNodeId(value: string): void {
if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`);
}
function record(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
function stringValue(value: unknown, path: string): string {
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
return value;
}
function optionalStringValue(value: unknown, path: string): string | null {
if (value === undefined || value === null) return null;
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string when set`);
return value;
}
function parseJsonObject(text: string): Record<string, unknown> {
const trimmed = text.trim();
if (trimmed.length === 0) return {};
try {
return record(JSON.parse(trimmed) as unknown);
} catch {
const start = trimmed.indexOf("{");
const end = trimmed.lastIndexOf("}");
if (start >= 0 && end > start) {
try {
return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
} catch {}
}
}
return {};
}
function shellQuote(value: string): string {
return `'${value.replace(/'/gu, `'"'"'`)}'`;
}