Merge pull request #1404 from pikasTech/fix/cicd-source-authority-all

fix: source authority uses k8s git-mirror snapshots
This commit is contained in:
Lyon
2026-07-01 19:04:20 +08:00
committed by GitHub
20 changed files with 465 additions and 57 deletions
+77 -3
View File
@@ -23,6 +23,20 @@ export interface AgentRunGitMirrorRepositorySpec {
readonly gitopsBranch?: string;
}
export interface AgentRunSourceAuthoritySpec {
readonly mode: "gitMirrorSnapshot";
readonly resolver: "k8s-git-mirror";
readonly allowHostGit: false;
readonly allowHostWorkspace: false;
readonly allowGithubDirectInPipeline: false;
}
export interface AgentRunSourceSnapshotSpec {
readonly stageRefPrefix: string;
readonly missingObjectPolicy: "fail-fast";
readonly refreshPolicy: "sync-before-snapshot";
}
export interface AgentRunSecretRef {
readonly namespace: string;
readonly name: string;
@@ -57,6 +71,8 @@ export interface AgentRunLaneSpec {
readonly statusMode: "host-worktree" | "k3s-git-mirror";
readonly repository: string;
readonly branch: string;
readonly sourceAuthority: AgentRunSourceAuthoritySpec | null;
readonly sourceSnapshot: AgentRunSourceSnapshotSpec | null;
readonly bootstrapFromBranch: string | null;
readonly bootstrapTimeoutSeconds: number;
readonly bootstrapPollSeconds: number;
@@ -278,6 +294,8 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record<string, unkn
repository: spec.source.repository,
branch: spec.source.branch,
statusMode: spec.source.statusMode,
sourceAuthority: spec.source.sourceAuthority,
sourceSnapshot: spec.source.sourceSnapshot,
bootstrapFromBranch: spec.source.bootstrapFromBranch,
bootstrapTimeoutSeconds: spec.source.bootstrapTimeoutSeconds,
bootstrapPollSeconds: spec.source.bootstrapPollSeconds,
@@ -426,6 +444,17 @@ export function agentRunPipelineRunName(spec: AgentRunLaneSpec, sourceCommit: st
return `${spec.ci.pipelineRunPrefix}-${sourceCommit.slice(0, 12)}`;
}
export function agentRunSourceSnapshotStageRefPrefix(spec: AgentRunLaneSpec): string {
const snapshot = spec.source.sourceSnapshot;
if (snapshot === null) throw new Error(`config/agentrun.yaml controlPlane.lanes.${spec.lane}.source.sourceSnapshot is required for k8s git-mirror snapshot source authority`);
return snapshot.stageRefPrefix.replaceAll("{branch}", spec.source.branch);
}
export function agentRunSourceSnapshotRef(spec: AgentRunLaneSpec, sourceCommit: string): string {
if (!/^[0-9a-f]{40}$/iu.test(sourceCommit)) throw new Error(`sourceCommit must be a 40-hex git SHA for source snapshot ref: ${sourceCommit}`);
return `${agentRunSourceSnapshotStageRefPrefix(spec).replace(/\/+$/u, "")}/${sourceCommit.toLowerCase()}`;
}
function readAgentRunControlPlaneConfig(env: NodeJS.ProcessEnv): AgentRunControlPlaneConfig {
const configPath = env.AGENTRUN_CONTROL_PLANE_CONFIG ?? rootPath(AGENTRUN_CONFIG_PATH);
const root = readYamlRecord<Record<string, unknown>>(configPath);
@@ -492,16 +521,20 @@ function parseLane(lane: string, node: AgentRunNodeSpec, input: Record<string, u
const deployment = recordField(input, "deployment", path);
const gitMirror = recordField(input, "gitMirror", path);
const database = recordField(input, "database", path);
return {
const version = stringField(input, "version", path);
const statusMode = sourceStatusModeField(optionalStringField(source, "statusMode", `${path}.source`) ?? (version === "v0.2" ? "k3s-git-mirror" : "host-worktree"), `${path}.source.statusMode`);
const spec: AgentRunLaneSpec = {
lane,
nodeId: node.id,
nodeRoute: node.route,
nodeKubeRoute: node.kubeRoute,
version: stringField(input, "version", path),
version,
source: {
statusMode: sourceStatusModeField(optionalStringField(source, "statusMode", `${path}.source`) ?? "host-worktree", `${path}.source.statusMode`),
statusMode,
repository: stringField(source, "repository", `${path}.source`),
branch: stringField(source, "branch", `${path}.source`),
sourceAuthority: sourceAuthorityConfig(source.sourceAuthority, `${path}.source.sourceAuthority`),
sourceSnapshot: sourceSnapshotConfig(source.sourceSnapshot, `${path}.source.sourceSnapshot`),
bootstrapFromBranch: optionalStringField(source, "bootstrapFromBranch", `${path}.source`) ?? null,
bootstrapTimeoutSeconds: integerField(source, "bootstrapTimeoutSeconds", `${path}.source`),
bootstrapPollSeconds: integerField(source, "bootstrapPollSeconds", `${path}.source`),
@@ -552,6 +585,8 @@ function parseLane(lane: string, node: AgentRunNodeSpec, input: Record<string, u
database: parseDatabase(database, `${path}.database`),
secrets: arrayField(input, "secrets", path).map((secret, index) => parseLaneSecret(secret, `${path}.secrets[${index}]`)),
};
validateAgentRunLaneSourceAuthority(spec, path);
return spec;
}
function sourceStatusModeField(value: string, path: string): "host-worktree" | "k3s-git-mirror" {
@@ -559,6 +594,45 @@ function sourceStatusModeField(value: string, path: string): "host-worktree" | "
return value;
}
function sourceAuthorityConfig(value: unknown, path: string): AgentRunSourceAuthoritySpec | null {
if (value === undefined) return null;
const raw = asRecord(value, path);
return {
mode: enumField(raw, "mode", path, ["gitMirrorSnapshot"]),
resolver: enumField(raw, "resolver", path, ["k8s-git-mirror"]),
allowHostGit: falseBooleanField(raw, "allowHostGit", path),
allowHostWorkspace: falseBooleanField(raw, "allowHostWorkspace", path),
allowGithubDirectInPipeline: falseBooleanField(raw, "allowGithubDirectInPipeline", path),
};
}
function sourceSnapshotConfig(value: unknown, path: string): AgentRunSourceSnapshotSpec | null {
if (value === undefined) return null;
const raw = asRecord(value, path);
const stageRefPrefix = stringField(raw, "stageRefPrefix", path);
if (!stageRefPrefix.startsWith("refs/")) throw new Error(`${path}.stageRefPrefix must start with refs/`);
if (stageRefPrefix.includes("..") || /\s/u.test(stageRefPrefix)) throw new Error(`${path}.stageRefPrefix must not contain whitespace or ..`);
if (!stageRefPrefix.includes("{branch}")) throw new Error(`${path}.stageRefPrefix must include {branch}`);
return {
stageRefPrefix,
missingObjectPolicy: enumField(raw, "missingObjectPolicy", path, ["fail-fast"]),
refreshPolicy: enumField(raw, "refreshPolicy", path, ["sync-before-snapshot"]),
};
}
function falseBooleanField(obj: Record<string, unknown>, key: string, path: string): false {
const value = booleanField(obj, key, path);
if (value !== false) throw new Error(`${path}.${key} must be false`);
return false;
}
function validateAgentRunLaneSourceAuthority(spec: AgentRunLaneSpec, path: string): void {
if (spec.version !== "v0.2") return;
if (spec.source.statusMode !== "k3s-git-mirror") throw new Error(`${path}.source.statusMode must be k3s-git-mirror for AgentRun v0.2`);
if (spec.source.sourceAuthority === null) throw new Error(`${path}.source.sourceAuthority is required for AgentRun v0.2`);
if (spec.source.sourceSnapshot === null) throw new Error(`${path}.source.sourceSnapshot is required for AgentRun v0.2`);
}
function parseDeployment(input: Record<string, unknown>, path: string): AgentRunLaneSpec["deployment"] {
const argocd = recordField(input, "argocd", path);
const manager = recordField(input, "manager", path);
+24 -3
View File
@@ -201,11 +201,31 @@ export async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig
async function resolveTriggerCurrentSource(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise<Record<string, unknown> & { ok: boolean; sourceCommit?: string | null; sourcePayload?: Record<string, unknown> }> {
if (spec.source.statusMode === "k3s-git-mirror") {
progressEvent("agentrun.yaml-lane.source-snapshot.progress", {
node: spec.nodeId,
lane: spec.lane,
statusMode: spec.source.statusMode,
status: "syncing",
});
const sourceSync = await runYamlLaneGitMirrorSyncJob(config, spec);
if (sourceSync.ok !== true) {
return {
ok: false,
command: "agentrun control-plane trigger-current",
mode: waited ? "confirmed-waited" : "confirmed-trigger",
configPath,
target: agentRunLaneSummary(spec),
phase: "source-snapshot-sync",
degradedReason: "yaml-lane-k3s-source-snapshot-sync-failed",
result: sourceSync,
valuesPrinted: false,
};
}
progressEvent("agentrun.yaml-lane.source-status.progress", {
node: spec.nodeId,
lane: spec.lane,
statusMode: spec.source.statusMode,
status: "probing",
status: "probing-snapshot",
});
const sourceStatus = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)]);
const sourcePayload = captureJsonPayload(sourceStatus);
@@ -218,13 +238,14 @@ async function resolveTriggerCurrentSource(config: UniDeskConfig, spec: AgentRun
configPath,
target: agentRunLaneSummary(spec),
phase: "source-status",
degradedReason: "yaml-lane-k3s-source-status-failed",
degradedReason: stringOrNull(sourcePayload.degradedReason) ?? "yaml-lane-k3s-source-status-failed",
gitMirrorSync: sourceSync,
result: sourcePayload,
capture: compactCapture(sourceStatus, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
return { ok: true, sourceCommit, sourcePayload, valuesPrinted: false };
return { ok: true, sourceCommit, sourcePayload: { ...sourcePayload, gitMirrorSync: sourceSync, valuesPrinted: false }, valuesPrinted: false };
}
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
+11 -1
View File
@@ -234,7 +234,7 @@ export async function controlPlanePlan(_config: UniDeskConfig, options: StatusOp
target: agentRunLaneSummary(spec),
plannedChecks: [
"source-branch-exists",
"source-worktree-exists-and-clean",
spec.source.statusMode === "k3s-git-mirror" ? "source-git-mirror-snapshot-present" : "source-worktree-exists-and-clean",
"git-mirror-services-ready",
"ci-namespace-pipeline-serviceaccount",
"argo-application-alignment",
@@ -369,9 +369,11 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio
...(sourceWorktreeDetached ? ["source-worktree-detached"] : []),
...(sourceBranchAdvanced ? ["source-branch-advanced-after-target"] : []),
];
const sourceSnapshotRequired = spec.source.statusMode === "k3s-git-mirror";
const blockers = [
...(sourceWorkspaceRequired && sourcePayload.workspaceExists !== true ? ["source-worktree-missing"] : []),
...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]),
...(sourceSnapshotRequired && sourcePayload.snapshotPresent !== true ? ["source-snapshot-missing"] : []),
...(sourceWorkspaceRequired && sourcePayload.workspaceClean !== true && sourcePayload.workspaceExists === true ? ["source-worktree-dirty"] : []),
...(mirrorPayload.readReady === true ? [] : ["git-mirror-read-not-ready"]),
...(mirrorPayload.writeReady === true ? [] : ["git-mirror-write-not-ready"]),
@@ -426,11 +428,15 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio
: { code: "inspect-full-status", summary: "alignment is inconclusive; inspect full status details", command: statusFullCommand };
const compactSourceStatus = {
statusMode: spec.source.statusMode,
sourceAuthority: sourcePayload.sourceAuthority ?? spec.source.sourceAuthority?.mode ?? null,
workspaceExists: sourcePayload.workspaceExists ?? false,
workspaceClean: sourcePayload.workspaceClean ?? null,
branch: sourcePayload.branch ?? null,
remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
sourceRef: sourcePayload.sourceRef ?? null,
sourceStageRef: sourcePayload.sourceStageRef ?? null,
snapshotPresent: sourcePayload.snapshotPresent ?? null,
workspaceDetached: sourceWorktreeDetached,
};
const compactGitMirrorStatus = {
@@ -499,6 +505,10 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio
localHead: sourcePayload.localHead ?? null,
remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
sourceAuthority: sourcePayload.sourceAuthority ?? spec.source.sourceAuthority?.mode ?? null,
sourceRef: sourcePayload.sourceRef ?? null,
sourceStageRef: sourcePayload.sourceStageRef ?? null,
snapshotPresent: sourcePayload.snapshotPresent ?? null,
},
gitMirror: {
readReady: mirrorPayload.readReady ?? false,
+6 -1
View File
@@ -69,6 +69,11 @@ export function renderAgentRunControlPlaneStatusSummary(result: Record<string, u
const timings = record(result.timings);
const blockers = Array.isArray(summary.blockers) ? summary.blockers.map(String) : [];
const warnings = Array.isArray(summary.warnings) ? summary.warnings.map(String) : [];
const sourceSnapshotMode = source.sourceAuthority === "git-mirror-snapshot" || source.snapshotPresent !== null;
const sourceStatus = sourceSnapshotMode ? yesNo(source.snapshotPresent) : yesNo(source.workspaceClean);
const sourceDetail = sourceSnapshotMode
? `branch=${displayValue(source.branch)} commit=${shortSha(source.remoteBranchCommit)} snapshot=${yesNo(source.snapshotPresent)} stage=${displayValue(source.sourceStageRef ?? "-")}`
: `branch=${displayValue(source.branch)} commit=${shortSha(source.remoteBranchCommit)} detached=${yesNo(source.workspaceDetached)}`;
const lines = [
"AGENTRUN CONTROL-PLANE STATUS",
renderTable(
@@ -88,7 +93,7 @@ export function renderAgentRunControlPlaneStatusSummary(result: Record<string, u
renderTable(
["COMPONENT", "STATUS", "DETAIL"],
[
["source", yesNo(source.workspaceClean), `branch=${displayValue(source.branch)} commit=${shortSha(source.remoteBranchCommit)} detached=${yesNo(source.workspaceDetached)}`],
["source", sourceStatus, sourceDetail],
["git-mirror", yesNo(gitMirror.alreadySynced), `read=${yesNo(gitMirror.readReady)} write=${yesNo(gitMirror.writeReady)} gitops=${shortSha(gitMirror.gitopsCommit)}`],
["ci", yesNo(ciRun.status === "True"), `run=${displayValue(ciRun.name)} status=${displayValue(ciRun.status)} reason=${displayValue(ciRun.reason)} evidenceMissing=${yesNo(ci.evidenceMissing)}`],
["argo", yesNo(argo.syncedToGitops), `sync=${displayValue(argo.syncStatus)} health=${displayValue(argo.healthStatus)} revision=${shortSha(argo.revision)}`],
+12 -2
View File
@@ -18,6 +18,8 @@ import {
agentRunLaneSummary,
agentRunPipelineRunName,
agentRunProviderCredentialRefs,
agentRunSourceSnapshotRef,
agentRunSourceSnapshotStageRefPrefix,
resolveAgentRunLaneTarget,
type AgentRunCancelLifecycleSpec,
type AgentRunLaneSpec,
@@ -294,6 +296,7 @@ export async function runYamlLaneGitMirrorJob(config: UniDeskConfig, spec: Agent
}
export function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string {
const sourceStageRef = spec.source.statusMode === "k3s-git-mirror" ? agentRunSourceSnapshotRef(spec, sourceCommit) : null;
const manifest = {
apiVersion: "tekton.dev/v1",
kind: "PipelineRun",
@@ -307,6 +310,10 @@ export function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCo
"agentrun.pikastech.local/source-commit": sourceCommit,
"agentrun.pikastech.local/trigger": "unidesk-yaml-only",
},
annotations: sourceStageRef === null ? undefined : {
"agentrun.pikastech.local/source-authority": "git-mirror-snapshot",
"agentrun.pikastech.local/source-stage-ref": sourceStageRef,
},
},
spec: {
pipelineRef: { name: spec.ci.pipeline },
@@ -545,6 +552,7 @@ export function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string {
...yamlLaneGitMirrorSshSetupShellLines(spec),
`repository=${shQuote(spec.source.repository)}`,
`source_branch=${shQuote(spec.source.branch)}`,
`source_stage_ref_prefix=${shQuote(agentRunSourceSnapshotStageRefPrefix(spec))}`,
`gitops_branch=${shQuote(spec.gitops.branch)}`,
"repo=\"/cache/${repository}.git\"",
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
@@ -562,6 +570,8 @@ export function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string {
"git --git-dir=\"$repo\" config http.receivepack true",
"timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"",
"source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")",
"source_stage_ref=\"${source_stage_ref_prefix%/}/$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"$source_stage_ref\" \"$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"",
"gitops_fetch_err=$(mktemp)",
"gitops_remote_missing=false",
@@ -580,8 +590,8 @@ export function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string {
" fi",
"fi",
"git --git-dir=\"$repo\" update-server-info",
"SOURCE_SHA=\"$source_sha\" GITOPS_SHA=\"$gitops_sha\" GITOPS_REMOTE_MISSING=\"$gitops_remote_missing\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, localSource: process.env.SOURCE_SHA, localGitops: process.env.GITOPS_SHA || null, gitopsRemoteMissing: process.env.GITOPS_REMOTE_MISSING === 'true', valuesPrinted: false }));",
"SOURCE_SHA=\"$source_sha\" SOURCE_STAGE_REF=\"$source_stage_ref\" GITOPS_SHA=\"$gitops_sha\" GITOPS_REMOTE_MISSING=\"$gitops_remote_missing\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, sourceAuthority: 'git-mirror-snapshot', localSource: process.env.SOURCE_SHA, sourceStageRef: process.env.SOURCE_STAGE_REF, localGitops: process.env.GITOPS_SHA || null, gitopsRemoteMissing: process.env.GITOPS_REMOTE_MISSING === 'true', valuesPrinted: false }));",
"NODE",
].join("\n");
}
+30 -8
View File
@@ -42,7 +42,7 @@ import { pathValue } from "./render";
import { startAsyncAgentRunJob } from "./rest-bridge";
import { collectLaneSecretSources, readSecretSourceValue, restartYamlLaneScript, secretSyncScript } from "./secrets";
import { capture, captureJsonPayload, compactCapture, isGitSha, stringOrNull } from "./utils";
import { yamlLaneSourceBootstrapProbeScript } from "./yaml-lane";
import { yamlLaneK3sSourceStatusScript, yamlLaneSourceBootstrapProbeScript } from "./yaml-lane";
export function renderNextObjectLines(next: Record<string, unknown>): string[] {
return Object.values(next)
@@ -409,18 +409,34 @@ export async function triggerCurrent(config: UniDeskConfig, options: TriggerOpti
export async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise<Record<string, unknown>> {
const spec = target.spec;
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapProbeScript(spec)]);
const probe = spec.source.statusMode === "k3s-git-mirror"
? await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)])
: await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapProbeScript(spec)]);
const source = captureJsonPayload(probe);
const sourceCommit = stringOrNull(source.sourceCommit);
const remoteBranchExists = source.remoteBranchExists === true;
const pipelineRun = sourceCommit !== null && isGitSha(sourceCommit) ? agentRunPipelineRunName(spec, sourceCommit) : null;
const placeholderImage = sourceCommit === null ? null : placeholderAgentRunImage(spec, sourceCommit);
const renderedFiles = placeholderImage === null ? [] : renderAgentRunGitopsFiles(spec, { sourceCommit, image: placeholderImage });
const plan = {
node: spec.nodeId,
lane: spec.lane,
version: spec.version,
source: {
const sourcePlan = spec.source.statusMode === "k3s-git-mirror"
? {
statusMode: spec.source.statusMode,
sourceAuthority: spec.source.sourceAuthority,
sourceSnapshot: spec.source.sourceSnapshot,
repository: spec.source.repository,
branch: spec.source.branch,
remoteBranchExists,
remoteBranchCommit: stringOrNull(source.remoteBranchCommit),
sourceCommit,
sourceRef: stringOrNull(source.sourceRef),
sourceStageRef: stringOrNull(source.sourceStageRef),
snapshotPresent: source.snapshotPresent === true,
degradedReason: stringOrNull(source.degradedReason),
next: source.snapshotPresent === true ? null : `bun scripts/cli.ts agentrun git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
valuesPrinted: false,
}
: {
statusMode: spec.source.statusMode,
workspace: spec.source.workspace,
remote: spec.source.remote,
branch: spec.source.branch,
@@ -429,7 +445,13 @@ export async function triggerCurrentYamlLane(config: UniDeskConfig, options: Tri
bootstrapPollSeconds: spec.source.bootstrapPollSeconds,
remoteBranchExists,
sourceCommit,
},
valuesPrinted: false,
};
const plan = {
node: spec.nodeId,
lane: spec.lane,
version: spec.version,
source: sourcePlan,
deploymentFormat: spec.deployment.format,
deploymentTruth: "config/agentrun.yaml",
removedServiceDeployJson: true,
+36 -8
View File
@@ -18,6 +18,8 @@ import {
agentRunLaneSummary,
agentRunPipelineRunName,
agentRunProviderCredentialRefs,
agentRunSourceSnapshotRef,
agentRunSourceSnapshotStageRefPrefix,
resolveAgentRunLaneTarget,
type AgentRunCancelLifecycleSpec,
type AgentRunLaneSpec,
@@ -390,21 +392,41 @@ export function yamlLaneK3sSourceStatusScript(spec: AgentRunLaneSpec): string {
`read_deployment=${shQuote(spec.gitMirror.readDeployment)}`,
`repository=${shQuote(spec.source.repository)}`,
`source_branch=${shQuote(spec.source.branch)}`,
`source_stage_ref_prefix=${shQuote(agentRunSourceSnapshotStageRefPrefix(spec))}`,
"repo_path=\"/cache/${repository}.git\"",
"kubectl -n \"$namespace\" get deploy \"$read_deployment\" >/tmp/agentrun-source-read-deploy.txt 2>/dev/null",
"read_exit=$?",
"source_commit=''",
"source_ref=\"refs/mirror-stage/heads/$source_branch\"",
"source_stage_ref=''",
"snapshot_commit=''",
"if [ \"$read_exit\" -eq 0 ]; then",
" source_commit=$(kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; git --git-dir=\"$repo_path\" rev-parse --verify \"refs/heads/$source_branch^{commit}\" 2>/dev/null || true' sh \"$repo_path\" \"$source_branch\" 2>/dev/null | tail -n 1 | tr -d '\\r')",
" refs=$(kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; stage_ref_prefix=\"$3\"; source_ref=\"refs/mirror-stage/heads/$source_branch\"; source_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_ref^{commit}\" 2>/dev/null || true); source_stage_ref=\"\"; snapshot_commit=\"\"; if printf \"%s\" \"$source_commit\" | grep -Eq \"^[0-9a-fA-F]{40}$\"; then source_stage_ref=\"${stage_ref_prefix%/}/$source_commit\"; snapshot_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_stage_ref^{commit}\" 2>/dev/null || true); fi; printf \"source_commit=%s\\nsource_stage_ref=%s\\nsnapshot_commit=%s\\n\" \"$source_commit\" \"$source_stage_ref\" \"$snapshot_commit\"' sh \"$repo_path\" \"$source_branch\" \"$source_stage_ref_prefix\" 2>/dev/null)",
" source_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_commit=//p' | tail -n 1 | tr -d '\\r')",
" source_stage_ref=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_stage_ref=//p' | tail -n 1 | tr -d '\\r')",
" snapshot_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^snapshot_commit=//p' | tail -n 1 | tr -d '\\r')",
"fi",
"export namespace read_deployment repository source_branch read_exit source_commit",
"export namespace read_deployment repository source_branch read_exit source_commit source_ref source_stage_ref snapshot_commit",
"python3 - <<'PY'",
"import json, os",
"source_commit = os.environ.get('source_commit') or None",
"import json, os, re",
"def sha(value):",
" return (value or '').lower() if re.match(r'^[0-9a-f]{40}$', value or '', re.I) else None",
"source_commit = sha(os.environ.get('source_commit'))",
"snapshot_commit = sha(os.environ.get('snapshot_commit'))",
"source_stage_ref = os.environ.get('source_stage_ref') or None",
"read_ready = os.environ.get('read_exit') == '0'",
"snapshot_present = source_commit is not None and snapshot_commit == source_commit",
"degraded_reason = None",
"if not read_ready:",
" degraded_reason = 'git-mirror-read-not-ready'",
"elif source_commit is None:",
" degraded_reason = 'source-branch-missing'",
"elif not snapshot_present:",
" degraded_reason = 'source-snapshot-missing'",
"print(json.dumps({",
" 'ok': read_ready and source_commit is not None,",
" 'ok': read_ready and snapshot_present,",
" 'statusMode': 'k3s-git-mirror',",
" 'sourceAuthority': 'git-mirror-snapshot',",
" 'expectedWorkspace': None,",
" 'actualWorkspace': None,",
" 'workspaceExists': None,",
@@ -415,12 +437,18 @@ export function yamlLaneK3sSourceStatusScript(spec: AgentRunLaneSpec): string {
" 'remoteBranch': os.environ.get('source_branch'),",
" 'remoteBranchExists': source_commit is not None,",
" 'remoteBranchCommit': source_commit,",
" 'sourceCommit': source_commit,",
" 'sourceCommit': snapshot_commit if snapshot_present else source_commit,",
" 'sourceRef': os.environ.get('source_ref'),",
" 'sourceStageRef': source_stage_ref,",
" 'snapshotPresent': snapshot_present,",
" 'degradedReason': degraded_reason,",
" 'gitMirror': {",
" 'namespace': os.environ.get('namespace'),",
" 'readDeployment': os.environ.get('read_deployment'),",
" 'readReady': read_ready,",
" 'repository': os.environ.get('repository'),",
" 'sourceRef': os.environ.get('source_ref'),",
" 'sourceStageRef': source_stage_ref,",
" },",
" 'statusShort': None,",
" 'valuesPrinted': False,",
@@ -1168,12 +1196,12 @@ function yamlLaneK3sBuildSourceShell(spec: AgentRunLaneSpec, sourceCommit: strin
return [
"set -eu",
`read_url=${shQuote(spec.gitMirror.readUrl)}`,
`source_branch=${shQuote(spec.source.branch)}`,
`source_commit=${shQuote(sourceCommit)}`,
`source_stage_ref=${shQuote(agentRunSourceSnapshotRef(spec, sourceCommit))}`,
"rm -rf /workspace/repo",
"git clone --no-checkout \"$read_url\" /workspace/repo",
"cd /workspace/repo",
"git fetch origin \"refs/heads/$source_branch:refs/remotes/origin/$source_branch\"",
"git fetch origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"",
"git checkout --detach \"$source_commit\"",
"actual=$(git rev-parse HEAD)",
"test \"$actual\" = \"$source_commit\"",
+16 -1
View File
@@ -294,7 +294,22 @@ export interface ControlPlaneTargetSpec {
enabled: boolean;
ciNamespace: string;
runtimeNamespace: string;
source: { repository: string; branch: string };
source: {
repository: string;
branch: string;
sourceAuthority: {
mode: "gitMirrorSnapshot";
resolver: "k8s-git-mirror";
allowHostGit: false;
allowHostWorkspace: false;
allowGithubDirectInPipeline: false;
};
sourceSnapshot: {
stageRefPrefix: string;
missingObjectPolicy: "fail-fast";
refreshPolicy: "sync-before-snapshot";
};
};
gitops: { branch: string; path: string };
gitMirror: {
namespace: string;
+55 -6
View File
@@ -1898,6 +1898,8 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
const ciNamespace = stringField(raw, "ciNamespace", path);
const runtimeNamespace = stringField(raw, "runtimeNamespace", path);
const argoNamespace = stringField(argo, "namespace", `${path}.argo`);
const sourceAuthority = controlPlaneSourceAuthoritySpec(asRecord(source.sourceAuthority, `${path}.source.sourceAuthority`), `${path}.source.sourceAuthority`);
const sourceSnapshot = controlPlaneSourceSnapshotSpec(asRecord(source.sourceSnapshot, `${path}.source.sourceSnapshot`), `${path}.source.sourceSnapshot`);
return {
id: stringField(raw, "id", path),
node,
@@ -1905,7 +1907,7 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
enabled: booleanField(raw, "enabled", path),
ciNamespace,
runtimeNamespace,
source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`) },
source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`), sourceAuthority, sourceSnapshot },
gitops: { branch: stringField(gitops, "branch", `${path}.gitops`), path: stringField(gitops, "path", `${path}.gitops`) },
gitMirror: {
namespace: gitMirrorNamespace,
@@ -1948,6 +1950,38 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
};
}
function controlPlaneSourceAuthoritySpec(raw: Record<string, unknown>, path: string): ControlPlaneTargetSpec["source"]["sourceAuthority"] {
const mode = stringField(raw, "mode", path);
const resolver = stringField(raw, "resolver", path);
if (mode !== "gitMirrorSnapshot") throw new Error(`${path}.mode must be gitMirrorSnapshot`);
if (resolver !== "k8s-git-mirror") throw new Error(`${path}.resolver must be k8s-git-mirror`);
return {
mode,
resolver,
allowHostGit: falseBooleanField(raw, "allowHostGit", path),
allowHostWorkspace: falseBooleanField(raw, "allowHostWorkspace", path),
allowGithubDirectInPipeline: falseBooleanField(raw, "allowGithubDirectInPipeline", path),
};
}
function controlPlaneSourceSnapshotSpec(raw: Record<string, unknown>, path: string): ControlPlaneTargetSpec["source"]["sourceSnapshot"] {
const stageRefPrefix = stringField(raw, "stageRefPrefix", path);
if (!stageRefPrefix.startsWith("refs/")) throw new Error(`${path}.stageRefPrefix must start with refs/`);
if (stageRefPrefix.includes("..") || /\s/u.test(stageRefPrefix)) throw new Error(`${path}.stageRefPrefix must not contain whitespace or ..`);
if (!stageRefPrefix.includes("{branch}")) throw new Error(`${path}.stageRefPrefix must include {branch}`);
const missingObjectPolicy = stringField(raw, "missingObjectPolicy", path);
const refreshPolicy = stringField(raw, "refreshPolicy", path);
if (missingObjectPolicy !== "fail-fast") throw new Error(`${path}.missingObjectPolicy must be fail-fast`);
if (refreshPolicy !== "sync-before-snapshot") throw new Error(`${path}.refreshPolicy must be sync-before-snapshot`);
return { stageRefPrefix, missingObjectPolicy, refreshPolicy };
}
function falseBooleanField(obj: Record<string, unknown>, key: string, path: string): false {
const value = booleanField(obj, key, path);
if (value !== false) throw new Error(`${path}.${key} must be false`);
return false;
}
function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record<string, unknown>[] {
const labels = {
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
@@ -1977,7 +2011,7 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa
kind: "ConfigMap",
metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } },
data: {
"repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2),
"repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, sourceStageRefPrefix: target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch), gitopsBranch: target.gitops.branch }], null, 2),
"server.js": gitMirrorServerJs(),
"status.sh": gitMirrorStatusShell(),
"sync.sh": gitMirrorSyncShell(_node, target),
@@ -2280,7 +2314,7 @@ function service(name: string, namespace: string, labels: Record<string, string>
function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
return sha256Short(JSON.stringify({
repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }],
repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, sourceStageRefPrefix: target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch), gitopsBranch: target.gitops.branch }],
ports: { read: target.gitMirror.readContainerPort, write: target.gitMirror.writeContainerPort },
githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport),
runtimeProxy: runtimeHostProxyConfig(node, gitMirrorRuntimeProxySpec(node, target)),
@@ -2467,6 +2501,9 @@ for (const spec of repositories) {
const repoPath = '/cache/' + spec.repository + '.git';
const localSource = rev(repoPath, 'refs/heads/' + spec.sourceBranch);
const githubSource = rev(repoPath, 'refs/mirror-stage/heads/' + spec.sourceBranch);
const sourceStageRefPrefix = spec.sourceStageRefPrefix || ('refs/unidesk/snapshots/hwlab-node-runtime/' + spec.sourceBranch);
const sourceStageRef = githubSource ? sourceStageRefPrefix.replace(/\/+$/, '') + '/' + githubSource : null;
const sourceSnapshot = sourceStageRef ? rev(repoPath, sourceStageRef) : null;
const localGitops = rev(repoPath, 'refs/heads/' + spec.gitopsBranch);
const githubGitops = rev(repoPath, 'refs/mirror-stage/heads/' + spec.gitopsBranch);
items[spec.key] = {
@@ -2474,10 +2511,13 @@ for (const spec of repositories) {
sourceBranch: spec.sourceBranch,
localSource,
githubSource,
sourceAuthority: 'git-mirror-snapshot',
sourceStageRef,
sourceSnapshot,
gitopsBranch: spec.gitopsBranch,
localGitops,
githubGitops,
sourceInSync: Boolean(localSource && githubSource && localSource === githubSource),
sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource),
gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops),
pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)),
};
@@ -2487,11 +2527,15 @@ const pendingFlush = Object.values(items).some((item) => Boolean(item.pendingFlu
console.log(JSON.stringify({
localSource: first.localSource || null,
githubSource: first.githubSource || null,
sourceAuthority: first.sourceAuthority || null,
sourceStageRef: first.sourceStageRef || null,
sourceSnapshot: first.sourceSnapshot || null,
localGitops: first.localGitops || null,
githubGitops: first.githubGitops || null,
refSources: {
localSource: 'refs/heads/' + (first.sourceBranch || ''),
githubSource: 'refs/mirror-stage/heads/' + (first.sourceBranch || ''),
sourceSnapshot: first.sourceStageRef || null,
localGitops: 'refs/heads/' + (first.gitopsBranch || ''),
githubGitops: 'refs/mirror-stage/heads/' + (first.gitopsBranch || ''),
githubFieldsAreMirrorStageCache: true
@@ -2557,6 +2601,7 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT
useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`,
`repository=${shQuote(target.source.repository)}`,
`source_branch=${shQuote(target.source.branch)}`,
`source_stage_ref_prefix=${shQuote(target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch))}`,
`gitops_branch=${shQuote(target.gitops.branch)}`,
"repo=\"/cache/${repository}.git\"",
].filter(Boolean);
@@ -2678,6 +2723,8 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg
"git --git-dir=\"$repo\" config http.receivepack true",
"timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"",
"source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")",
"source_stage_ref=\"${source_stage_ref_prefix%/}/$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"$source_stage_ref\" \"$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"",
"discarded_stale_gitops=false",
"if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then",
@@ -2694,7 +2741,7 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg
" fi",
"fi",
"git --git-dir=\"$repo\" update-server-info",
"export repository source_branch gitops_branch started_at discarded_stale_gitops",
"export repository source_branch source_stage_ref gitops_branch started_at discarded_stale_gitops",
"node <<'NODE' | tee /cache/HWLAB.last-sync.json",
"const { execFileSync } = require('node:child_process');",
"const repository = process.env.repository;",
@@ -2704,10 +2751,12 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg
"function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }",
"const localSource = rev(`refs/heads/${sourceBranch}`);",
"const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);",
"const sourceStageRef = process.env.source_stage_ref;",
"const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;",
"const localGitops = rev(`refs/heads/${gitopsBranch}`);",
"const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);",
"const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));",
"console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));",
"console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), sourceAuthority: 'git-mirror-snapshot', localSource, githubSource, sourceStageRef, sourceSnapshot, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));",
"NODE",
"cat /cache/HWLAB.last-sync.json",
"",
+68
View File
@@ -323,6 +323,20 @@ export interface HwlabRuntimeImageBuildSpec {
readonly dockerNetworkMode: "default" | "host";
}
export interface HwlabRuntimeSourceAuthoritySpec {
readonly mode: "gitMirrorSnapshot";
readonly resolver: "k8s-git-mirror";
readonly allowHostGit: false;
readonly allowHostWorkspace: false;
readonly allowGithubDirectInPipeline: false;
}
export interface HwlabRuntimeSourceSnapshotSpec {
readonly stageRefPrefix: string;
readonly missingObjectPolicy: "fail-fast";
readonly refreshPolicy: "sync-before-snapshot";
}
export interface HwlabRuntimePublicExposureFrpcProxySpec {
readonly name: string;
readonly localIP: string;
@@ -450,6 +464,8 @@ export interface HwlabRuntimeLaneSpec {
readonly minor: number;
readonly version: string;
readonly sourceBranch: string;
readonly sourceAuthority?: HwlabRuntimeSourceAuthoritySpec;
readonly sourceSnapshot?: HwlabRuntimeSourceSnapshotSpec;
readonly workspace: string;
readonly cicdRepo: string;
readonly cicdRepoLock: string;
@@ -498,6 +514,16 @@ export function hwlabRuntimeActiveExternalPostgres(spec: HwlabRuntimeLaneSpec):
return spec.runtimeStore?.postgres?.mode === "platform-service" ? spec.externalPostgres : undefined;
}
export function hwlabRuntimeSourceSnapshotStageRefPrefix(spec: HwlabRuntimeLaneSpec): string {
if (spec.sourceSnapshot === undefined) throw new Error(`${HWLAB_NODE_LANE_CONFIG_PATH} lanes.${spec.lane}.sourceSnapshot is required for k8s git-mirror snapshot source authority`);
return spec.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", spec.sourceBranch);
}
export function hwlabRuntimeSourceSnapshotRef(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
if (!/^[0-9a-f]{40}$/iu.test(sourceCommit)) throw new Error(`sourceCommit must be a 40-hex git SHA for source snapshot ref: ${sourceCommit}`);
return `${hwlabRuntimeSourceSnapshotStageRefPrefix(spec).replace(/\/+$/u, "")}/${sourceCommit.toLowerCase()}`;
}
export const HWLAB_NODE_LANE_CONFIG_PATH = "config/hwlab-node-lanes.yaml";
interface HwlabLaneConfig {
@@ -507,6 +533,8 @@ interface HwlabLaneConfig {
readonly minor: number;
readonly version: string;
readonly sourceBranch: string;
readonly sourceAuthority?: HwlabRuntimeSourceAuthoritySpec;
readonly sourceSnapshot?: HwlabRuntimeSourceSnapshotSpec;
readonly workspace: string;
readonly cicdRepo: string;
readonly cicdRepoLock: string;
@@ -735,6 +763,8 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record<string, unknown>): HwlabLa
const minor = numberField(raw, "minor", `lanes.${id}`);
const version = stringField(raw, "version", `lanes.${id}`);
if (version !== `v0.${minor}`) throw new Error(`lanes.${id}.version must equal v0.${minor}`);
if (minor >= 3 && raw.sourceAuthority === undefined) throw new Error(`lanes.${id}.sourceAuthority is required for HWLAB runtime v0.3+`);
if (minor >= 3 && raw.sourceSnapshot === undefined) throw new Error(`lanes.${id}.sourceSnapshot is required for HWLAB runtime v0.3+`);
return {
id,
node: stringField(raw, "node", `lanes.${id}`),
@@ -742,6 +772,8 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record<string, unknown>): HwlabLa
minor,
version,
sourceBranch: stringField(raw, "sourceBranch", `lanes.${id}`),
sourceAuthority: sourceAuthorityConfig(raw.sourceAuthority, `lanes.${id}.sourceAuthority`),
sourceSnapshot: sourceSnapshotConfig(raw.sourceSnapshot, `lanes.${id}.sourceSnapshot`),
workspace: stringField(raw, "workspace", `lanes.${id}`),
cicdRepo: stringField(raw, "cicdRepo", `lanes.${id}`),
cicdRepoLock: stringField(raw, "cicdRepoLock", `lanes.${id}`),
@@ -805,6 +837,8 @@ function laneTargetConfig(id: HwlabRuntimeLane, nodeId: string, baseRaw: Record<
bootstrapAdmin: mergeOptionalRecord(baseRaw.bootstrapAdmin, targetRaw.bootstrapAdmin),
codeAgentProvider: mergeOptionalRecord(baseRaw.codeAgentProvider, targetRaw.codeAgentProvider),
codeAgentRuntime: mergeOptionalRecord(baseRaw.codeAgentRuntime, targetRaw.codeAgentRuntime),
sourceAuthority: mergeOptionalRecord(baseRaw.sourceAuthority, targetRaw.sourceAuthority),
sourceSnapshot: mergeOptionalRecord(baseRaw.sourceSnapshot, targetRaw.sourceSnapshot),
sourceWorkspace: mergeOptionalRecord(baseRaw.sourceWorkspace, targetRaw.sourceWorkspace),
externalPostgres: mergeOptionalRecord(baseRaw.externalPostgres, targetRaw.externalPostgres),
runtimeStore: mergeOptionalRecord(baseRaw.runtimeStore, targetRaw.runtimeStore),
@@ -817,6 +851,38 @@ function laneTargetConfig(id: HwlabRuntimeLane, nodeId: string, baseRaw: Record<
return laneConfig(id, merged);
}
function sourceAuthorityConfig(value: unknown, path: string): HwlabRuntimeSourceAuthoritySpec | undefined {
if (value === undefined) return undefined;
const raw = asRecord(value, path);
return {
mode: enumStringField(raw, "mode", path, ["gitMirrorSnapshot"]),
resolver: enumStringField(raw, "resolver", path, ["k8s-git-mirror"]),
allowHostGit: falseBooleanField(raw, "allowHostGit", path),
allowHostWorkspace: falseBooleanField(raw, "allowHostWorkspace", path),
allowGithubDirectInPipeline: falseBooleanField(raw, "allowGithubDirectInPipeline", path),
};
}
function sourceSnapshotConfig(value: unknown, path: string): HwlabRuntimeSourceSnapshotSpec | undefined {
if (value === undefined) return undefined;
const raw = asRecord(value, path);
const stageRefPrefix = stringField(raw, "stageRefPrefix", path);
if (!stageRefPrefix.startsWith("refs/")) throw new Error(`${path}.stageRefPrefix must start with refs/`);
if (stageRefPrefix.includes("..") || /\s/u.test(stageRefPrefix)) throw new Error(`${path}.stageRefPrefix must not contain whitespace or ..`);
if (!stageRefPrefix.includes("{branch}")) throw new Error(`${path}.stageRefPrefix must include {branch}`);
return {
stageRefPrefix,
missingObjectPolicy: enumStringField(raw, "missingObjectPolicy", path, ["fail-fast"]),
refreshPolicy: enumStringField(raw, "refreshPolicy", path, ["sync-before-snapshot"]),
};
}
function falseBooleanField(obj: Record<string, unknown>, key: string, path: string): false {
const value = booleanField(obj, key, path);
if (value !== false) throw new Error(`${path}.${key} must be false`);
return false;
}
function buildkitConfig(value: unknown, path: string): HwlabRuntimeBuildkitSpec | undefined {
if (value === undefined) return undefined;
const raw = asRecord(value, path);
@@ -1682,6 +1748,8 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec {
minor: config.minor,
version: config.version,
sourceBranch: config.sourceBranch,
...(config.sourceAuthority === undefined ? {} : { sourceAuthority: config.sourceAuthority }),
...(config.sourceSnapshot === undefined ? {} : { sourceSnapshot: config.sourceSnapshot }),
workspace: config.workspace,
cicdRepo: config.cicdRepo,
cicdRepoLock: config.cicdRepoLock,
+13 -10
View File
@@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, hwlabRuntimeSourceSnapshotStageRefPrefix, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -110,25 +110,28 @@ export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { source
`read_deploy=${shellQuote(mirror.serviceReadName)}`,
`repo_path=${shellQuote(`/cache/${mirror.sourceRepository}.git`)}`,
`source_branch=${shellQuote(mirror.sourceBranch)}`,
`source_stage_ref_prefix=${shellQuote(hwlabRuntimeSourceSnapshotStageRefPrefix(spec))}`,
"read_ref() {",
" ref=\"$1\"",
" kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'repo_path=$1; ref=$2; git --git-dir=\"$repo_path\" rev-parse --verify \"$ref^{commit}\" 2>/dev/null' sh \"$repo_path\" \"$ref\" 2>/dev/null",
"}",
"mirror_stage=$(read_ref \"refs/mirror-stage/heads/$source_branch\")",
"mirror_stage_rc=$?",
"local_head=$(read_ref \"refs/heads/$source_branch\")",
"local_head_rc=$?",
"source_commit=\"$mirror_stage\"",
"source_ref=\"refs/mirror-stage/heads/$source_branch\"",
"if ! printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then",
" source_commit=\"$local_head\"",
" source_ref=\"refs/heads/$source_branch\"",
"source_stage_ref=''",
"snapshot_commit=''",
"if printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then",
" source_stage_ref=\"${source_stage_ref_prefix%/}/$source_commit\"",
" snapshot_commit=$(read_ref \"$source_stage_ref\")",
"fi",
"node - \"$mirror_stage_rc\" \"$local_head_rc\" \"$mirror_stage\" \"$local_head\" \"$source_commit\" \"$source_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'",
"const [mirrorStageRc, localHeadRc, mirrorStage, localHead, sourceCommit, sourceRef, repoPath, branch] = process.argv.slice(2);",
"node - \"$mirror_stage_rc\" \"$mirror_stage\" \"$snapshot_commit\" \"$source_commit\" \"$source_ref\" \"$source_stage_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'",
"const [mirrorStageRc, mirrorStage, snapshotCommit, sourceCommit, sourceRef, sourceStageRef, repoPath, branch] = process.argv.slice(2);",
"const isSha = (value) => /^[0-9a-f]{40}$/i.test(value || '');",
"const ok = isSha(sourceCommit);",
"console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-cache', sourceAuthority: 'git-mirror-cache', sourceCommit: ok ? sourceCommit.toLowerCase() : null, sourceRef: ok ? sourceRef : null, mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, localHead: isSha(localHead) ? localHead.toLowerCase() : null, mirrorStageRc: Number(mirrorStageRc), localHeadRc: Number(localHeadRc), branch, repoPath, valuesRedacted: true }));",
"const normalizedSource = isSha(sourceCommit) ? sourceCommit.toLowerCase() : null;",
"const normalizedSnapshot = isSha(snapshotCommit) ? snapshotCommit.toLowerCase() : null;",
"const ok = normalizedSource !== null && normalizedSnapshot === normalizedSource;",
"console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-snapshot', sourceAuthority: 'git-mirror-snapshot', sourceCommit: ok ? normalizedSnapshot : normalizedSource, sourceRef: normalizedSource ? sourceRef : null, sourceStageRef: sourceStageRef || null, snapshotPresent: ok, degradedReason: ok ? null : normalizedSource === null ? 'source-branch-missing' : 'source-snapshot-missing', mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, sourceSnapshot: normalizedSnapshot, mirrorStageRc: Number(mirrorStageRc), branch, repoPath, valuesRedacted: true }));",
"NODE",
].join("\n");
const result = runNodeK3sScript(spec, script, 45);
+3 -2
View File
@@ -456,15 +456,16 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string,
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
["runtime", runtime.ready === true ? "ok" : "failed", runtimeDetail],
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(diagnostic.kind ?? diagnostic.message), 80)],
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `snapshot=${webObserveText(gitMirror.sourceSnapshotReady)} pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
],
),
"",
webObserveTable(
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS"],
["LOCAL_SOURCE", "GITHUB_SOURCE", "SNAPSHOT", "LOCAL_GITOPS", "GITHUB_GITOPS"],
[[
shortValue(gitMirror.localSource),
shortValue(gitMirror.githubSource),
shortValue(gitMirror.sourceSnapshot),
shortValue(gitMirror.localGitops),
shortValue(gitMirror.githubGitops),
]],
+16 -1
View File
@@ -231,6 +231,11 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
: "pipelinerun-not-succeeded";
const publicReady = publicProbes.ready === true;
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
const gitMirrorDegradedReason = gitMirrorCompact.sourceSnapshotReady === false
? "source-snapshot-missing"
: gitMirrorCompact.pendingFlush === true
? "git-mirror-pending-flush"
: "git-mirror-not-in-sync";
const fullStatus = {
ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady,
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
@@ -296,7 +301,7 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
? argoReady
? pipelineRunReady
? publicReady
? gitMirrorReady ? undefined : "git-mirror-pending-flush"
? gitMirrorReady ? undefined : gitMirrorDegradedReason
: "public-probe-not-ready"
: pipelineRunDegradedReason
: "argo-not-synced-healthy"
@@ -574,6 +579,10 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
ready: gitMirror.ready === true,
localSource: gitMirrorCompact.localSource ?? null,
githubSource: gitMirrorCompact.githubSource ?? null,
sourceAuthority: gitMirrorCompact.sourceAuthority ?? null,
sourceStageRef: gitMirrorCompact.sourceStageRef ?? null,
sourceSnapshot: gitMirrorCompact.sourceSnapshot ?? null,
sourceSnapshotReady: gitMirrorCompact.sourceSnapshotReady === true,
localGitops: gitMirrorCompact.localGitops ?? null,
githubGitops: gitMirrorCompact.githubGitops ?? null,
pendingFlush: gitMirrorCompact.pendingFlush === true,
@@ -618,9 +627,15 @@ export function nodeRuntimeStatusNextAction(status: Record<string, unknown>, sco
if (reason === "public-probe-not-ready") {
return `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
}
if (reason === "source-snapshot-missing") {
return `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`;
}
if (reason === "git-mirror-pending-flush") {
return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`;
}
if (reason === "git-mirror-not-in-sync") {
return `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane} --full`;
}
return `${nodeRuntimeStatusCommand(scoped)} --full`;
}
+1
View File
@@ -146,6 +146,7 @@ export function nodeRuntimeGitMirrorRefSources(scoped: ReturnType<typeof parseNo
return {
localSource: `refs/heads/${mirror.sourceBranch}`,
githubSource: `refs/mirror-stage/heads/${mirror.sourceBranch}`,
sourceSnapshotPrefix: `refs/unidesk/snapshots/hwlab-node-runtime/${mirror.sourceBranch}`,
localGitops: `refs/heads/${mirror.gitopsBranch}`,
githubGitops: `refs/mirror-stage/heads/${mirror.gitopsBranch}`,
githubFieldsAreMirrorStageCache: true,
+18 -7
View File
@@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, hwlabRuntimeSourceSnapshotStageRefPrefix, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -68,6 +68,7 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
`cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`,
`source_repository=${shellQuote(mirror.sourceRepository)}`,
`source_branch=${shellQuote(mirror.sourceBranch)}`,
`source_stage_ref_prefix=${shellQuote(hwlabRuntimeSourceSnapshotStageRefPrefix(spec))}`,
`gitops_branch=${shellQuote(mirror.gitopsBranch)}`,
`github_transport_mode=${shellQuote(mirror.githubTransport.mode)}`,
`github_ssh_secret=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.secretName : "")}`,
@@ -125,38 +126,42 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
" value = {}",
"sys.exit(0 if isinstance(value, dict) and value.get('localSource') else 1)",
"PY",
" summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc \"source_repository=\\$1 source_branch=\\$2 gitops_branch=\\$3 node <<'NODE'",
" summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc \"source_repository=\\$1 source_branch=\\$2 source_stage_ref_prefix=\\$3 gitops_branch=\\$4 node <<'NODE'",
"const { execFileSync } = require('node:child_process');",
"const { readFileSync, existsSync } = require('node:fs');",
"const repository = process.env.source_repository;",
"const sourceBranch = process.env.source_branch;",
"const sourceStageRefPrefix = process.env.source_stage_ref_prefix;",
"const gitopsBranch = process.env.gitops_branch;",
"const repoPath = '/cache/' + repository + '.git';",
"function readJson(path) { try { return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null; } catch { return null; } }",
"function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }",
"const localSource = rev('refs/heads/' + sourceBranch);",
"const githubSource = rev('refs/mirror-stage/heads/' + sourceBranch);",
"const sourceStageRef = githubSource ? sourceStageRefPrefix.replace(/\\/+$/, '') + '/' + githubSource : null;",
"const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;",
"const localGitops = rev('refs/heads/' + gitopsBranch);",
"const githubGitops = rev('refs/mirror-stage/heads/' + gitopsBranch);",
"const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));",
"console.log(JSON.stringify({",
" localSource, githubSource, localGitops, githubGitops,",
" localSource, githubSource, sourceAuthority: 'git-mirror-snapshot', sourceStageRef, sourceSnapshot, localGitops, githubGitops,",
" refSources: {",
" localSource: 'refs/heads/' + sourceBranch,",
" githubSource: 'refs/mirror-stage/heads/' + sourceBranch,",
" sourceSnapshot: sourceStageRef,",
" localGitops: 'refs/heads/' + gitopsBranch,",
" githubGitops: 'refs/mirror-stage/heads/' + gitopsBranch,",
" githubFieldsAreMirrorStageCache: true",
" },",
" pendingFlush,",
" flushNeeded: pendingFlush,",
" githubInSync: Boolean(localSource && githubSource && localSource === githubSource && localGitops && githubGitops && localGitops === githubGitops),",
" githubInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource && localGitops && githubGitops && localGitops === githubGitops),",
" statusSource: 'cache-ref-fallback',",
" lastSync: readJson('/cache/HWLAB.last-sync.json'),",
" lastFlush: readJson('/cache/HWLAB.last-flush.json')",
"}));",
"NODE",
"\" sh \"$source_repository\" \"$source_branch\" \"$gitops_branch\" 2>>/tmp/hwlab-node-gitmirror-status.err || true)",
"\" sh \"$source_repository\" \"$source_branch\" \"$source_stage_ref_prefix\" \"$gitops_branch\" 2>>/tmp/hwlab-node-gitmirror-status.err || true)",
"fi",
"if [ -z \"$summary_json\" ]; then summary_json='{}'; fi",
"read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")",
@@ -180,7 +185,8 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
" return os.environ.get(name) == 'true'",
"summary = load_env_json('SUMMARY_JSON')",
"github_transport = load_env_json('GITHUB_TRANSPORT_JSON')",
"ok = truth('read_deployment_ready') and truth('write_deployment_ready') and truth('read_service_exists') and truth('write_service_exists') and truth('read_endpoints_ready') and truth('write_endpoints_ready') and (truth('cache_pvc_exists') or truth('cache_host_path_exists')) and github_transport.get('ready') is not False and bool(summary.get('localSource'))",
"source_snapshot_ready = bool(summary.get('githubSource')) and summary.get('sourceSnapshot') == summary.get('githubSource')",
"ok = truth('read_deployment_ready') and truth('write_deployment_ready') and truth('read_service_exists') and truth('write_service_exists') and truth('read_endpoints_ready') and truth('write_endpoints_ready') and (truth('cache_pvc_exists') or truth('cache_host_path_exists')) and github_transport.get('ready') is not False and bool(summary.get('localSource')) and source_snapshot_ready",
"print(json.dumps({",
" 'ok': bool(ok),",
" 'resources': {",
@@ -727,10 +733,11 @@ export function withNodeRuntimeGitMirrorRendered(result: Record<string, unknown>
Object.keys(summary).length === 0
? "REFS\n-"
: webObserveTable(
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
["LOCAL_SOURCE", "GITHUB_SOURCE", "SNAPSHOT", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
[[
shortValue(summary.localSource),
shortValue(summary.githubSource),
shortValue(summary.sourceSnapshot),
shortValue(summary.localGitops),
shortValue(summary.githubGitops),
webObserveText(summary.pendingFlush),
@@ -783,6 +790,10 @@ export function compactNodeRuntimeGitMirrorStatus(status: Record<string, unknown
ok: status.ok === true,
localSource: summary.localSource ?? null,
githubSource: summary.githubSource ?? null,
sourceAuthority: summary.sourceAuthority ?? null,
sourceStageRef: summary.sourceStageRef ?? null,
sourceSnapshot: summary.sourceSnapshot ?? null,
sourceSnapshotReady: Boolean(summary.githubSource && summary.sourceSnapshot === summary.githubSource),
localGitops: summary.localGitops ?? null,
githubGitops: summary.githubGitops ?? null,
pendingFlush: summary.pendingFlush === true,