mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use k8s git mirror source snapshots

Browse Source
This commit is contained in:
Codex
2026-07-01 11:03:18 +00:00
parent b972ebd290
commit d687bdf668
20 changed files with 465 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node/cleanup.ts
+13 -10
View File
@@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, hwlabRuntimeSourceSnapshotStageRefPrefix, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -110,25 +110,28 @@ export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { source
`read_deploy=${shellQuote(mirror.serviceReadName)}`,
`repo_path=${shellQuote(`/cache/${mirror.sourceRepository}.git`)}`,
`source_branch=${shellQuote(mirror.sourceBranch)}`,
`source_stage_ref_prefix=${shellQuote(hwlabRuntimeSourceSnapshotStageRefPrefix(spec))}`,
"read_ref() {",
" ref=\"$1\"",
" kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'repo_path=$1; ref=$2; git --git-dir=\"$repo_path\" rev-parse --verify \"$ref^{commit}\" 2>/dev/null' sh \"$repo_path\" \"$ref\" 2>/dev/null",
"}",
"mirror_stage=$(read_ref \"refs/mirror-stage/heads/$source_branch\")",
"mirror_stage_rc=$?",
"local_head=$(read_ref \"refs/heads/$source_branch\")",
"local_head_rc=$?",
"source_commit=\"$mirror_stage\"",
"source_ref=\"refs/mirror-stage/heads/$source_branch\"",
"if ! printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then",
" source_commit=\"$local_head\"",
" source_ref=\"refs/heads/$source_branch\"",
"source_stage_ref=''",
"snapshot_commit=''",
"if printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then",
" source_stage_ref=\"${source_stage_ref_prefix%/}/$source_commit\"",
" snapshot_commit=$(read_ref \"$source_stage_ref\")",
"fi",
"node - \"$mirror_stage_rc\" \"$local_head_rc\" \"$mirror_stage\" \"$local_head\" \"$source_commit\" \"$source_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'",
"const [mirrorStageRc, localHeadRc, mirrorStage, localHead, sourceCommit, sourceRef, repoPath, branch] = process.argv.slice(2);",
"node - \"$mirror_stage_rc\" \"$mirror_stage\" \"$snapshot_commit\" \"$source_commit\" \"$source_ref\" \"$source_stage_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'",
"const [mirrorStageRc, mirrorStage, snapshotCommit, sourceCommit, sourceRef, sourceStageRef, repoPath, branch] = process.argv.slice(2);",
"const isSha = (value) => /^[0-9a-f]{40}$/i.test(value || '');",
"const ok = isSha(sourceCommit);",
"console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-cache', sourceAuthority: 'git-mirror-cache', sourceCommit: ok ? sourceCommit.toLowerCase() : null, sourceRef: ok ? sourceRef : null, mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, localHead: isSha(localHead) ? localHead.toLowerCase() : null, mirrorStageRc: Number(mirrorStageRc), localHeadRc: Number(localHeadRc), branch, repoPath, valuesRedacted: true }));",
"const normalizedSource = isSha(sourceCommit) ? sourceCommit.toLowerCase() : null;",
"const normalizedSnapshot = isSha(snapshotCommit) ? snapshotCommit.toLowerCase() : null;",
"const ok = normalizedSource !== null && normalizedSnapshot === normalizedSource;",
"console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-snapshot', sourceAuthority: 'git-mirror-snapshot', sourceCommit: ok ? normalizedSnapshot : normalizedSource, sourceRef: normalizedSource ? sourceRef : null, sourceStageRef: sourceStageRef || null, snapshotPresent: ok, degradedReason: ok ? null : normalizedSource === null ? 'source-branch-missing' : 'source-snapshot-missing', mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, sourceSnapshot: normalizedSnapshot, mirrorStageRc: Number(mirrorStageRc), branch, repoPath, valuesRedacted: true }));",
"NODE",
].join("\n");
const result = runNodeK3sScript(spec, script, 45);
scripts/src/hwlab-node/git-mirror.ts
+3 -2
View File
@@ -456,15 +456,16 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string,
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
["runtime", runtime.ready === true ? "ok" : "failed", runtimeDetail],
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(diagnostic.kind ?? diagnostic.message), 80)],
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `snapshot=${webObserveText(gitMirror.sourceSnapshotReady)} pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
],
),
"",
webObserveTable(
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS"],
["LOCAL_SOURCE", "GITHUB_SOURCE", "SNAPSHOT", "LOCAL_GITOPS", "GITHUB_GITOPS"],
[[
shortValue(gitMirror.localSource),
shortValue(gitMirror.githubSource),
shortValue(gitMirror.sourceSnapshot),
shortValue(gitMirror.localGitops),
shortValue(gitMirror.githubGitops),
]],
scripts/src/hwlab-node/render.ts
+16 -1
View File
@@ -231,6 +231,11 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
: "pipelinerun-not-succeeded";
const publicReady = publicProbes.ready === true;
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
const gitMirrorDegradedReason = gitMirrorCompact.sourceSnapshotReady === false
? "source-snapshot-missing"
: gitMirrorCompact.pendingFlush === true
? "git-mirror-pending-flush"
: "git-mirror-not-in-sync";
const fullStatus = {
ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady,
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
@@ -296,7 +301,7 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
? argoReady
? pipelineRunReady
? publicReady
? gitMirrorReady ? undefined : "git-mirror-pending-flush"
? gitMirrorReady ? undefined : gitMirrorDegradedReason
: "public-probe-not-ready"
: pipelineRunDegradedReason
: "argo-not-synced-healthy"
@@ -574,6 +579,10 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
ready: gitMirror.ready === true,
localSource: gitMirrorCompact.localSource ?? null,
githubSource: gitMirrorCompact.githubSource ?? null,
sourceAuthority: gitMirrorCompact.sourceAuthority ?? null,
sourceStageRef: gitMirrorCompact.sourceStageRef ?? null,
sourceSnapshot: gitMirrorCompact.sourceSnapshot ?? null,
sourceSnapshotReady: gitMirrorCompact.sourceSnapshotReady === true,
localGitops: gitMirrorCompact.localGitops ?? null,
githubGitops: gitMirrorCompact.githubGitops ?? null,
pendingFlush: gitMirrorCompact.pendingFlush === true,
@@ -618,9 +627,15 @@ export function nodeRuntimeStatusNextAction(status: Record<string, unknown>, sco
if (reason === "public-probe-not-ready") {
return `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
}
if (reason === "source-snapshot-missing") {
return `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`;
}
if (reason === "git-mirror-pending-flush") {
return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`;
}
if (reason === "git-mirror-not-in-sync") {
return `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane} --full`;
}
return `${nodeRuntimeStatusCommand(scoped)} --full`;
}
scripts/src/hwlab-node/runtime-common.ts
+1
View File
@@ -146,6 +146,7 @@ export function nodeRuntimeGitMirrorRefSources(scoped: ReturnType<typeof parseNo
return {
localSource: `refs/heads/${mirror.sourceBranch}`,
githubSource: `refs/mirror-stage/heads/${mirror.sourceBranch}`,
sourceSnapshotPrefix: `refs/unidesk/snapshots/hwlab-node-runtime/${mirror.sourceBranch}`,
localGitops: `refs/heads/${mirror.gitopsBranch}`,
githubGitops: `refs/mirror-stage/heads/${mirror.gitopsBranch}`,
githubFieldsAreMirrorStageCache: true,
scripts/src/hwlab-node/status.ts
+18 -7
View File
@@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, hwlabRuntimeSourceSnapshotStageRefPrefix, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -68,6 +68,7 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
`cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`,
`source_repository=${shellQuote(mirror.sourceRepository)}`,
`source_branch=${shellQuote(mirror.sourceBranch)}`,
`source_stage_ref_prefix=${shellQuote(hwlabRuntimeSourceSnapshotStageRefPrefix(spec))}`,
`gitops_branch=${shellQuote(mirror.gitopsBranch)}`,
`github_transport_mode=${shellQuote(mirror.githubTransport.mode)}`,
`github_ssh_secret=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.secretName : "")}`,
@@ -125,38 +126,42 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
" value = {}",
"sys.exit(0 if isinstance(value, dict) and value.get('localSource') else 1)",
"PY",
" summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc \"source_repository=\\$1 source_branch=\\$2 gitops_branch=\\$3 node <<'NODE'",
" summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc \"source_repository=\\$1 source_branch=\\$2 source_stage_ref_prefix=\\$3 gitops_branch=\\$4 node <<'NODE'",
"const { execFileSync } = require('node:child_process');",
"const { readFileSync, existsSync } = require('node:fs');",
"const repository = process.env.source_repository;",
"const sourceBranch = process.env.source_branch;",
"const sourceStageRefPrefix = process.env.source_stage_ref_prefix;",
"const gitopsBranch = process.env.gitops_branch;",
"const repoPath = '/cache/' + repository + '.git';",
"function readJson(path) { try { return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null; } catch { return null; } }",
"function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }",
"const localSource = rev('refs/heads/' + sourceBranch);",
"const githubSource = rev('refs/mirror-stage/heads/' + sourceBranch);",
"const sourceStageRef = githubSource ? sourceStageRefPrefix.replace(/\\/+$/, '') + '/' + githubSource : null;",
"const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;",
"const localGitops = rev('refs/heads/' + gitopsBranch);",
"const githubGitops = rev('refs/mirror-stage/heads/' + gitopsBranch);",
"const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));",
"console.log(JSON.stringify({",
" localSource, githubSource, localGitops, githubGitops,",
" localSource, githubSource, sourceAuthority: 'git-mirror-snapshot', sourceStageRef, sourceSnapshot, localGitops, githubGitops,",
" refSources: {",
" localSource: 'refs/heads/' + sourceBranch,",
" githubSource: 'refs/mirror-stage/heads/' + sourceBranch,",
" sourceSnapshot: sourceStageRef,",
" localGitops: 'refs/heads/' + gitopsBranch,",
" githubGitops: 'refs/mirror-stage/heads/' + gitopsBranch,",
" githubFieldsAreMirrorStageCache: true",
" },",
" pendingFlush,",
" flushNeeded: pendingFlush,",
" githubInSync: Boolean(localSource && githubSource && localSource === githubSource && localGitops && githubGitops && localGitops === githubGitops),",
" githubInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource && localGitops && githubGitops && localGitops === githubGitops),",
" statusSource: 'cache-ref-fallback',",
" lastSync: readJson('/cache/HWLAB.last-sync.json'),",
" lastFlush: readJson('/cache/HWLAB.last-flush.json')",
"}));",
"NODE",
"\" sh \"$source_repository\" \"$source_branch\" \"$gitops_branch\" 2>>/tmp/hwlab-node-gitmirror-status.err || true)",
"\" sh \"$source_repository\" \"$source_branch\" \"$source_stage_ref_prefix\" \"$gitops_branch\" 2>>/tmp/hwlab-node-gitmirror-status.err || true)",
"fi",
"if [ -z \"$summary_json\" ]; then summary_json='{}'; fi",
"read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")",
@@ -180,7 +185,8 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
" return os.environ.get(name) == 'true'",
"summary = load_env_json('SUMMARY_JSON')",
"github_transport = load_env_json('GITHUB_TRANSPORT_JSON')",
"ok = truth('read_deployment_ready') and truth('write_deployment_ready') and truth('read_service_exists') and truth('write_service_exists') and truth('read_endpoints_ready') and truth('write_endpoints_ready') and (truth('cache_pvc_exists') or truth('cache_host_path_exists')) and github_transport.get('ready') is not False and bool(summary.get('localSource'))",
"source_snapshot_ready = bool(summary.get('githubSource')) and summary.get('sourceSnapshot') == summary.get('githubSource')",
"ok = truth('read_deployment_ready') and truth('write_deployment_ready') and truth('read_service_exists') and truth('write_service_exists') and truth('read_endpoints_ready') and truth('write_endpoints_ready') and (truth('cache_pvc_exists') or truth('cache_host_path_exists')) and github_transport.get('ready') is not False and bool(summary.get('localSource')) and source_snapshot_ready",
"print(json.dumps({",
" 'ok': bool(ok),",
" 'resources': {",
@@ -727,10 +733,11 @@ export function withNodeRuntimeGitMirrorRendered(result: Record<string, unknown>
Object.keys(summary).length === 0
? "REFS\n-"
: webObserveTable(
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
["LOCAL_SOURCE", "GITHUB_SOURCE", "SNAPSHOT", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
[[
shortValue(summary.localSource),
shortValue(summary.githubSource),
shortValue(summary.sourceSnapshot),
shortValue(summary.localGitops),
shortValue(summary.githubGitops),
webObserveText(summary.pendingFlush),
@@ -783,6 +790,10 @@ export function compactNodeRuntimeGitMirrorStatus(status: Record<string, unknown
ok: status.ok === true,
localSource: summary.localSource ?? null,
githubSource: summary.githubSource ?? null,
sourceAuthority: summary.sourceAuthority ?? null,
sourceStageRef: summary.sourceStageRef ?? null,
sourceSnapshot: summary.sourceSnapshot ?? null,
sourceSnapshotReady: Boolean(summary.githubSource && summary.sourceSnapshot === summary.githubSource),
localGitops: summary.localGitops ?? null,
githubGitops: summary.githubGitops ?? null,
pendingFlush: summary.pendingFlush === true,