mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: isolate v02 cicd source repo

Browse Source
This commit is contained in:
Codex
2026-05-31 17:00:24 +00:00
parent e929f4e570
commit 114d2e8208
2 changed files with 324 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/hwlab-g14-contract-test.ts
+38 -8
View File
@@ -1,4 +1,4 @@
import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02PipelineServiceIds } from "./src/hwlab-g14";
import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02PipelineServiceIds } from "./src/hwlab-g14";
function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
@@ -44,7 +44,7 @@ const gitMirrorStatusRaw = [
'lastSync={"ok":true}',
"lastWrite=",
"lastFlush=",
'refs={"refs":{"localV02":"0cf79ecc9d8a784b7712b0b3a58d5e39025ba0dc","localG14":"1111111111111111111111111111111111111111","localGitops":"2222222222222222222222222222222222222222","githubGitops":"3333333333333333333333333333333333333333"},"pendingFlush":true}',
'refs={"refs":{"localV02":"0cf79ecc9d8a784b7712b0b3a58d5e39025ba0dc","githubV02":"0cf79ecc9d8a784b7712b0b3a58d5e39025ba0dc","localG14":"1111111111111111111111111111111111111111","githubG14":"1111111111111111111111111111111111111111","localGitops":"2222222222222222222222222222222222222222","githubGitops":"3333333333333333333333333333333333333333"},"pendingFlush":true}',
].join("\n");
const parsedGitMirrorRefs = parseGitMirrorStatusRefs(gitMirrorStatusRaw);
assertCondition(
@@ -53,6 +53,11 @@ assertCondition(
parsedGitMirrorRefs,
);
assertCondition(parsedGitMirrorRefs.pendingFlush === true, "git mirror status parser must preserve pending flush signal", parsedGitMirrorRefs);
assertCondition(
parsedGitMirrorRefs.refs.githubV02 === "0cf79ecc9d8a784b7712b0b3a58d5e39025ba0dc",
"git mirror status parser must expose GitHub source branch staging ref",
parsedGitMirrorRefs,
);
assertCondition(
gitMirrorV02SyncRequirement("0cf79ecc9d8a784b7712b0b3a58d5e39025ba0dc", gitMirrorStatusRaw).required === false,
"trigger-current must not sync mirror when local v0.2 already matches source commit",
@@ -68,15 +73,16 @@ assertCondition(
gitMirrorSummary,
);
assertCondition(gitMirrorSummary.githubInSync === false, "git mirror status summary must expose GitHub GitOps drift", gitMirrorSummary);
assertCondition(gitMirrorSummary.sourceInSync === true && gitMirrorSummary.gitopsInSync === false, "git mirror status must split source and gitops GitHub sync state", gitMirrorSummary);
const renderScript = v02ControlPlaneRenderScript("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
assertCondition(
renderScript.includes("git worktree add --detach") && renderScript.includes("/tmp/hwlab-v02-control-plane-source-aaaaaaaaaaaa"),
"v0.2 control-plane render must use a detached temp worktree",
renderScript.includes("git --git-dir=\"$cicd_repo\" worktree add --detach") && renderScript.includes("/tmp/hwlab-v02-control-plane-source-aaaaaaaaaaaa"),
"v0.2 control-plane render must use a detached temp worktree from the CI/CD repo",
renderScript,
);
assertCondition(
!renderScript.includes("git merge --ff-only origin/v0.2") && !renderScript.includes("git checkout v0.2"),
"v0.2 control-plane render must not require fixed workspace checkout/merge and clean status",
renderScript.includes("cicd_repo='/root/hwlab-v02-cicd.git'") && !renderScript.includes("git -C /root/hwlab-v02") && !renderScript.includes("git checkout v0.2"),
"v0.2 control-plane render must not use the fixed workspace checkout or its clean status",
renderScript,
);
assertCondition(
@@ -85,6 +91,29 @@ assertCondition(
v02PipelineServiceIds(),
);
const staleSuccessAlignment = v02CommitAlignment({
expectedSourceHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
sourceHeads: {
cicdRepo: "/root/hwlab-v02-cicd.git",
cicdSourceHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
originHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
workspace: { path: "/root/hwlab-v02", head: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", originHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", dirty: true, dirtyCount: 1 },
},
gitMirrorSummary: { localV02: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", githubV02: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", sourceInSync: false, gitopsInSync: true },
pipelineRun: { pipelineRun: "hwlab-v02-ci-poll-aaaaaaaaaaaa", status: null },
recentPipelineRuns: { items: [{ name: "hwlab-v02-ci-poll-bbbbbbbbbbbb", sourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", status: "True", reason: "Completed" }] },
runtimeWorkloads: { items: [] },
webAssets: { apiRevision: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" },
});
assertCondition(
staleSuccessAlignment.aligned === false
&& staleSuccessAlignment.state === "stale-success"
&& JSON.stringify(staleSuccessAlignment.staleReasons).includes("mirror-source-stale")
&& JSON.stringify(staleSuccessAlignment.workspaceWarnings).includes("workspace-dirty-but-isolated-from-cicd"),
"v0.2 commit alignment must call out stale-success while keeping dirty workspace isolated from CI source selection",
staleSuccessAlignment,
);
const falseGreenPassed = v02FalseGreenGuard({
sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
pipelineRun: { exists: true, status: "True" },
@@ -261,8 +290,9 @@ console.log(JSON.stringify({
"git mirror sync is a manual devops-infra Job, not a CronJob",
"git mirror flush is a manual devops-infra Job, not a CronJob",
"trigger-current can decide whether v0.2 git mirror pre-sync is required",
"git mirror status exposes pending flush and controlled flush command",
"v0.2 control-plane render uses a detached temp worktree",
"git mirror status exposes source and gitops GitHub sync state plus controlled flush command",
"v0.2 control-plane render uses a detached temp worktree from a CI/CD dedicated bare repo",
"v0.2 status alignment reports stale-success without coupling CI to dirty workspace state",
"v0.2 PipelineRun service matrix excludes hwlab-cli",
"v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance",
"rollout brief includes natural-language changelog before automatic diff summary",
scripts/src/hwlab-g14.ts
+286 -46
View File
@@ -11,6 +11,7 @@ const G14_PROVIDER = "G14";
const G14_WORKSPACE = "/root/hwlab";
const V02_SOURCE_BRANCH = "v0.2";
const V02_WORKSPACE = "/root/hwlab-v02";
const V02_CICD_REPO = "/root/hwlab-v02-cicd.git";
const DEV_NAMESPACE = "hwlab-dev";
const CI_NAMESPACE = "hwlab-ci";
const ARGO_NAMESPACE = "argocd";
@@ -488,16 +489,39 @@ function precheckWorkspace(): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:${G14_WORKSPACE}`, "script", "--", "pwd; git fetch origin G14 --prune; git status --short --branch; git remote -v | sed -n '1,4p'"], 120_000);
}
function v02WorkspaceScript(script: string, timeoutMs = 120_000): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:${V02_WORKSPACE}`, "script", "--", script], timeoutMs);
function g14HostScript(script: string, timeoutMs = 120_000): CommandJsonResult {
return cliJson(["ssh", G14_PROVIDER, "script", "--", script], timeoutMs);
}
function g14K3s(args: string[], timeoutMs = 60_000): CommandJsonResult {
return cliJson(["ssh", `${G14_PROVIDER}:k3s`, ...args], timeoutMs);
}
function v02CicdRepoEnsureScript(): string {
return [
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
`cicd_url=${shellQuote(V02_GIT_URL)}`,
"mkdir -p \"$(dirname \"$cicd_repo\")\"",
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
" :",
"elif [ -e \"$cicd_repo\" ]; then",
" echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
" exit 41",
"else",
" git clone --bare \"$cicd_url\" \"$cicd_repo\"",
"fi",
"git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"",
"git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
"git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune",
].join("\n");
}
function getV02Head(): string | null {
const result = v02WorkspaceScript("git fetch origin v0.2 --prune >/dev/null 2>&1; git rev-parse origin/v0.2", 120_000);
const result = g14HostScript([
"set -eu",
v02CicdRepoEnsureScript(),
"git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2",
].join("\n"), 180_000);
if (!isCommandSuccess(result)) return null;
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
const match = /[0-9a-f]{40}/iu.exec(output);
@@ -693,7 +717,7 @@ function shellSectionOk(section: ShellSection | undefined): boolean {
function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): CommandJsonResult {
const sourceCommitLine = sourceCommit === undefined
? `source_commit=$(git -C ${shellQuote(V02_WORKSPACE)} rev-parse origin/v0.2 2>/dev/null || true)`
? `source_commit=$(git --git-dir=${shellQuote(V02_CICD_REPO)} rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)`
: `source_commit=${shellQuote(sourceCommit ?? "")}`;
const script = [
"set +e",
@@ -709,6 +733,7 @@ function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): C
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
"}",
"section sourceCommit printf '%s\\n' \"$source_commit\"",
`section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())}`,
"section queryNow date -u +%Y-%m-%dT%H:%M:%SZ",
`section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`,
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
@@ -718,11 +743,31 @@ function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): C
`if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`,
`section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`,
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
`section gitMirrorCache kubectl exec -n ${shellQuote(GIT_MIRROR_NAMESPACE)} deploy/git-mirror-http -- sh -lc ${shellQuote(gitMirrorCacheProbeScript())}`,
`section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
].join("\n");
return g14K3s(["script", "--", script], 60_000);
}
function v02SourceHeadsProbeScript(): string {
return [
"set +e",
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
`workspace=${shellQuote(V02_WORKSPACE)}`,
"rev_cicd() { git --git-dir=\"$cicd_repo\" rev-parse \"$1\" 2>/dev/null || true; }",
"rev_workspace() { git -C \"$workspace\" rev-parse \"$1\" 2>/dev/null || true; }",
"printf 'cicdRepo\\t%s\\n' \"$cicd_repo\"",
"printf 'cicdRepoExists\\t%s\\n' \"$([ -d \"$cicd_repo/objects\" ] && printf yes || printf no)\"",
"printf 'cicdSourceHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"",
"printf 'originHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"",
"printf 'workspacePath\\t%s\\n' \"$workspace\"",
"printf 'workspaceBranch\\t%s\\n' \"$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)\"",
"printf 'workspaceHead\\t%s\\n' \"$(rev_workspace HEAD)\"",
"printf 'workspaceOriginHead\\t%s\\n' \"$(rev_workspace refs/remotes/origin/v0.2)\"",
"printf 'workspaceDirtyCount\\t%s\\n' \"$(git -C \"$workspace\" status --porcelain=v1 2>/dev/null | wc -l | tr -d ' ')\"",
].join("\n");
}
function v02PlanArtifactsLogScript(): string {
return [
"pipeline_run=\"$1\"",
@@ -981,6 +1026,38 @@ function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode:
};
}
function keyValueLinesFromText(text: string): Record<string, string> {
const fields: Record<string, string> = {};
for (const line of text.split(/\r?\n/u)) {
const [key = "", ...rest] = line.split("\t");
if (key.trim().length > 0) fields[key.trim()] = rest.join("\t").trim();
}
return fields;
}
function v02SourceHeadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
const fields = keyValueLinesFromText(text);
const dirtyCount = numericField(fields.workspaceDirtyCount);
return {
ok: commandOk,
cicdRepo: fields.cicdRepo || V02_CICD_REPO,
cicdRepoExists: fields.cicdRepoExists === "yes",
cicdSourceHead: fields.cicdSourceHead || null,
originHead: fields.originHead || null,
workspace: {
path: fields.workspacePath || V02_WORKSPACE,
branch: fields.workspaceBranch || null,
head: fields.workspaceHead || null,
originHead: fields.workspaceOriginHead || null,
dirtyCount,
dirty: typeof dirtyCount === "number" ? dirtyCount > 0 : null,
isolatedFromCicd: true,
},
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
};
}
export function v02FalseGreenGuard(input: {
sourceCommit: string | null;
pipelineRun: Record<string, unknown> | null;
@@ -1057,6 +1134,118 @@ export function v02FalseGreenGuard(input: {
};
}
export function v02CommitAlignment(input: {
expectedSourceHead: string | null;
sourceHeads: Record<string, unknown>;
gitMirrorSummary: Record<string, unknown>;
pipelineRun: Record<string, unknown> | null;
recentPipelineRuns: Record<string, unknown>;
planArtifacts?: Record<string, unknown>;
runtimeWorkloads: Record<string, unknown>;
webAssets: Record<string, unknown>;
}): Record<string, unknown> {
const expectedSourceHead = input.expectedSourceHead;
const cicdSourceHead = stringOrNull(input.sourceHeads.cicdSourceHead);
const originHead = stringOrNull(input.sourceHeads.originHead) ?? expectedSourceHead;
const workspace = record(input.sourceHeads.workspace);
const workspaceHead = stringOrNull(workspace.head);
const workspaceOriginHead = stringOrNull(workspace.originHead);
const mirrorSourceHead = stringOrNull(input.gitMirrorSummary.localV02);
const mirrorGithubSourceHead = stringOrNull(input.gitMirrorSummary.githubV02);
const recentItems = Array.isArray(input.recentPipelineRuns.items)
? input.recentPipelineRuns.items.map((item) => record(item))
: [];
const latestPipelineRun = recentItems[0] ?? null;
const latestPipelineSourceCommit = latestPipelineRun === null ? null : stringOrNull(latestPipelineRun.sourceCommit);
const currentPipelineStatus = stringOrNull(input.pipelineRun?.status);
const apiRevision = stringOrNull(input.webAssets.apiRevision);
const planArtifacts = record(input.planArtifacts);
const planSourceCommit = stringOrNull(planArtifacts.sourceCommitId);
const rolloutServices = stringArray(planArtifacts.rolloutServices);
const apiRevisionRequired = expectedSourceHead !== null && planSourceCommit === expectedSourceHead && rolloutServices.includes("hwlab-cloud-api");
const workloadItems = Array.isArray(input.runtimeWorkloads.items)
? input.runtimeWorkloads.items.map((item) => record(item))
: [];
const serviceSourceCommits = Object.fromEntries(workloadItems
.filter((item) => typeof item.serviceId === "string" && item.serviceId.length > 0)
.map((item) => [String(item.serviceId), stringOrNull(item.sourceCommit) ?? stringOrNull(item.artifactSourceCommit)]));
const staleReasons: string[] = [];
if (expectedSourceHead === null) staleReasons.push("expected-source-head-unresolved");
if (expectedSourceHead !== null && originHead !== null && originHead !== expectedSourceHead) staleReasons.push("origin-head-mismatch");
if (expectedSourceHead !== null && cicdSourceHead !== null && cicdSourceHead !== expectedSourceHead) staleReasons.push("cicd-source-repo-stale");
if (expectedSourceHead !== null && mirrorSourceHead !== expectedSourceHead) staleReasons.push("mirror-source-stale");
if (expectedSourceHead !== null && latestPipelineSourceCommit !== expectedSourceHead) staleReasons.push("latest-pipelinerun-not-current");
if (apiRevisionRequired && apiRevision !== expectedSourceHead) staleReasons.push("runtime-api-revision-stale");
const workspaceWarnings: string[] = [];
if (expectedSourceHead !== null && workspaceHead !== null && workspaceHead !== expectedSourceHead) workspaceWarnings.push("workspace-head-differs-from-latest-source-but-isolated");
if (expectedSourceHead !== null && workspaceOriginHead !== null && workspaceOriginHead !== expectedSourceHead) workspaceWarnings.push("workspace-origin-ref-stale-but-isolated");
if (workspace.dirty === true) workspaceWarnings.push("workspace-dirty-but-isolated-from-cicd");
const runtimeWarnings: string[] = [];
if (!apiRevisionRequired && expectedSourceHead !== null && apiRevision !== null && apiRevision !== expectedSourceHead) {
runtimeWarnings.push("api-revision-differs-without-current-cloud-api-rollout");
}
const latestPipelineSucceeded = latestPipelineRun !== null && latestPipelineRun.status === "True";
const aligned = staleReasons.length === 0;
const state = aligned
? "aligned"
: latestPipelineSucceeded
? "stale-success"
: currentPipelineStatus === "Unknown"
? "in-progress"
: "stale";
return {
aligned,
state,
expectedSourceHead,
originHead,
cicdSourceHead,
cicdRepo: input.sourceHeads.cicdRepo ?? V02_CICD_REPO,
mirrorSourceHead,
mirrorGithubSourceHead,
latestPipelineRun: latestPipelineRun === null
? null
: {
name: latestPipelineRun.name ?? null,
sourceCommit: latestPipelineSourceCommit,
status: latestPipelineRun.status ?? null,
reason: latestPipelineRun.reason ?? null,
createdAt: latestPipelineRun.createdAt ?? null,
durationSeconds: latestPipelineRun.durationSeconds ?? null,
},
latestPipelineSourceCommit,
currentPipelineRun: input.pipelineRun === null
? null
: {
name: input.pipelineRun.pipelineRun ?? null,
status: input.pipelineRun.status ?? null,
reason: input.pipelineRun.reason ?? null,
},
runtimeSourceCommit: apiRevision,
apiRevision,
apiRevisionRequired,
planSourceCommit,
rolloutServices,
serviceSourceCommits,
sourceInSync: input.gitMirrorSummary.sourceInSync ?? null,
gitopsInSync: input.gitMirrorSummary.gitopsInSync ?? null,
staleReasons,
workspace: {
path: workspace.path ?? V02_WORKSPACE,
branch: workspace.branch ?? null,
head: workspaceHead,
originHead: workspaceOriginHead,
dirty: workspace.dirty ?? null,
dirtyCount: workspace.dirtyCount ?? null,
isolatedFromCicd: true,
},
workspaceWarnings,
runtimeWarnings,
summary: aligned
? `v0.2 CI/CD source, mirror, PipelineRun, and runtime align with ${shortSha(expectedSourceHead ?? "")}`
: `v0.2 CI/CD alignment state=${state}; staleReasons=${staleReasons.join(",")}`,
};
}
function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null {
if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null;
const activeItems = activePipelineRuns.map((item) => record(item));
@@ -1380,23 +1569,24 @@ export function v02ControlPlaneRenderScript(sourceCommit: string): string {
const worktreeDir = v02RenderWorktreeDir(sourceCommit);
return [
"set -eu",
v02CicdRepoEnsureScript(),
`render_dir=${shellQuote(renderDir)}`,
`worktree_dir=${shellQuote(worktreeDir)}`,
"cleanup_render_worktree() { git worktree remove --force \"$worktree_dir\" >/dev/null 2>&1 || rm -rf \"$worktree_dir\"; }",
"cleanup_render_worktree() { git --git-dir=\"$cicd_repo\" worktree remove --force \"$worktree_dir\" >/dev/null 2>&1 || rm -rf \"$worktree_dir\"; }",
"trap cleanup_render_worktree EXIT",
"git fetch origin v0.2 --prune",
`test "$(git rev-parse origin/v0.2)" = ${shellQuote(sourceCommit)}`,
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/v0.2)" = ${shellQuote(sourceCommit)}`,
"cleanup_render_worktree",
"git --git-dir=\"$cicd_repo\" worktree prune >/dev/null 2>&1 || true",
"rm -rf \"$render_dir\"",
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
`git worktree add --detach "$worktree_dir" ${shellQuote(sourceCommit)}`,
`git --git-dir="$cicd_repo" worktree add --detach "$worktree_dir" ${shellQuote(sourceCommit)}`,
"cd \"$worktree_dir\"",
`node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
].join("\n");
}
function runV02RenderToTemp(sourceCommit: string): CommandJsonResult {
return v02WorkspaceScript(v02ControlPlaneRenderScript(sourceCommit), 180_000);
return g14HostScript(v02ControlPlaneRenderScript(sourceCommit), 180_000);
}
function v02RenderDir(sourceCommit: string): string {
@@ -1563,6 +1753,7 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null;
const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit);
const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now();
const sourceHeadsSection = sections.sourceHeads;
const controlPlane = sections.controlPlane;
const obsoleteCronJobs = sections.obsoleteCronJobs;
const argo = sections.argo;
@@ -1570,6 +1761,7 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
const taskRunsSection = sections.taskRuns;
const planArtifactsSection = sections.planArtifacts;
const runtimeWorkloadsSection = sections.runtimeWorkloads;
const gitMirrorCacheSection = sections.gitMirrorCache;
const webAssetsSection = sections.webAssets;
const recentPipelineRuns = listV02PipelineRunsCompactFromText(
sections.recentPipelineRuns?.stdout ?? "",
@@ -1582,6 +1774,12 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
);
const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
const sourceHeads = v02SourceHeadsFromText(
sourceHeadsSection?.stdout ?? "",
shellSectionOk(sourceHeadsSection),
sourceHeadsSection?.exitCode ?? null,
bundle.stderr,
);
const pipelineRunInfo = pipelineRun === null
? null
: pipelineRunCompactFromText(
@@ -1621,14 +1819,36 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
bundle.stderr,
activePipelineRuns,
);
const gitMirror = {
ok: shellSectionOk(gitMirrorCacheSection),
summary: gitMirrorStatusSummary(String(gitMirrorCacheSection?.stdout ?? "").trim()),
raw: String(gitMirrorCacheSection?.stdout ?? "").trim(),
exitCode: gitMirrorCacheSection?.exitCode ?? null,
stderr: shellSectionOk(gitMirrorCacheSection) ? "" : bundle.stderr.trim().slice(0, 2000),
};
const commitAlignment = v02CommitAlignment({
expectedSourceHead: sourceCommit,
sourceHeads,
gitMirrorSummary: record(gitMirror.summary),
pipelineRun: pipelineRunInfo,
recentPipelineRuns,
planArtifacts,
runtimeWorkloads,
webAssets,
});
const baseOk = sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo);
return {
ok: sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo),
ok: baseOk && commitAlignment.aligned !== false,
command: "hwlab g14 control-plane status --lane v02",
lane: "v02",
state: commitAlignment.state,
degradedReason: commitAlignment.aligned === false ? commitAlignment.state : undefined,
sourceCommit,
sourceCommitSource: "cached origin/v0.2; write operations fetch before acting",
sourceCommitSource: "G14 CI/CD dedicated bare repo refs/remotes/origin/v0.2; workspace checkout is observable but isolated",
expected: {
sourceRepo: V02_CICD_REPO,
workspace: V02_WORKSPACE,
workspaceIsolation: "workspace dirty/stale state must not select CI/CD source commits",
branch: V02_SOURCE_BRANCH,
namespace: CI_NAMESPACE,
runtimeNamespace: "hwlab-v02",
@@ -1636,6 +1856,9 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
trigger: "manual UniDesk CLI PipelineRun creation",
argoApplication: V02_APP,
},
commitAlignment,
sourceHeads,
gitMirror,
controlPlane: {
ok: shellSectionOk(controlPlane),
names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
@@ -1675,12 +1898,12 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
const sourceCommit = options.action === "status" ? undefined : getV02Head();
const sourceCommit = getV02Head();
if (sourceCommit === null) {
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE };
}
if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit);
if (options.action === "status") return v02ControlPlaneStatus();
if (options.action === "status") return v02ControlPlaneStatus(sourceCommit);
if (options.action === "apply") {
const render = runV02RenderToTemp(sourceCommit);
if (!isCommandSuccess(render)) {
@@ -1933,7 +2156,9 @@ export function parseGitMirrorStatusRefs(raw: string): { refs: Record<string, st
return {
refs: {
localV02: stringOrNull(refs.localV02),
githubV02: stringOrNull(refs.githubV02),
localG14: stringOrNull(refs.localG14),
githubG14: stringOrNull(refs.githubG14),
localGitops: stringOrNull(refs.localGitops),
githubGitops: stringOrNull(refs.githubGitops),
},
@@ -1961,11 +2186,17 @@ export function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
const lastSync = parseLabeledJsonLine(raw, "lastSync");
const lastWrite = parseLabeledJsonLine(raw, "lastWrite");
const lastFlush = parseLabeledJsonLine(raw, "lastFlush");
const localV02 = refs.refs.localV02 ?? null;
const githubV02 = refs.refs.githubV02 ?? null;
const localGitops = refs.refs.localGitops ?? null;
const githubGitops = refs.refs.githubGitops ?? null;
const sourceInSync = Boolean(localV02 && githubV02 && localV02 === githubV02);
const gitopsInSync = Boolean(localGitops && githubGitops && localGitops === githubGitops);
return {
localV02: refs.refs.localV02 ?? null,
localV02,
githubV02,
localG14: refs.refs.localG14 ?? null,
githubG14: refs.refs.githubG14 ?? null,
localGitops,
githubGitops,
pendingFlush: refs.pendingFlush,
@@ -1986,7 +2217,9 @@ export function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
},
flushNeeded: refs.pendingFlush === true,
flushCommand: refs.pendingFlush === true ? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm" : null,
githubInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops),
sourceInSync,
gitopsInSync,
githubInSync: sourceInSync && gitopsInSync,
};
}
@@ -2018,7 +2251,9 @@ function compactGitMirrorStatus(status: Record<string, unknown>, sourceCommit: s
required: requirement.required,
sourceCommit,
localV02: requirement.localV02 ?? null,
githubV02: requirement.refs.githubV02 ?? null,
pendingFlush: requirement.pendingFlush ?? null,
sourceInSync: Boolean(requirement.refs.localV02 && requirement.refs.githubV02 && requirement.refs.localV02 === requirement.refs.githubV02),
reason: requirement.reason,
cacheOk: nested(status, ["cache", "ok"]) === true,
cacheStderr: tailText(nested(status, ["cache", "stderr"]), 1000),
@@ -2042,6 +2277,39 @@ function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unk
};
}
function gitMirrorCacheProbeScript(): string {
return [
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
"printf 'refs='",
"node - <<'NODE' 2>/dev/null || true",
"const { execFileSync } = require('node:child_process');",
"const repo = '/cache/pikasTech/HWLAB.git';",
"function rev(ref) {",
" try {",
" return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim();",
" } catch {",
" return null;",
" }",
"}",
"const localGitops = rev('refs/heads/v0.2-gitops');",
"const githubGitops = rev('refs/mirror-stage/heads/v0.2-gitops');",
"console.log(JSON.stringify({",
" refs: {",
" localV02: rev('refs/heads/v0.2'),",
" githubV02: rev('refs/mirror-stage/heads/v0.2'),",
" localG14: rev('refs/heads/G14'),",
" githubG14: rev('refs/mirror-stage/heads/G14'),",
" localGitops,",
" githubGitops,",
" },",
" pendingFlush: Boolean(localGitops && githubGitops && localGitops !== githubGitops),",
"}));",
"NODE",
].join("\n");
}
function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
const statusStartMs = Date.now();
printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
@@ -2113,34 +2381,6 @@ function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlane
function runGitMirrorStatus(): Record<string, unknown> {
const startedAtMs = Date.now();
const cacheScript = [
"printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
"printf 'lastWrite='; cat /cache/HWLAB.last-write.json 2>/dev/null || true; printf '\\n'",
"printf 'lastFlush='; cat /cache/HWLAB.last-flush.json 2>/dev/null || true; printf '\\n'",
"printf 'refs='",
"node - <<'NODE' 2>/dev/null || true",
"const { execFileSync } = require('node:child_process');",
"const repo = '/cache/pikasTech/HWLAB.git';",
"function rev(ref) {",
" try {",
" return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', ref], { encoding: 'utf8' }).trim();",
" } catch {",
" return null;",
" }",
"}",
"const localGitops = rev('refs/heads/v0.2-gitops');",
"const githubGitops = rev('refs/mirror-stage/heads/v0.2-gitops');",
"console.log(JSON.stringify({",
" refs: {",
" localV02: rev('refs/heads/v0.2'),",
" localG14: rev('refs/heads/G14'),",
" localGitops,",
" githubGitops,",
" },",
" pendingFlush: Boolean(localGitops && githubGitops && localGitops !== githubGitops),",
"}));",
"NODE",
].join("\n");
const script = [
"set +e",
"section() {",
@@ -2176,7 +2416,7 @@ function runGitMirrorStatus(): Record<string, unknown> {
`-n ${shellQuote(GIT_MIRROR_NAMESPACE)}`,
"deploy/git-mirror-http",
"-- sh -lc",
shellQuote(cacheScript),
shellQuote(gitMirrorCacheProbeScript()),
].join(" "),
].join("\n");
const bundle = g14K3s(["script", "--", script], 60_000);
@@ -2226,7 +2466,7 @@ function runGitMirrorStatus(): Record<string, unknown> {
function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
const sourceCommit = getV02Head();
if (sourceCommit === null) {
return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE };
}
const render = runV02RenderToTemp(sourceCommit);
if (!isCommandSuccess(render)) {