mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: expose node git mirror flush partial success

Browse Source
This commit is contained in:
Codex
2026-06-16 06:19:52 +00:00
parent 6f8fe9e098
commit b9ae7b4798
6 changed files with 126 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-control-plane.ts
+41 -3
View File
@@ -1085,6 +1085,13 @@ console.log(JSON.stringify({
githubSource: first.githubSource || null,
localGitops: first.localGitops || null,
githubGitops: first.githubGitops || null,
refSources: {
localSource: 'refs/heads/' + (first.sourceBranch || ''),
githubSource: 'refs/mirror-stage/heads/' + (first.sourceBranch || ''),
localGitops: 'refs/heads/' + (first.gitopsBranch || ''),
githubGitops: 'refs/mirror-stage/heads/' + (first.gitopsBranch || ''),
githubFieldsAreMirrorStageCache: true
},
pendingFlush,
flushNeeded: pendingFlush,
githubInSync: Object.values(items).every((item) => item.sourceInSync === true && item.gitopsInSync === true),
@@ -1203,18 +1210,49 @@ function gitMirrorFlushShell(node: ControlPlaneNodeSpec, target: ControlPlaneTar
"test -d \"$repo/objects\"",
"git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"",
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
"push_status=skipped",
"push_exit=0",
"fetch_status=skipped",
"fetch_exit=0",
"if [ -n \"$local_gitops\" ]; then",
" set +e",
" git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"",
" git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"",
" push_exit=$?",
" set -e",
" if [ \"$push_exit\" = \"0\" ]; then",
" push_status=succeeded",
" set +e",
" git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"",
" fetch_exit=$?",
" set -e",
" if [ \"$fetch_exit\" = \"0\" ]; then fetch_status=succeeded; else fetch_status=failed; fi",
" else",
" push_status=failed",
" fi",
"fi",
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
"export repository gitops_branch started_at local_gitops github_gitops pending",
"status=succeeded",
"partial_success=",
"degraded_reason=",
"exit_code=0",
"if [ \"$push_status\" = \"failed\" ]; then",
" status=failed",
" degraded_reason=git-mirror-push-failed",
" exit_code=$push_exit",
"elif [ \"$push_status\" = \"succeeded\" ] && [ \"$fetch_status\" = \"failed\" ]; then",
" status=partial-success",
" partial_success=push-succeeded-fetch-failed",
" degraded_reason=git-mirror-post-push-fetch-failed",
" exit_code=44",
"fi",
"export repository gitops_branch started_at local_gitops github_gitops pending push_status push_exit fetch_status fetch_exit status partial_success degraded_reason",
"node <<'NODE' | tee /cache/HWLAB.last-flush.json",
"const payload = { event: 'git-mirror-flush', repo: process.env.repository, status: 'succeeded', startedAt: process.env.started_at, flushedAt: new Date().toISOString(), gitopsBranch: process.env.gitops_branch, localGitops: process.env.local_gitops || null, githubGitops: process.env.github_gitops || null, pendingFlush: process.env.pending === 'true' };",
"const payload = { event: 'git-mirror-flush', repo: process.env.repository, status: process.env.status || 'failed', partialSuccess: process.env.partial_success || null, degradedReason: process.env.degraded_reason || null, startedAt: process.env.started_at, flushedAt: new Date().toISOString(), gitopsBranch: process.env.gitops_branch, localGitops: process.env.local_gitops || null, githubGitops: process.env.github_gitops || null, pendingFlush: process.env.pending === 'true', stages: { push: process.env.push_status || null, pushExitCode: Number.parseInt(process.env.push_exit || '0', 10), postPushFetch: process.env.fetch_status || null, postPushFetchExitCode: Number.parseInt(process.env.fetch_exit || '0', 10) } };",
"console.log(JSON.stringify(payload));",
"NODE",
"cat /cache/HWLAB.last-flush.json",
"if [ \"$exit_code\" != \"0\" ]; then exit \"$exit_code\"; fi",
"",
].join("\n");
}
scripts/src/hwlab-node.ts
+67 -2
View File
@@ -495,6 +495,63 @@ function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
};
}
function parseLastJsonLineObject(text: string): Record<string, unknown> {
for (const line of text.split(/\r?\n/u).reverse()) {
const trimmed = line.trim();
if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) continue;
try {
return record(JSON.parse(trimmed) as unknown);
} catch {
// Keep scanning: command tails can mix kubectl status, git output, and JSON payloads.
}
}
return {};
}
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
}
function nodeRuntimeGitMirrorRefSources(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
return {
localSource: `refs/heads/${mirror.sourceBranch}`,
githubSource: `refs/mirror-stage/heads/${mirror.sourceBranch}`,
localGitops: `refs/heads/${mirror.gitopsBranch}`,
githubGitops: `refs/mirror-stage/heads/${mirror.gitopsBranch}`,
githubFieldsAreMirrorStageCache: true,
cacheRefresh: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
};
}
function nodeRuntimeGitMirrorFlushPartialSuccess(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, mirror: NodeRuntimeGitMirrorTargetSpec, result: CommandResult): Record<string, unknown> | null {
if (scoped.action !== "flush") return null;
const text = `${result.stdout}\n${result.stderr}`;
const payload = parseLastJsonLineObject(text);
const partialSuccess = typeof payload.partialSuccess === "string" && payload.partialSuccess.length > 0
? payload.partialSuccess
: null;
const payloadStatus = typeof payload.status === "string" && payload.status.length > 0 ? payload.status : null;
const branch = escapeRegExp(mirror.gitopsBranch);
const pushSucceededInGitOutput = new RegExp(`\\b${branch}\\s*->\\s*${branch}\\b`, "u").test(text);
const postPushFetchFailed = /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|fatal:.*fetch|fetch-pack|early EOF/iu.test(text);
if (partialSuccess !== "push-succeeded-fetch-failed" && !(result.exitCode !== 0 && pushSucceededInGitOutput && postPushFetchFailed)) {
return null;
}
return {
status: payloadStatus ?? "partial-success",
partialSuccess: "push-succeeded-fetch-failed",
degradedReason: "node-runtime-git-mirror-flush-post-push-fetch-failed",
retryable: true,
message: "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Refresh mirror-stage cache before deciding another flush is needed.",
payload: Object.keys(payload).length > 0 ? payload : null,
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
next: {
sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
},
};
}
function sshTcpPoolDiagnosticsFromCommand(spec: HwlabRuntimeLaneSpec, result: CommandResult): Record<string, unknown> | null {
if (isCommandSuccess(result)) return null;
const failureKind = classifySshTcpPoolFailure(`${result.stderr}\n${result.stdout}`);
@@ -1347,6 +1404,7 @@ function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDel
gitopsBranch: mirror.gitopsBranch,
resources,
summary,
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
result: compactRuntimeCommand(result),
degradedReason: ok ? undefined : "node-runtime-git-mirror-not-ready",
valuesPrinted: false,
@@ -1392,6 +1450,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const status = scoped.dryRun || !isCommandSuccess(result) ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
return {
ok: isCommandSuccess(result) && (status === undefined || status.ok === true),
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
@@ -1404,8 +1463,14 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
manifest: scoped.dryRun ? manifest : undefined,
result: compactRuntimeCommand(result),
status,
degradedReason: isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
next: scoped.dryRun
partialSuccess: partialSuccess?.partialSuccess ?? undefined,
recovery: partialSuccess ?? undefined,
degradedReason: partialSuccess !== null
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
: isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
next: partialSuccess !== null
? partialSuccess.next
: scoped.dryRun
? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` }
: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
};
scripts/src/jobs.ts
+13 -1
View File
@@ -230,7 +230,11 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name);
const agentRunYamlLaneTriggerWorkflow = /^agentrun_v[0-9]{2}_trigger_current$/u.test(job.name);
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync"
|| job.name === "hwlab_g14_git_mirror_flush"
|| job.name === "agentrun_v01_git_mirror_sync"
|| job.name === "agentrun_v01_git_mirror_flush"
|| /^hwlab_nodes_v[0-9]{2}_git-mirror_(sync|flush)$/u.test(job.name);
const ciInstallWorkflow = job.name === "ci_install";
if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs);
if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
@@ -318,7 +322,10 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder
firstMatch(stdoutTail, new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u")) ??
firstMatch(stdoutTail.replaceAll('\\"', '"').replaceAll("\\n", "\n"), new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u"));
const pendingFlush = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"pendingFlush"\s*:\s*(true|false)/u);
const partialSuccess = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"partialSuccess"\s*:\s*"([^"]+)"/u);
const jobName = firstMatch(stdoutTail, /"jobName"\s*:\s*"([^"]+)"/u);
const node = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"node"\s*:\s*"([^"]+)"/u);
const lane = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"lane"\s*:\s*"(v[0-9]+)"/u) ?? firstMatch(job.name, /^hwlab_nodes_(v[0-9]+)_/u);
const warnings = jobProgressWarnings({
job,
eventsObserved: 0,
@@ -349,6 +356,7 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder
summary: [
job.status,
`git-mirror-${action}${status ? `:${status}` : ""}`,
partialSuccess ? `partialSuccess=${partialSuccess}` : null,
jobName ? `job=${jobName}` : null,
pendingFlush !== null ? `pendingFlush=${pendingFlush}` : null,
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
@@ -359,6 +367,10 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
: job.name.startsWith("agentrun_")
? "bun scripts/cli.ts agentrun git-mirror status"
: job.name.startsWith("hwlab_nodes_") && node !== null && lane !== null
? partialSuccess === "push-succeeded-fetch-failed"
? `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm --wait`
: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`
: "bun scripts/cli.ts hwlab g14 git-mirror status",
};
}