fix(web-probe): report provenance performance and retry flush
This commit is contained in:
Codex
+117
-40
@@ -2852,43 +2852,92 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
||||
if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`);
|
||||
const spec = scoped.spec;
|
||||
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
||||
const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
|
||||
const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName);
|
||||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`namespace=${shellQuote(mirror.namespace)}`,
|
||||
`job=${shellQuote(jobName)}`,
|
||||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||||
"manifest_path=\"/tmp/$job.json\"",
|
||||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||||
scoped.dryRun
|
||||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||||
: [
|
||||
"kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null",
|
||||
"kubectl create -f \"$manifest_path\"",
|
||||
`deadline=$(( $(date +%s) + ${scoped.timeoutSeconds} ))`,
|
||||
"while :; do",
|
||||
" status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)",
|
||||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||||
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi",
|
||||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi",
|
||||
" sleep 2",
|
||||
"done",
|
||||
"kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
|
||||
].join("\n"),
|
||||
].join("\n");
|
||||
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
||||
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
|
||||
const retryableFailure = !isCommandSuccess(result)
|
||||
? nodeRuntimeGitMirrorRetryableFailure(scoped, mirror, result, partialSuccess)
|
||||
: null;
|
||||
const actionSucceeded = isCommandSuccess(result);
|
||||
const retryMaxAttempts = !scoped.dryRun && scoped.action === "flush" ? 5 : 1;
|
||||
const attempts: Record<string, unknown>[] = [];
|
||||
let finalAttempt: {
|
||||
attempt: number;
|
||||
retryLabel: string;
|
||||
jobName: string;
|
||||
manifest: Record<string, unknown>;
|
||||
result: CommandResult;
|
||||
partialSuccess: Record<string, unknown> | null;
|
||||
retryableFailure: Record<string, unknown> | null;
|
||||
actionSucceeded: boolean;
|
||||
} | null = null;
|
||||
|
||||
for (let attempt = 1; attempt <= retryMaxAttempts; attempt += 1) {
|
||||
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
||||
const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
|
||||
const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName);
|
||||
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`namespace=${shellQuote(mirror.namespace)}`,
|
||||
`job=${shellQuote(jobName)}`,
|
||||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||||
"manifest_path=\"/tmp/$job.json\"",
|
||||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||||
scoped.dryRun
|
||||
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
||||
: [
|
||||
"kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null",
|
||||
"kubectl create -f \"$manifest_path\"",
|
||||
`deadline=$(( $(date +%s) + ${scoped.timeoutSeconds} ))`,
|
||||
"while :; do",
|
||||
" status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)",
|
||||
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
||||
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
||||
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
||||
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi",
|
||||
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi",
|
||||
" sleep 2",
|
||||
"done",
|
||||
"kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
|
||||
].join("\n"),
|
||||
].join("\n");
|
||||
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
||||
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
|
||||
const actionSucceeded = isCommandSuccess(result);
|
||||
const retryableFailure = !actionSucceeded
|
||||
? nodeRuntimeGitMirrorRetryableFailure(scoped, mirror, result, partialSuccess, attempt, retryMaxAttempts)
|
||||
: null;
|
||||
const retryable = retryableFailure?.retryable === true;
|
||||
const exhausted = retryable && attempt >= retryMaxAttempts;
|
||||
attempts.push({
|
||||
attempt,
|
||||
retryLabel,
|
||||
jobName,
|
||||
ok: actionSucceeded,
|
||||
exitCode: result.exitCode,
|
||||
retryable,
|
||||
exhausted,
|
||||
partialSuccess: partialSuccess?.partialSuccess ?? null,
|
||||
degradedReason: partialSuccess !== null
|
||||
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
||||
: actionSucceeded ? null : `node-runtime-git-mirror-${scoped.action}-failed`,
|
||||
result: compactRuntimeCommand(result),
|
||||
valuesPrinted: false,
|
||||
});
|
||||
finalAttempt = { attempt, retryLabel, jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded };
|
||||
if (actionSucceeded || scoped.dryRun || !retryable || exhausted) break;
|
||||
const backoffMs = nodeRuntimeGitMirrorRetryDelayMs(attempt);
|
||||
printNodeRuntimeTriggerProgress(spec, {
|
||||
stage: `git-mirror-${scoped.action}-retry`,
|
||||
status: "waiting",
|
||||
retryLabel,
|
||||
nextRetryLabel: `${attempt + 1}/${retryMaxAttempts}`,
|
||||
backoffMs,
|
||||
jobName,
|
||||
});
|
||||
sleepSync(backoffMs);
|
||||
}
|
||||
|
||||
if (finalAttempt === null) throw new Error("git-mirror run produced no attempts");
|
||||
const { jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded } = finalAttempt;
|
||||
const status = scoped.dryRun || !actionSucceeded
|
||||
? undefined
|
||||
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
||||
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
|
||||
return {
|
||||
ok: actionSucceeded && (status === undefined || status.ok === true),
|
||||
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
@@ -2901,6 +2950,14 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
||||
manifest: scoped.dryRun ? manifest : undefined,
|
||||
result: compactRuntimeCommand(result),
|
||||
status,
|
||||
retry: retryMaxAttempts > 1 ? {
|
||||
policy: "exponential-backoff",
|
||||
maxAttempts: retryMaxAttempts,
|
||||
attempts,
|
||||
exhausted: retryExhausted,
|
||||
stopped: retryExhausted,
|
||||
valuesPrinted: false,
|
||||
} : undefined,
|
||||
retryableFailure: retryableFailure ?? undefined,
|
||||
partialSuccess: partialSuccess?.partialSuccess ?? undefined,
|
||||
recovery: partialSuccess ?? undefined,
|
||||
@@ -2920,27 +2977,45 @@ function nodeRuntimeGitMirrorRetryableFailure(
|
||||
mirror: NodeRuntimeGitMirrorTargetSpec,
|
||||
result: CommandResult,
|
||||
partialSuccess: Record<string, unknown> | null,
|
||||
attempt: number,
|
||||
retryMaxAttempts: number,
|
||||
): Record<string, unknown> | null {
|
||||
const text = `${result.stdout}\n${result.stderr}`;
|
||||
const retryable = partialSuccess !== null
|
||||
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
||||
if (!retryable) return null;
|
||||
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
||||
const exhausted = attempt >= retryMaxAttempts;
|
||||
return {
|
||||
retryable: true,
|
||||
retryMaxAttempts: 5,
|
||||
retryLabel: "1/5",
|
||||
retryAttempt: attempt,
|
||||
retryMaxAttempts,
|
||||
retryLabel,
|
||||
retryExhausted: exhausted,
|
||||
stopped: exhausted,
|
||||
backoffPolicy: "exponential",
|
||||
nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt),
|
||||
reason: partialSuccess !== null
|
||||
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
||||
: "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.",
|
||||
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
||||
next: {
|
||||
retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
||||
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
},
|
||||
next: exhausted
|
||||
? {
|
||||
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
stopped: "retry budget exhausted; standard git-mirror stopped instead of silently continuing",
|
||||
}
|
||||
: {
|
||||
retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
||||
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorRetryDelayMs(attempt: number): number {
|
||||
return Math.min(15_000, 1000 * (2 ** Math.max(0, attempt - 1)));
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null {
|
||||
const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60);
|
||||
const value = result.stdout.trim();
|
||||
@@ -3091,6 +3166,8 @@ function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record
|
||||
jobName: result.jobName ?? null,
|
||||
partialSuccess: result.partialSuccess ?? null,
|
||||
fallback: result.fallback ?? null,
|
||||
retry: result.retry ?? null,
|
||||
retryableFailure: result.retryableFailure ?? null,
|
||||
degradedReason: result.degradedReason ?? null,
|
||||
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
|
||||
next: result.next ?? null,
|
||||
|
||||
Reference in New Issue
Block a user