fix: speed up hwlab v02 trigger reuse

This commit is contained in:
Codex
2026-06-04 17:22:55 +00:00
parent 7367488e28
commit f287720d75
2 changed files with 200 additions and 18 deletions
+41 -3
View File
@@ -1,4 +1,4 @@
import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02ExistingPipelineRunReuseDecision, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14";
import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, parseV02TriggerSnapshot, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02ExistingPipelineRunReuseDecision, v02FalseGreenGuard, v02GitMirrorPreSyncWaitMs, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14";
function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
@@ -157,6 +157,13 @@ assertCondition(
"v0.2 git mirror pre-sync marker must only reuse fresh same-commit markers written after the stale status observation",
{ reusableGitMirrorPreSyncMarker, preObservationGitMirrorMarker, staleGitMirrorPreSyncMarker },
);
assertCondition(
v02GitMirrorPreSyncWaitMs(10) === 10_000
&& v02GitMirrorPreSyncWaitMs(180) === 120_000
&& v02GitMirrorPreSyncWaitMs(0) === 120_000,
"v0.2 git mirror pre-sync wait must not impose a hidden 30s minimum latency",
{ tenSeconds: v02GitMirrorPreSyncWaitMs(10), capped: v02GitMirrorPreSyncWaitMs(180), default: v02GitMirrorPreSyncWaitMs(0) },
);
const gitMirrorSummary = gitMirrorStatusSummary(gitMirrorStatusRaw);
assertCondition(
gitMirrorSummary.flushNeeded === true && gitMirrorSummary.flushCommand === "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
@@ -223,15 +230,45 @@ const existingPipelineRunHeadAdvanced = v02ExistingPipelineRunReuseDecision({
before: { exists: true, status: "True" },
latestSourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
});
const existingPipelineRunHeadUnresolved = v02ExistingPipelineRunReuseDecision({
sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
before: { exists: true, status: "True" },
latestSourceCommit: null,
});
const triggerSnapshot = parseV02TriggerSnapshot({
ok: true,
command: ["test"],
exitCode: 0,
stdout: [
"sourceCommit\taaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"pipelineRun\thwlab-v02-ci-poll-aaaaaaaaaaaa",
"pipelineRunExitCode\t0",
"status\tTrue",
"reason\tCompleted",
"message\tTasks Completed: 9",
"pipelineRunStderr\t",
].join("\n"),
stderr: "",
parsed: null,
}, Date.parse("2026-06-04T16:00:00.000Z"));
assertCondition(
existingPipelineRunReuse.reusable === true
&& existingPipelineRunReuse.alreadyUsable === true
&& existingPipelineRunFailed.reusable === true
&& existingPipelineRunFailed.alreadyUsable === false
&& existingPipelineRunHeadAdvanced.reusable === false
&& existingPipelineRunHeadAdvanced.reason === "source-head-advanced-before-existing-pipelinerun-reuse",
&& existingPipelineRunHeadAdvanced.reason === "source-head-advanced-before-existing-pipelinerun-reuse"
&& existingPipelineRunHeadUnresolved.reusable === false
&& existingPipelineRunHeadUnresolved.reason === "source-head-recheck-unresolved-before-existing-pipelinerun-reuse",
"trigger-current must recheck latest v0.2 head before reusing an existing PipelineRun",
{ existingPipelineRunReuse, existingPipelineRunFailed, existingPipelineRunHeadAdvanced },
{ existingPipelineRunReuse, existingPipelineRunFailed, existingPipelineRunHeadAdvanced, existingPipelineRunHeadUnresolved },
);
assertCondition(
triggerSnapshot.sourceCommit === "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
&& triggerSnapshot.pipelineRun === "hwlab-v02-ci-poll-aaaaaaaaaaaa"
&& record(triggerSnapshot.before).status === "True",
"trigger-current fast snapshot must parse source head and PipelineRun status in one G14:k3s route while latest head is checked locally",
triggerSnapshot,
);
assertCondition(
!v02PipelineServiceIds().includes("hwlab-cli"),
@@ -513,6 +550,7 @@ console.log(JSON.stringify({
"v0.2 control-plane render uses an isolated temp clone from a CI/CD dedicated bare repo",
"v0.2 control-plane refresh marker only reuses recent same-contract refreshes",
"trigger-current rechecks latest v0.2 head before reusing an existing PipelineRun",
"trigger-current fast snapshot parses source head and PipelineRun status in one G14:k3s route while latest head is checked locally",
"v0.2 status alignment reports stale-success without coupling CI to dirty workspace state",
"v0.2 PipelineRun service matrix excludes hwlab-cli",
"v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance",
+159 -15
View File
@@ -66,7 +66,6 @@ const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000;
const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000;
const V02_GIT_MIRROR_PRESYNC_MIN_WAIT_MS = 30 * 1000;
const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000;
const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000;
const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000;
@@ -668,10 +667,90 @@ function getV02Head(): string | null {
return resolveV02Head().sourceCommit;
}
function resolveV02LatestRemoteHead(): { sourceCommit: string | null; result: CommandJsonResult } {
const result = commandJson(["git", "ls-remote", V02_GIT_URL, "refs/heads/v0.2"], 30_000);
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
const output = statusText(result);
const match = /^[0-9a-f]{40}/imu.exec(output);
return { sourceCommit: match?.[0] ?? null, result };
}
function v02PipelineRunName(sourceCommit: string): string {
return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`;
}
interface V02TriggerSnapshot {
sourceCommit: string | null;
pipelineRun: string | null;
before: Record<string, unknown> | null;
result: CommandJsonResult;
observedAtMs: number;
}
export function parseV02TriggerSnapshot(result: CommandJsonResult, observedAtMs: number): V02TriggerSnapshot {
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout);
const fields = keyValueLinesFromText(output);
const sourceCommit = stringOrNull(fields.sourceCommit?.match(/^[0-9a-f]{40}$/iu)?.[0]);
const pipelineRun = stringOrNull(fields.pipelineRun);
const pipelineRunExitCode = numericField(fields.pipelineRunExitCode);
const before = pipelineRun !== null && pipelineRunExitCode !== null
? pipelineRunCompactFromText(
pipelineRun,
[fields.status ?? "", fields.reason ?? "", fields.message ?? ""].join("\n"),
pipelineRunExitCode === 0,
"hwlab-v02-trigger-snapshot:pipelinerun",
pipelineRunExitCode,
fields.pipelineRunStderr ?? "",
)
: null;
return {
sourceCommit,
pipelineRun,
before,
result,
observedAtMs,
};
}
function getV02TriggerSnapshot(): V02TriggerSnapshot {
const script = [
"set +e",
"one_line() { tr '\\r\\n\\t' ' ' | sed 's/[[:space:]][[:space:]]*/ /g' | cut -c1-2000; }",
"tmp_dir=$(mktemp -d /tmp/hwlab-v02-trigger-snapshot.XXXXXX)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
"source_commit=",
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
" source_commit=$(git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2 2>/dev/null || true)",
"elif [ -e \"$cicd_repo\" ]; then",
" printf 'snapshotError\\t%s\\n' \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\"",
"fi",
"pipeline_run=",
`if [ -n "$source_commit" ]; then pipeline_run=${shellQuote(V02_PIPELINERUN_PREFIX)}-$(printf '%s' "$source_commit" | cut -c1-12); fi`,
"if [ -n \"$pipeline_run\" ]; then",
` (kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}' >\"$tmp_dir/pipelinerun.out\" 2>\"$tmp_dir/pipelinerun.err\"; printf '%s' "$?" >\"$tmp_dir/pipelinerun.status\") &`,
" pipeline_pid=$!",
"else",
" printf '' >\"$tmp_dir/pipelinerun.out\"",
" printf '' >\"$tmp_dir/pipelinerun.err\"",
" printf '' >\"$tmp_dir/pipelinerun.status\"",
" pipeline_pid=",
"fi",
"if [ -n \"$pipeline_pid\" ]; then wait \"$pipeline_pid\" 2>/dev/null || true; pipeline_pid=; fi",
"status_text=$(cat \"$tmp_dir/pipelinerun.out\" 2>/dev/null)",
"pipeline_status=$(cat \"$tmp_dir/pipelinerun.status\" 2>/dev/null)",
"printf 'sourceCommit\\t%s\\n' \"$source_commit\"",
"printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"",
"printf 'pipelineRunExitCode\\t%s\\n' \"$pipeline_status\"",
"printf 'status\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '1p')\"",
"printf 'reason\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '2p')\"",
"printf 'message\\t%s\\n' \"$(printf '%s\\n' \"$status_text\" | sed -n '3p')\"",
"printf 'pipelineRunStderr\\t%s\\n' \"$(one_line < \"$tmp_dir/pipelinerun.err\")\"",
].join("\n");
const result = g14K3s(["script", "--", script], 180_000);
return parseV02TriggerSnapshot(result, Date.now());
}
export function v02ExistingPipelineRunReuseDecision(input: {
sourceCommit: string;
before: Record<string, unknown>;
@@ -681,6 +760,16 @@ export function v02ExistingPipelineRunReuseDecision(input: {
const exists = input.before.exists === true;
const latestSourceCommit = input.latestSourceCommit;
const alreadyUsable = exists && (status === "True" || status === "Unknown");
if (latestSourceCommit === null) {
return {
reusable: false,
alreadyUsable,
reason: "source-head-recheck-unresolved-before-existing-pipelinerun-reuse",
sourceCommit: input.sourceCommit,
latestSourceCommit,
previousPipelineRun: v02PipelineRunName(input.sourceCommit),
};
}
if (latestSourceCommit !== null && latestSourceCommit !== input.sourceCommit) {
return {
reusable: false,
@@ -2542,8 +2631,34 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
if (options.action === "status" && options.pipelineRun !== undefined) return v02ControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run" });
if (options.action === "status" && options.sourceCommit !== undefined) return v02ControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit" });
const head = resolveV02Head();
const snapshot = options.action === "trigger-current" ? getV02TriggerSnapshot() : null;
const latestHead = options.action === "trigger-current" ? resolveV02LatestRemoteHead() : null;
const head = snapshot === null
? resolveV02Head()
: snapshot.sourceCommit === null
? resolveV02Head()
: { sourceCommit: snapshot.sourceCommit, result: snapshot.result };
let sourceCommit = head.sourceCommit;
if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) {
const refreshedHead = resolveV02Head();
sourceCommit = refreshedHead.sourceCommit;
if (sourceCommit !== latestHead.sourceCommit) {
return {
ok: false,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
phase: "source-head-refresh",
sourceCommit,
latestSourceHeadProbe: {
sourceCommit: latestHead.sourceCommit,
result: compactCommandResult(latestHead.result),
},
refreshedHeadProbe: compactCommandResult(refreshedHead.result),
degradedReason: "v02-head-refresh-did-not-reach-latest-source",
};
}
}
if (sourceCommit === null) {
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) };
}
@@ -2580,7 +2695,9 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
};
}
let before = getPipelineRunCompact(v02PipelineRunName(sourceCommit));
let before = snapshot !== null && snapshot.sourceCommit === sourceCommit && snapshot.before !== null
? snapshot.before
: getPipelineRunCompact(v02PipelineRunName(sourceCommit));
if (options.dryRun) {
const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
return {
@@ -2602,8 +2719,8 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
};
}
if (before.exists === true && before.status !== null) {
const latestHead = resolveV02Head();
const reuseDecision = v02ExistingPipelineRunReuseDecision({ sourceCommit, before, latestSourceCommit: latestHead.sourceCommit });
const reuseHead = latestHead ?? resolveV02LatestRemoteHead();
const reuseDecision = v02ExistingPipelineRunReuseDecision({ sourceCommit, before, latestSourceCommit: reuseHead.sourceCommit });
if (reuseDecision.reusable === true) {
const alreadyUsable = reuseDecision.alreadyUsable === true;
return {
@@ -2621,16 +2738,16 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse",
};
}
if (latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) {
if (reuseHead.sourceCommit !== null && reuseHead.sourceCommit !== sourceCommit) {
printProgressEvent("hwlab.v02.trigger.progress", {
stage: "source-head-recheck",
status: "advanced",
sourceCommit,
latestSourceCommit: latestHead.sourceCommit,
latestSourceCommit: reuseHead.sourceCommit,
previousPipelineRun: v02PipelineRunName(sourceCommit),
latestPipelineRun: v02PipelineRunName(latestHead.sourceCommit),
latestPipelineRun: v02PipelineRunName(reuseHead.sourceCommit),
});
sourceCommit = latestHead.sourceCommit;
sourceCommit = reuseHead.sourceCommit;
before = getPipelineRunCompact(v02PipelineRunName(sourceCommit));
if (before.exists === true && before.status !== null) {
const advancedAlreadyUsable = before.status === "True" || before.status === "Unknown";
@@ -2665,9 +2782,34 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
before,
reuseDecision,
degradedReason: "existing-pipelinerun-reuse-rejected",
latestSourceHeadProbe: {
sourceCommit: reuseHead.sourceCommit,
exitCode: reuseHead.result.exitCode,
stderr: reuseHead.result.stderr.trim().slice(0, 2000),
},
};
}
}
if (latestHead !== null && latestHead.sourceCommit !== sourceCommit) {
return {
ok: false,
command: "hwlab g14 control-plane trigger-current --lane v02",
lane: "v02",
mode: "confirmed-trigger",
phase: "source-head-recheck",
sourceCommit,
pipelineRun: v02PipelineRunName(sourceCommit),
before,
latestSourceHeadProbe: {
sourceCommit: latestHead.sourceCommit,
exitCode: latestHead.result.exitCode,
stderr: latestHead.result.stderr.trim().slice(0, 2000),
},
degradedReason: latestHead.sourceCommit === null
? "source-head-recheck-unresolved-before-pipelinerun-create"
: "source-head-advanced-before-pipelinerun-create",
};
}
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
const controlPlaneRefresh = refreshV02ControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds);
printProgressEvent("hwlab.v02.trigger.progress", {
@@ -3127,10 +3269,10 @@ function v02GitMirrorPreSyncLockDir(sourceCommit: string): string {
return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.lock`);
}
function v02GitMirrorPreSyncWaitMs(timeoutSeconds: number): number {
export function v02GitMirrorPreSyncWaitMs(timeoutSeconds: number): number {
const requestedMs = Math.max(0, Math.floor(timeoutSeconds * 1000));
if (requestedMs === 0) return V02_GIT_MIRROR_PRESYNC_MIN_WAIT_MS;
return Math.min(V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, Math.max(V02_GIT_MIRROR_PRESYNC_MIN_WAIT_MS, requestedMs));
if (requestedMs === 0) return V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS;
return Math.min(V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, requestedMs);
}
export function v02ReusableGitMirrorPreSyncMarker(marker: unknown, sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
@@ -3167,12 +3309,14 @@ function writeV02GitMirrorPreSyncMarker(sourceCommit: string, summary: Record<st
return marker;
}
function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number): { acquired: boolean; lockDir: string; waitedMs: number } {
function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number, minSyncedAtMs = 0): { acquired: boolean; lockDir: string; waitedMs: number; marker?: V02GitMirrorPreSyncMarker } {
const stateDir = v02GitMirrorPreSyncStateDir();
mkdirSync(stateDir, { recursive: true });
const lockDir = v02GitMirrorPreSyncLockDir(sourceCommit);
const startedAtMs = Date.now();
for (;;) {
const marker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), minSyncedAtMs);
if (marker !== null) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs, marker };
try {
mkdirSync(lockDir);
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
@@ -3364,9 +3508,9 @@ function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlane
waitMs,
};
}
const lock = acquireV02GitMirrorPreSyncLock(sourceCommit, waitMs);
const lock = acquireV02GitMirrorPreSyncLock(sourceCommit, waitMs, statusStartMs);
if (!lock.acquired) {
const markerAfterLockTimeout = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
const markerAfterLockTimeout = lock.marker ?? readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
if (markerAfterLockTimeout !== null) {
return {
ok: true,