mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: clarify code queue split-brain liveness

Browse Source
This commit is contained in:
Codex
2026-05-20 01:41:16 +00:00
parent b01907739c
commit ada6da3da6
11 changed files with 264 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/code-queue.ts
+52 -1
View File
@@ -239,7 +239,14 @@ function upstreamError(response: unknown): string {
if (typeof bodyError === "string") {
const requestId = typeof body?.requestId === "string" ? ` requestId=${body.requestId}` : "";
const providerId = typeof body?.providerId === "string" ? ` providerId=${body.providerId}` : "";
return `${bodyError}${providerId}${requestId}`;
const observationHint = /\b(microservice|proxy|provider|tunnel|k3sctl|adapter|backend-core|offline|unavailable|timed out|timeout)\b/iu.test(bodyError)
? " The stable code-queue proxy failure is observation-path evidence only; from the supervisor or main-server CLI environment, also check `bun scripts/cli.ts codex queues`, `codex tasks`, or `codex task <taskId>` before declaring the work unevaluable or stopped."
: "";
return `${bodyError}${providerId}${requestId}${observationHint}`;
}
if (typeof record.codeQueueObservationNote === "string") {
const commands = Array.isArray(record.recommendedCommands) ? ` recommendedCommands=${JSON.stringify(record.recommendedCommands)}` : "";
return `${record.codeQueueObservationNote}${commands}`;
}
const status = typeof record.status === "number" ? `HTTP ${record.status}` : "upstream request failed";
return `${status}: ${JSON.stringify(response).slice(0, 1200)}`;
@@ -335,13 +342,56 @@ function compactSchedulerHeartbeat(value: unknown): Record<string, unknown> | nu
};
}
function splitBrainLiveFromDiagnostics(record: Record<string, unknown>): boolean {
if (typeof record.splitBrainLive === "boolean") return record.splitBrainLive;
const state = String(record.state ?? record.health ?? "").toLowerCase();
const riskTaskIds = Array.from(new Set([
...stringList(record.heartbeatRiskTaskIds),
...stringList(record.heartbeatExpiredTaskIds),
...stringList(record.heartbeatMissingTaskIds),
...stringList(record.staleRecoveryCandidateTaskIds),
]));
return state === "split-brain" && stringList(record.heartbeatFreshTaskIds).length > 0 && riskTaskIds.length === 0;
}
function effectiveLivenessFromDiagnostics(record: Record<string, unknown>): string {
if (typeof record.effectiveLiveness === "string" && record.effectiveLiveness.length > 0) return record.effectiveLiveness;
if (stringList(record.heartbeatRiskTaskIds).length > 0
|| stringList(record.heartbeatExpiredTaskIds).length > 0
|| stringList(record.heartbeatMissingTaskIds).length > 0
|| stringList(record.staleRecoveryCandidateTaskIds).length > 0) {
return "at-risk";
}
if (splitBrainLiveFromDiagnostics(record)) return "live";
return String(record.state ?? record.health ?? "unknown") === "healthy" ? "healthy" : "degraded";
}
function recommendedActionFromDiagnostics(record: Record<string, unknown>): string {
if (typeof record.recommendedAction === "string" && record.recommendedAction.length > 0) return record.recommendedAction;
const effectiveLiveness = effectiveLivenessFromDiagnostics(record);
if (effectiveLiveness === "at-risk") return "investigate-heartbeat-risk";
if (effectiveLiveness === "live") return "continue-supervision";
if (effectiveLiveness === "degraded") return "observe-degraded";
return "none";
}
function compactExecutionDiagnostics(value: unknown): Record<string, unknown> | null {
const record = asRecord(value);
if (record === null) return null;
const heartbeatRiskTaskIds = Array.from(new Set([
...stringList(record.heartbeatRiskTaskIds),
...stringList(record.heartbeatExpiredTaskIds),
...stringList(record.heartbeatMissingTaskIds),
...stringList(record.staleRecoveryCandidateTaskIds),
])).sort();
return {
state: record.state ?? record.health ?? null,
degraded: record.degraded ?? null,
splitBrain: record.splitBrain ?? null,
splitBrainLive: splitBrainLiveFromDiagnostics(record),
effectiveLiveness: effectiveLivenessFromDiagnostics({ ...record, heartbeatRiskTaskIds }),
recommendedAction: recommendedActionFromDiagnostics({ ...record, heartbeatRiskTaskIds }),
livenessSummary: record.livenessSummary ?? null,
executionStateSource: record.executionStateSource ?? null,
controlPlane: record.controlPlane ?? null,
databaseActiveTaskCount: record.databaseActiveTaskCount ?? null,
@@ -353,6 +403,7 @@ function compactExecutionDiagnostics(value: unknown): Record<string, unknown> |
heartbeatExpiredTaskIds: record.heartbeatExpiredTaskIds ?? [],
heartbeatMissingTaskIds: record.heartbeatMissingTaskIds ?? [],
staleRecoveryCandidateTaskIds: record.staleRecoveryCandidateTaskIds ?? [],
heartbeatRiskTaskIds,
traceGapTaskIds: record.traceGapTaskIds ?? [],
traceGapNotStaleTaskIds: record.traceGapNotStaleTaskIds ?? [],
lastSchedulerHeartbeatAt: record.lastSchedulerHeartbeatAt ?? null,