mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: clarify code queue split-brain liveness

Browse Source
This commit is contained in:
Codex
2026-05-20 01:41:16 +00:00
parent b01907739c
commit ada6da3da6
11 changed files with 264 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/microservices/code-queue-mgr/src-rs/main.rs
+41
View File
@@ -737,6 +737,14 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
.filter(|task_id| heartbeat_expired_set.contains(*task_id))
.cloned()
.collect();
let mut heartbeat_risk_task_ids: Vec<String> = heartbeat_expired_task_ids
.iter()
.chain(heartbeat_missing_task_ids.iter())
.chain(stale_recovery_candidate_task_ids.iter())
.cloned()
.collect();
heartbeat_risk_task_ids.sort();
heartbeat_risk_task_ids.dedup();
let mut trace_gap_task_ids: Vec<String> = active_heartbeats
.iter()
.filter(|(task_id, heartbeat)| {
@@ -754,6 +762,7 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
.collect();
let split_brain = !database_active_task_ids.is_empty() && !heartbeat_fresh_task_ids.is_empty();
let split_brain_live = split_brain && heartbeat_risk_task_ids.is_empty();
let stale_active = !stale_recovery_candidate_task_ids.is_empty();
let degraded = split_brain
|| stale_active
@@ -769,6 +778,33 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
} else {
"healthy"
};
let effective_liveness = if !heartbeat_risk_task_ids.is_empty() {
"at-risk"
} else if split_brain_live {
"live"
} else if degraded {
"degraded"
} else {
"healthy"
};
let recommended_action = if !heartbeat_risk_task_ids.is_empty() {
"investigate-heartbeat-risk"
} else if split_brain_live {
"continue-supervision"
} else if degraded {
"observe-degraded"
} else {
"none"
};
let liveness_summary = if !heartbeat_risk_task_ids.is_empty() {
"Heartbeat is expired, missing, or stale-recovery eligible for at least one database-active task; investigate before assuming the task is still live."
} else if split_brain_live {
"PostgreSQL and the local control-plane view are split, but scheduler-owned heartbeat is fresh; treat the task as live and continue supervision."
} else if degraded {
"Execution diagnostics are degraded; inspect the listed reasons together with heartbeat and trace progress."
} else {
"Execution diagnostics are healthy."
};
let mut reasons = Vec::new();
if split_brain {
reasons.push("postgres control-plane has database-active tasks while its local active slots are empty, but scheduler heartbeat is fresh");
@@ -795,6 +831,10 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
"health": state,
"degraded": degraded,
"splitBrain": split_brain,
"splitBrainLive": split_brain_live,
"effectiveLiveness": effective_liveness,
"recommendedAction": recommended_action,
"livenessSummary": liveness_summary,
"executionStateSource": "postgres-control-plane",
"controlPlane": "master-code-queue-mgr",
"databaseActiveTaskIds": database_active_task_ids,
@@ -812,6 +852,7 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
"heartbeatExpiredTaskIds": heartbeat_expired_task_ids,
"heartbeatMissingTaskIds": heartbeat_missing_task_ids,
"staleRecoveryCandidateTaskIds": stale_recovery_candidate_task_ids,
"heartbeatRiskTaskIds": heartbeat_risk_task_ids,
"traceGapTaskIds": trace_gap_task_ids,
"traceGapNotStaleTaskIds": trace_gap_not_stale_task_ids,
"schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,
src/components/microservices/code-queue-mgr/src/index.ts
+20
View File
@@ -859,7 +859,13 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
const staleRecoveryCandidateTaskIds = heartbeatExpiredTaskIds.slice();
const traceGapTaskIds = heartbeatRows.filter((row) => traceGap(row.task, row.heartbeat, nowMs)).map((row) => row.task.id).sort();
const traceGapNotStaleTaskIds = traceGapTaskIds.filter((taskId) => heartbeatFreshSet.has(taskId));
const heartbeatRiskTaskIds = Array.from(new Set([
...heartbeatExpiredTaskIds,
...heartbeatMissingTaskIds,
...staleRecoveryCandidateTaskIds,
])).sort();
const splitBrain = databaseActiveTaskIds.length > 0 && heartbeatFreshTaskIds.length > 0;
const splitBrainLive = splitBrain && heartbeatRiskTaskIds.length === 0;
const staleActive = staleRecoveryCandidateTaskIds.length > 0;
const degraded = splitBrain
|| staleActive
@@ -867,6 +873,15 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
|| heartbeatExpiredTaskIds.length > 0
|| traceGapTaskIds.length > 0;
const state = splitBrain ? "split-brain" : staleActive ? "stale-active" : degraded ? "degraded" : "healthy";
const effectiveLiveness = heartbeatRiskTaskIds.length > 0 ? "at-risk" : splitBrainLive ? "live" : degraded ? "degraded" : "healthy";
const recommendedAction = heartbeatRiskTaskIds.length > 0 ? "investigate-heartbeat-risk" : splitBrainLive ? "continue-supervision" : degraded ? "observe-degraded" : "none";
const livenessSummary = heartbeatRiskTaskIds.length > 0
? "Heartbeat is expired, missing, or stale-recovery eligible for at least one database-active task; investigate before assuming the task is still live."
: splitBrainLive
? "PostgreSQL and the local control-plane view are split, but scheduler-owned heartbeat is fresh; treat the task as live and continue supervision."
: degraded
? "Execution diagnostics are degraded; inspect the listed reasons together with heartbeat and trace progress."
: "Execution diagnostics are healthy.";
const reasons: string[] = [];
if (splitBrain) reasons.push("postgres control-plane has database-active tasks while its local active slots are empty, but scheduler heartbeat is fresh");
if (staleActive) reasons.push("owner heartbeat is expired and scheduler has no local active run for at least one database-active task");
@@ -877,6 +892,10 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
health: state,
degraded,
splitBrain,
splitBrainLive,
effectiveLiveness,
recommendedAction,
livenessSummary,
executionStateSource: "postgres-control-plane",
controlPlane: "master-code-queue-mgr",
databaseActiveTaskIds,
@@ -894,6 +913,7 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
heartbeatExpiredTaskIds,
heartbeatMissingTaskIds,
staleRecoveryCandidateTaskIds,
heartbeatRiskTaskIds,
traceGapTaskIds,
traceGapNotStaleTaskIds,
schedulerHeartbeatStaleMs,
src/components/microservices/code-queue/src/execution-diagnostics.ts
+32
View File
@@ -158,10 +158,16 @@ export function buildExecutionDiagnostics(input: ExecutionDiagnosticsInput): Cod
.sort();
const traceGapNotStaleTaskIds = traceGapTaskIds.filter((taskId) => heartbeatFreshSet.has(taskId));
const schedulerLocalMissingIds = databaseActiveTaskIds.filter((taskId) => !schedulerActiveTaskSet.has(taskId));
const heartbeatRiskTaskIds = Array.from(new Set([
...heartbeatExpiredTaskIds,
...heartbeatMissingTaskIds,
...staleRecoveryCandidateTaskIds,
])).sort();
const splitBrain = input.executionStateSource === "postgres-control-plane"
&& databaseActiveTaskIds.length > 0
&& schedulerActiveTaskIds.length === 0
&& heartbeatFreshTaskIds.length > 0;
const splitBrainLive = splitBrain && heartbeatRiskTaskIds.length === 0;
const staleActive = staleRecoveryCandidateTaskIds.length > 0;
const degraded = splitBrain
|| staleActive
@@ -171,6 +177,27 @@ export function buildExecutionDiagnostics(input: ExecutionDiagnosticsInput): Cod
|| traceGapTaskIds.length > 0
|| (typeof input.oaPublisher === "object" && input.oaPublisher !== null && !Array.isArray(input.oaPublisher) && (Number((input.oaPublisher as Record<string, JsonValue>).pending ?? 0) > 0 || (input.oaPublisher as Record<string, JsonValue>).lastError !== null));
const state = splitBrain ? "split-brain" : staleActive ? "stale-active" : degraded ? "degraded" : "healthy";
const effectiveLiveness = heartbeatRiskTaskIds.length > 0
? "at-risk"
: splitBrainLive
? "live"
: degraded
? "degraded"
: "healthy";
const recommendedAction = heartbeatRiskTaskIds.length > 0
? "investigate-heartbeat-risk"
: splitBrainLive
? "continue-supervision"
: degraded
? "observe-degraded"
: "none";
const livenessSummary = heartbeatRiskTaskIds.length > 0
? "Heartbeat is expired, missing, or stale-recovery eligible for at least one database-active task; investigate before assuming the task is still live."
: splitBrainLive
? "PostgreSQL and the local control-plane view are split, but scheduler-owned heartbeat is fresh; treat the task as live and continue supervision."
: degraded
? "Execution diagnostics are degraded; inspect the listed reasons together with heartbeat and trace progress."
: "Execution diagnostics are healthy.";
const lastSchedulerHeartbeatAt = maxTimestamp(activeHeartbeats.map((heartbeat) => heartbeat.lastLocalHeartbeatAt));
const lastObservedAgentEventAt = maxTimestamp(activeHeartbeats.map((heartbeat) => heartbeat.lastObservedAgentEventAt));
const lastPersistedTraceAt = maxTimestamp(activeHeartbeats.map((heartbeat) => heartbeat.lastPersistedTraceAt));
@@ -190,6 +217,10 @@ export function buildExecutionDiagnostics(input: ExecutionDiagnosticsInput): Cod
health: state,
degraded,
splitBrain,
splitBrainLive,
effectiveLiveness,
recommendedAction,
livenessSummary,
executionStateSource: input.executionStateSource,
controlPlane: input.controlPlane,
databaseActiveTaskIds,
@@ -207,6 +238,7 @@ export function buildExecutionDiagnostics(input: ExecutionDiagnosticsInput): Cod
heartbeatExpiredTaskIds,
heartbeatMissingTaskIds,
staleRecoveryCandidateTaskIds,
heartbeatRiskTaskIds,
traceGapTaskIds,
traceGapNotStaleTaskIds,
schedulerHeartbeatStaleMs: staleMs,
src/components/microservices/code-queue/src/types.ts
+7
View File
@@ -33,6 +33,8 @@ export type TranscriptKind = "ran" | "explored" | "edited" | "plan" | "message"
export type CodeQueueExecutionPlane = "scheduler-execution-plane" | "postgres-control-plane";
export type CodeQueueExecutionHealth = "healthy" | "degraded" | "split-brain" | "stale-active";
export type CodeQueueEffectiveLiveness = "healthy" | "live" | "degraded" | "at-risk";
export type CodeQueueRecommendedAction = "none" | "continue-supervision" | "observe-degraded" | "investigate-heartbeat-risk";
export type CodeQueueRecoveryCandidateReason =
| "scheduler-local-active"
@@ -64,6 +66,10 @@ export interface CodeQueueExecutionDiagnostics {
health: CodeQueueExecutionHealth;
degraded: boolean;
splitBrain: boolean;
splitBrainLive: boolean;
effectiveLiveness: CodeQueueEffectiveLiveness;
recommendedAction: CodeQueueRecommendedAction;
livenessSummary: string;
executionStateSource: CodeQueueExecutionPlane;
controlPlane: string;
databaseActiveTaskIds: string[];
@@ -81,6 +87,7 @@ export interface CodeQueueExecutionDiagnostics {
heartbeatExpiredTaskIds: string[];
heartbeatMissingTaskIds: string[];
staleRecoveryCandidateTaskIds: string[];
heartbeatRiskTaskIds: string[];
traceGapTaskIds: string[];
traceGapNotStaleTaskIds: string[];
schedulerHeartbeatStaleMs: number;