fix: clarify live split brain liveness

This commit is contained in:
Codex
2026-05-20 06:27:34 +00:00
parent d2eef85aad
commit d7c043ad5c
5 changed files with 416 additions and 49 deletions
File diff suppressed because one or more lines are too long
+22 -4
View File
@@ -2173,13 +2173,18 @@ input:focus, select:focus, textarea:focus { border-color: var(--accent-2); }
border-color: rgba(78, 183, 168, 0.50);
color: var(--accent-2);
}
.codex-trace-status-chip.liveness.warn {
.codex-trace-status-chip.liveness.warn,
.codex-trace-status-chip.liveness.degraded-live {
border-color: rgba(215, 161, 58, 0.55);
color: #ffe0a2;
background:
linear-gradient(135deg, rgba(215, 161, 58, 0.13), rgba(78, 183, 168, 0.06)),
rgba(0,0,0,0.18);
}
.codex-trace-status-chip.liveness.degraded-live {
border-color: rgba(215, 161, 58, 0.62);
box-shadow: inset 0 0 0 1px rgba(78, 183, 168, 0.10);
}
.codex-trace-status-chip.liveness.failed {
border-color: rgba(255, 98, 98, 0.58);
color: #ffb2b2;
@@ -2204,9 +2209,15 @@ input:focus, select:focus, textarea:focus { border-color: var(--accent-2); }
linear-gradient(135deg, rgba(78, 183, 168, 0.08), rgba(255,255,255,0.015)),
rgba(0,0,0,0.16);
}
.codex-liveness-metric.warn {
.codex-liveness-metric.warn,
.codex-liveness-metric.degraded-live {
border-color: rgba(215, 161, 58, 0.44);
}
.codex-liveness-metric.degraded-live {
background:
linear-gradient(135deg, rgba(215, 161, 58, 0.12), rgba(78, 183, 168, 0.05)),
rgba(0,0,0,0.16);
}
.codex-liveness-metric.failed {
border-color: rgba(255, 98, 98, 0.46);
}
@@ -2252,16 +2263,23 @@ input:focus, select:focus, textarea:focus { border-color: var(--accent-2); }
min-width: 0;
overflow-wrap: anywhere;
}
.codex-liveness-advisory.warn {
.codex-liveness-advisory.warn,
.codex-liveness-advisory.degraded-live {
border-color: rgba(215, 161, 58, 0.42);
background: rgba(215, 161, 58, 0.07);
}
.codex-liveness-advisory.warn b { color: var(--warn); }
.codex-liveness-advisory.warn b,
.codex-liveness-advisory.degraded-live b { color: var(--warn); }
.codex-liveness-advisory.failed {
border-color: rgba(255, 98, 98, 0.46);
background: rgba(207, 106, 84, 0.09);
}
.codex-liveness-advisory.failed b { color: #ffb2b2; }
.codex-liveness-advisory code {
max-width: 100%;
color: var(--text);
overflow-wrap: anywhere;
}
.codex-liveness-reasons {
display: flex;
flex-wrap: wrap;
+12 -6
View File
@@ -249,11 +249,12 @@ function diagnosticsHeartbeatRiskTaskIds(diagnostics: any): string[] {
}
function splitBrainLiveDiagnostics(diagnostics: any): boolean {
if (diagnosticsHeartbeatRiskTaskIds(diagnostics).length > 0) return false;
if (typeof diagnostics?.splitBrainLive === "boolean") return diagnostics.splitBrainLive;
const state = String(diagnostics?.state || diagnostics?.health || "").toLowerCase();
const effective = String(diagnostics?.effectiveLiveness || "").toLowerCase();
return state === "split-brain"
&& stringArray(diagnostics?.heartbeatFreshTaskIds).length > 0
&& diagnosticsHeartbeatRiskTaskIds(diagnostics).length === 0;
&& (effective === "live" || stringArray(diagnostics?.heartbeatFreshTaskIds).length > 0);
}
function diagnosticsEffectiveLiveness(diagnostics: any): string {
@@ -267,11 +268,15 @@ function diagnosticsEffectiveLiveness(diagnostics: any): string {
function diagnosticsTone(diagnostics: any): string {
const effective = diagnosticsEffectiveLiveness(diagnostics);
const heartbeatRiskTaskIds = diagnosticsHeartbeatRiskTaskIds(diagnostics);
const state = String(diagnostics?.state || diagnostics?.health || "unknown").toLowerCase();
if (heartbeatRiskTaskIds.length > 0 || effective === "at-risk" || state === "stale-active") return "failed";
if (splitBrainLiveDiagnostics(diagnostics)) return "degraded-live";
if (effective === "live" || effective === "degraded") return "warn";
if (effective === "at-risk") return "failed";
if (["dead", "failed", "stale", "unhealthy"].includes(effective)) return "failed";
const value = String(diagnostics?.state || diagnostics?.health || "unknown").toLowerCase();
if (value === "healthy") return "ok";
if (value === "stale-active") return "failed";
if (value === "split-brain") return "warn";
if (value === "degraded") return "warn";
return "unknown";
}
@@ -1878,7 +1883,7 @@ function CodeQueueLivenessPanel({ diagnostics, queue, onRaw }: AnyRecord) {
const effectiveLiveness = diagnosticsEffectiveLiveness(diagnostics);
const recommendedAction = String(diagnostics?.recommendedAction || (heartbeatRiskTaskIds.length > 0 ? "investigate-heartbeat-risk" : splitBrainLive ? "continue-supervision" : effectiveLiveness === "degraded" ? "observe-degraded" : "none"));
const livenessText = String(splitBrainLive
? "执行面 heartbeat 新鲜,任务仍应继续监督。"
? "控制面/执行面观测分裂,但 heartbeat/trace 新鲜,继续监督。"
: heartbeatRiskTaskIds.length > 0
? "存在 expired/missing/stale heartbeat 风险,请先确认执行面状态。"
: diagnostics?.livenessSummary || (effectiveLiveness === "degraded"
@@ -1906,7 +1911,7 @@ function CodeQueueLivenessPanel({ diagnostics, queue, onRaw }: AnyRecord) {
h(LivenessMetric, { label: "PostgreSQL active", value: String(diagnostics?.databaseActiveTaskCount ?? queue?.databaseActiveTaskCount ?? 0), hint: compactIdList(diagnostics?.databaseActiveTaskIds ?? queue?.databaseActiveTaskIds) }),
h(LivenessMetric, { label: "Scheduler active", value: String(diagnostics?.schedulerActiveRunSlotCount ?? queue?.activeRunSlotCount ?? 0), hint: compactIdList(diagnostics?.schedulerActiveTaskIds ?? queue?.activeTaskIds) }),
h(LivenessMetric, { label: "Fresh heartbeat", value: String(stringArray(diagnostics?.heartbeatFreshTaskIds).length), hint: compactIdList(diagnostics?.heartbeatFreshTaskIds) }),
h(LivenessMetric, { tone: heartbeatRiskTaskIds.length > 0 ? "failed" : splitBrainLive ? "warn" : "", label: "Heartbeat risk", value: String(heartbeatRiskTaskIds.length), hint: heartbeatRiskTaskIds.length > 0 ? compactIdList(heartbeatRiskTaskIds) : splitBrainLive ? "fresh heartbeat: keep supervising" : "--" }),
h(LivenessMetric, { tone: heartbeatRiskTaskIds.length > 0 ? "failed" : splitBrainLive ? "degraded-live" : "", label: "Heartbeat risk", value: String(heartbeatRiskTaskIds.length), hint: heartbeatRiskTaskIds.length > 0 ? compactIdList(heartbeatRiskTaskIds) : splitBrainLive ? "fresh heartbeat/trace: keep supervising" : "--" }),
h(LivenessMetric, { tone: stringArray(diagnostics?.traceGapNotStaleTaskIds).length > 0 ? "warn" : "", label: "Trace gap", value: String(stringArray(diagnostics?.traceGapTaskIds).length), hint: compactIdList(diagnostics?.traceGapNotStaleTaskIds) }),
h(LivenessMetric, { tone: stringArray(diagnostics?.staleRecoveryCandidateTaskIds).length > 0 ? "failed" : "", label: "Stale candidates", value: String(stringArray(diagnostics?.staleRecoveryCandidateTaskIds).length), hint: compactIdList(diagnostics?.staleRecoveryCandidateTaskIds) }),
h(LivenessMetric, { label: "Last scheduler heartbeat", value: fmtRelativeAge(diagnostics?.lastSchedulerHeartbeatAt), hint: String(diagnostics?.lastSchedulerHeartbeatAt || "--") }),
@@ -1917,6 +1922,7 @@ function CodeQueueLivenessPanel({ diagnostics, queue, onRaw }: AnyRecord) {
h("div", { className: `codex-liveness-advisory ${tone}` },
h("b", null, splitBrainLive ? "Observing split" : heartbeatRiskTaskIds.length > 0 ? "Heartbeat risk" : "Liveness note"),
h("span", null, livenessText),
h("code", null, recommendedAction),
),
reasons.length > 0 ? h("div", { className: "codex-liveness-reasons" }, reasons.map((reason: string) => h("span", { key: reason }, reason))) : null,
);
@@ -55,6 +55,9 @@ COPY package.json /app/package.json
COPY bun.lock /app/bun.lock
RUN bun install
COPY src/components/shared /app/src/components/shared
COPY src/components/frontend/package.json /app/src/components/frontend/package.json
WORKDIR /app/src/components/frontend
RUN test -d node_modules/react || bun install
WORKDIR /app/src/components/microservices/code-queue
COPY src/components/microservices/code-queue/tsconfig.json ./tsconfig.json
COPY src/components/microservices/code-queue/src ./src