mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: clarify code queue active diagnostics

Browse Source
This commit is contained in:
Codex
2026-05-22 16:57:11 +00:00
parent 77b577a2cc
commit 4f432fd58f
5 changed files with 166 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/code-queue-pr-preflight-contract-test.ts
+25 -1
View File
@@ -339,6 +339,9 @@ async function main(): Promise<void> {
assertCondition(fallback.runnerDisposition === "ready", "remote fallback should stay ready", fallback);
assertCondition(fallback.controlPlane && asRecord(fallback.controlPlane).remoteFallbackUsed === true, "remote fallback should be marked", fallback.controlPlane);
assertCondition(fallback.failureKind === null, "remote fallback should not invent a failure kind when remote control plane is healthy", fallback);
const fallbackLocalGap = asRecord(fallback.localObservationGap);
assertCondition(fallbackLocalGap.kind === "runner-local-observation-gap", "healthy remote fallback should classify local backend-core absence as runner-local observation gap", fallbackLocalGap);
assertCondition(fallbackLocalGap.schedulerStoppage === false, "local observation gap must not imply scheduler stoppage", fallbackLocalGap);
const fallbackPreflight = asRecord(fallback.preflight);
assertCondition(fallbackPreflight.ok === true, "remote fallback preflight should stay ready", fallbackPreflight);
assertCondition(asRecord(fallbackPreflight.tokenCoverage).source === "GH_TOKEN", "token source should be GH_TOKEN", fallbackPreflight.tokenCoverage);
@@ -352,6 +355,11 @@ async function main(): Promise<void> {
assertCondition(remoteControlPlaneMissingRecord.ok === false, "missing control plane should fail", remoteControlPlaneMissingRecord);
assertCondition(remoteControlPlaneMissingRecord.failureKind === "control-plane-missing", "missing control plane should classify as control-plane-missing", remoteControlPlaneMissingRecord);
assertCondition(remoteControlPlaneMissingRecord.degradedReason === "remote-control-plane-unreachable", "missing control plane should classify as remote-control-plane-unreachable", remoteControlPlaneMissingRecord);
assertCondition(remoteControlPlaneMissingRecord.runnerDisposition === "infra-blocked", "missing remote control plane keeps legacy runnerDisposition compatibility", remoteControlPlaneMissingRecord);
assertCondition(remoteControlPlaneMissingRecord.blockingDisposition === "control-plane-observation-gap", "missing remote control plane should expose observation gap blocking disposition", remoteControlPlaneMissingRecord);
const remoteControlPlaneGap = asRecord(remoteControlPlaneMissingRecord.observationGap);
assertCondition(remoteControlPlaneGap.kind === "control-plane-observation-gap", "missing remote control plane should expose control-plane observation gap", remoteControlPlaneGap);
assertCondition(remoteControlPlaneGap.schedulerStoppage === false, "control-plane observation gap must not imply scheduler stoppage", remoteControlPlaneGap);
assertCondition(asRecord(remoteControlPlaneMissingRecord.controlPlane).localBackendCoreMissing === true, "local backend-core absence should remain evidence only", remoteControlPlaneMissingRecord.controlPlane);
const directAuthMissing = await codexPrPreflightQueryForTest(["--remote"], {
@@ -389,6 +397,8 @@ async function main(): Promise<void> {
assertCondition(directAuthMissingRecord.ok === false, "auth-missing remote result should fail", directAuthMissingRecord);
assertCondition(directAuthMissingRecord.failureKind === "auth-missing", "missing token should classify as auth-missing", directAuthMissingRecord);
assertCondition(directAuthMissingRecord.degradedReason === "GH_TOKEN/GITHUB_TOKEN missing", "auth missing should state token gap", directAuthMissingRecord);
const directAuthObservationGap = asRecord(directAuthMissingRecord.observationGap);
assertCondition(directAuthObservationGap.kind === "runner-local-observation-gap", "auth missing after remote fallback should keep local backend-core absence scoped as runner-local observation gap", directAuthObservationGap);
const directAuthScopeBoundary = asRecord(directAuthMissingRecord.scopeBoundary);
const directAuthActiveRunner = asRecord(directAuthMissingRecord.activeRunnerDevContainer);
assertCondition(directAuthScopeBoundary.scopesAreIndependent === true, "remote auth-missing must distinguish scheduler env from active runner dev container", directAuthScopeBoundary);
@@ -405,6 +415,19 @@ async function main(): Promise<void> {
const gitRemoteGapRecord = asRecord(gitRemoteGap);
assertCondition(gitRemoteGapRecord.failureKind === "git-remote-gap", "git probe failures should stay structured", gitRemoteGapRecord);
const localOnlyObservationGap = await codexPrPreflightQueryForTest([], {
config: null,
coreFetch: () => localBackendCoreMissingFixture(),
});
const localOnlyObservationGapRecord = asRecord(localOnlyObservationGap);
assertCondition(localOnlyObservationGapRecord.ok === false, "local-only backend-core absence should fail the preflight", localOnlyObservationGapRecord);
assertCondition(localOnlyObservationGapRecord.failureKind === "target-stack-not-running", "local-only backend-core absence should preserve target-stack evidence", localOnlyObservationGapRecord);
assertCondition(localOnlyObservationGapRecord.runnerDisposition === "infra-blocked", "local-only backend-core absence keeps legacy runnerDisposition compatibility", localOnlyObservationGapRecord);
assertCondition(localOnlyObservationGapRecord.blockingDisposition === "runner-local-observation-gap", "local-only backend-core absence should expose runner-local blocking disposition", localOnlyObservationGapRecord);
const localOnlyGap = asRecord(localOnlyObservationGapRecord.observationGap);
assertCondition(localOnlyGap.kind === "runner-local-observation-gap", "local-only backend-core absence should include observationGap detail", localOnlyGap);
assertCondition(localOnlyGap.schedulerStoppage === false, "local-only backend-core absence must not imply scheduler stoppage", localOnlyGap);
const proxyGap = await codexPrPreflightQueryForTest(["--remote"], {
config: null,
coreFetch: () => ({
@@ -741,7 +764,8 @@ async function main(): Promise<void> {
checks: [
"runner-like local target-stack absence does not block remote fallback",
"remote control plane fallback preserves ready preflight",
"missing remote control plane returns control-plane-missing",
"missing remote control plane returns control-plane-observation-gap",
"local backend-core absence returns runner-local-observation-gap",
"auth missing returns auth-missing with broker/auth-broker-needed",
"proxy failures return proxy-gap",
"git remote failures return git-remote-gap",
scripts/code-queue-queues-shape-contract-test.ts
+9 -2
View File
@@ -36,8 +36,8 @@ function fixtureResponse(): JsonRecord {
heartbeatFreshTaskIds: ["task-running"],
databaseActiveTaskCount: 1,
databaseActiveTaskIds: ["task-running"],
schedulerActiveRunSlotCount: 1,
schedulerActiveTaskIds: ["task-running"],
schedulerActiveRunSlotCount: 0,
schedulerActiveTaskIds: [],
},
},
queues: [
@@ -93,6 +93,13 @@ function assertQueuesShape(label: string, result: unknown, expectedView: string)
assertCondition(diagnostics.splitBrainLive === true, `${label} split-brain live should remain explicitly true`, diagnostics);
assertCondition(diagnostics.effectiveLiveness === "live", `${label} diagnostics should retain derived liveness`, diagnostics);
assertCondition(diagnostics.recommendedAction === "continue-supervision", `${label} split-brain live should continue supervision`, diagnostics);
const liveness = asRecord(diagnostics.liveness);
assertCondition(liveness.effectiveLiveness === "live", `${label} liveness summary should foreground effective live state`, liveness);
assertCondition(liveness.recommendedAction === "continue-supervision", `${label} liveness summary should foreground recommended action`, liveness);
assertCondition(liveness.activeHeartbeatCount === 1, `${label} liveness summary should derive active heartbeat count from fresh heartbeat ids`, liveness);
assertCondition(liveness.schedulerActiveRunSlotCount === 0, `${label} liveness summary should keep master active slot zero visible`, liveness);
assertCondition(asArray(liveness.heartbeatFreshTaskIds).length === 1, `${label} liveness summary should include bounded fresh heartbeat task ids`, liveness);
assertCondition(String(liveness.interpretation ?? "").includes("heartbeat is fresh"), `${label} liveness interpretation should explain slot-zero split-brain`, liveness);
assertCondition(Array.isArray(queues.activeTaskIds), `${label} activeTaskIds should be present`, queues);
assertCondition(Array.isArray(queues.queuedTaskIds), `${label} queuedTaskIds should be present`, queues);
}
scripts/code-queue-supervisor-disclosure-contract-test.ts
+7
View File
@@ -157,6 +157,7 @@ export function runCodeQueueSupervisorDisclosureContract(): JsonRecord {
const completedUnread = asRecord(supervisorView.completedUnread);
const fullTasks = asRecord(asRecord(full).tasks);
const diagnostics = asRecord(supervisorView.executionDiagnostics);
const liveness = asRecord(diagnostics.liveness);
const listBudget = asRecord(diagnostics.listBudget);
const omittedCounts = asRecord(listBudget.omittedCounts);
@@ -167,6 +168,12 @@ export function runCodeQueueSupervisorDisclosureContract(): JsonRecord {
assertCondition(recentItems.every((item) => asRecord(item).unreadTerminal === false), "recentCompleted should not duplicate unread terminal tasks", { recentItems });
assertCondition(asArray(diagnostics.databaseActiveTaskIds).length === 12, "diagnostic task id lists should be capped", diagnostics);
assertCondition(omittedCounts.databaseActiveTaskIds === 68, "diagnostic omitted counts should preserve full visibility metadata", omittedCounts);
assertCondition(liveness.effectiveLiveness === "live", "supervisor liveness summary should keep split-brain live explicit", liveness);
assertCondition(liveness.recommendedAction === "continue-supervision", "supervisor liveness summary should recommend continued supervision", liveness);
assertCondition(liveness.splitBrainLive === true, "supervisor liveness summary should mark splitBrainLive", liveness);
assertCondition(liveness.activeHeartbeatCount === 80, "supervisor liveness summary should foreground active heartbeat count", liveness);
assertCondition(asArray(liveness.heartbeatFreshTaskIds).length === 12, "supervisor liveness summary should keep heartbeatFreshTaskIds bounded", liveness);
assertCondition(String(liveness.interpretation ?? "").includes("continue supervision"), "supervisor liveness interpretation should not imply scheduler stoppage", liveness);
assertCondition(asArray(diagnostics.reasons).length === 6, "diagnostic reasons should be capped", diagnostics);
assertCondition(diagnostics.livenessSummaryTruncated === true, "long diagnostic liveness summary should be previewed", diagnostics);
assertCondition(listBudget.truncated === true && typeof listBudget.rawCommand === "string", "diagnostic list budget should disclose raw command", listBudget);
scripts/src/code-queue.ts
+122 -4
View File
@@ -260,6 +260,7 @@ interface CodexPrPreflightOptions {
}
type CodeQueuePrPreflightFailureKind = "auth-missing" | "proxy-gap" | "git-remote-gap" | "control-plane-missing" | "target-stack-not-running";
type CodeQueueObservationGapKind = "runner-local-observation-gap" | "control-plane-observation-gap" | null;
interface CodeQueuePrPreflightTransport {
config?: UniDeskConfig | null;
@@ -1015,6 +1016,42 @@ function recommendedActionFromDiagnostics(record: Record<string, unknown>): stri
return "none";
}
function activeHeartbeatCountFromDiagnostics(record: Record<string, unknown>, activeHeartbeatTaskIds: { count: number }, heartbeatFreshTaskIds: { count: number }): number {
const explicit = asNumber(record.activeHeartbeatCount, Number.NaN);
return Number.isFinite(explicit) ? explicit : Math.max(activeHeartbeatTaskIds.count, heartbeatFreshTaskIds.count);
}
function compactLivenessDecision(record: Record<string, unknown>, lists: {
activeHeartbeatTaskIds: { items: string[]; count: number; truncated: boolean; omitted: number };
heartbeatFreshTaskIds: { items: string[]; count: number; truncated: boolean; omitted: number };
heartbeatRiskTaskIds: { items: string[]; count: number };
databaseActiveTaskIds: { count: number };
}): Record<string, unknown> {
const splitBrainLive = splitBrainLiveFromDiagnostics(record);
const effectiveLiveness = effectiveLivenessFromDiagnostics(record);
const recommendedAction = recommendedActionFromDiagnostics(record);
const activeHeartbeatCount = activeHeartbeatCountFromDiagnostics(record, lists.activeHeartbeatTaskIds, lists.heartbeatFreshTaskIds);
return {
effectiveLiveness,
recommendedAction,
splitBrainLive,
activeHeartbeatCount,
heartbeatFreshTaskCount: lists.heartbeatFreshTaskIds.count,
heartbeatFreshTaskIds: lists.heartbeatFreshTaskIds.items,
heartbeatFreshTaskIdsTruncated: lists.heartbeatFreshTaskIds.truncated,
databaseActiveTaskCount: asNumber(record.databaseActiveTaskCount, lists.databaseActiveTaskIds.count),
schedulerActiveRunSlotCount: record.schedulerActiveRunSlotCount ?? null,
heartbeatRiskTaskCount: lists.heartbeatRiskTaskIds.count,
interpretation: splitBrainLive
? "scheduler heartbeat is fresh; treat active task count from heartbeat as live and continue supervision"
: effectiveLiveness === "at-risk"
? "heartbeat risk is present; investigate heartbeat freshness before recovery"
: effectiveLiveness === "degraded"
? "diagnostics are degraded; cross-check heartbeat, trace and control-plane sources"
: "diagnostics indicate healthy liveness",
};
}
function boundedUniqueStringList(value: unknown, limit = diagnosticsIdPreviewLimit): { items: string[]; count: number; omitted: number; truncated: boolean } {
const all = Array.from(new Set(stringList(value))).sort();
const items = all.slice(0, limit);
@@ -1059,6 +1096,12 @@ function compactExecutionDiagnostics(value: unknown): Record<string, unknown> |
const allReasons = stringList(record.reasons);
const reasons = allReasons.slice(0, diagnosticsReasonPreviewLimit).map((reason) => boundedInlineString(reason, 240).text).filter((reason): reason is string => reason !== null);
const livenessSummary = boundedInlineString(record.livenessSummary, 420);
const liveness = compactLivenessDecision(record, {
activeHeartbeatTaskIds,
heartbeatFreshTaskIds,
heartbeatRiskTaskIds,
databaseActiveTaskIds,
});
const omittedCounts = {
databaseActiveTaskIds: databaseActiveTaskIds.omitted,
schedulerActiveTaskIds: schedulerActiveTaskIds.omitted,
@@ -1078,8 +1121,9 @@ function compactExecutionDiagnostics(value: unknown): Record<string, unknown> |
degraded: record.degraded ?? null,
splitBrain: record.splitBrain ?? null,
splitBrainLive: splitBrainLiveFromDiagnostics(record),
effectiveLiveness: effectiveLivenessFromDiagnostics({ ...record, heartbeatRiskTaskIds: fullHeartbeatRiskTaskIds }),
recommendedAction: recommendedActionFromDiagnostics({ ...record, heartbeatRiskTaskIds: fullHeartbeatRiskTaskIds }),
effectiveLiveness: liveness.effectiveLiveness,
recommendedAction: liveness.recommendedAction,
liveness,
livenessSummary: livenessSummary.text,
livenessSummaryChars: livenessSummary.chars,
livenessSummaryTruncated: livenessSummary.truncated,
@@ -1089,7 +1133,7 @@ function compactExecutionDiagnostics(value: unknown): Record<string, unknown> |
databaseActiveTaskIds: databaseActiveTaskIds.items,
schedulerActiveRunSlotCount: record.schedulerActiveRunSlotCount ?? null,
schedulerActiveTaskIds: schedulerActiveTaskIds.items,
activeHeartbeatCount: record.activeHeartbeatCount ?? activeHeartbeatTaskIds.count,
activeHeartbeatCount: liveness.activeHeartbeatCount,
activeHeartbeatTaskIds: activeHeartbeatTaskIds.items,
heartbeatFreshTaskIds: heartbeatFreshTaskIds.items,
heartbeatExpiredTaskIds: heartbeatExpiredTaskIds.items,
@@ -2875,6 +2919,30 @@ function decoratePrPreflightScopeBoundary(record: Record<string, unknown>): Reco
};
}
function prPreflightObservationGap(kind: Exclude<CodeQueueObservationGapKind, null>, detail: {
reason: string;
localBackendCoreMissing?: boolean;
remoteFallbackUsed?: boolean;
remoteControlPlaneReachable?: boolean | null;
}): Record<string, unknown> {
const controlPlaneGap = kind === "control-plane-observation-gap";
return {
kind,
blockingDisposition: kind,
scope: controlPlaneGap ? "control-plane" : "runner-local",
reason: detail.reason,
localBackendCoreMissing: detail.localBackendCoreMissing === true,
remoteFallbackUsed: detail.remoteFallbackUsed === true,
remoteControlPlaneReachable: detail.remoteControlPlaneReachable ?? null,
schedulerStoppage: false,
schedulerStateMachineChanged: false,
recommendedAction: controlPlaneGap ? "cross-check-control-plane" : "retry-from-control-plane-or-remote-fallback",
note: controlPlaneGap
? "The control-plane observation path is unavailable; this is not evidence that the scheduler stopped executing active tasks."
: "The current runner/local backend-core observation path is unavailable; this is not evidence that active Code Queue execution stopped.",
};
}
function compactPrRuntimePreflight(preflight: Record<string, unknown>, options: CodexPrPreflightOptions): Record<string, unknown> {
const pull = asRecord(preflight.pullRequestDelivery) ?? {};
const tools = asRecord(pull.tools) ?? {};
@@ -3056,9 +3124,16 @@ function queryRemoteMainServerPrPreflight(optionArgs: string[], config: UniDeskC
return {
ok: false,
runnerDisposition: "infra-blocked",
blockingDisposition: "control-plane-observation-gap",
failureKind: "control-plane-missing",
degradedReason: "remote-control-plane-unreachable",
message,
observationGap: prPreflightObservationGap("control-plane-observation-gap", {
reason: "remote control plane CLI returned a structured error",
localBackendCoreMissing: false,
remoteFallbackUsed: true,
remoteControlPlaneReachable: false,
}),
controlPlane: {
mode: "remote-frontend",
host: config.network.publicHost,
@@ -3081,9 +3156,16 @@ function queryRemoteMainServerPrPreflight(optionArgs: string[], config: UniDeskC
return {
ok: false,
runnerDisposition: "infra-blocked",
blockingDisposition: "control-plane-observation-gap",
failureKind: "control-plane-missing",
degradedReason: "remote-control-plane-unreachable",
message,
observationGap: prPreflightObservationGap("control-plane-observation-gap", {
reason: "remote control plane CLI could not be reached or returned non-JSON output",
localBackendCoreMissing: false,
remoteFallbackUsed: true,
remoteControlPlaneReachable: false,
}),
controlPlane: {
mode: "remote-frontend",
host: config.network.publicHost,
@@ -3128,6 +3210,18 @@ function codeQueuePrPreflight(optionArgs: string[] = [], transport: CodeQueuePrP
if (remoteRecord.ok === false) {
return decoratePrPreflightScopeBoundary({
...remoteRecord,
observationGap: prPreflightObservationGap(
remoteRecord.failureKind === "control-plane-missing" ? "control-plane-observation-gap" : "runner-local-observation-gap",
{
reason: remoteRecord.failureKind === "control-plane-missing"
? "remote control plane could not be observed after local backend-core target-stack absence"
: "local backend-core target-stack absence was bypassed by remote fallback, but the remote result still failed",
localBackendCoreMissing: true,
remoteFallbackUsed: true,
remoteControlPlaneReachable: remoteRecord.failureKind === "control-plane-missing" ? false : true,
},
),
blockingDisposition: remoteRecord.failureKind === "control-plane-missing" ? "control-plane-observation-gap" : remoteRecord.blockingDisposition ?? remoteRecord.runnerDisposition ?? "infra-blocked",
controlPlane: {
...(asRecord(remoteRecord.controlPlane) ?? {}),
mode: "remote-frontend",
@@ -3142,6 +3236,12 @@ function codeQueuePrPreflight(optionArgs: string[] = [], transport: CodeQueuePrP
}
return decoratePrPreflightScopeBoundary({
...remoteRecord,
localObservationGap: prPreflightObservationGap("runner-local-observation-gap", {
reason: "local backend-core target-stack absence was bypassed by healthy remote control-plane fallback",
localBackendCoreMissing: true,
remoteFallbackUsed: true,
remoteControlPlaneReachable: true,
}),
controlPlane: {
...(asRecord(remoteRecord.controlPlane) ?? {}),
mode: "remote-frontend",
@@ -3163,9 +3263,16 @@ function codeQueuePrPreflight(optionArgs: string[] = [], transport: CodeQueuePrP
...(localRecord ?? {}),
ok: false,
runnerDisposition: "infra-blocked",
blockingDisposition: "control-plane-observation-gap",
failureKind,
degradedReason,
message: "remote control plane unreachable; local backend-core target-stack absence is evidence only",
message: "remote control plane unreachable; local backend-core target-stack absence is observation-gap evidence only",
observationGap: prPreflightObservationGap("control-plane-observation-gap", {
reason: "local backend-core target stack is missing and no remote control plane could be observed",
localBackendCoreMissing: true,
remoteFallbackUsed: false,
remoteControlPlaneReachable: false,
}),
controlPlane: {
mode: "local-backend-core",
localBackendCoreMissing: true,
@@ -3183,9 +3290,20 @@ function codeQueuePrPreflight(optionArgs: string[] = [], transport: CodeQueuePrP
...(localRecord ?? {}),
ok: false,
runnerDisposition: localRecord?.runnerDisposition ?? "infra-blocked",
blockingDisposition: localTargetStackMissing ? "runner-local-observation-gap" : localRecord?.blockingDisposition ?? localRecord?.runnerDisposition ?? "infra-blocked",
failureKind: (localRecord?.failureKind as CodeQueuePrPreflightFailureKind | undefined) ?? "proxy-gap",
degradedReason: localRecord?.degradedReason ?? "backend-core-proxy-unavailable",
message: localRecord?.message ?? localRecord?.stderrTail ?? localRecord?.stdoutTail ?? "Code Queue runtime preflight could not be observed",
...(localTargetStackMissing
? {
observationGap: prPreflightObservationGap("runner-local-observation-gap", {
reason: "local backend-core target stack is missing in this runner observation path",
localBackendCoreMissing: true,
remoteFallbackUsed: false,
remoteControlPlaneReachable: null,
}),
}
: {}),
controlPlane: {
mode: "local-backend-core",
localBackendCoreMissing: localTargetStackMissing,