fix: classify transient GitHub connectivity
Merge PR #102 after rebasing onto current master and validating focused GitHub transient/Code Queue preflight behavior. GitHub DNS/API failures now classify as retryable github-transient with retry/backoff or keep-fresh-heartbeat-task guidance.
This commit is contained in:
Lyon
@@ -320,7 +320,7 @@ interface CodexSkillsSyncOptions {
|
||||
full: boolean;
|
||||
}
|
||||
|
||||
type CodeQueuePrPreflightFailureKind = "auth-missing" | "runner-skills-blocker" | "proxy-gap" | "git-remote-gap" | "control-plane-missing" | "target-stack-not-running";
|
||||
type CodeQueuePrPreflightFailureKind = "auth-missing" | "runner-skills-blocker" | "github-transient" | "proxy-gap" | "git-remote-gap" | "control-plane-missing" | "target-stack-not-running";
|
||||
type CodeQueueObservationGapKind = "runner-local-observation-gap" | "control-plane-observation-gap" | null;
|
||||
|
||||
interface CodeQueuePrPreflightTransport {
|
||||
@@ -3817,6 +3817,64 @@ function compactCommandProbe(value: unknown): Record<string, unknown> | null {
|
||||
};
|
||||
}
|
||||
|
||||
function commandProbeFailureText(value: unknown): string {
|
||||
const probe = asRecord(value);
|
||||
if (probe === null || probe.ok === true) return "";
|
||||
const args = Array.isArray(probe.args) ? probe.args.map(String).join(" ") : "";
|
||||
return [
|
||||
typeof probe.command === "string" ? probe.command : "",
|
||||
args,
|
||||
typeof probe.error === "string" ? probe.error : "",
|
||||
typeof probe.stderr === "string" ? probe.stderr : "",
|
||||
typeof probe.stdout === "string" ? probe.stdout : "",
|
||||
].filter((item) => item.length > 0).join("\n");
|
||||
}
|
||||
|
||||
function commandProbeLooksLikeGithubTransient(value: unknown): boolean {
|
||||
const text = commandProbeFailureText(value);
|
||||
if (text.length === 0) return false;
|
||||
if (/\bproxy\b/iu.test(text) && /could not resolve|connect|tunnel|proxy/iu.test(text)) return false;
|
||||
return /temporary failure in name resolution|error connecting to api\.github\.com|getaddrinfo|could not resolve host:? (github\.com|api\.github\.com)|name or service not known|eai_again|enotfound|etimedout|econnreset|ehostunreach|enetunreach|failed to connect to (github\.com|api\.github\.com)|network is unreachable/iu.test(text);
|
||||
}
|
||||
|
||||
function githubTransientEvidence(pull: Record<string, unknown>): Record<string, unknown> | null {
|
||||
const egress = asRecord(pull.egress) ?? {};
|
||||
const remote = asRecord(pull.remote);
|
||||
const probes: Array<[string, unknown]> = [
|
||||
["egress.githubDefault", egress.githubDefault],
|
||||
["egress.apiDefault", egress.apiDefault],
|
||||
["egress.issueApi", egress.issueApi],
|
||||
];
|
||||
if (remote !== null) {
|
||||
probes.push(
|
||||
["remote.gitLsRemote", remote.gitLsRemote],
|
||||
["remote.gitHttpsLsRemote", remote.gitHttpsLsRemote],
|
||||
["remote.ghAuthStatus", remote.ghAuthStatus],
|
||||
["remote.ghRepoView", remote.ghRepoView],
|
||||
["remote.ghIssueView", remote.ghIssueView],
|
||||
["remote.ghPrReadOnly", remote.ghPrReadOnly],
|
||||
);
|
||||
}
|
||||
const failedProbes = probes
|
||||
.map(([name, probe]) => ({ name, probe, text: commandProbeFailureText(probe) }))
|
||||
.filter((item) => commandProbeLooksLikeGithubTransient(item.probe))
|
||||
.slice(0, 4);
|
||||
if (failedProbes.length === 0) return null;
|
||||
return {
|
||||
kind: "github-transient",
|
||||
retryable: true,
|
||||
scope: "github-dns-api",
|
||||
failedProbes: failedProbes.map((item) => ({
|
||||
name: item.name,
|
||||
preview: compactInlinePreview(item.text, 240),
|
||||
})),
|
||||
commanderAction: "retry/backoff; if Code Queue heartbeat or trace is fresh, keep the task running and continue supervision",
|
||||
notAuthMissing: true,
|
||||
notPrSemanticFailure: true,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactToolStatus(value: unknown): Record<string, unknown> {
|
||||
const tool = asRecord(value) ?? {};
|
||||
return {
|
||||
@@ -4183,21 +4241,26 @@ function compactPrRuntimePreflight(preflight: Record<string, unknown>, options:
|
||||
const limitations = Array.isArray(pull.limitations) ? pull.limitations.map(String) : [];
|
||||
const risks = Array.isArray(pull.risks) ? pull.risks.map(String) : [];
|
||||
const ok = preflight.ok === true && tokenCoverage.ok === true;
|
||||
const githubTransient = githubTransientEvidence(pull);
|
||||
const skillsBlocked = skills !== null && skills.ok !== true;
|
||||
const skillsSyncBlocked = skillsSync !== null && skillsSync.ok !== true;
|
||||
const failureKind = !tokenCoverage.ok
|
||||
? "auth-missing"
|
||||
: skillsBlocked || skillsSyncBlocked
|
||||
? "runner-skills-blocker"
|
||||
: limitations.some((item) => item.includes("git ls-remote") || item.includes("git push --dry-run failed"))
|
||||
? "git-remote-gap"
|
||||
: !preflight.ok
|
||||
? "proxy-gap"
|
||||
: null;
|
||||
: githubTransient !== null
|
||||
? "github-transient"
|
||||
: limitations.some((item) => item.includes("git ls-remote") || item.includes("git push --dry-run failed"))
|
||||
? "git-remote-gap"
|
||||
: !preflight.ok
|
||||
? "proxy-gap"
|
||||
: null;
|
||||
const degradedReason = failureKind === "auth-missing"
|
||||
? "auth-broker-needed"
|
||||
: failureKind === "runner-skills-blocker"
|
||||
? typeof skillsSync?.blocker === "string" ? skillsSync.blocker : typeof skills?.blocker === "string" ? skills.blocker : "runner-skills-degraded"
|
||||
: failureKind === "github-transient"
|
||||
? "github-dns-api-transient"
|
||||
: failureKind === "git-remote-gap"
|
||||
? "git remote probe failed"
|
||||
: failureKind === "proxy-gap"
|
||||
@@ -4326,12 +4389,19 @@ function compactPrRuntimePreflight(preflight: Record<string, unknown>, options:
|
||||
limitations,
|
||||
risks,
|
||||
runnerDisposition: ok ? "ready" : "infra-blocked",
|
||||
...(githubTransient === null ? {} : {
|
||||
retryable: true,
|
||||
commanderAction: "retry-backoff-or-keep-running-if-heartbeat-fresh",
|
||||
githubTransient,
|
||||
}),
|
||||
activeRunnerDevContainer,
|
||||
recoveryHint: ok
|
||||
? tokenCoverage.source === "auth-broker"
|
||||
? "Runner PR workflow has auth-broker coverage for GitHub REST preflight; real PR creation still requires commander authorization."
|
||||
: "Runner PR workflow has env-token coverage for the scheduler."
|
||||
: "Scheduler preflight lacks GitHub auth coverage for scheduler-scoped admission only. First verify the active task with repo-native gh auth status and PR create dry-run; long-term configure auth-broker or inject GH_TOKEN/GITHUB_TOKEN into the scheduler runtime secret.",
|
||||
: githubTransient !== null
|
||||
? "GitHub DNS/API connectivity failed transiently. Retry with backoff; if the task heartbeat or trace is fresh, keep supervising the running task instead of closing or requeueing business work."
|
||||
: "Scheduler preflight lacks GitHub auth coverage for scheduler-scoped admission only. First verify the active task with repo-native gh auth status and PR create dry-run; long-term configure auth-broker or inject GH_TOKEN/GITHUB_TOKEN into the scheduler runtime secret.",
|
||||
commands: {
|
||||
local: "bun scripts/cli.ts gh auth status --repo pikasTech/unidesk",
|
||||
runner: "bun scripts/cli.ts codex pr-preflight --remote",
|
||||
@@ -4582,6 +4652,8 @@ function codeQueuePrPreflight(optionArgs: string[] = [], transport: CodeQueuePrP
|
||||
runnerDisposition: compact.runnerDisposition,
|
||||
failureKind: compact.failureKind ?? null,
|
||||
degradedReason: compact.degradedReason ?? null,
|
||||
...(compact.retryable === true ? { retryable: true } : {}),
|
||||
...(typeof compact.commanderAction === "string" ? { commanderAction: compact.commanderAction } : {}),
|
||||
authScopeSummary: compact.authScopeSummary,
|
||||
scopeBoundary: compact.scopeBoundary,
|
||||
activeRunnerDevContainer: compact.activeRunnerDevContainer,
|
||||
@@ -4638,6 +4710,8 @@ export async function codexPrPreflightQueryAsync(optionArgs: string[], fetcher:
|
||||
runnerDisposition: compact.runnerDisposition,
|
||||
failureKind: compact.failureKind ?? null,
|
||||
degradedReason: compact.degradedReason ?? null,
|
||||
...(compact.retryable === true ? { retryable: true } : {}),
|
||||
...(typeof compact.commanderAction === "string" ? { commanderAction: compact.commanderAction } : {}),
|
||||
authScopeSummary: compact.authScopeSummary,
|
||||
scopeBoundary: compact.scopeBoundary,
|
||||
activeRunnerDevContainer: compact.activeRunnerDevContainer,
|
||||
|
||||
+108
-2
@@ -77,6 +77,7 @@ type GitHubDegradedReason =
|
||||
| "missing-token"
|
||||
| "auth-failed"
|
||||
| "egress-failed"
|
||||
| "github-transient"
|
||||
| "network-proxy-failed"
|
||||
| "permission-denied"
|
||||
| "repo-not-found"
|
||||
@@ -359,6 +360,8 @@ interface GitHubErrorPayload {
|
||||
runnerDisposition: RunnerDisposition;
|
||||
status?: number;
|
||||
message: string;
|
||||
retryable?: boolean;
|
||||
commanderAction?: string;
|
||||
details?: unknown;
|
||||
scopes?: {
|
||||
accepted: string | null;
|
||||
@@ -1347,6 +1350,97 @@ function sanitizedErrorDetails(parsed: unknown): unknown {
|
||||
return details;
|
||||
}
|
||||
|
||||
function errorProperty(value: unknown, key: string): string | null {
|
||||
if (!isRecord(value)) return null;
|
||||
const item = value[key];
|
||||
return typeof item === "string" && item.length > 0 ? item : null;
|
||||
}
|
||||
|
||||
function errorChainText(error: unknown): string {
|
||||
const parts: string[] = [];
|
||||
let current: unknown = error;
|
||||
const seen = new Set<unknown>();
|
||||
while (current !== null && current !== undefined && !seen.has(current)) {
|
||||
seen.add(current);
|
||||
if (current instanceof Error) {
|
||||
parts.push(current.name, current.message);
|
||||
current = (current as Error & { cause?: unknown }).cause;
|
||||
continue;
|
||||
}
|
||||
if (isRecord(current)) {
|
||||
for (const key of ["name", "message", "code", "errno", "syscall", "hostname", "host"]) {
|
||||
const value = errorProperty(current, key);
|
||||
if (value !== null) parts.push(value);
|
||||
}
|
||||
current = current.cause;
|
||||
continue;
|
||||
}
|
||||
parts.push(String(current));
|
||||
break;
|
||||
}
|
||||
return parts.join(" ").replace(/\s+/gu, " ").trim();
|
||||
}
|
||||
|
||||
function firstErrorProperty(error: unknown, key: string): string | null {
|
||||
let current: unknown = error;
|
||||
const seen = new Set<unknown>();
|
||||
while (current !== null && current !== undefined && !seen.has(current)) {
|
||||
seen.add(current);
|
||||
const value = errorProperty(current, key);
|
||||
if (value !== null) return value;
|
||||
current = isRecord(current) ? current.cause : current instanceof Error ? (current as Error & { cause?: unknown }).cause : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function classifyGitHubFetchFailure(error: unknown): {
|
||||
reason: GitHubDegradedReason;
|
||||
message: string;
|
||||
retryable?: boolean;
|
||||
commanderAction?: string;
|
||||
details: Record<string, unknown>;
|
||||
} {
|
||||
const text = errorChainText(error);
|
||||
const lower = text.toLowerCase();
|
||||
const isProxyFailure = /\bproxy\b/u.test(lower) && /could not resolve|connect|tunnel|proxy/i.test(text);
|
||||
const isGitHubTransient = !isProxyFailure && (
|
||||
/temporary failure in name resolution/iu.test(text)
|
||||
|| /\b(eai_again|enotfound|etimedout|econnreset|econnrefused|ehostunreach|enetunreach)\b/iu.test(text)
|
||||
|| /getaddrinfo|could not resolve host|name or service not known|failed to connect|connection timed out|connection reset|connection closed|other side closed|network is unreachable|socket connection/iu.test(text)
|
||||
);
|
||||
if (!isGitHubTransient) {
|
||||
return {
|
||||
reason: "network-proxy-failed",
|
||||
message: text.length > 0 ? text : "GitHub request failed before receiving an HTTP response",
|
||||
details: {
|
||||
scope: "github-api",
|
||||
transient: false,
|
||||
errorName: error instanceof Error ? error.name : firstErrorProperty(error, "name"),
|
||||
errorCode: firstErrorProperty(error, "code"),
|
||||
host: firstErrorProperty(error, "hostname") ?? firstErrorProperty(error, "host"),
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
reason: "github-transient",
|
||||
message: text.length > 0 ? text : "GitHub DNS/API connection failed before receiving an HTTP response",
|
||||
retryable: true,
|
||||
commanderAction: "retry-backoff-or-keep-running-if-heartbeat-fresh",
|
||||
details: {
|
||||
scope: "github-api",
|
||||
transient: true,
|
||||
retryable: true,
|
||||
authFailure: false,
|
||||
semanticFailure: false,
|
||||
errorName: error instanceof Error ? error.name : firstErrorProperty(error, "name"),
|
||||
errorCode: firstErrorProperty(error, "code"),
|
||||
host: firstErrorProperty(error, "hostname") ?? firstErrorProperty(error, "host"),
|
||||
commanderAction: "retry/backoff; if the Code Queue task heartbeat or trace is fresh, keep supervising instead of closing or requeueing business work",
|
||||
valuesPrinted: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function errorPayload(reason: GitHubDegradedReason, message: string, extra: Omit<GitHubErrorPayload, "ok" | "degradedReason" | "runnerDisposition" | "message"> = {}): GitHubErrorPayload {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -1431,7 +1525,13 @@ async function githubRequest<T>(
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
} catch (error) {
|
||||
return errorPayload("network-proxy-failed", error instanceof Error ? error.message : String(error), { request: { method, path } });
|
||||
const classified = classifyGitHubFetchFailure(error);
|
||||
return errorPayload(classified.reason, classified.message, {
|
||||
request: { method, path },
|
||||
details: classified.details,
|
||||
retryable: classified.retryable,
|
||||
commanderAction: classified.commanderAction,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
@@ -1473,7 +1573,13 @@ async function githubGraphqlRequest<T>(
|
||||
body: JSON.stringify({ query, variables }),
|
||||
});
|
||||
} catch (error) {
|
||||
return errorPayload("network-proxy-failed", error instanceof Error ? error.message : String(error), { request: { method: "POST", path: "/graphql" } });
|
||||
const classified = classifyGitHubFetchFailure(error);
|
||||
return errorPayload(classified.reason, classified.message, {
|
||||
request: { method: "POST", path: "/graphql" },
|
||||
details: classified.details,
|
||||
retryable: classified.retryable,
|
||||
commanderAction: classified.commanderAction,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user