fix: classify D601 provider health signals
This commit is contained in:
@@ -1080,6 +1080,29 @@ function asBool(value: string | undefined): boolean {
|
||||
return value === "true";
|
||||
}
|
||||
|
||||
function registryHealthDecision(checks: Record<string, boolean>, commandOk: boolean): Record<string, unknown> {
|
||||
const scopeChecks: Array<[string, boolean]> = [
|
||||
["systemd", checks.systemctlAvailable === true && checks.unitExists === true && checks.unitActive === true],
|
||||
["docker", checks.dockerAvailable === true],
|
||||
["registry-container", checks.containerRunning === true],
|
||||
["loopback-listener", checks.loopbackOnly === true],
|
||||
["registry-api", checks.v2Ok === true],
|
||||
["storage", checks.storageExists === true],
|
||||
["rendered-config", checks.configHashMatches === true && checks.composeHashMatches === true && checks.unitHashMatches === true],
|
||||
["registry-image", checks.imageMatches === true],
|
||||
];
|
||||
const healthyScopes = scopeChecks.filter(([, ok]) => ok).map(([scope]) => scope);
|
||||
const failedScopes = scopeChecks.filter(([, ok]) => !ok).map(([scope]) => scope);
|
||||
const runtimeApiHealthy = checks.containerRunning === true && checks.loopbackOnly === true && checks.v2Ok === true;
|
||||
return {
|
||||
decision: commandOk && failedScopes.length === 0 ? "healthy" : commandOk || runtimeApiHealthy ? "service-degraded" : "retryable-transient",
|
||||
retryable: true,
|
||||
healthyScopes,
|
||||
failedScopes,
|
||||
runtimeApiHealthy,
|
||||
};
|
||||
}
|
||||
|
||||
function commandTail(result: CommandResult): Record<string, unknown> {
|
||||
return {
|
||||
command: result.command.length > 7 ? [...result.command.slice(0, 7), "<readonly-script>"] : result.command,
|
||||
@@ -1247,11 +1270,13 @@ function statusFromValues(options: ArtifactRegistryOptions, values: Record<strin
|
||||
&& checks.configHashMatches
|
||||
&& checks.composeHashMatches
|
||||
&& checks.unitHashMatches;
|
||||
const decision = registryHealthDecision(checks, commandOk);
|
||||
return {
|
||||
ok: healthMode ? healthy : commandOk,
|
||||
readonly: true,
|
||||
installed,
|
||||
healthy,
|
||||
...decision,
|
||||
checks,
|
||||
observed: {
|
||||
unit: { path: values.unit_path, active: values.unit_active, enabled: values.unit_enabled },
|
||||
@@ -1293,6 +1318,11 @@ function runReadonlyStatus(options: ArtifactRegistryOptions, healthMode: boolean
|
||||
readonly: true,
|
||||
installed: false,
|
||||
healthy: false,
|
||||
decision: "retryable-transient",
|
||||
retryable: true,
|
||||
healthyScopes: [],
|
||||
failedScopes: ["provider-ssh-command"],
|
||||
runtimeApiHealthy: false,
|
||||
checks: {},
|
||||
expected: {
|
||||
endpoint: `http://${options.host}:${options.port}`,
|
||||
|
||||
@@ -25,8 +25,10 @@ describe("provider triage contract", () => {
|
||||
], "2026-05-20T00:00:00.000Z");
|
||||
|
||||
expect(result.blockingDisposition).toBe("runner-local-observation-gap");
|
||||
expect(result.decision).toBe("retryable-transient");
|
||||
expect(result.retryable).toBe(true);
|
||||
expect(result.scope).toBe("runner-local");
|
||||
expect(result.failedScopes).toContain("runner-local");
|
||||
expect(result.contract.singlePathProviderOfflineIsGlobalBlocker).toBe(false);
|
||||
});
|
||||
|
||||
@@ -38,6 +40,7 @@ describe("provider triage contract", () => {
|
||||
], "2026-05-20T00:00:00.000Z");
|
||||
|
||||
expect(result.blockingDisposition).toBe("global-blocker");
|
||||
expect(result.decision).toBe("global-offline");
|
||||
expect(result.retryable).toBe(false);
|
||||
expect(result.failedIndependentScopes).toContain("provider-gateway");
|
||||
expect(result.failedIndependentScopes).toContain("ssh");
|
||||
@@ -52,8 +55,26 @@ describe("provider triage contract", () => {
|
||||
], "2026-05-20T00:00:00.000Z");
|
||||
|
||||
expect(result.blockingDisposition).toBe("service-degraded");
|
||||
expect(result.decision).toBe("service-degraded");
|
||||
expect(result.retryable).toBe(true);
|
||||
expect(result.failedScopes).toContain("registry");
|
||||
expect(result.healthyScopes).toContain("provider-gateway");
|
||||
expect(result.healthyIndependentScopes).toContain("provider-gateway");
|
||||
expect(result.healthyIndependentScopes).toContain("ssh");
|
||||
});
|
||||
|
||||
test("registry systemd inactive with listener and api healthy is service degraded", () => {
|
||||
const result = buildProviderTriageResult("D601", [
|
||||
signal("backend-core-node", "provider-gateway", "ok"),
|
||||
signal("host-ssh-probe", "ssh", "ok"),
|
||||
signal("artifact-registry-health", "registry", "degraded"),
|
||||
signal("k3sctl-adapter-health", "k3s", "ok"),
|
||||
], "2026-05-20T00:00:00.000Z");
|
||||
|
||||
expect(result.decision).toBe("service-degraded");
|
||||
expect(result.blockingDisposition).toBe("service-degraded");
|
||||
expect(result.retryable).toBe(true);
|
||||
expect(result.degradedScopes).toContain("registry");
|
||||
expect(result.healthyScopes).toEqual(expect.arrayContaining(["k3s", "provider-gateway", "ssh"]));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,6 +24,12 @@ export type ProviderBlockingDisposition =
|
||||
| "service-degraded"
|
||||
| "global-blocker";
|
||||
|
||||
export type ProviderTriageDecision =
|
||||
| "healthy"
|
||||
| "retryable-transient"
|
||||
| "service-degraded"
|
||||
| "global-offline";
|
||||
|
||||
export interface ProviderTriageSignal {
|
||||
id: string;
|
||||
scope: ProviderSignalScope;
|
||||
@@ -36,11 +42,15 @@ export interface ProviderTriageSignal {
|
||||
|
||||
export interface ProviderTriageClassification {
|
||||
scope: ProviderSignalScope;
|
||||
decision: ProviderTriageDecision;
|
||||
observedAt: string;
|
||||
retryable: boolean;
|
||||
recommendedCrossChecks: string[];
|
||||
blockingDisposition: ProviderBlockingDisposition;
|
||||
rationale: string[];
|
||||
failedScopes: ProviderSignalScope[];
|
||||
degradedScopes: ProviderSignalScope[];
|
||||
healthyScopes: ProviderSignalScope[];
|
||||
failedIndependentScopes: ProviderSignalScope[];
|
||||
healthyIndependentScopes: ProviderSignalScope[];
|
||||
}
|
||||
@@ -193,8 +203,23 @@ function sshSignal(result: unknown, providerId: string): ProviderTriageSignal {
|
||||
function registrySignal(result: unknown): ProviderTriageSignal {
|
||||
const record = asRecord(result);
|
||||
if (record === null) return signal("artifact-registry-health", "registry", "unknown", "artifact registry health returned non-object", result);
|
||||
const status: ProviderSignalStatus = record.ok === true && record.healthy !== false ? "ok" : record.ok === false ? "failed" : "degraded";
|
||||
return signal("artifact-registry-health", "registry", status, `artifact registry health ok=${String(record.ok)} healthy=${String(record.healthy)}`, {
|
||||
const checks = asRecord(record.checks) ?? {};
|
||||
const runtimeApiHealthy = checks.containerRunning === true && checks.loopbackOnly === true && checks.v2Ok === true;
|
||||
const status: ProviderSignalStatus = record.ok === true && record.healthy !== false
|
||||
? "ok"
|
||||
: runtimeApiHealthy
|
||||
? "degraded"
|
||||
: record.ok === false
|
||||
? "failed"
|
||||
: "degraded";
|
||||
return signal("artifact-registry-health", "registry", status, [
|
||||
`artifact registry health ok=${String(record.ok)}`,
|
||||
`healthy=${String(record.healthy)}`,
|
||||
`unitActive=${String(checks.unitActive)}`,
|
||||
`containerRunning=${String(checks.containerRunning)}`,
|
||||
`loopbackOnly=${String(checks.loopbackOnly)}`,
|
||||
`v2Ok=${String(checks.v2Ok)}`,
|
||||
].join(" "), {
|
||||
ok: record.ok,
|
||||
installed: record.installed ?? null,
|
||||
healthy: record.healthy ?? null,
|
||||
@@ -273,9 +298,9 @@ export function providerTriageRecommendedCrossChecks(providerId: string): string
|
||||
];
|
||||
}
|
||||
|
||||
function uniqueScopes(signals: ProviderTriageSignal[], statuses: ProviderSignalStatus[]): ProviderSignalScope[] {
|
||||
function uniqueScopes(signals: ProviderTriageSignal[], statuses: ProviderSignalStatus[], independentOnly = true): ProviderSignalScope[] {
|
||||
return Array.from(new Set(signals
|
||||
.filter((item) => item.independentPath)
|
||||
.filter((item) => !independentOnly || item.independentPath)
|
||||
.filter((item) => statuses.includes(item.status))
|
||||
.map((item) => item.scope)))
|
||||
.sort();
|
||||
@@ -292,10 +317,11 @@ function primaryScope(signals: ProviderTriageSignal[]): ProviderSignalScope {
|
||||
}
|
||||
|
||||
export function classifyProviderTriage(providerId: string, signals: ProviderTriageSignal[], observedAt = isoNow()): ProviderTriageClassification {
|
||||
const failedScopes = uniqueScopes(signals, ["failed"]);
|
||||
const degradedScopes = uniqueScopes(signals, ["degraded"]);
|
||||
const failedScopes = uniqueScopes(signals, ["failed"], false);
|
||||
const degradedScopes = uniqueScopes(signals, ["degraded"], false);
|
||||
const healthyScopes = uniqueScopes(signals, ["ok"]);
|
||||
const independentFailedScopes = failedScopes.filter((scope) => scope !== "runner-local");
|
||||
const independentFailedScopes = uniqueScopes(signals, ["failed"]).filter((scope) => scope !== "runner-local");
|
||||
const independentDegradedScopes = uniqueScopes(signals, ["degraded"]);
|
||||
const failedCriticalScopes = independentFailedScopes.filter((scope) => criticalScopes.has(scope));
|
||||
const runnerLocalObservedFailure = signals.some((signal) => signal.scope === "runner-local" && signal.status === "failed");
|
||||
const serviceOnlyFailure = independentFailedScopes.length > 0 && independentFailedScopes.every((scope) => scope === "registry" || scope === "service-proxy" || scope === "microservice" || scope === "k3s");
|
||||
@@ -312,7 +338,7 @@ export function classifyProviderTriage(providerId: string, signals: ProviderTria
|
||||
} else if (serviceOnlyFailure && hasIndependentHealthy) {
|
||||
blockingDisposition = "service-degraded";
|
||||
rationale.push("service-scoped path failed while at least one provider-level path remains healthy");
|
||||
} else if (failedCriticalScopes.length > 0 || degradedScopes.some((scope) => criticalScopes.has(scope))) {
|
||||
} else if (failedCriticalScopes.length > 0 || independentDegradedScopes.some((scope) => criticalScopes.has(scope))) {
|
||||
blockingDisposition = hasIndependentHealthy ? "provider-degraded" : "transient";
|
||||
rationale.push(hasIndependentHealthy
|
||||
? "provider-critical path is degraded but cross-checks still show independent healthy evidence"
|
||||
@@ -327,15 +353,27 @@ export function classifyProviderTriage(providerId: string, signals: ProviderTria
|
||||
|
||||
if (runnerLocalObservedFailure) rationale.push("runner-local observation failed but is not counted as an independent global blocker by contract");
|
||||
if (hasIndependentHealthy) rationale.push(`healthy independent scopes: ${healthyScopes.join(", ")}`);
|
||||
if (failedScopes.length > 0) rationale.push(`failed independent scopes: ${failedScopes.join(", ")}`);
|
||||
if (failedScopes.length > 0) rationale.push(`failed scopes: ${failedScopes.join(", ")}`);
|
||||
const hasAnyIssueSignal = signals.some((item) => item.status === "failed" || item.status === "degraded");
|
||||
const decision: ProviderTriageDecision = blockingDisposition === "global-blocker"
|
||||
? "global-offline"
|
||||
: blockingDisposition === "service-degraded"
|
||||
? "service-degraded"
|
||||
: hasAnyIssueSignal
|
||||
? "retryable-transient"
|
||||
: "healthy";
|
||||
|
||||
return {
|
||||
scope: runnerLocalObservedFailure && failedScopes.length === 0 && degradedScopes.length === 0 ? "runner-local" : primaryScope(signals),
|
||||
scope: runnerLocalObservedFailure && independentFailedScopes.length === 0 && independentDegradedScopes.length === 0 ? "runner-local" : primaryScope(signals),
|
||||
decision,
|
||||
observedAt,
|
||||
retryable: blockingDisposition !== "global-blocker",
|
||||
recommendedCrossChecks: providerTriageRecommendedCrossChecks(providerId),
|
||||
blockingDisposition,
|
||||
rationale,
|
||||
failedScopes,
|
||||
degradedScopes,
|
||||
healthyScopes,
|
||||
failedIndependentScopes: independentFailedScopes,
|
||||
healthyIndependentScopes: healthyScopes,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user