fix(code-queue): add issue 3 gateway diagnostics
This commit is contained in:
Codex
@@ -0,0 +1,106 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { codeQueueContainerdImagePreflight, codeQueueManifestImagePreflight } from "./src/deploy";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
function assertCondition(condition: unknown, message: string, detail: JsonRecord = {}): void {
|
||||
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
|
||||
}
|
||||
|
||||
function gatewayCheck(diagnostics: JsonRecord): JsonRecord {
|
||||
const checks = diagnostics.checks as JsonRecord | undefined;
|
||||
const deployment = checks?.deployment as JsonRecord | undefined;
|
||||
const endpoint = checks?.endpoint as JsonRecord | undefined;
|
||||
const target = checks?.targetService as JsonRecord | undefined;
|
||||
const deploymentAvailable = deployment?.available === true;
|
||||
const endpointNonEmpty = Number(endpoint?.readyAddressCount ?? 0) > 0;
|
||||
return {
|
||||
ok: diagnostics.ok === true && deploymentAvailable && endpointNonEmpty && target?.ok === true,
|
||||
deploymentAvailable,
|
||||
endpointNonEmpty,
|
||||
targetServiceOk: target?.ok === true,
|
||||
};
|
||||
}
|
||||
|
||||
function schedulerStorageCheck(schedulerHealth: JsonRecord): JsonRecord {
|
||||
const queue = schedulerHealth.queue as JsonRecord | undefined;
|
||||
const storage = queue?.storage as JsonRecord | undefined;
|
||||
const lastError = storage?.lastError ?? null;
|
||||
return {
|
||||
ok: storage?.postgresReady === true && lastError === null,
|
||||
postgresReady: storage?.postgresReady === true,
|
||||
lastError,
|
||||
};
|
||||
}
|
||||
|
||||
function staleReconcileCheck(schedulerHealth: JsonRecord): JsonRecord {
|
||||
const queue = schedulerHealth.queue as JsonRecord | undefined;
|
||||
const reconcile = queue?.reconcile as JsonRecord | undefined;
|
||||
const recoverable = Number(reconcile?.recoverableOrphanedActiveTaskCount ?? queue?.orphanedActiveTaskCount ?? 0);
|
||||
return {
|
||||
ok: recoverable === 0,
|
||||
recoverableOrphanedActiveTaskCount: recoverable,
|
||||
retryWaitTaskCount: Number(reconcile?.retryWaitTaskCount ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
function runIssue3Regression(): JsonRecord {
|
||||
const missingEndpointGateway = gatewayCheck({
|
||||
ok: false,
|
||||
checks: {
|
||||
deployment: { ok: true, available: true, availableReplicas: 1 },
|
||||
endpoint: { ok: false, readyAddressCount: 0 },
|
||||
targetService: { ok: false },
|
||||
},
|
||||
});
|
||||
assertCondition(missingEndpointGateway.ok === false, "microservice:code-queue-egress-gateway-health must fail on empty endpoint", missingEndpointGateway);
|
||||
assertCondition(missingEndpointGateway.deploymentAvailable === true, "gateway deployment availability should be reported separately", missingEndpointGateway);
|
||||
assertCondition(missingEndpointGateway.endpointNonEmpty === false, "gateway endpoint non-empty check should be explicit", missingEndpointGateway);
|
||||
|
||||
const storageFailure = schedulerStorageCheck({
|
||||
queue: {
|
||||
storage: {
|
||||
postgresReady: false,
|
||||
lastError: "CONNECT_TIMEOUT d601-tcp-egress-gateway.unidesk.svc.cluster.local:15432",
|
||||
},
|
||||
},
|
||||
});
|
||||
assertCondition(storageFailure.ok === false, "diagnostics must fail when scheduler storage.lastError reports PostgreSQL route failure", storageFailure);
|
||||
assertCondition(String(storageFailure.lastError).includes("CONNECT_TIMEOUT"), "storage failure detail should preserve route error", storageFailure);
|
||||
|
||||
const staleReconcile = staleReconcileCheck({
|
||||
queue: {
|
||||
orphanedActiveTaskCount: 1,
|
||||
reconcile: {
|
||||
recoverableOrphanedActiveTaskCount: 1,
|
||||
retryWaitTaskCount: 2,
|
||||
},
|
||||
},
|
||||
});
|
||||
assertCondition(staleReconcile.ok === false, "code-queue:stale-active-reconcile must fail while recoverable active tasks remain", staleReconcile);
|
||||
assertCondition(staleReconcile.retryWaitTaskCount === 2, "retry_wait reconcile count should be visible", staleReconcile);
|
||||
|
||||
const missingImage = codeQueueContainerdImagePreflight("docker.io/library/busybox:latest\n", "unidesk-code-queue:d601");
|
||||
assertCondition(missingImage.ok === false, "code-queue:containerd-image-preflight must fail when k3s containerd lacks unidesk-code-queue:d601", missingImage);
|
||||
const presentImage = codeQueueContainerdImagePreflight("docker.io/library/busybox:latest\nunidesk-code-queue:d601 application/vnd.oci.image.manifest.v1+json\n", "unidesk-code-queue:d601");
|
||||
assertCondition(presentImage.ok === true, "code-queue:containerd-image-preflight should pass when the tag is present", presentImage);
|
||||
|
||||
const manifestPath = "src/components/microservices/k3sctl-adapter/k3s/code-queue.k8s.yaml";
|
||||
const manifest = readFileSync(manifestPath, "utf8");
|
||||
const manifestImage = codeQueueManifestImagePreflight(manifest, "unidesk-code-queue:d601");
|
||||
assertCondition(manifestImage.ok === true, "Code Queue k8s manifest deployments must all use unidesk-code-queue:d601", manifestImage);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
checks: [
|
||||
{ name: "microservice:code-queue-egress-gateway-health", ok: true },
|
||||
{ name: "microservice:code-queue-postgres-route-health", ok: true },
|
||||
{ name: "code-queue:containerd-image-preflight", ok: true },
|
||||
{ name: "code-queue:stale-active-reconcile", ok: true },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
process.stdout.write(`${JSON.stringify(runIssue3Regression(), null, 2)}\n`);
|
||||
}
|
||||
@@ -236,6 +236,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
|
||||
fileItem("src/components/microservices/code-queue-mgr/src/index.ts"),
|
||||
fileItem("src/components/microservices/code-queue-mgr/src/prompt-observation.ts"),
|
||||
fileItem("scripts/src/deploy.ts"),
|
||||
fileItem("scripts/code-queue-issue3-regression-test.ts"),
|
||||
fileItem("scripts/src/ci.ts"),
|
||||
fileItem("scripts/src/e2e.ts"),
|
||||
fileItem("scripts/code-queue-prompt-observation-test.ts"),
|
||||
@@ -248,9 +249,11 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
|
||||
if (options.scriptsTypecheck) {
|
||||
items.push(commandItem("typescript:scripts", ["bunx", "tsc", "-p", "scripts/tsconfig.json", "--noEmit", "--pretty", "false"], 120_000));
|
||||
items.push(commandItem("code-queue:prompt-observation-contract", ["bun", "scripts/code-queue-prompt-observation-test.ts"], 30_000));
|
||||
items.push(commandItem("code-queue:issue3-diagnostics-and-image-preflight", ["bun", "scripts/code-queue-issue3-regression-test.ts"], 30_000));
|
||||
} else {
|
||||
items.push(skippedItem("typescript:scripts", "scripts TypeScript typecheck is opt-in", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("code-queue:prompt-observation-contract", "prompt observation contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("code-queue:issue3-diagnostics-and-image-preflight", "Code Queue issue #3 regression fixtures are opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
}
|
||||
if (options.logs) {
|
||||
items.push(unifiedLogRotationItem());
|
||||
|
||||
+103
-1
@@ -853,6 +853,40 @@ function buildImageTag(service: UniDeskMicroserviceConfig): string {
|
||||
return `unidesk-${service.id}:${service.providerId.toLowerCase()}`;
|
||||
}
|
||||
|
||||
export function codeQueueContainerdImagePreflight(imageListText: string, expectedImage: string): { ok: boolean; expectedImage: string; matchedLine: string | null; error: string | null } {
|
||||
const matchedLine = imageListText
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line.length > 0 && line.includes(expectedImage)) ?? null;
|
||||
return matchedLine === null
|
||||
? {
|
||||
ok: false,
|
||||
expectedImage,
|
||||
matchedLine: null,
|
||||
error: `native k3s containerd is missing required image tag: ${expectedImage}`,
|
||||
}
|
||||
: { ok: true, expectedImage, matchedLine, error: null };
|
||||
}
|
||||
|
||||
export function codeQueueManifestImagePreflight(manifestText: string, expectedImage: string): { ok: boolean; expectedImage: string; objects: Array<{ kind: string; name: string; images: string[]; ok: boolean }>; errors: string[] } {
|
||||
const requiredObjectNames = new Set(["code-queue", "code-queue-read", "code-queue-write", "d601-provider-egress-proxy", "d601-tcp-egress-gateway"]);
|
||||
const objects: Array<{ kind: string; name: string; images: string[]; ok: boolean }> = [];
|
||||
for (const documentText of manifestText.split(/^---\s*$/mu)) {
|
||||
const kind = documentText.match(/^kind:\s*(\S+)\s*$/mu)?.[1] ?? "";
|
||||
if (kind !== "Deployment") continue;
|
||||
const deploymentName = documentText.match(/^metadata:\s*$[\s\S]*?^\s{2}name:\s*([A-Za-z0-9_.-]+)\s*$/mu)?.[1] ?? "";
|
||||
if (!requiredObjectNames.has(deploymentName)) continue;
|
||||
const images = Array.from(documentText.matchAll(/^\s*image:\s*(?:"([^"]+)"|([^\s#]+))/gmu)).map((match) => match[1] ?? match[2] ?? "");
|
||||
objects.push({ kind, name: deploymentName, images, ok: images.length > 0 && images.every((image) => image === expectedImage) });
|
||||
}
|
||||
const presentNames = new Set(objects.map((object) => object.name));
|
||||
const errors = [
|
||||
...Array.from(requiredObjectNames).filter((name) => !presentNames.has(name)).map((name) => `required deployment missing from manifest: ${name}`),
|
||||
...objects.filter((object) => !object.ok).map((object) => `deployment ${object.name} must use only ${expectedImage}; found ${object.images.join(",") || "<none>"}`),
|
||||
];
|
||||
return { ok: errors.length === 0, expectedImage, objects, errors };
|
||||
}
|
||||
|
||||
function directComposeFile(service: UniDeskMicroserviceConfig): string {
|
||||
return targetIsMain(service)
|
||||
? rootPath("docker-compose.yml")
|
||||
@@ -1554,17 +1588,54 @@ function ensureNativeK3sScript(): string {
|
||||
function importK3sImageScript(service: UniDeskMicroserviceConfig): string {
|
||||
const image = buildImageTag(service);
|
||||
const archive = `/tmp/unidesk-${safeId(service.id)}-k3s-image.tar`;
|
||||
const manifest = `${targetWorkDir(service)}/${k8sManifestPath(service)}`;
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
...rootAccessPrelude(),
|
||||
`image=${shellQuote(image)}`,
|
||||
`archive=${shellQuote(archive)}`,
|
||||
`manifest=${shellQuote(manifest)}`,
|
||||
"docker image inspect \"$image\" >/dev/null",
|
||||
"if [ -f \"$manifest\" ]; then",
|
||||
" bad_manifest_images=$(python3 - \"$manifest\" \"$image\" <<'PY'",
|
||||
"import re, sys",
|
||||
"path, expected = sys.argv[1], sys.argv[2]",
|
||||
"required = {'code-queue','code-queue-read','code-queue-write','d601-provider-egress-proxy','d601-tcp-egress-gateway'}",
|
||||
"text = open(path, encoding='utf-8').read()",
|
||||
"bad = []",
|
||||
"seen = set()",
|
||||
"for doc in re.split(r'(?m)^---\\s*$', text):",
|
||||
" if not re.search(r'(?m)^kind:\\s*Deployment\\s*$', doc):",
|
||||
" continue",
|
||||
" match = re.search(r'(?ms)^metadata:\\s*$.*?^ name:\\s*([A-Za-z0-9_.-]+)\\s*$', doc)",
|
||||
" name = match.group(1) if match else ''",
|
||||
" if name not in required:",
|
||||
" continue",
|
||||
" seen.add(name)",
|
||||
" images = re.findall(r'(?m)^\\s*image:\\s*\"?([^\"\\s#]+)\"?', doc)",
|
||||
" if not images or any(image != expected for image in images):",
|
||||
" found = ','.join(images) if images else '<none>'",
|
||||
" bad.append(f'{name}:{found}')",
|
||||
"missing = sorted(required - seen)",
|
||||
"bad.extend(f'{name}:<missing>' for name in missing)",
|
||||
"print('\\n'.join(bad))",
|
||||
"PY",
|
||||
" )",
|
||||
" if [ -n \"$bad_manifest_images\" ]; then",
|
||||
" printf 'code_queue_manifest_image_preflight_failed image=%s\\n%s\\n' \"$image\" \"$bad_manifest_images\" >&2",
|
||||
" exit 1",
|
||||
" fi",
|
||||
" echo code_queue_manifest_image_preflight=ok image=$image",
|
||||
"fi",
|
||||
"rm -f \"$archive\"",
|
||||
"docker save \"$image\" -o \"$archive\"",
|
||||
`root_exec ctr --address ${shellQuote(nativeK3sCtrAddress)} -n k8s.io images import "$archive"`,
|
||||
"rm -f \"$archive\"",
|
||||
`root_exec ctr --address ${shellQuote(nativeK3sCtrAddress)} -n k8s.io images ls | grep -F "$image" || true`,
|
||||
`if ! root_exec ctr --address ${shellQuote(nativeK3sCtrAddress)} -n k8s.io images ls | grep -F "$image"; then`,
|
||||
" printf 'native_k3s_containerd_image_missing=%s\\n' \"$image\" >&2",
|
||||
" exit 1",
|
||||
"fi",
|
||||
"printf 'native_k3s_containerd_image_present=%s\\n' \"$image\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -1621,6 +1692,35 @@ function applyK8sScript(service: UniDeskMicroserviceConfig): string {
|
||||
].filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
function verifyK8sImagesScript(service: UniDeskMicroserviceConfig): string {
|
||||
const namespace = k8sNamespaceForService(service);
|
||||
const image = buildImageTag(service);
|
||||
const deployments = k8sDeploymentsForService(service);
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`image=${shellQuote(image)}`,
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`deployments=(${deployments.map(shellQuote).join(" ")})`,
|
||||
"for deployment in \"${deployments[@]}\"; do",
|
||||
` actual=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n "$namespace" get deployment "$deployment" -o jsonpath='{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}')`,
|
||||
" if [ -z \"$actual\" ]; then",
|
||||
" echo \"deployment_image_missing=$deployment\" >&2",
|
||||
" exit 1",
|
||||
" fi",
|
||||
" while IFS= read -r actual_image; do",
|
||||
" [ -z \"$actual_image\" ] && continue",
|
||||
" if [ \"$actual_image\" != \"$image\" ]; then",
|
||||
" printf 'deployment_image_mismatch deployment=%s expected=%s actual=%s\\n' \"$deployment\" \"$image\" \"$actual_image\" >&2",
|
||||
" exit 1",
|
||||
" fi",
|
||||
" done <<EOF",
|
||||
"$actual",
|
||||
"EOF",
|
||||
" printf 'deployment_image_ok deployment=%s image=%s\\n' \"$deployment\" \"$image\"",
|
||||
"done",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function stampK8sScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, resolvedCommit: string): string {
|
||||
const deployments = k8sDeploymentsForService(service).map((name) => `deployment/${name}`);
|
||||
const namespace = k8sNamespaceForService(service);
|
||||
@@ -2223,6 +2323,8 @@ async function applyOneService(config: UniDeskConfig, service: UniDeskMicroservi
|
||||
if (!pushStep(steps, imageImport)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const apply = await step(config, service, "kubectl-apply", applyK8sScript(service), targetWorkDir(service), 60_000, false);
|
||||
if (!pushStep(steps, apply)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const k8sImages = await step(config, service, "k8s-image-preflight", verifyK8sImagesScript(service), targetWorkDir(service), 60_000, false);
|
||||
if (!pushStep(steps, k8sImages)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const stamp = await step(config, service, "stamp-deploy-commit", stampK8sScript(service, desired, resolvedCommit), targetWorkDir(service), 60_000, false);
|
||||
if (!pushStep(steps, stamp)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const rollout = await step(config, service, "rollout", rolloutK8sScript(service), targetWorkDir(service), 240_000, true);
|
||||
|
||||
Reference in New Issue
Block a user