329 lines
15 KiB
TypeScript
329 lines
15 KiB
TypeScript
import {
|
|
artifactRegistryReadonlyAutoRemotePlan,
|
|
artifactRegistryReadonlyResultFromCommand,
|
|
buildArtifactRegistryReadonlyProbe,
|
|
parseArtifactRegistryOptions,
|
|
runArtifactRegistryCommand,
|
|
} from "./src/artifact-registry";
|
|
import type { CommandResult } from "./src/command";
|
|
|
|
type JsonRecord = Record<string, unknown>;
|
|
|
|
function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
|
|
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
|
|
}
|
|
|
|
function asRecord(value: unknown, label: string): JsonRecord {
|
|
assertCondition(typeof value === "object" && value !== null && !Array.isArray(value), `${label} must be an object`, { value });
|
|
return value as JsonRecord;
|
|
}
|
|
|
|
function asStringArray(value: unknown, label: string): string[] {
|
|
assertCondition(Array.isArray(value), `${label} must be an array`, { value });
|
|
return (value as unknown[]).map(String);
|
|
}
|
|
|
|
function command(overrides: Partial<CommandResult>): CommandResult {
|
|
return {
|
|
command: ["frontend", "/api/dispatch", "D601", "host.ssh", "health"],
|
|
cwd: ".",
|
|
exitCode: 0,
|
|
stdout: "",
|
|
stderr: "",
|
|
signal: null,
|
|
timedOut: false,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
const options = parseArtifactRegistryOptions(["--provider-id", "D601"]);
|
|
const probe = buildArtifactRegistryReadonlyProbe("health", options);
|
|
assertCondition(Buffer.byteLength(probe.script, "utf8") < 4000, "readonly registry probe script must fit provider-gateway host.ssh command length limit", {
|
|
bytes: Buffer.byteLength(probe.script, "utf8"),
|
|
remoteCommandShape: probe.remoteCommandShape,
|
|
});
|
|
const hashOutputIndex = probe.script.indexOf('kv config_hash "$config_hash"');
|
|
const matchOutputIndex = probe.script.indexOf("kv config_hash_matches");
|
|
assertCondition(matchOutputIndex > 0 && hashOutputIndex > matchOutputIndex, "readonly registry probe must emit match booleans before long hash values", {
|
|
hashOutputIndex,
|
|
matchOutputIndex,
|
|
});
|
|
|
|
const missing = asRecord(artifactRegistryReadonlyResultFromCommand(probe, command({
|
|
exitCode: 1,
|
|
stderr: "provider does not declare host.ssh capability: D601\n",
|
|
})), "missing provider ssh result");
|
|
assertCondition(missing.ok === false, "missing provider ssh command should fail", missing);
|
|
assertCondition(missing.failureClassification === "provider-ssh-command-missing", "missing provider ssh command classification mismatch", missing);
|
|
assertCondition(asStringArray(missing.failedScopes, "missing.failedScopes").includes("provider-ssh-command"), "missing provider ssh command scope should be reported", missing);
|
|
assertCondition(typeof missing.recommendedAction === "string" && missing.recommendedAction.length > 0, "missing provider ssh command should include recommendedAction", missing);
|
|
assertCondition(typeof missing.remoteCommandShape === "string" && missing.remoteCommandShape.includes("bash -lc"), "missing provider ssh command should include remoteCommandShape", missing);
|
|
|
|
const commandShape = asRecord(artifactRegistryReadonlyResultFromCommand(probe, command({
|
|
exitCode: 1,
|
|
stderr: "Error: host SSH command is too long: 4039 bytes\n",
|
|
})), "command shape result");
|
|
assertCondition(commandShape.ok === false, "oversized host.ssh command should fail", commandShape);
|
|
assertCondition(commandShape.failureClassification === "ssh-helper-command-shape-incompatible", "oversized host.ssh command classification mismatch", commandShape);
|
|
assertCondition(asStringArray(commandShape.failedScopes, "commandShape.failedScopes").includes("ssh-helper-command-shape"), "oversized host.ssh command scope should be reported", commandShape);
|
|
|
|
const timeout = asRecord(artifactRegistryReadonlyResultFromCommand(probe, command({
|
|
exitCode: null,
|
|
stderr: "host.ssh task task_contract did not finish within 30000ms\n",
|
|
timedOut: true,
|
|
})), "timeout result");
|
|
assertCondition(timeout.ok === false, "remote timeout should fail", timeout);
|
|
assertCondition(timeout.failureClassification === "remote-command-timeout", "remote timeout classification mismatch", timeout);
|
|
assertCondition(asStringArray(timeout.failedScopes, "timeout.failedScopes").includes("remote-command-timeout"), "remote timeout scope should be reported", timeout);
|
|
assertCondition(timeout.retryable === true, "remote timeout should be retryable", timeout);
|
|
|
|
const successStdout = [
|
|
"readonly=true",
|
|
"unit_path=/etc/systemd/system/unidesk-artifact-registry.service",
|
|
"compose_path=/home/ubuntu/.unidesk/artifact-registry/compose.yml",
|
|
"config_path=/home/ubuntu/.unidesk/artifact-registry/config.yml",
|
|
"storage_path=/home/ubuntu/.unidesk/registry-storage",
|
|
"unit_exists=true",
|
|
"compose_exists=true",
|
|
"config_exists=true",
|
|
"storage_exists=true",
|
|
"systemctl_available=true",
|
|
"unit_active=active",
|
|
"unit_enabled=enabled",
|
|
"docker_available=true",
|
|
"container_running=true",
|
|
"container_status=running",
|
|
"container_image=registry:2.8.3",
|
|
"container_restart_policy=unless-stopped",
|
|
"listener_count=1",
|
|
"bad_listener_count=0",
|
|
"loopback_only=true",
|
|
"curl_available=true",
|
|
"v2_http_code=200",
|
|
"config_hash=contract-config",
|
|
"compose_hash=contract-compose",
|
|
"unit_hash=contract-unit",
|
|
"expected_config_hash=contract-config",
|
|
"expected_compose_hash=contract-compose",
|
|
"expected_unit_hash=contract-unit",
|
|
"config_hash_matches=true",
|
|
"compose_hash_matches=true",
|
|
"unit_hash_matches=true",
|
|
"image_matches=true",
|
|
"",
|
|
].join("\n");
|
|
|
|
const success = asRecord(artifactRegistryReadonlyResultFromCommand(probe, command({
|
|
stdout: successStdout,
|
|
})), "success result");
|
|
assertCondition(success.ok === true, "healthy registry command should pass", success);
|
|
assertCondition(success.failureClassification === null, "healthy registry should not have a failure classification", success);
|
|
assertCondition(asStringArray(success.failedScopes, "success.failedScopes").length === 0, "healthy registry should not have failed scopes", success);
|
|
assertCondition(success.recommendedAction === "none", "healthy registry recommendedAction should be none", success);
|
|
assertCondition(success.remoteCommandShape === probe.remoteCommandShape, "healthy registry should echo remote command shape", success);
|
|
|
|
const compactedSuccessStdout = [
|
|
"readonly=true",
|
|
"unit_exists=true",
|
|
"compose_exists=true",
|
|
"config_exists=true",
|
|
"storage_exists=true",
|
|
"systemctl_available=true",
|
|
"unit_active=active",
|
|
"unit_enabled=enabled",
|
|
"docker_available=true",
|
|
"container_running=true",
|
|
"container_status=running",
|
|
"container_image=registry:2.8.3",
|
|
"container_restart_policy=unless-stopped",
|
|
"listener_count=1",
|
|
"bad_listener_count=0",
|
|
"loopback_only=true",
|
|
"curl_available=true",
|
|
"v2_http_code=200",
|
|
"config_hash_matches=true",
|
|
"compose_hash_matches=true",
|
|
"unit_hash_matches=true",
|
|
"image_matches=true",
|
|
"config_hash=contract-config",
|
|
"compose_hash=contract-compose",
|
|
"unit_hash=contract-unit...<truncated:713>",
|
|
"",
|
|
].join("\n");
|
|
const compactedSuccess = asRecord(artifactRegistryReadonlyResultFromCommand(probe, command({
|
|
stdout: compactedSuccessStdout,
|
|
})), "compacted success result");
|
|
assertCondition(compactedSuccess.ok === true, "registry health must survive backend-core task detail compaction when match flags are present", compactedSuccess);
|
|
assertCondition(asStringArray(compactedSuccess.failedScopes, "compactedSuccess.failedScopes").length === 0, "compacted healthy registry should not have failed scopes", compactedSuccess);
|
|
|
|
const driftStdout = [
|
|
"readonly=true",
|
|
"unit_exists=true",
|
|
"compose_exists=true",
|
|
"config_exists=true",
|
|
"storage_exists=true",
|
|
"systemctl_available=true",
|
|
"unit_active=active",
|
|
"unit_enabled=enabled",
|
|
"docker_available=true",
|
|
"container_running=true",
|
|
"container_status=running",
|
|
"container_image=registry:2.8.2",
|
|
"container_restart_policy=unless-stopped",
|
|
"listener_count=1",
|
|
"bad_listener_count=0",
|
|
"loopback_only=true",
|
|
"curl_available=true",
|
|
"v2_http_code=200",
|
|
"config_hash=old-config",
|
|
"compose_hash=old-compose",
|
|
"unit_hash=old-unit",
|
|
"config_hash_matches=false",
|
|
"compose_hash_matches=false",
|
|
"unit_hash_matches=false",
|
|
"image_matches=false",
|
|
"",
|
|
].join("\n");
|
|
|
|
const drift = asRecord(artifactRegistryReadonlyResultFromCommand(probe, command({
|
|
stdout: driftStdout,
|
|
})), "registry drift result");
|
|
assertCondition(drift.ok === false, "health should fail when rendered config/image drift exists", drift);
|
|
assertCondition(drift.failureClassification === "registry-unhealthy", "registry drift should classify as registry-unhealthy", drift);
|
|
const driftScopes = asStringArray(drift.failedScopes, "drift.failedScopes");
|
|
assertCondition(driftScopes.includes("rendered-config"), "registry drift should include rendered-config scope", drift);
|
|
assertCondition(driftScopes.includes("registry-image"), "registry drift should include registry-image scope", drift);
|
|
assertCondition(!driftScopes.includes("control-plane-missing"), "registry drift must not be classified as control-plane missing", drift);
|
|
|
|
async function main(): Promise<void> {
|
|
const localMissingWithNoRemote = await runArtifactRegistryCommand(["health", "--provider-id", "D601"], {
|
|
env: {
|
|
CODE_QUEUE_DEV_CONTAINER_MASTER_HOST: "203.0.113.10",
|
|
CODE_QUEUE_SERVICE_ROLE: "scheduler",
|
|
},
|
|
runRemoteScriptForTest: () => command({
|
|
exitCode: 1,
|
|
stderr: "Error response from daemon: No such container: unidesk-backend-core\n",
|
|
}),
|
|
runCliForTest: () => command({
|
|
exitCode: 1,
|
|
stdout: JSON.stringify({
|
|
ok: false,
|
|
command: "artifact-registry health --provider-id D601",
|
|
data: {
|
|
transport: "frontend",
|
|
readonly: true,
|
|
dispatch: { ok: false, status: 502, body: { ok: false, error: "backend-core proxy unavailable" } },
|
|
wait: null,
|
|
result: {
|
|
ok: false,
|
|
readonly: true,
|
|
installed: false,
|
|
healthy: false,
|
|
decision: "infra-blocked",
|
|
retryable: true,
|
|
runnerDisposition: "infra-blocked",
|
|
failureClassification: "control-plane-missing",
|
|
failedScopes: ["control-plane-missing", "backend-core-api"],
|
|
runtimeApiHealthy: false,
|
|
},
|
|
},
|
|
}),
|
|
}),
|
|
});
|
|
const controlPlaneMissing = asRecord(localMissingWithNoRemote, "control-plane missing result");
|
|
assertCondition(controlPlaneMissing.ok === false, "missing local and remote control planes should fail", controlPlaneMissing);
|
|
assertCondition(controlPlaneMissing.failureClassification === "control-plane-missing", "missing remote control plane should classify control-plane-missing", controlPlaneMissing);
|
|
assertCondition(asStringArray(controlPlaneMissing.failedScopes, "controlPlaneMissing.failedScopes").includes("control-plane-missing"), "control-plane missing scope should be reported", controlPlaneMissing);
|
|
assertCondition(asRecord(controlPlaneMissing.controlPlane, "controlPlane").localBackendCoreMissing === true, "local backend-core absence should remain evidence", controlPlaneMissing);
|
|
|
|
const remoteFallback = await runArtifactRegistryCommand(["health", "--provider-id", "D601"], {
|
|
env: {
|
|
CODE_QUEUE_DEV_CONTAINER_MASTER_HOST: "74.48.78.17",
|
|
},
|
|
runRemoteScriptForTest: () => command({
|
|
exitCode: 1,
|
|
stderr: "Error response from daemon: No such container: unidesk-backend-core\n",
|
|
}),
|
|
runCliForTest: () => command({
|
|
stdout: JSON.stringify({
|
|
ok: true,
|
|
command: "artifact-registry health --provider-id D601",
|
|
data: {
|
|
transport: "frontend",
|
|
readonly: true,
|
|
result: success,
|
|
},
|
|
}),
|
|
}),
|
|
});
|
|
const remoteFallbackRecord = asRecord(remoteFallback, "remote fallback result");
|
|
assertCondition(remoteFallbackRecord.ok === true, "remote fallback should return the remote registry result", remoteFallbackRecord);
|
|
const fallbackControlPlane = asRecord(remoteFallbackRecord.controlPlane, "remote fallback controlPlane");
|
|
assertCondition(fallbackControlPlane.remoteFallbackUsed === true, "remote fallback should be marked", fallbackControlPlane);
|
|
assertCondition(fallbackControlPlane.localBackendCoreMissing === true, "local backend-core absence should remain evidence only", fallbackControlPlane);
|
|
assertCondition(asStringArray(remoteFallbackRecord.failedScopes, "remoteFallback.failedScopes").length === 0, "remote fallback should preserve registry scopes", remoteFallbackRecord);
|
|
|
|
let remoteFirstLocalSshCalls = 0;
|
|
const remoteFirst = await runArtifactRegistryCommand(["health", "--provider-id", "D601"], {
|
|
env: {
|
|
CODE_QUEUE_SERVICE_ROLE: "scheduler",
|
|
CODE_QUEUE_DEV_CONTAINER_MASTER_HOST: "74.48.78.17",
|
|
},
|
|
runRemoteScriptForTest: () => {
|
|
remoteFirstLocalSshCalls += 1;
|
|
return command({ exitCode: 1, stderr: "unexpected local ssh path" });
|
|
},
|
|
runCliForTest: () => command({
|
|
stdout: JSON.stringify({
|
|
ok: true,
|
|
command: "artifact-registry health --provider-id D601",
|
|
data: {
|
|
transport: "frontend",
|
|
readonly: true,
|
|
result: success,
|
|
},
|
|
}),
|
|
}),
|
|
});
|
|
const remoteFirstRecord = asRecord(remoteFirst, "remote first result");
|
|
assertCondition(remoteFirstRecord.ok === true, "runner-like env should succeed through remote frontend first", remoteFirstRecord);
|
|
assertCondition(remoteFirstLocalSshCalls === 0, "runner-like env should not require local backend-core before remote frontend", { remoteFirstLocalSshCalls });
|
|
assertCondition(asRecord(remoteFirstRecord.controlPlane, "remoteFirst.controlPlane").remoteFirst === true, "remote-first controlPlane should be marked", remoteFirstRecord.controlPlane);
|
|
|
|
const autoPlan = artifactRegistryReadonlyAutoRemotePlan("health", options, {
|
|
CODE_QUEUE_SERVICE_ROLE: "scheduler",
|
|
CODE_QUEUE_DEV_CONTAINER_MASTER_HOST: "74.48.78.17",
|
|
});
|
|
assertCondition(autoPlan.enabled === true, "runner-like env should auto-select remote frontend for readonly registry health", autoPlan);
|
|
assertCondition(String(autoPlan.command ?? "").includes("--main-server-ip 74.48.78.17"), "auto remote plan should expose command shape", autoPlan);
|
|
|
|
process.stdout.write(`${JSON.stringify({
|
|
ok: true,
|
|
checks: [
|
|
"provider-ssh-command missing is classified distinctly",
|
|
"oversized host.ssh command shape is classified distinctly",
|
|
"remote host.ssh timeout is classified distinctly",
|
|
"successful registry readonly probe has no failed scopes",
|
|
"runner-like env uses remote frontend before local backend-core",
|
|
"local backend-core absence can fall back to remote frontend control plane",
|
|
"missing local and remote control planes classify as control-plane-missing",
|
|
"rendered-config and registry-image drift classify as registry-unhealthy",
|
|
],
|
|
classifications: {
|
|
missing: missing.failureClassification,
|
|
commandShape: commandShape.failureClassification,
|
|
timeout: timeout.failureClassification,
|
|
success: success.failureClassification,
|
|
drift: drift.failureClassification,
|
|
controlPlaneMissing: controlPlaneMissing.failureClassification,
|
|
remoteFallback: remoteFallbackRecord.failureClassification,
|
|
remoteFirst: remoteFirstRecord.failureClassification,
|
|
},
|
|
}, null, 2)}\n`);
|
|
}
|
|
|
|
if (import.meta.main) {
|
|
await main();
|
|
}
|