fix: add backend-core publish preflight

2026-05-21 09:39:25 +00:00
parent 1a722412cd
commit 43ce0ee051
6 changed files with 449 additions and 63 deletions
@@ -56,8 +56,17 @@ export interface ArtifactRegistryReadonlyProbe {
  timeoutMs: number;
  healthMode: boolean;
  options: ArtifactRegistryOptions;
+  remoteCommandShape: string;
 }

+export type ArtifactRegistryFailureClassification =
+  | "local-docker-required"
+  | "provider-ssh-command-missing"
+  | "ssh-helper-command-shape-incompatible"
+  | "remote-command-timeout"
+  | "registry-not-installed"
+  | "registry-unhealthy";
+
 const defaultOptions: ArtifactRegistryOptions = {
  environment: null,
  providerId: "D601",
@@ -1033,10 +1042,6 @@ container=${shellQuote(options.containerName)}
 expected_image=${shellQuote(options.image)}
 port=${options.port}
 kv readonly true
-kv unit_path "$unit"
-kv compose_path "$compose"
-kv config_path "$config"
-kv storage_path "$storage"
 kv unit_exists "$(bool_file "$unit")"
 kv compose_exists "$(bool_file "$compose")"
 kv config_exists "$(bool_file "$config")"
@@ -1092,9 +1097,6 @@ unit_hash="$(hash_file "$unit")"
 kv config_hash "$config_hash"
 kv compose_hash "$compose_hash"
 kv unit_hash "$unit_hash"
-kv expected_config_hash ${shellQuote(hashes[bundle.paths.config] ?? "")}
-kv expected_compose_hash ${shellQuote(hashes[bundle.paths.compose] ?? "")}
-kv expected_unit_hash ${shellQuote(hashes[bundle.paths.unit] ?? "")}
 kv config_hash_matches "$([ -n "$config_hash" ] && [ "$config_hash" = ${shellQuote(hashes[bundle.paths.config] ?? "")} ] && printf true || printf false)"
 kv compose_hash_matches "$([ -n "$compose_hash" ] && [ "$compose_hash" = ${shellQuote(hashes[bundle.paths.compose] ?? "")} ] && printf true || printf false)"
 kv unit_hash_matches "$([ -n "$unit_hash" ] && [ "$unit_hash" = ${shellQuote(hashes[bundle.paths.unit] ?? "")} ] && printf true || printf false)"
@@ -1115,6 +1117,56 @@ function asBool(value: string | undefined): boolean {
  return value === "true";
 }

+function registryRecommendedAction(classification: ArtifactRegistryFailureClassification | null): string {
+  if (classification === null) return "none";
+  if (classification === "local-docker-required") return "run the read-only check through --main-server-ip <host> or from a main-server CLI with backend-core available";
+  if (classification === "provider-ssh-command-missing") return "restore D601 provider-gateway host.ssh dispatch/capability before retrying artifact registry health";
+  if (classification === "ssh-helper-command-shape-incompatible") return "upgrade the CLI/control-plane host.ssh helper shape so it can run bash -lc readonly probes";
+  if (classification === "remote-command-timeout") return "rerun artifact-registry health and inspect D601 host.ssh latency if the timeout repeats";
+  if (classification === "registry-not-installed") return "install the D601 artifact registry through the controlled artifact-registry install path, then rerun health";
+  return "restore the D601 artifact registry service/container/API until registry health passes";
+}
+
+function readonlyRemoteCommandShape(action: "status" | "health", options: ArtifactRegistryOptions): string {
+  return `host.ssh provider=${options.providerId} mode=exec argv=bash -lc <artifact-registry-${action}-readonly-script> timeoutMs=${options.timeoutMs}`;
+}
+
+function classifyProviderSshCommandFailure(command: CommandResult): ArtifactRegistryFailureClassification {
+  const output = `${command.stderr}\n${command.stdout}`.toLowerCase();
+  if (command.timedOut || output.includes("timed out") || output.includes("timeout")) return "remote-command-timeout";
+  if (output.includes("no such container: unidesk-backend-core") || output.includes("no such container: unidesk-database") || output.includes("backend-core bridge unavailable")) {
+    return "local-docker-required";
+  }
+  if (output.includes("provider does not declare host.ssh") || output.includes("provider is offline") || output.includes("host.ssh capability")) {
+    return "provider-ssh-command-missing";
+  }
+  if (output.includes("remote frontend transport does not support") || output.includes("ssh helper") || output.includes("unsupported command") || output.includes("bad substitution") || output.includes("command is too long")) {
+    return "ssh-helper-command-shape-incompatible";
+  }
+  if (command.exitCode === 126 || command.exitCode === 127 || output.includes("command not found") || output.includes("exec format error")) {
+    return "ssh-helper-command-shape-incompatible";
+  }
+  return "provider-ssh-command-missing";
+}
+
+function providerSshCommandFailureScopes(classification: ArtifactRegistryFailureClassification): string[] {
+  const scopes = ["provider-ssh-command"];
+  if (classification === "remote-command-timeout") scopes.push("remote-command-timeout");
+  if (classification === "local-docker-required") scopes.push("local-docker-control-plane");
+  if (classification === "ssh-helper-command-shape-incompatible") scopes.push("ssh-helper-command-shape");
+  return scopes;
+}
+
+function registryHealthFailureClassification(checks: Record<string, boolean>, failedScopes: string[]): ArtifactRegistryFailureClassification | null {
+  if (failedScopes.length === 0) return null;
+  const installed = checks.unitExists === true
+    || checks.composeExists === true
+    || checks.configExists === true
+    || checks.storageExists === true
+    || checks.containerRunning === true;
+  return installed ? "registry-unhealthy" : "registry-not-installed";
+}
+
 function registryHealthDecision(checks: Record<string, boolean>, commandOk: boolean): Record<string, unknown> {
  const scopeChecks: Array<[string, boolean]> = [
    ["systemd", checks.systemctlAvailable === true && checks.unitExists === true && checks.unitActive === true],
@@ -1129,11 +1181,14 @@ function registryHealthDecision(checks: Record<string, boolean>, commandOk: bool
  const healthyScopes = scopeChecks.filter(([, ok]) => ok).map(([scope]) => scope);
  const failedScopes = scopeChecks.filter(([, ok]) => !ok).map(([scope]) => scope);
  const runtimeApiHealthy = checks.containerRunning === true && checks.loopbackOnly === true && checks.v2Ok === true;
+  const failureClassification = commandOk ? registryHealthFailureClassification(checks, failedScopes) : "provider-ssh-command-missing";
  return {
    decision: commandOk && failedScopes.length === 0 ? "healthy" : commandOk || runtimeApiHealthy ? "service-degraded" : "retryable-transient",
    retryable: true,
+    failureClassification,
    healthyScopes,
    failedScopes,
+    recommendedAction: registryRecommendedAction(failureClassification),
    runtimeApiHealthy,
  };
 }
@@ -1273,8 +1328,40 @@ function runRemoteScript(options: ArtifactRegistryOptions, script: string, timeo
  return runCommand(command, repoRoot, { timeoutMs });
 }

+function readonlyCommandFailureResult(
+  options: ArtifactRegistryOptions,
+  command: CommandResult,
+  action: "status" | "health",
+): Record<string, unknown> {
+  const bundle = renderBundle(options);
+  const failureClassification = classifyProviderSshCommandFailure(command);
+  return {
+    ok: false,
+    readonly: true,
+    installed: false,
+    healthy: false,
+    decision: failureClassification === "local-docker-required" ? "infra-blocked" : "retryable-transient",
+    retryable: failureClassification !== "ssh-helper-command-shape-incompatible",
+    failureClassification,
+    recommendedAction: registryRecommendedAction(failureClassification),
+    remoteCommandShape: readonlyRemoteCommandShape(action, options),
+    healthyScopes: [],
+    failedScopes: providerSshCommandFailureScopes(failureClassification),
+    runtimeApiHealthy: false,
+    checks: {},
+    expected: {
+      endpoint: `http://${options.host}:${options.port}`,
+      image: options.image,
+      paths: bundle.paths,
+    },
+    command: artifactRegistryCommandTail(command),
+  };
+}
+
 function statusFromValues(options: ArtifactRegistryOptions, values: Record<string, string>, command: CommandResult, healthMode: boolean): Record<string, unknown> {
  const commandOk = command.exitCode === 0 && !command.timedOut;
+  const bundle = renderBundle(options);
+  const hashes = Object.fromEntries(bundle.files.map((item) => [item.path, item.sha256]));
  const checks = {
    systemctlAvailable: asBool(values.systemctl_available),
    dockerAvailable: asBool(values.docker_available),
@@ -1316,12 +1403,13 @@ function statusFromValues(options: ArtifactRegistryOptions, values: Record<strin
    installed,
    healthy,
    ...decision,
+    remoteCommandShape: readonlyRemoteCommandShape(healthMode ? "health" : "status", options),
    checks,
    observed: {
-      unit: { path: values.unit_path, active: values.unit_active, enabled: values.unit_enabled },
-      compose: { path: values.compose_path, sha256: values.compose_hash },
-      config: { path: values.config_path, sha256: values.config_hash },
-      storage: { path: values.storage_path },
+      unit: { path: bundle.paths.unit, active: values.unit_active, enabled: values.unit_enabled },
+      compose: { path: bundle.paths.compose, sha256: values.compose_hash ?? null },
+      config: { path: bundle.paths.config, sha256: values.config_hash ?? null },
+      storage: { path: bundle.paths.storage },
      container: {
        name: options.containerName,
        running: values.container_running,
@@ -1337,9 +1425,9 @@ function statusFromValues(options: ArtifactRegistryOptions, values: Record<strin
      registryApi: { v2HttpCode: values.v2_http_code },
    },
    expected: {
-      unitHash: values.expected_unit_hash,
-      composeHash: values.expected_compose_hash,
-      configHash: values.expected_config_hash,
+      unitHash: hashes[bundle.paths.unit] ?? "",
+      composeHash: hashes[bundle.paths.compose] ?? "",
+      configHash: hashes[bundle.paths.config] ?? "",
      image: options.image,
      endpoint: `http://${options.host}:${options.port}`,
    },
@@ -1352,24 +1440,7 @@ function runReadonlyStatus(options: ArtifactRegistryOptions, healthMode: boolean
  const script = statusScript(options, bundle);
  const result = runRemoteScript(options, script);
  if (result.exitCode !== 0 || result.timedOut) {
-    return {
-      ok: false,
-      readonly: true,
-      installed: false,
-      healthy: false,
-      decision: "retryable-transient",
-      retryable: true,
-      healthyScopes: [],
-      failedScopes: ["provider-ssh-command"],
-      runtimeApiHealthy: false,
-      checks: {},
-      expected: {
-        endpoint: `http://${options.host}:${options.port}`,
-        image: options.image,
-        paths: bundle.paths,
-      },
-      command: artifactRegistryCommandTail(result),
-    };
+    return readonlyCommandFailureResult(options, result, healthMode ? "health" : "status");
  }
  return statusFromValues(options, parseKeyValueOutput(result.stdout), result, healthMode);
 }
@@ -1384,6 +1455,7 @@ export function buildArtifactRegistryReadonlyProbe(action: "status" | "health",
    timeoutMs: options.timeoutMs,
    healthMode,
    options,
+    remoteCommandShape: readonlyRemoteCommandShape(action, options),
  };
 }

@@ -1392,25 +1464,7 @@ export function artifactRegistryReadonlyResultFromCommand(
  command: CommandResult,
 ): Record<string, unknown> {
  if (command.exitCode !== 0 || command.timedOut) {
-    const bundle = renderBundle(probe.options);
-    return {
-      ok: false,
-      readonly: true,
-      installed: false,
-      healthy: false,
-      decision: "retryable-transient",
-      retryable: true,
-      healthyScopes: [],
-      failedScopes: ["provider-ssh-command"],
-      runtimeApiHealthy: false,
-      checks: {},
-      expected: {
-        endpoint: `http://${probe.options.host}:${probe.options.port}`,
-        image: probe.options.image,
-        paths: bundle.paths,
-      },
-      command: artifactRegistryCommandTail(command),
-    };
+    return readonlyCommandFailureResult(probe.options, command, probe.action);
  }
  return statusFromValues(probe.options, parseKeyValueOutput(command.stdout), command, probe.healthMode);
 }
@@ -87,7 +87,7 @@ interface DispatchResult {
  raw: unknown;
 }

-type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required";
+type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required" | "registry-not-installed" | "registry-unhealthy" | "remote-command-timeout" | "ssh-helper-command-shape-incompatible";
 type PublishPreflightControlChannel = "backend-core" | "database" | "provider" | "registry";
 type PublishPreflightDetailedChannel = "backend-core-api" | "provider-dispatch" | "provider-host-ssh" | "database" | "artifact-registry";

@@ -120,6 +120,8 @@ interface PublishPreflight {
  channels: PublishPreflightChannelProbe[];
  registry: unknown;
  controlPlane: Record<string, unknown>;
+  recommendedAction: string;
+  remoteCommandShape: string;
  next: string[];
  boundary: string;
 }
@@ -435,10 +437,36 @@ function classifyPublishPreflightFailure(
  if (kind !== "local-docker" && !responseOk(overview)) return "remote-proxy-missing";
  if (!sshProbe.ok) return "provider-unreachable";
  const registryRecord = asRecord(registry);
-  if (registryRecord?.ok === false) return "provider-unreachable";
+  if (registryRecord?.failureClassification === "local-docker-required") return "local-docker-required";
+  if (registryRecord?.failureClassification === "provider-ssh-command-missing") return "provider-unreachable";
+  if (registryRecord?.failureClassification === "ssh-helper-command-shape-incompatible") return "ssh-helper-command-shape-incompatible";
+  if (registryRecord?.failureClassification === "remote-command-timeout") return "remote-command-timeout";
+  if (registryRecord?.failureClassification === "registry-not-installed") return "registry-not-installed";
+  if (registryRecord?.failureClassification === "registry-unhealthy") return "registry-unhealthy";
+  if (registryRecord?.ok === false) return kind === "local-docker" ? "provider-unreachable" : "registry-unhealthy";
  return null;
 }

+function recommendedPublishPreflightAction(
+  failureClassification: PublishPreflightFailureClassification | null,
+  registry: unknown,
+  missingControlChannels: PublishPreflightControlChannel[],
+): string {
+  if (failureClassification === null && missingControlChannels.length === 0) return "none";
+  const registryRecommendedAction = asString(asRecord(registry)?.recommendedAction);
+  if (registryRecommendedAction.length > 0 && registryRecommendedAction !== "none") return registryRecommendedAction;
+  if (failureClassification === "local-docker-required") return "rerun this read-only preflight through --main-server-ip <host> or from a main-server CLI with backend-core available";
+  if (failureClassification === "auth-missing") return "restore frontend control-plane authentication before rerunning the dry-run preflight";
+  if (failureClassification === "remote-proxy-missing") return "restore frontend to backend-core proxy reachability before rerunning the dry-run preflight";
+  if (failureClassification === "provider-unreachable") return "restore D601 provider-gateway host.ssh reachability before rerunning the dry-run preflight";
+  if (failureClassification === "ssh-helper-command-shape-incompatible") return "upgrade or shorten the host.ssh readonly command shape before rerunning the dry-run preflight";
+  if (failureClassification === "remote-command-timeout") return "rerun artifact-registry health and inspect D601 host.ssh latency if the timeout repeats";
+  if (failureClassification === "registry-not-installed") return "install the D601 artifact registry through the controlled artifact-registry install path, then rerun health";
+  if (failureClassification === "registry-unhealthy") return "restore the D601 artifact registry service/container/API until artifact-registry health passes";
+  if (missingControlChannels.includes("registry")) return "restore or install the D601 artifact registry until artifact-registry health passes";
+  return `restore missing control channel(s): ${missingControlChannels.join(", ") || "unknown"}`;
+}
+
 function publishPreflightControlPlane(
  transport: PublishPreflightTransport,
  failureClassification: PublishPreflightFailureClassification | null,
@@ -462,6 +490,13 @@ function publishPreflightControlPlane(
  };
 }

+function publishPreflightFailedScopes(preflight: PublishPreflight): string[] {
+  if (preflight.ok) return [];
+  const registryScopes = asRecord(preflight.registry)?.failedScopes;
+  if (Array.isArray(registryScopes)) return registryScopes.map(String);
+  return preflight.missingChannels;
+}
+
 function dispatchPreflightFailure(command: string, result: DispatchResult): DispatchResult {
  return {
    ok: false,
@@ -1430,6 +1465,7 @@ async function publishUserServicePreflight(
  const missingControlChannels = controlChannels.filter((item) => !item.ok).map((item) => item.channel);
  const ready = missingChannels.length === 0;
  const failureClassification = ready ? null : classifyPublishPreflightFailure(transport, overview, sshProbe, registry);
+  const recommendedAction = recommendedPublishPreflightAction(failureClassification, registry, missingControlChannels);
  return {
    ok: ready,
    runnerDisposition: ready ? "ready" : "infra-blocked",
@@ -1444,6 +1480,8 @@ async function publishUserServicePreflight(
    channels,
    registry,
    controlPlane: publishPreflightControlPlane(transport, failureClassification),
+    recommendedAction,
+    remoteCommandShape: registryProbe.remoteCommandShape,
    next: ready
      ? [
        `bun scripts/cli.ts ci publish-user-service --service ${options.serviceId} --commit ${options.commit} --wait-ms 1200000`,
@@ -1634,6 +1672,9 @@ async function publishUserServiceArtifact(config: UniDeskConfig, options: CiPubl
      controlChannels: preflight.controlChannels,
      channels: preflight.channels,
      failureClassification: preflight.failureClassification,
+      failedScopes: publishPreflightFailedScopes(preflight),
+      recommendedAction: preflight.recommendedAction,
+      remoteCommandShape: preflight.remoteCommandShape,
      controlPlane: preflight.controlPlane,
      registry: preflight.registry,
      sourceHostPath: options.sourceHostPath,
@@ -1761,6 +1802,9 @@ export async function runCiPublishUserServiceDryRunPreflight(
    controlChannels: preflight.controlChannels,
    channels: preflight.channels,
    failureClassification: preflight.failureClassification,
+    failedScopes: publishPreflightFailedScopes(preflight),
+    recommendedAction: preflight.recommendedAction,
+    remoteCommandShape: preflight.remoteCommandShape,
    controlPlane: preflight.controlPlane,
    registry: preflight.registry,
    sourceHostPath: options.sourceHostPath,