fix: complete provider upgrade after final heartbeat

src/components/provider-gateway/src/index.ts

@@ -90,6 +90,7 @@ let upgradeSleepUntil = 0;
 let upgradeSleepTimer: ReturnType<typeof setTimeout> | null = null;
 let selfContainerGuard: ProviderGatewayContainerGuardState | null = null;
 let selfContainerGuardLastCheckMs = 0;
+const reportedProviderUpgradeCompletionTasks = new Set<string>();
 
 interface MicroserviceHttpCacheEntry {
   createdAt: number;
@@ -1017,12 +1018,74 @@ function sendRegister(): void {
 }
 
 function sendHeartbeat(): void {
+  const labels = currentLabels();
   sendJson({
     type: "heartbeat",
     providerId: config.providerId,
-    labels: currentLabels(),
+    labels,
     at: new Date().toISOString(),
   });
+  reportPendingProviderUpgradeCompletion(labels);
+}
+
+function providerUpgradeCompletionPath(): string {
+  return `${config.upgradeWorkspacePath.replace(/\/+$/, "")}/.state/provider-upgrade-final-${safeProviderSlug(config.providerId)}.json`;
+}
+
+function readPendingProviderUpgradeCompletion(): Record<string, JsonValue> | null {
+  const path = providerUpgradeCompletionPath();
+  if (!existsSync(path)) return null;
+  try {
+    const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
+    return parsed as Record<string, JsonValue>;
+  } catch (error) {
+    logger("warn", "provider_upgrade_completion_file_invalid", { path, error: error instanceof Error ? error.message : String(error) });
+    return null;
+  }
+}
+
+function reportPendingProviderUpgradeCompletion(labels: ProviderLabels): void {
+  const completion = readPendingProviderUpgradeCompletion();
+  if (completion === null) return;
+  const taskId = typeof completion.taskId === "string" ? completion.taskId : "";
+  if (!/^task_[A-Za-z0-9_.-]+$/u.test(taskId)) return;
+  if (reportedProviderUpgradeCompletionTasks.has(taskId)) return;
+  if (completion.providerId !== config.providerId) return;
+  const finalContainerId = typeof completion.finalContainerId === "string" ? completion.finalContainerId : "";
+  const currentContainerId = typeof labels.providerGatewayContainerId === "string" ? labels.providerGatewayContainerId : "";
+  if (finalContainerId.length > 0 && currentContainerId.length > 0 && finalContainerId !== currentContainerId) {
+    logger("warn", "provider_upgrade_completion_container_mismatch", { taskId, finalContainerId, currentContainerId });
+    return;
+  }
+  const heartbeatTimestamp = new Date().toISOString();
+  sendTaskStatus(taskId, "succeeded", "provider upgrade final promoted container verified by provider heartbeat", {
+    ok: true,
+    mode: "schedule",
+    taskId,
+    providerId: config.providerId,
+    providerGatewayVersion: gatewayMetadata.version,
+    targetProviderGatewayVersion: completion.version ?? gatewayMetadata.version,
+    candidateValidationWasTerminal: false,
+    finalPromotedContainer: {
+      containerId: currentContainerId || finalContainerId || null,
+      containerName: labels.providerGatewayContainerName ?? completion.finalContainerName ?? null,
+      version: gatewayMetadata.version,
+      restartPolicy: labels.providerGatewayRestartPolicy ?? completion.restartPolicy ?? null,
+      pidMode: labels.providerGatewayPidMode ?? completion.pidMode ?? null,
+      heartbeatTimestamp,
+    },
+    heartbeat: {
+      providerGatewayRuntimeGuardOk: labels.providerGatewayRuntimeGuardOk ?? null,
+      providerGatewayRuntimeGuardError: labels.providerGatewayRuntimeGuardError ?? null,
+      providerGatewayRestartPolicyOk: labels.providerGatewayRestartPolicyOk ?? null,
+      providerGatewayPidModeOk: labels.providerGatewayPidModeOk ?? null,
+      providerGatewayStartedAt: labels.providerGatewayStartedAt ?? null,
+    },
+    completionFile: providerUpgradeCompletionPath(),
+  });
+  reportedProviderUpgradeCompletionTasks.add(taskId);
+  logger("info", "provider_upgrade_completion_reported", { taskId, finalContainerId: currentContainerId || finalContainerId || null });
 }
 
 async function sendDockerStatus(): Promise<void> {
@@ -2127,6 +2190,7 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
   ];
   const updaterName = `unidesk-provider-upgrader-${safeDockerName(taskId)}`;
   const candidateName = candidateGatewayName(taskId);
+  const upgradeCompletionPath = `${workspaceRoot}/.state/provider-upgrade-final-${safeProviderSlug(config.providerId)}.json`;
   const validationNeedleOpen = `"message":"connect_open"`;
   const validationNeedleAck = `"requestId":"register"`;
   const validationNeedleOk = `"ok":true`;
@@ -2227,6 +2291,7 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
     `mount_args=""`,
     `extra_host_args=""`,
     `candidate_env_file=${shellQuote(`/tmp/${candidateName}.env`)}`,
+    `upgrade_completion_file=${shellQuote(upgradeCompletionPath)}`,
     `network_names=""`,
     `first_network=""`,
     `network_arg=""`,
@@ -2285,6 +2350,21 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
     `final_validated=0`,
     `while [ "$final_attempt" -lt "${finalValidationAttempts}" ]; do final_logs=$(docker logs "$final_container" 2>&1 || true); final_has_open=0; final_has_ack=0; final_has_ok=0; case "$final_logs" in *${shellQuote(validationNeedleOpen)}*) final_has_open=1;; esac; case "$final_logs" in *${shellQuote(validationNeedleAck)}*) final_has_ack=1;; esac; case "$final_logs" in *${shellQuote(validationNeedleOk)}*) final_has_ok=1;; esac; if [ "$final_has_open" = "1" ] && [ "$final_has_ack" = "1" ] && [ "$final_has_ok" = "1" ]; then final_validated=1; break; fi; final_running=$(docker inspect --format '{{.State.Running}}' "$final_container" 2>/dev/null || true); if [ "$final_running" != "true" ]; then break; fi; final_attempt=$((final_attempt + 1)); sleep 2; done`,
     `if [ "$final_validated" != "1" ]; then echo "final provider-gateway compose container did not reconnect to backend-core before local guard timeout; backend-core must not treat candidate validation as success" >&2; docker logs "$final_container" >&2 || true; rollback_to_last_known_good "final-promoted-container-reconnect-timeout" || true; rm -f "$candidate_env_file"; exit 1; fi`,
+    `mkdir -p "$(dirname "$upgrade_completion_file")"`,
+    `upgrade_completion_tmp="$upgrade_completion_file.tmp.$$"`,
+    [
+      `{`,
+      `  printf '{"taskId":"%s",' ${shellQuote(taskId)};`,
+      `  printf '"providerId":"%s",' ${shellQuote(config.providerId)};`,
+      `  printf '"finalContainerId":"%s",' "$final_container_id";`,
+      `  printf '"finalContainerName":"%s",' "$final_container";`,
+      `  printf '"version":"%s",' ${shellQuote(targetGatewayMetadata.version)};`,
+      `  printf '"restartPolicy":"%s",' "$final_restart";`,
+      `  printf '"pidMode":"%s",' "$final_pid_mode";`,
+      `  printf '"writtenAt":"%s"}\\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)";`,
+      `} > "$upgrade_completion_tmp"`,
+    ].join(" "),
+    `mv "$upgrade_completion_tmp" "$upgrade_completion_file"`,
     `rm -f "$candidate_env_file"`,
     `echo "candidate provider-gateway validated; final promoted compose container reconnected locally: container=$final_container id=$final_container_id restart=$final_restart pid=$final_pid_mode"`,
   ].join("\n");
@@ -2381,6 +2461,7 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
     finalPromotedContainer: unavailableFinalPromotedContainer,
     compactBuildContextCandidates,
     workspaceStateMount: `${config.upgradeHostProjectRoot.replace(/\/+$/, "")}/.state:${workspaceRoot}/.state`,
+    upgradeCompletionPath,
   };
 }