diff --git a/src/components/provider-gateway/src/index.ts b/src/components/provider-gateway/src/index.ts index 3023ce57..56538813 100644 --- a/src/components/provider-gateway/src/index.ts +++ b/src/components/provider-gateway/src/index.ts @@ -90,6 +90,7 @@ let upgradeSleepUntil = 0; let upgradeSleepTimer: ReturnType | null = null; let selfContainerGuard: ProviderGatewayContainerGuardState | null = null; let selfContainerGuardLastCheckMs = 0; +const reportedProviderUpgradeCompletionTasks = new Set(); interface MicroserviceHttpCacheEntry { createdAt: number; @@ -1017,12 +1018,74 @@ function sendRegister(): void { } function sendHeartbeat(): void { + const labels = currentLabels(); sendJson({ type: "heartbeat", providerId: config.providerId, - labels: currentLabels(), + labels, at: new Date().toISOString(), }); + reportPendingProviderUpgradeCompletion(labels); +} + +function providerUpgradeCompletionPath(): string { + return `${config.upgradeWorkspacePath.replace(/\/+$/, "")}/.state/provider-upgrade-final-${safeProviderSlug(config.providerId)}.json`; +} + +function readPendingProviderUpgradeCompletion(): Record | null { + const path = providerUpgradeCompletionPath(); + if (!existsSync(path)) return null; + try { + const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null; + return parsed as Record; + } catch (error) { + logger("warn", "provider_upgrade_completion_file_invalid", { path, error: error instanceof Error ? error.message : String(error) }); + return null; + } +} + +function reportPendingProviderUpgradeCompletion(labels: ProviderLabels): void { + const completion = readPendingProviderUpgradeCompletion(); + if (completion === null) return; + const taskId = typeof completion.taskId === "string" ? completion.taskId : ""; + if (!/^task_[A-Za-z0-9_.-]+$/u.test(taskId)) return; + if (reportedProviderUpgradeCompletionTasks.has(taskId)) return; + if (completion.providerId !== config.providerId) return; + const finalContainerId = typeof completion.finalContainerId === "string" ? completion.finalContainerId : ""; + const currentContainerId = typeof labels.providerGatewayContainerId === "string" ? labels.providerGatewayContainerId : ""; + if (finalContainerId.length > 0 && currentContainerId.length > 0 && finalContainerId !== currentContainerId) { + logger("warn", "provider_upgrade_completion_container_mismatch", { taskId, finalContainerId, currentContainerId }); + return; + } + const heartbeatTimestamp = new Date().toISOString(); + sendTaskStatus(taskId, "succeeded", "provider upgrade final promoted container verified by provider heartbeat", { + ok: true, + mode: "schedule", + taskId, + providerId: config.providerId, + providerGatewayVersion: gatewayMetadata.version, + targetProviderGatewayVersion: completion.version ?? gatewayMetadata.version, + candidateValidationWasTerminal: false, + finalPromotedContainer: { + containerId: currentContainerId || finalContainerId || null, + containerName: labels.providerGatewayContainerName ?? completion.finalContainerName ?? null, + version: gatewayMetadata.version, + restartPolicy: labels.providerGatewayRestartPolicy ?? completion.restartPolicy ?? null, + pidMode: labels.providerGatewayPidMode ?? completion.pidMode ?? null, + heartbeatTimestamp, + }, + heartbeat: { + providerGatewayRuntimeGuardOk: labels.providerGatewayRuntimeGuardOk ?? null, + providerGatewayRuntimeGuardError: labels.providerGatewayRuntimeGuardError ?? null, + providerGatewayRestartPolicyOk: labels.providerGatewayRestartPolicyOk ?? null, + providerGatewayPidModeOk: labels.providerGatewayPidModeOk ?? null, + providerGatewayStartedAt: labels.providerGatewayStartedAt ?? null, + }, + completionFile: providerUpgradeCompletionPath(), + }); + reportedProviderUpgradeCompletionTasks.add(taskId); + logger("info", "provider_upgrade_completion_reported", { taskId, finalContainerId: currentContainerId || finalContainerId || null }); } async function sendDockerStatus(): Promise { @@ -2127,6 +2190,7 @@ function upgradePlan(taskId: string): Record { ]; const updaterName = `unidesk-provider-upgrader-${safeDockerName(taskId)}`; const candidateName = candidateGatewayName(taskId); + const upgradeCompletionPath = `${workspaceRoot}/.state/provider-upgrade-final-${safeProviderSlug(config.providerId)}.json`; const validationNeedleOpen = `"message":"connect_open"`; const validationNeedleAck = `"requestId":"register"`; const validationNeedleOk = `"ok":true`; @@ -2227,6 +2291,7 @@ function upgradePlan(taskId: string): Record { `mount_args=""`, `extra_host_args=""`, `candidate_env_file=${shellQuote(`/tmp/${candidateName}.env`)}`, + `upgrade_completion_file=${shellQuote(upgradeCompletionPath)}`, `network_names=""`, `first_network=""`, `network_arg=""`, @@ -2285,6 +2350,21 @@ function upgradePlan(taskId: string): Record { `final_validated=0`, `while [ "$final_attempt" -lt "${finalValidationAttempts}" ]; do final_logs=$(docker logs "$final_container" 2>&1 || true); final_has_open=0; final_has_ack=0; final_has_ok=0; case "$final_logs" in *${shellQuote(validationNeedleOpen)}*) final_has_open=1;; esac; case "$final_logs" in *${shellQuote(validationNeedleAck)}*) final_has_ack=1;; esac; case "$final_logs" in *${shellQuote(validationNeedleOk)}*) final_has_ok=1;; esac; if [ "$final_has_open" = "1" ] && [ "$final_has_ack" = "1" ] && [ "$final_has_ok" = "1" ]; then final_validated=1; break; fi; final_running=$(docker inspect --format '{{.State.Running}}' "$final_container" 2>/dev/null || true); if [ "$final_running" != "true" ]; then break; fi; final_attempt=$((final_attempt + 1)); sleep 2; done`, `if [ "$final_validated" != "1" ]; then echo "final provider-gateway compose container did not reconnect to backend-core before local guard timeout; backend-core must not treat candidate validation as success" >&2; docker logs "$final_container" >&2 || true; rollback_to_last_known_good "final-promoted-container-reconnect-timeout" || true; rm -f "$candidate_env_file"; exit 1; fi`, + `mkdir -p "$(dirname "$upgrade_completion_file")"`, + `upgrade_completion_tmp="$upgrade_completion_file.tmp.$$"`, + [ + `{`, + ` printf '{"taskId":"%s",' ${shellQuote(taskId)};`, + ` printf '"providerId":"%s",' ${shellQuote(config.providerId)};`, + ` printf '"finalContainerId":"%s",' "$final_container_id";`, + ` printf '"finalContainerName":"%s",' "$final_container";`, + ` printf '"version":"%s",' ${shellQuote(targetGatewayMetadata.version)};`, + ` printf '"restartPolicy":"%s",' "$final_restart";`, + ` printf '"pidMode":"%s",' "$final_pid_mode";`, + ` printf '"writtenAt":"%s"}\\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)";`, + `} > "$upgrade_completion_tmp"`, + ].join(" "), + `mv "$upgrade_completion_tmp" "$upgrade_completion_file"`, `rm -f "$candidate_env_file"`, `echo "candidate provider-gateway validated; final promoted compose container reconnected locally: container=$final_container id=$final_container_id restart=$final_restart pid=$final_pid_mode"`, ].join("\n"); @@ -2381,6 +2461,7 @@ function upgradePlan(taskId: string): Record { finalPromotedContainer: unavailableFinalPromotedContainer, compactBuildContextCandidates, workspaceStateMount: `${config.upgradeHostProjectRoot.replace(/\/+$/, "")}/.state:${workspaceRoot}/.state`, + upgradeCompletionPath, }; }