fix: enforce final provider upgrade heartbeat
This commit is contained in:
Codex
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@unidesk/provider-gateway",
|
||||
"version": "0.2.26",
|
||||
"version": "0.2.27",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -1671,7 +1671,24 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
const validationNeedleAck = `"requestId":"register"`;
|
||||
const validationNeedleOk = `"ok":true`;
|
||||
const validationAttempts = Math.max(1, Math.ceil(validationTimeoutMs / 2000));
|
||||
const finalPromotedReconnectTimeoutMs = 180_000;
|
||||
const finalPromotedHeartbeatTimeoutMs = 240_000;
|
||||
const finalValidationAttempts = Math.max(1, Math.ceil(finalPromotedReconnectTimeoutMs / 2000));
|
||||
const targetGatewayMetadata = readTargetGatewayMetadata(workspace);
|
||||
const unavailableFinalPromotedContainer = {
|
||||
containerId: null,
|
||||
containerIdUnavailableReason: "mode=plan or schedule has not yet observed backend-core final promoted heartbeat",
|
||||
containerName: null,
|
||||
containerNameUnavailableReason: "mode=plan or schedule has not yet observed backend-core final promoted heartbeat",
|
||||
version: null,
|
||||
versionUnavailableReason: "mode=plan or schedule has not yet observed backend-core final promoted heartbeat",
|
||||
restartPolicy: null,
|
||||
restartPolicyUnavailableReason: "mode=plan or schedule has not yet observed backend-core final promoted heartbeat",
|
||||
pidMode: null,
|
||||
pidModeUnavailableReason: "mode=plan or schedule has not yet observed backend-core final promoted heartbeat",
|
||||
heartbeatTimestamp: null,
|
||||
heartbeatTimestampUnavailableReason: "mode=plan or schedule has not yet observed backend-core final promoted heartbeat",
|
||||
};
|
||||
const compactBuildContextRefreshScript = [
|
||||
`provider_src=${shellQuote(`${workspaceRoot}/src/components/provider-gateway`)}`,
|
||||
`shared_src=${shellQuote(`${workspaceRoot}/src/components/shared`)}`,
|
||||
@@ -1693,6 +1710,33 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
` fi`,
|
||||
`done`,
|
||||
].join("\n");
|
||||
const rollbackToLastKnownGoodScript = [
|
||||
`rollback_to_last_known_good() {`,
|
||||
` reason="$1"`,
|
||||
` echo "rollback_to_last_known_good: $reason" >&2`,
|
||||
` if [ -n "$final_container" ]; then docker rm -f "$final_container" >/dev/null 2>&1 || true; fi`,
|
||||
` docker rm -f ${shellQuote(candidateName)} >/dev/null 2>&1 || true`,
|
||||
` if [ -z "$old_name" ] || [ -z "$old_image" ]; then echo "rollback_to_last_known_good unavailable: old_name=$old_name old_image=$old_image" >&2; return 1; fi`,
|
||||
` docker rm -f "$old_name" >/dev/null 2>&1 || true`,
|
||||
[
|
||||
` docker run -d`,
|
||||
`--name "$old_name"`,
|
||||
`--restart always`,
|
||||
`--pid "\${old_pid_mode:-host}"`,
|
||||
`$network_arg`,
|
||||
`--env-file "$candidate_env_file"`,
|
||||
`--label ${shellQuote(`com.docker.compose.project=${config.upgradeComposeProject}`)}`,
|
||||
`--label ${shellQuote(`com.docker.compose.service=${config.upgradeService}`)}`,
|
||||
`--label ${shellQuote("com.docker.compose.oneoff=False")}`,
|
||||
`--label ${shellQuote(`unidesk.upgrade.rollback=${taskId}`)}`,
|
||||
`$mount_args`,
|
||||
`$extra_host_args`,
|
||||
`"$old_image" >/dev/null || return 1`,
|
||||
].join(" "),
|
||||
` if [ -n "$network_names" ]; then for net in $network_names; do if [ "$net" != "$first_network" ]; then docker network connect "$net" "$old_name" >/dev/null 2>&1 || true; fi; done; fi`,
|
||||
` echo "rollback_to_last_known_good restored old gateway container name=$old_name image=$old_image reason=$reason" >&2`,
|
||||
`}`,
|
||||
].join("\n");
|
||||
const script = [
|
||||
"set -eu",
|
||||
`cd ${shellQuote(workspace)}`,
|
||||
@@ -1701,16 +1745,24 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
`first_old=""`,
|
||||
`for old_id in $old_ids; do first_old="$old_id"; break; done`,
|
||||
`old_name=""`,
|
||||
`old_image=""`,
|
||||
`old_restart=""`,
|
||||
`old_pid_mode=""`,
|
||||
`final_container=""`,
|
||||
`mount_args=""`,
|
||||
`extra_host_args=""`,
|
||||
`candidate_env_file=${shellQuote(`/tmp/${candidateName}.env`)}`,
|
||||
`network_names=""`,
|
||||
`first_network=""`,
|
||||
`network_arg=""`,
|
||||
rollbackToLastKnownGoodScript,
|
||||
`if [ -n "$old_ids" ]; then docker update --restart always $old_ids >/dev/null 2>&1 || true; fi`,
|
||||
`if [ -z "$first_old" ]; then echo "no existing provider-gateway compose container found; cannot perform safe in-place upgrade" >&2; exit 1; fi`,
|
||||
`old_name=$(docker inspect --format '{{.Name}}' "$first_old")`,
|
||||
`old_name="\${old_name#/}"`,
|
||||
`old_image=$(docker inspect --format '{{.Image}}' "$first_old")`,
|
||||
`old_restart=$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$first_old")`,
|
||||
`old_pid_mode=$(docker inspect --format '{{.HostConfig.PidMode}}' "$first_old")`,
|
||||
`docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$first_old" > "$candidate_env_file"`,
|
||||
`mount_args=$(docker inspect --format '{{range .Mounts}}{{printf "-v %s:%s" .Source .Destination}}{{if not .RW}}{{printf ":ro"}}{{end}}{{printf "\\n"}}{{end}}' "$first_old")`,
|
||||
`extra_host_args=$(docker inspect --format '{{range .HostConfig.ExtraHosts}}{{printf "--add-host %s\\n" .}}{{end}}' "$first_old")`,
|
||||
@@ -1745,12 +1797,18 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
`docker rm -f ${shellQuote(candidateName)} >/dev/null 2>&1 || true`,
|
||||
composeUpCommand.map(shellQuote).join(" "),
|
||||
`final_container="$old_name"`,
|
||||
`if [ -z "$final_container" ]; then final_container=$(${listServiceContainersCommand.map(shellQuote).join(" ")} | head -n 1); fi`,
|
||||
`if [ -z "$final_container" ] || ! docker inspect "$final_container" >/dev/null 2>&1; then final_container=$(${listServiceContainersCommand.map(shellQuote).join(" ")} | head -n 1); fi`,
|
||||
`if [ -z "$final_container" ]; then echo "final provider-gateway compose container not found after promotion" >&2; rollback_to_last_known_good "final-compose-container-missing" || true; rm -f "$candidate_env_file"; exit 1; fi`,
|
||||
`final_container_id=$(docker inspect --format '{{.Id}}' "$final_container")`,
|
||||
`final_restart=$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$final_container")`,
|
||||
`final_pid_mode=$(docker inspect --format '{{.HostConfig.PidMode}}' "$final_container")`,
|
||||
`if [ "$final_restart" != "always" ] || [ "$final_pid_mode" != "host" ]; then echo "final provider-gateway runtime guard failed: restart=$final_restart pid=$final_pid_mode" >&2; exit 1; fi`,
|
||||
`if [ "$final_restart" != "always" ] || [ "$final_pid_mode" != "host" ]; then echo "final provider-gateway runtime guard failed: restart=$final_restart pid=$final_pid_mode" >&2; rollback_to_last_known_good "final-runtime-guard-failed" || true; rm -f "$candidate_env_file"; exit 1; fi`,
|
||||
`final_attempt=0`,
|
||||
`final_validated=0`,
|
||||
`while [ "$final_attempt" -lt "${finalValidationAttempts}" ]; do final_logs=$(docker logs "$final_container" 2>&1 || true); final_has_open=0; final_has_ack=0; final_has_ok=0; case "$final_logs" in *${shellQuote(validationNeedleOpen)}*) final_has_open=1;; esac; case "$final_logs" in *${shellQuote(validationNeedleAck)}*) final_has_ack=1;; esac; case "$final_logs" in *${shellQuote(validationNeedleOk)}*) final_has_ok=1;; esac; if [ "$final_has_open" = "1" ] && [ "$final_has_ack" = "1" ] && [ "$final_has_ok" = "1" ]; then final_validated=1; break; fi; final_running=$(docker inspect --format '{{.State.Running}}' "$final_container" 2>/dev/null || true); if [ "$final_running" != "true" ]; then break; fi; final_attempt=$((final_attempt + 1)); sleep 2; done`,
|
||||
`if [ "$final_validated" != "1" ]; then echo "final provider-gateway compose container did not reconnect to backend-core before local guard timeout; backend-core must not treat candidate validation as success" >&2; docker logs "$final_container" >&2 || true; rollback_to_last_known_good "final-promoted-container-reconnect-timeout" || true; rm -f "$candidate_env_file"; exit 1; fi`,
|
||||
`rm -f "$candidate_env_file"`,
|
||||
`echo "candidate provider-gateway validated and promoted"`,
|
||||
`echo "candidate provider-gateway validated; final promoted compose container reconnected locally: container=$final_container id=$final_container_id restart=$final_restart pid=$final_pid_mode"`,
|
||||
].join("\n");
|
||||
const dockerRunCommand = [
|
||||
"docker",
|
||||
@@ -1805,16 +1863,23 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
buildBeforeCandidate: true,
|
||||
oldGatewaySleepMs: sleepMs,
|
||||
validationTimeoutMs,
|
||||
finalPromotedReconnectTimeoutMs,
|
||||
finalPromotedHeartbeatTimeoutMs,
|
||||
oldGatewayRestartPolicyBeforeSleep: "always",
|
||||
promoteOnlyAfterCandidateValidation: true,
|
||||
candidateRestartPolicyAfterPromotion: "always",
|
||||
candidateFinalRestartPolicyValidation: true,
|
||||
candidateUsesOldContainerMounts: true,
|
||||
candidateUsesOldContainerNetworks: true,
|
||||
candidateUsesOldContainerExtraHosts: true,
|
||||
candidateUsesOldContainerEnvironment: true,
|
||||
candidateUsesHostPidNamespace: true,
|
||||
startupSelfHealsRestartPolicy: true,
|
||||
candidateValidationIsNotTerminalSuccess: true,
|
||||
finalPromotedComposeHeartbeatRequired: true,
|
||||
finalPromotedContainerMustNotBeCandidate: true,
|
||||
localRollbackToLastKnownGoodOnFinalReconnectFailure: true,
|
||||
backendCoreOwnsFinalTerminalStatus: true,
|
||||
candidateRestartPolicyAfterPromotion: "always",
|
||||
candidateFinalRestartPolicyValidation: true,
|
||||
candidateUsesOldContainerMounts: true,
|
||||
candidateUsesOldContainerNetworks: true,
|
||||
candidateUsesOldContainerExtraHosts: true,
|
||||
candidateUsesOldContainerEnvironment: true,
|
||||
candidateUsesHostPidNamespace: true,
|
||||
startupSelfHealsRestartPolicy: true,
|
||||
dockerStatusReportsRestartPolicyAndPidMode: true,
|
||||
refreshesCompactBuildContextBeforeBuild: true,
|
||||
removeScope: {
|
||||
@@ -1825,6 +1890,17 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
namedVolumesPreserved: true,
|
||||
},
|
||||
dockerRunCommand,
|
||||
finalPromotedHeartbeatConfirmation: {
|
||||
required: true,
|
||||
providerId: config.providerId,
|
||||
targetProviderGatewayVersion: targetGatewayMetadata.version,
|
||||
backendCoreNodeRegistry: "/api/nodes",
|
||||
heartbeatTimeoutMs: finalPromotedHeartbeatTimeoutMs,
|
||||
finalContainerMustNotEqualCandidate: candidateName,
|
||||
requiredResultFields: ["containerId", "version", "restartPolicy", "pidMode", "heartbeatTimestamp"],
|
||||
unavailableResult: unavailableFinalPromotedContainer,
|
||||
},
|
||||
finalPromotedContainer: unavailableFinalPromotedContainer,
|
||||
compactBuildContextCandidates,
|
||||
workspaceStateMount: `${config.upgradeHostProjectRoot.replace(/\/+$/, "")}/.state:${workspaceRoot}/.state`,
|
||||
};
|
||||
@@ -1843,12 +1919,16 @@ async function runProviderUpgrade(taskId: string, payload: Record<string, JsonVa
|
||||
setTimeout(() => enterUpgradeSleep(sleepMs), 250).unref?.();
|
||||
return {
|
||||
mode,
|
||||
message: "provider gateway upgrade scheduled with sleep-and-validate rollback guard",
|
||||
message: "provider gateway upgrade scheduled; backend-core must wait for final promoted compose heartbeat before terminal success",
|
||||
providerId: config.providerId,
|
||||
providerName: config.providerName,
|
||||
providerGatewayVersion: gatewayMetadata.version,
|
||||
targetProviderGatewayVersion: (plan.targetProviderGatewayVersion as string | undefined) ?? gatewayMetadata.version,
|
||||
updaterContainerId: result.stdout.trim(),
|
||||
terminalStatusManagedByBackendCore: true,
|
||||
candidateValidationIsNotTerminalSuccess: true,
|
||||
finalPromotedHeartbeatConfirmation: plan.finalPromotedHeartbeatConfirmation,
|
||||
finalPromotedContainer: plan.finalPromotedContainer,
|
||||
plan,
|
||||
stderr: result.stderr.slice(0, 500),
|
||||
};
|
||||
@@ -2145,7 +2225,15 @@ async function handleDispatch(message: CoreDispatchMessage): Promise<void> {
|
||||
}
|
||||
if (message.command === "provider.upgrade") {
|
||||
const result = await runProviderUpgrade(message.taskId, message.payload);
|
||||
await sendTaskStatus(message.taskId, "succeeded", "provider upgrade command completed", result);
|
||||
const scheduleMode = message.payload.mode === "schedule";
|
||||
await sendTaskStatus(
|
||||
message.taskId,
|
||||
scheduleMode ? "running" : "succeeded",
|
||||
scheduleMode
|
||||
? "provider upgrade scheduled; awaiting backend-core final promoted heartbeat"
|
||||
: "provider upgrade command completed",
|
||||
result,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (message.command === "host.ssh") {
|
||||
|
||||
Reference in New Issue
Block a user