fix: 终止失败的 PikaOA migration Job 等待
This commit is contained in:
@@ -74,9 +74,15 @@ if printf '%s' "$args" | grep -q ' rollout status '; then
|
||||
[ ! -f "$root/slow" ] || sleep 0.4
|
||||
exit 0
|
||||
fi
|
||||
if printf '%s' "$args" | grep -q ' wait '; then
|
||||
[ ! -f "$root/slow" ] || sleep 0.4
|
||||
if [ -f "$root/fail-migration" ]; then printf '%s\n' 'migration failed' >&2; exit 1; fi
|
||||
if printf '%s' "$args" | grep -q ' get job/pikaoa-migrate '; then
|
||||
if [ -f "$root/fail-migration-query" ]; then printf '%s\n' 'job query failed' >&2; exit 1; fi
|
||||
if [ -f "$root/failure-target-migration" ]; then
|
||||
printf '%s\n' '{"status":{"conditions":[{"type":"FailureTarget","status":"True","reason":"BackoffLimitExceeded"}]}}'
|
||||
elif [ -f "$root/fail-migration" ]; then
|
||||
printf '%s\n' '{"status":{"conditions":[{"type":"Failed","status":"True","reason":"BackoffLimitExceeded"}]}}'
|
||||
else
|
||||
printf '%s\n' '{"status":{"conditions":[{"type":"Complete","status":"True"}]}}'
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
if printf '%s' "$args" | grep -Eq '(^| )get namespace '; then
|
||||
@@ -257,10 +263,30 @@ exit 64
|
||||
await projection("start", "runtime", true, "migration");
|
||||
const failed = await waitFor("runtime", "start-failed", "migration");
|
||||
const failedTask = ((failed.status as Record<string, unknown>).task as Record<string, unknown>);
|
||||
assert.equal(failedTask.code, "migration-wait-failed");
|
||||
assert.equal(failedTask.code, "migration-job-failed");
|
||||
assert.equal(failedTask.exitCode, 46);
|
||||
rmSync(join(fakeRoot, "fail-migration"));
|
||||
|
||||
writeFileSync(join(fakeRoot, "failure-target-migration"), "1\n");
|
||||
await projectionWithCommit("start", "runtime", concurrentCommit, "migration");
|
||||
const failureTarget = await waitFor("runtime", "start-failed", "migration");
|
||||
const failureTargetTask = ((failureTarget.status as Record<string, unknown>).task as Record<string, unknown>);
|
||||
assert.equal(failureTargetTask.code, "migration-job-failure-target");
|
||||
assert.equal(failureTargetTask.exitCode, 46);
|
||||
rmSync(join(fakeRoot, "failure-target-migration"));
|
||||
|
||||
writeFileSync(join(fakeRoot, "fail-migration-query"), "1\n");
|
||||
await projectionWithCommit("start", "runtime", "abcdef0123456789", "migration");
|
||||
const migrationQueryFailed = await waitFor("runtime", "start-failed", "migration");
|
||||
const migrationQueryFailedTask = ((migrationQueryFailed.status as Record<string, unknown>).task as Record<string, unknown>);
|
||||
assert.equal(migrationQueryFailedTask.code, "migration-job-query-failed");
|
||||
assert.equal(migrationQueryFailedTask.exitCode, 46);
|
||||
rmSync(join(fakeRoot, "fail-migration-query"));
|
||||
|
||||
await projectionWithCommit("start", "runtime", "1234567890abcdef", "migration");
|
||||
const migrationSucceeded = await waitFor("runtime", "start-succeeded", "migration");
|
||||
assert.equal((((migrationSucceeded.status as Record<string, unknown>).task as Record<string, unknown>).exitCode), 0);
|
||||
|
||||
writeFileSync(join(fakeRoot, "slow"), "1\n");
|
||||
await projection("start", "runtime", true, "web");
|
||||
const stopped = await projection("stop", "runtime", false, "web");
|
||||
@@ -303,6 +329,9 @@ exit 64
|
||||
workerMissingReset,
|
||||
workerMissingSucceeded,
|
||||
failed,
|
||||
failureTarget,
|
||||
migrationQueryFailed,
|
||||
migrationSucceeded,
|
||||
stopped,
|
||||
foundationSubmitted,
|
||||
foundationSucceeded,
|
||||
|
||||
@@ -1206,8 +1206,45 @@ if [ "$step" = all ] || [ "$step" = migration ]; then
|
||||
if ! kubectl apply --server-side --force-conflicts --field-manager=${shQuote(target.fieldManager)} -f "$tmp/migration.yaml" >"$tmp/migration-apply.out" 2>"$tmp/migration-apply.err"; then failure_code=migration-apply-failed; exit 45; fi
|
||||
current_phase=migration-wait
|
||||
write_status running "$current_phase" start-running null
|
||||
check_canceled
|
||||
if ! kubectl -n "$namespace" wait --for=condition=complete job/${target.migration.jobName} --timeout=${target.waitTimeoutSeconds}s >/dev/null 2>"$tmp/migration-wait.err"; then failure_code=migration-wait-failed; exit 46; fi
|
||||
migration_deadline=$(( $(date +%s) + ${target.waitTimeoutSeconds} ))
|
||||
while :; do
|
||||
check_canceled
|
||||
if ! kubectl -n "$namespace" get job/${target.migration.jobName} --request-timeout=${Math.min(target.taskState.statusRequestTimeoutSeconds, 5)}s -o json >"$tmp/migration-job.json" 2>"$tmp/migration-wait.err"; then
|
||||
failure_code=migration-job-query-failed
|
||||
exit 46
|
||||
fi
|
||||
if ! migration_condition="$(python3 - "$tmp/migration-job.json" <<'PY'
|
||||
import json, pathlib, sys
|
||||
payload = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
conditions = {
|
||||
item.get("type"): item.get("status")
|
||||
for item in payload.get("status", {}).get("conditions", [])
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
if conditions.get("Complete") == "True":
|
||||
print("complete")
|
||||
elif conditions.get("FailureTarget") == "True":
|
||||
print("failure-target")
|
||||
elif conditions.get("Failed") == "True":
|
||||
print("failed")
|
||||
else:
|
||||
print("running")
|
||||
PY
|
||||
)"; then
|
||||
failure_code=migration-job-condition-invalid
|
||||
exit 46
|
||||
fi
|
||||
case "$migration_condition" in
|
||||
complete) break ;;
|
||||
failure-target) failure_code=migration-job-failure-target; exit 46 ;;
|
||||
failed) failure_code=migration-job-failed; exit 46 ;;
|
||||
esac
|
||||
if [ "$(date +%s)" -ge "$migration_deadline" ]; then
|
||||
failure_code=migration-job-timeout
|
||||
exit 46
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
for component in api worker web; do
|
||||
if [ "$step" != all ] && [ "$step" != "$component" ]; then continue; fi
|
||||
|
||||
Reference in New Issue
Block a user