fix: speed up native branch follower refresh

scripts/native/cicd/native-job.mjs

@@ -4,7 +4,8 @@ import https from "node:https";
 const namespace = process.env.NAMESPACE || "";
 const jobName = process.env.JOB_NAME || "";
 const logContainer = process.env.LOG_CONTAINER || "";
-const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || "60");
+const timeoutSeconds = requiredPositiveNumber("TIMEOUT_SECONDS");
+const pollIntervalSeconds = requiredPositiveNumber("POLL_INTERVAL_SECONDS");
 const host = process.env.KUBERNETES_SERVICE_HOST;
 const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
 const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim();
@@ -75,7 +76,23 @@ async function logsTail() {
 
 let created = false;
 let reused = false;
-const existing = await getJob();
+let replacedFailed = false;
+let existing = await getJob();
+if (existing && condition(existing, "Failed")) {
+  await deleteJob();
+  replacedFailed = true;
+  existing = null;
+  const deleteDeadline = Date.now() + timeoutSeconds * 1000;
+  let deleted = false;
+  while (Date.now() <= deleteDeadline) {
+    if (await getJob() === null) {
+      deleted = true;
+      break;
+    }
+    await delay(pollIntervalSeconds * 1000);
+  }
+  if (!deleted) throw new Error(`timed out deleting failed job ${jobName}`);
+}
 if (existing) {
   reused = true;
 } else {
@@ -89,7 +106,7 @@ if (existing) {
 }
 
 const startedAt = Date.now();
-const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000;
+const deadline = startedAt + timeoutSeconds * 1000;
 let polls = 0;
 let latest = await getJob();
 while (Date.now() <= deadline) {
@@ -97,7 +114,7 @@ while (Date.now() <= deadline) {
   const failed = condition(latest, "Failed");
   if (complete || failed) break;
   polls += 1;
-  await delay(2000);
+  await delay(pollIntervalSeconds * 1000);
   latest = await getJob();
 }
 
@@ -112,6 +129,7 @@ const output = {
   timedOut,
   created,
   reused,
+  replacedFailed,
   jobName,
   namespace,
   polls,
@@ -125,3 +143,15 @@ const output = {
 };
 process.stdout.write(JSON.stringify(output));
 if (!output.ok) process.exit(1);
+
+async function deleteJob() {
+  const result = await request("DELETE", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs/${encodeURIComponent(jobName)}?propagationPolicy=Background`);
+  if (result.status === 404) return;
+  if (result.status < 200 || result.status >= 300) throw new Error(result.text || `kube api DELETE job status ${result.status}`);
+}
+
+function requiredPositiveNumber(name) {
+  const value = Number(process.env[name]);
+  if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`);
+  return value;
+}