fix: complete Release A migration execution

This commit is contained in:
AgentRun Codex
2026-07-13 00:25:44 +00:00
parent 9a32ead76f
commit 3a63da0fa8
9 changed files with 83 additions and 20 deletions
+30 -1
View File
@@ -37,7 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration";
import { releaseAMonitorManifests, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor";
import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor";
import {
arrayAt,
arrayAtNullable,
@@ -195,6 +195,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract<WebProbeSentinelOptions, { kind: "migration" }>): RenderedCliResult {
const sources = resolvedMigrationSources(spec, options.sentinelId);
const command = `web-probe sentinel migration ${options.action}`;
if (options.action === "job-plan" || options.action === "job-run") return runSentinelMigrationJob(spec, options, sources, command);
if (options.action === "plan") {
const projection = planMonitorSqliteSources(sources);
return rendered(true, command, renderMigrationCompact(projection), projection);
@@ -228,6 +229,34 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract<WebPr
return rendered(projection.ok === true, command, renderMigrationCompact(projection), projection);
}
function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract<WebProbeSentinelOptions, { kind: "migration" }>, sources: readonly MonitorSqliteSource[], command: string): RenderedCliResult {
if (sources.length !== 1) throw new Error(`web-probe sentinel migration ${options.action} requires --sentinel when more than one resolved registry source exists`);
const state = loadSentinelCicdState(spec, sources[0]!.sentinelId, options.timeoutSeconds, "cached");
const manifest = releaseAMonitorMigrationJob(spec, state.image.ref, sources[0]!, { enabled: true, nameSuffix: Date.now().toString(36) });
if (manifest === null) throw new Error("Release A migration Job helper did not render an enabled Job");
const metadata = manifest.metadata as Record<string, unknown>;
const namespace = String(metadata.namespace);
const jobName = String(metadata.name);
const plan = { ok: true, command, mutation: false, namespace, jobName, image: state.image.ref, source: sources[0], manifest: { apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { name: jobName, namespace } }, valuesRedacted: true };
if (options.action === "job-plan") return rendered(true, command, renderMigrationCompact(plan), plan);
if (!options.confirm || !options.wait) throw new Error("web-probe sentinel migration job-run requires --confirm --wait");
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 });
if (created.exitCode !== 0) return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase: "create-failed", create: compactCommand(created) });
const startedAt = Date.now();
let lastProbe = created;
while (Date.now() - startedAt < Math.max(5_000, Math.min(options.timeoutSeconds * 1000, 120_000))) {
lastProbe = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 });
const probe = resolveSentinelChildJson(lastProbe, "web-probe-sentinel-migration-job-probe").parsed ?? {};
if (probe.succeeded === true || probe.failed === true) {
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
const ok = probe.succeeded === true && payload.ok === true;
return rendered(ok, command, `ok=${ok}`, { ...plan, ok, mutation: true, phase: probe.succeeded === true ? "succeeded" : "failed", result: payload, valuesRedacted: true });
}
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
}
return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase: "timeout", probe: compactCommand(lastProbe), valuesRedacted: true });
}
function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean {
return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath);
}