fix: verify code queue deploy commit

This commit is contained in:
Codex
2026-05-16 04:18:25 +00:00
parent 88516cec6a
commit 860b2a76b1
7 changed files with 108 additions and 13 deletions
+84 -6
View File
@@ -165,6 +165,17 @@ function coreBody(response: unknown): Record<string, unknown> | null {
return asRecord(record?.body);
}
function dispatchStdout(raw: unknown): string {
const task = asRecord(raw);
const result = asRecord(task?.result);
return asString(result?.stdout);
}
function parseFullCommit(value: string): string {
const match = value.match(/\b[0-9a-f]{40}\b/iu);
return match?.[0]?.toLowerCase() ?? "";
}
async function dispatchSsh(
config: UniDeskConfig,
providerId: string,
@@ -378,6 +389,15 @@ function prepareSourceScript(options: DeployCliOptions, exportDir: string): stri
].join("\n");
}
function resolveCommitScript(options: DeployCliOptions): string {
return [
"set -euo pipefail",
`repo=${shellQuote(options.sourceRepoDir)}`,
`commit=${shellQuote(options.commitId)}`,
"git -C \"$repo\" rev-parse --verify \"$commit^{commit}\"",
].join("\n");
}
function syncDeployScript(options: DeployCliOptions, exportDir: string): string {
return [
"set -euo pipefail",
@@ -429,6 +449,23 @@ function applyManifestCommand(options: DeployCliOptions): string {
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl apply -f ${shellQuote(manifest)}`;
}
function stampDeployCommitScript(options: DeployCliOptions, resolvedCommit: string): string {
return [
"set -euo pipefail",
`kubeconfig=${shellQuote(options.kubeconfigPath)}`,
`namespace=${shellQuote(k8sNamespace)}`,
`deployment=${shellQuote(k8sDeployment)}`,
`tcp_deployment=${shellQuote(tcpEgressDeployment)}`,
`resolved_commit=${shellQuote(resolvedCommit)}`,
`requested_commit=${shellQuote(options.commitId)}`,
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" set env \"deployment/$tcp_deployment\" \"deployment/$deployment\" CODE_QUEUE_DEPLOY_COMMIT=\"$resolved_commit\" CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=\"$requested_commit\"",
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" annotate \"deployment/$tcp_deployment\" \"deployment/$deployment\" unidesk.ai/deploy-commit=\"$resolved_commit\" unidesk.ai/deploy-requested-commit=\"$requested_commit\" --overwrite",
"current=$(KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"CODE_QUEUE_DEPLOY_COMMIT\")].value}')",
"test \"$current\" = \"$resolved_commit\"",
"printf 'deployment_commit=%s\\nrequested_commit=%s\\n' \"$current\" \"$requested_commit\"",
].join("\n");
}
function rolloutRestartCommand(options: DeployCliOptions): string {
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl -n ${shellQuote(k8sNamespace)} rollout restart deployment/${shellQuote(tcpEgressDeployment)} deployment/${shellQuote(k8sDeployment)}`;
}
@@ -447,23 +484,44 @@ function rolloutStatusScript(options: DeployCliOptions): string {
].join("\n");
}
async function microserviceHealthStep(config: UniDeskConfig, timeoutMs: number): Promise<StepResult> {
function healthDeployCommit(body: Record<string, unknown> | null): string {
const deploy = asRecord(body?.deploy);
return asString(deploy?.commit).toLowerCase();
}
function healthProbeSummary(body: Record<string, unknown> | null): Record<string, unknown> | null {
if (body === null) return null;
return {
ok: body.ok,
service: body.service,
instanceId: body.instanceId,
deploy: body.deploy,
status: body.status,
databaseReady: body.databaseReady,
startedAt: body.startedAt,
};
}
async function microserviceHealthStep(config: UniDeskConfig, expectedCommit: string, timeoutMs: number): Promise<StepResult> {
const step = "unidesk-health";
const startedAt = nowIso();
const startedMs = Date.now();
const deadline = Date.now() + timeoutMs;
let latest: unknown = null;
let latestCommit = "";
while (Date.now() < deadline) {
latest = coreInternalFetch("/api/microservices/code-queue/health");
const record = asRecord(latest);
const body = asRecord(record?.body);
const ok = record?.ok === true && body?.ok !== false;
progressLine(step, "health probe", { ok, status: record?.status ?? null, body });
latestCommit = healthDeployCommit(body);
const commitMatches = latestCommit === expectedCommit;
const ok = record?.ok === true && body?.ok !== false && commitMatches;
progressLine(step, "health probe", { ok, status: record?.status ?? null, expectedCommit, deployedCommit: latestCommit || null, commitMatches, body: healthProbeSummary(body) });
if (ok) {
return {
step,
ok: true,
detail: `Code Queue health passed in ${elapsedMs(startedMs)}ms`,
detail: `Code Queue health passed with deployed commit ${latestCommit} in ${elapsedMs(startedMs)}ms`,
startedAt,
finishedAt: nowIso(),
raw: latest,
@@ -474,7 +532,7 @@ async function microserviceHealthStep(config: UniDeskConfig, timeoutMs: number):
return {
step,
ok: false,
detail: `Code Queue health did not pass within ${timeoutMs}ms`,
detail: `Code Queue health did not expose expected commit ${expectedCommit} within ${timeoutMs}ms; latest deployed commit=${latestCommit || "none"}`,
startedAt,
finishedAt: nowIso(),
raw: latest,
@@ -511,6 +569,22 @@ export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptio
const prepare = await backgroundStep(config, options, "prepare-source", prepareSourceScript(options, exportDir), remainingTimeout(deadline, 180_000), null);
if (!pushStep(prepare)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const resolveCommit = await directStep(config, options, "resolve-commit", bashScriptCommand(resolveCommitScript(options)), null, 60_000, 45_000);
if (!pushStep(resolveCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const resolvedCommit = parseFullCommit(dispatchStdout(resolveCommit.raw) || resolveCommit.detail);
if (resolvedCommit.length !== 40) {
const parseFailure = {
step: "resolve-commit-parse",
ok: false,
detail: `remote commit did not resolve to a full 40-character SHA; output=${resolveCommit.detail}`,
startedAt: nowIso(),
finishedAt: nowIso(),
};
pushStep(parseFailure);
return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
}
progressLine("resolve-commit", "resolved requested commit", { requestedCommit: options.commitId, resolvedCommit });
const sync = await directStep(config, options, "sync-deploy-tree", bashScriptCommand(syncDeployScript(options, exportDir)), null, 60_000, 45_000);
if (!pushStep(sync)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
@@ -527,13 +601,16 @@ export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptio
const apply = await directStep(config, options, "kubectl-apply", applyManifestCommand(options), options.deployDir, 60_000, 45_000);
if (!pushStep(apply)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const stampCommit = await directStep(config, options, "stamp-deploy-commit", bashScriptCommand(stampDeployCommitScript(options, resolvedCommit)), options.deployDir, 60_000, 45_000);
if (!pushStep(stampCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, resolvedCommit, steps };
const restart = await directStep(config, options, "rollout-restart", rolloutRestartCommand(options), options.deployDir, 60_000, 45_000);
if (!pushStep(restart)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const rollout = await backgroundStep(config, options, "rollout-status", rolloutStatusScript(options), remainingTimeout(deadline, 240_000));
if (!pushStep(rollout)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const health = await microserviceHealthStep(config, remainingTimeout(deadline, 90_000));
const health = await microserviceHealthStep(config, resolvedCommit, remainingTimeout(deadline, 90_000));
pushStep(health);
return {
@@ -541,6 +618,7 @@ export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptio
startedAt,
finishedAt: nowIso(),
options,
resolvedCommit,
steps,
statusCommands: {
health: "bun scripts/cli.ts microservice health code-queue",