fix: verify code queue deploy commit
This commit is contained in:
@@ -165,6 +165,17 @@ function coreBody(response: unknown): Record<string, unknown> | null {
|
||||
return asRecord(record?.body);
|
||||
}
|
||||
|
||||
function dispatchStdout(raw: unknown): string {
|
||||
const task = asRecord(raw);
|
||||
const result = asRecord(task?.result);
|
||||
return asString(result?.stdout);
|
||||
}
|
||||
|
||||
function parseFullCommit(value: string): string {
|
||||
const match = value.match(/\b[0-9a-f]{40}\b/iu);
|
||||
return match?.[0]?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
async function dispatchSsh(
|
||||
config: UniDeskConfig,
|
||||
providerId: string,
|
||||
@@ -378,6 +389,15 @@ function prepareSourceScript(options: DeployCliOptions, exportDir: string): stri
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function resolveCommitScript(options: DeployCliOptions): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`repo=${shellQuote(options.sourceRepoDir)}`,
|
||||
`commit=${shellQuote(options.commitId)}`,
|
||||
"git -C \"$repo\" rev-parse --verify \"$commit^{commit}\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function syncDeployScript(options: DeployCliOptions, exportDir: string): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
@@ -429,6 +449,23 @@ function applyManifestCommand(options: DeployCliOptions): string {
|
||||
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl apply -f ${shellQuote(manifest)}`;
|
||||
}
|
||||
|
||||
function stampDeployCommitScript(options: DeployCliOptions, resolvedCommit: string): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`kubeconfig=${shellQuote(options.kubeconfigPath)}`,
|
||||
`namespace=${shellQuote(k8sNamespace)}`,
|
||||
`deployment=${shellQuote(k8sDeployment)}`,
|
||||
`tcp_deployment=${shellQuote(tcpEgressDeployment)}`,
|
||||
`resolved_commit=${shellQuote(resolvedCommit)}`,
|
||||
`requested_commit=${shellQuote(options.commitId)}`,
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" set env \"deployment/$tcp_deployment\" \"deployment/$deployment\" CODE_QUEUE_DEPLOY_COMMIT=\"$resolved_commit\" CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=\"$requested_commit\"",
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" annotate \"deployment/$tcp_deployment\" \"deployment/$deployment\" unidesk.ai/deploy-commit=\"$resolved_commit\" unidesk.ai/deploy-requested-commit=\"$requested_commit\" --overwrite",
|
||||
"current=$(KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"CODE_QUEUE_DEPLOY_COMMIT\")].value}')",
|
||||
"test \"$current\" = \"$resolved_commit\"",
|
||||
"printf 'deployment_commit=%s\\nrequested_commit=%s\\n' \"$current\" \"$requested_commit\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function rolloutRestartCommand(options: DeployCliOptions): string {
|
||||
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl -n ${shellQuote(k8sNamespace)} rollout restart deployment/${shellQuote(tcpEgressDeployment)} deployment/${shellQuote(k8sDeployment)}`;
|
||||
}
|
||||
@@ -447,23 +484,44 @@ function rolloutStatusScript(options: DeployCliOptions): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function microserviceHealthStep(config: UniDeskConfig, timeoutMs: number): Promise<StepResult> {
|
||||
function healthDeployCommit(body: Record<string, unknown> | null): string {
|
||||
const deploy = asRecord(body?.deploy);
|
||||
return asString(deploy?.commit).toLowerCase();
|
||||
}
|
||||
|
||||
function healthProbeSummary(body: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
if (body === null) return null;
|
||||
return {
|
||||
ok: body.ok,
|
||||
service: body.service,
|
||||
instanceId: body.instanceId,
|
||||
deploy: body.deploy,
|
||||
status: body.status,
|
||||
databaseReady: body.databaseReady,
|
||||
startedAt: body.startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
async function microserviceHealthStep(config: UniDeskConfig, expectedCommit: string, timeoutMs: number): Promise<StepResult> {
|
||||
const step = "unidesk-health";
|
||||
const startedAt = nowIso();
|
||||
const startedMs = Date.now();
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let latest: unknown = null;
|
||||
let latestCommit = "";
|
||||
while (Date.now() < deadline) {
|
||||
latest = coreInternalFetch("/api/microservices/code-queue/health");
|
||||
const record = asRecord(latest);
|
||||
const body = asRecord(record?.body);
|
||||
const ok = record?.ok === true && body?.ok !== false;
|
||||
progressLine(step, "health probe", { ok, status: record?.status ?? null, body });
|
||||
latestCommit = healthDeployCommit(body);
|
||||
const commitMatches = latestCommit === expectedCommit;
|
||||
const ok = record?.ok === true && body?.ok !== false && commitMatches;
|
||||
progressLine(step, "health probe", { ok, status: record?.status ?? null, expectedCommit, deployedCommit: latestCommit || null, commitMatches, body: healthProbeSummary(body) });
|
||||
if (ok) {
|
||||
return {
|
||||
step,
|
||||
ok: true,
|
||||
detail: `Code Queue health passed in ${elapsedMs(startedMs)}ms`,
|
||||
detail: `Code Queue health passed with deployed commit ${latestCommit} in ${elapsedMs(startedMs)}ms`,
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
raw: latest,
|
||||
@@ -474,7 +532,7 @@ async function microserviceHealthStep(config: UniDeskConfig, timeoutMs: number):
|
||||
return {
|
||||
step,
|
||||
ok: false,
|
||||
detail: `Code Queue health did not pass within ${timeoutMs}ms`,
|
||||
detail: `Code Queue health did not expose expected commit ${expectedCommit} within ${timeoutMs}ms; latest deployed commit=${latestCommit || "none"}`,
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
raw: latest,
|
||||
@@ -511,6 +569,22 @@ export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptio
|
||||
const prepare = await backgroundStep(config, options, "prepare-source", prepareSourceScript(options, exportDir), remainingTimeout(deadline, 180_000), null);
|
||||
if (!pushStep(prepare)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const resolveCommit = await directStep(config, options, "resolve-commit", bashScriptCommand(resolveCommitScript(options)), null, 60_000, 45_000);
|
||||
if (!pushStep(resolveCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
const resolvedCommit = parseFullCommit(dispatchStdout(resolveCommit.raw) || resolveCommit.detail);
|
||||
if (resolvedCommit.length !== 40) {
|
||||
const parseFailure = {
|
||||
step: "resolve-commit-parse",
|
||||
ok: false,
|
||||
detail: `remote commit did not resolve to a full 40-character SHA; output=${resolveCommit.detail}`,
|
||||
startedAt: nowIso(),
|
||||
finishedAt: nowIso(),
|
||||
};
|
||||
pushStep(parseFailure);
|
||||
return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
}
|
||||
progressLine("resolve-commit", "resolved requested commit", { requestedCommit: options.commitId, resolvedCommit });
|
||||
|
||||
const sync = await directStep(config, options, "sync-deploy-tree", bashScriptCommand(syncDeployScript(options, exportDir)), null, 60_000, 45_000);
|
||||
if (!pushStep(sync)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
@@ -527,13 +601,16 @@ export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptio
|
||||
const apply = await directStep(config, options, "kubectl-apply", applyManifestCommand(options), options.deployDir, 60_000, 45_000);
|
||||
if (!pushStep(apply)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const stampCommit = await directStep(config, options, "stamp-deploy-commit", bashScriptCommand(stampDeployCommitScript(options, resolvedCommit)), options.deployDir, 60_000, 45_000);
|
||||
if (!pushStep(stampCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, resolvedCommit, steps };
|
||||
|
||||
const restart = await directStep(config, options, "rollout-restart", rolloutRestartCommand(options), options.deployDir, 60_000, 45_000);
|
||||
if (!pushStep(restart)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const rollout = await backgroundStep(config, options, "rollout-status", rolloutStatusScript(options), remainingTimeout(deadline, 240_000));
|
||||
if (!pushStep(rollout)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const health = await microserviceHealthStep(config, remainingTimeout(deadline, 90_000));
|
||||
const health = await microserviceHealthStep(config, resolvedCommit, remainingTimeout(deadline, 90_000));
|
||||
pushStep(health);
|
||||
|
||||
return {
|
||||
@@ -541,6 +618,7 @@ export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptio
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
options,
|
||||
resolvedCommit,
|
||||
steps,
|
||||
statusCommands: {
|
||||
health: "bun scripts/cli.ts microservice health code-queue",
|
||||
|
||||
Reference in New Issue
Block a user