diff --git a/scripts/artifact-registry-ssh-timeout-contract-test.ts b/scripts/artifact-registry-ssh-timeout-contract-test.ts index ac12e5fe..b88d4e23 100644 --- a/scripts/artifact-registry-ssh-timeout-contract-test.ts +++ b/scripts/artifact-registry-ssh-timeout-contract-test.ts @@ -16,7 +16,7 @@ assertCondition(source.includes("downloadRemoteFile(options, remoteArchive, loca assertCondition(source.includes("runRemoteScriptBackground(options, remoteScript"), "remote docker save must run as a background job"); assertCondition(source.includes('runRemoteScriptBackground(options, deployScript, Math.max(options.timeoutMs, 420_000), "d601-k3s-deploy")'), "D601 k3s deploy must use background polling"); assertCondition(source.includes('"ssh",\n options.providerId,\n "download"'), "download helper must route through UniDesk ssh download"); -assertCondition(!downloadRemoteFileSource.includes('"--chunk-bytes"'), "artifact ssh download must use the stable default bounded chunk size, not the largest chunk"); +assertCondition(downloadRemoteFileSource.includes('"--chunk-bytes",\n "64000"'), "artifact ssh download must use a mid-size bounded chunk, not the largest chunk"); assertCondition(source.includes("UNIDESK_SSH_CLIENT_TOKEN") && source.includes("UNIDESK_SSH_CLIENT_ROUTE_ALLOWLIST"), "dev frontend artifact deploy must sync scoped ssh runtime keys"); console.log(JSON.stringify({ @@ -26,7 +26,7 @@ console.log(JSON.stringify({ "no docker-save stdout stream over ssh", "compose artifact uses verified ssh download", "remote docker save and k3s deploy use background polling", - "artifact downloads use the stable default bounded ssh chunk size", + "artifact downloads use a mid-size bounded ssh chunk", "dev frontend artifact deploy syncs scoped ssh runtime keys" ] }, null, 2)); diff --git a/scripts/cli.ts b/scripts/cli.ts index a14dccbb..3f2e5f17 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -3,7 +3,7 @@ import { debugDispatch, debugHealth, debugTask, isDebugDispatchCommand, type Deb import { isRebuildableService, rebuildService, stackLogs, stackStatus, startStack, stopStack, unsupportedRebuildService } from "./src/docker"; import { parseE2ERunOptions, runE2E } from "./src/e2e"; import { emitError, emitJson } from "./src/output"; -import { jobWithTail, listJobs, listJobsSummary, readJob, runJob } from "./src/jobs"; +import { cancelJob, jobWithTail, listJobs, listJobsSummary, readJob, runJob } from "./src/jobs"; import { checkHelp, parseCheckOptions, runChecks, runRecoveryGuardrailsCheck } from "./src/check"; import { runSsh } from "./src/ssh"; import { autoRemoteCiPublishUserServiceDryRunPlan, extractRemoteCliOptions, runRemoteCli } from "./src/remote"; @@ -469,6 +469,11 @@ async function main(): Promise { emitJson(commandName, { job: jobWithTail(readJob(id), boundedNumberOption("--tail-bytes", 12000, 500_000)) }); return; } + if (sub === "cancel") { + if (!third) throw new Error("job cancel requires job id"); + emitJson(commandName, cancelJob(third)); + return; + } } if (top === "debug") { diff --git a/scripts/src/artifact-registry.ts b/scripts/src/artifact-registry.ts index 576b0123..c45b76a8 100644 --- a/scripts/src/artifact-registry.ts +++ b/scripts/src/artifact-registry.ts @@ -1677,6 +1677,8 @@ function downloadRemoteFile(options: ArtifactRegistryOptions, remotePath: string "ssh", options.providerId, "download", + "--chunk-bytes", + "64000", remotePath, localPath, ], repoRoot, { timeoutMs }); diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 0cf8bf26..96d02079 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -84,6 +84,7 @@ export function rootHelp(): unknown { { command: "codex (queues [--full|--all] | queue create | queue merge --into | move --queue )", description: "List low-noise queue summaries by default, including effective activity counts that distinguish scheduler-local queues, DB running tasks, and heartbeat-fresh runners; full queue rows require --full/--all." }, { command: "job list [--limit N] [--include-command]", description: "List async jobs from .state/jobs with a bounded default page and progress summaries." }, { command: "job status [--tail-bytes N]", description: "Show job state with a structured progress summary and bounded stdout/stderr tails." }, + { command: "job cancel ", description: "Cancel a queued/running async job through the .state/jobs control entry and keep a terminal canceled record." }, { command: "debug health", description: "Probe internal core, nodes, system/Docker status, frontend, provider ingress, and public boundary." }, { command: "debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]", description: "Submit a real internal-core dispatch request for CLI debugging." }, { command: "debug task ", description: "Read a dispatched task record from internal core for CLI debugging." }, @@ -480,13 +481,14 @@ function codexHelp(): unknown { function jobHelp(): unknown { return { - command: "job list|status", + command: "job list|status|cancel", output: "json", usage: [ "bun scripts/cli.ts job list [--limit N] [--include-command]", "bun scripts/cli.ts job status [--tail-bytes N]", + "bun scripts/cli.ts job cancel ", ], - description: "Inspect fire-and-forget job state from .state/jobs with structured progress summaries and bounded log tails.", + description: "Inspect or cancel fire-and-forget job state from .state/jobs with structured progress summaries and bounded log tails.", }; } diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index f61a4a06..152bc5d0 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { repoRoot, rootPath } from "./config"; import { runCommandToFiles, tailFile } from "./command"; -export type JobStatus = "queued" | "running" | "succeeded" | "failed"; +export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "canceled"; export interface JobRecord { id: string; @@ -68,6 +68,34 @@ export function readJob(id: string): JobRecord { return JSON.parse(readFileSync(path, "utf8")) as JobRecord; } +export function cancelJob(id: string): unknown { + const job = readJob(id); + if (job.status !== "queued" && job.status !== "running") { + return { ok: true, action: "cancel-job", id, alreadyTerminal: true, job }; + } + const actions: Array> = []; + if (job.runnerPid !== null && job.runnerPid !== undefined) { + try { + process.kill(-job.runnerPid, "SIGTERM"); + actions.push({ signal: "SIGTERM", target: "process-group", pid: job.runnerPid, ok: true }); + } catch (error) { + actions.push({ signal: "SIGTERM", target: "process-group", pid: job.runnerPid, ok: false, error: error instanceof Error ? error.message : String(error) }); + try { + process.kill(job.runnerPid, "SIGTERM"); + actions.push({ signal: "SIGTERM", target: "process", pid: job.runnerPid, ok: true }); + } catch (innerError) { + actions.push({ signal: "SIGTERM", target: "process", pid: job.runnerPid, ok: false, error: innerError instanceof Error ? innerError.message : String(innerError) }); + } + } + } + job.status = "canceled"; + job.finishedAt = new Date().toISOString(); + job.exitCode = 130; + writeFileSync(job.stderrFile, `${tailFile(job.stderrFile, 512_000)}${JSON.stringify({ event: "unidesk.job.cancel", at: job.finishedAt, jobId: id, actions })}\n`, "utf8"); + writeJob(job); + return { ok: true, action: "cancel-job", id, actions, job }; +} + export function listJobs(): JobRecord[] { return readdirSync(jobsDir()) .filter((name) => name.endsWith(".json")) @@ -323,32 +351,40 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder function genericJobProgress(job: JobRecord): JobProgressSummary { const nowMs = Date.now(); + const stderrTail = tailFile(job.stderrFile, 96_000); + const downloadEvents = parseJsonLineEvents(stderrTail, "unidesk.ssh.download.progress"); + const lastDownload = downloadEvents.at(-1) ?? null; + const lastDownloadAt = stringField(lastDownload?.at); + const lastEventAgeSeconds = lastDownloadAt === null ? null : secondsSince(lastDownloadAt, job.finishedAt ?? nowMs); const elapsedSeconds = jobElapsedSeconds(job, nowMs); const warnings = jobProgressWarnings({ job, - eventsObserved: 0, + eventsObserved: downloadEvents.length, elapsedSeconds, - stage: null, - stageStatus: null, + stage: lastDownload === null ? null : "ssh-download", + stageStatus: lastDownload === null ? null : "running", stageElapsedSeconds: null, - lastEventAgeSeconds: null, + lastEventAgeSeconds, }); + const downloadSummary = lastDownload === null + ? null + : ` ssh-download ${Number(lastDownload.actualBytes ?? 0)}/${Number(lastDownload.expectedBytes ?? 0)} bytes chunks=${Number(lastDownload.chunks ?? 0)}`; return { kind: "generic", - stage: null, - stageStatus: null, + stage: lastDownload === null ? null : "ssh-download", + stageStatus: lastDownload === null ? null : "running", sourceCommit: null, pipelineRun: null, pipelineCreated: null, elapsedSeconds, stageElapsedSeconds: null, - lastEventAt: null, - lastEventAgeSeconds: null, - eventsObserved: 0, + lastEventAt: lastDownloadAt, + lastEventAgeSeconds, + eventsObserved: downloadEvents.length, slow: warnings.length > 0, warnings, timings: {}, - summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`, + summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}${downloadSummary ?? ""}`, nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, }; }