fix: 补齐 cancel lifecycle 运行时治理 (#245)
This commit is contained in:
Lyon
+133
-17
@@ -3,10 +3,10 @@ import { Pool } from "pg";
|
||||
import type { PoolClient, QueryResultRow } from "pg";
|
||||
import { AgentRunError } from "../common/errors.js";
|
||||
import { redactJson } from "../common/redaction.js";
|
||||
import type { BackendProfile, BackendTurnResult, CommandRecord, CommandState, CreateCommandInput, CreateQueueTaskInput, CreateRunInput, EventType, FailureKind, JsonRecord, JsonValue, ListGcExpiredSessionsInput, QueueCommanderSnapshot, QueueReadCursorRecord, QueueStats, QueueTaskListResult, QueueTaskRecord, QueueTaskState, RunEvent, RunnerJobRecord, RunnerRecord, RunRecord, RunStatus, SessionEventPage, SessionListResult, SessionReadCursorRecord, SessionRecord, SessionRef, SessionStoragePatch, SessionSummary, TerminalStatus, UpsertSessionInput } from "../common/types.js";
|
||||
import type { BackendProfile, BackendTurnResult, CancelRequestRecord, CancelStage, CancelTargetKind, CommandRecord, CommandState, CreateCommandInput, CreateQueueTaskInput, CreateRunInput, EventType, FailureKind, JsonRecord, JsonValue, ListGcExpiredSessionsInput, QueueCommanderSnapshot, QueueReadCursorRecord, QueueStats, QueueTaskListResult, QueueTaskRecord, QueueTaskState, RunEvent, RunnerJobRecord, RunnerRecord, RunRecord, RunStatus, SessionEventPage, SessionListResult, SessionReadCursorRecord, SessionRecord, SessionRef, SessionStoragePatch, SessionSummary, TerminalStatus, UpsertSessionInput } from "../common/types.js";
|
||||
import { newId, nowIso, stableHash } from "../common/validation.js";
|
||||
import type { AgentRunStore, ListQueueTasksInput, ListSessionsInput, SaveRunnerJobInput, SessionEventPageInput, StoreHealth, UpdateQueueTaskAttemptInput } from "./store.js";
|
||||
import { assertSessionBoundary, buildQueueStats, buildQueueTaskSummary, buildSessionSummary, clampQueueLimit, clampSessionLimit, commandStateFromTerminal, isLeaseExpired, isSessionOutputEvent, isTerminalCommandState, isTerminalQueueTaskState, isTerminalRunStatus, parseQueueCursor, parseSessionCursor, queueTaskMatchesCommander, queueTaskPayloadHash, queueTaskSort, sessionListFilters, sessionMatchesListState, sessionRefFromRecord, sessionSort, sessionTitleFromCommand, statusFromTerminal, summarizeResourceBundleRef, summarizeSessionRef, titleFromMetadata } from "./store.js";
|
||||
import { assertSessionBoundary, buildQueueStats, buildQueueTaskSummary, buildSessionSummary, cancelStagePayload, clampQueueLimit, clampSessionLimit, commandStateFromTerminal, fenceLateEventForCancelledRun, isLeaseExpired, isSessionOutputEvent, isTerminalCommandState, isTerminalQueueTaskState, isTerminalRunStatus, lateWriteRejectedPayload, parseQueueCursor, parseSessionCursor, queueTaskMatchesCommander, queueTaskPayloadHash, queueTaskSort, sessionListFilters, sessionMatchesListState, sessionRefFromRecord, sessionSort, sessionTitleFromCommand, statusFromTerminal, summarizeResourceBundleRef, summarizeSessionRef, titleFromMetadata } from "./store.js";
|
||||
import { backendCapabilitiesSqlValues, mergeBackendCapability } from "../common/backend-profiles.js";
|
||||
import { normalizeRunEventPayload, requireEventType } from "../common/events.js";
|
||||
|
||||
@@ -335,6 +335,39 @@ ALTER TABLE agentrun_queue_tasks
|
||||
ADD COLUMN IF NOT EXISTS session_ref jsonb;
|
||||
`;
|
||||
|
||||
const cancelLifecycleMigrationSql = `
|
||||
ALTER TABLE agentrun_runs ADD COLUMN IF NOT EXISTS cancel_epoch integer NOT NULL DEFAULT 0;
|
||||
ALTER TABLE agentrun_runs ADD COLUMN IF NOT EXISTS cancel_request_id text;
|
||||
ALTER TABLE agentrun_runs ADD COLUMN IF NOT EXISTS cancel_requested_at timestamptz;
|
||||
ALTER TABLE agentrun_runs ADD COLUMN IF NOT EXISTS cancel_reason text;
|
||||
|
||||
ALTER TABLE agentrun_commands ADD COLUMN IF NOT EXISTS cancel_epoch integer NOT NULL DEFAULT 0;
|
||||
ALTER TABLE agentrun_commands ADD COLUMN IF NOT EXISTS cancel_request_id text;
|
||||
ALTER TABLE agentrun_commands ADD COLUMN IF NOT EXISTS cancel_requested_at timestamptz;
|
||||
ALTER TABLE agentrun_commands ADD COLUMN IF NOT EXISTS cancel_reason text;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS agentrun_cancel_requests (
|
||||
id text PRIMARY KEY,
|
||||
target_kind text NOT NULL,
|
||||
target_id text NOT NULL,
|
||||
run_id text REFERENCES agentrun_runs(id) ON DELETE CASCADE,
|
||||
command_id text REFERENCES agentrun_commands(id) ON DELETE SET NULL,
|
||||
session_id text,
|
||||
task_id text,
|
||||
reason text NOT NULL,
|
||||
requested_by text,
|
||||
epoch integer NOT NULL,
|
||||
stage text NOT NULL,
|
||||
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS agentrun_cancel_requests_target_idx ON agentrun_cancel_requests (target_kind, target_id, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS agentrun_cancel_requests_run_idx ON agentrun_cancel_requests (run_id, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS agentrun_cancel_requests_command_idx ON agentrun_cancel_requests (command_id, created_at DESC);
|
||||
`;
|
||||
|
||||
const postgresMigrations: MigrationDefinition[] = [
|
||||
{
|
||||
id: "001_v01_initial_durable_store",
|
||||
@@ -386,13 +419,18 @@ const postgresMigrations: MigrationDefinition[] = [
|
||||
checksum: checksumSql(queueSessionRefMigrationSql),
|
||||
sql: queueSessionRefMigrationSql,
|
||||
},
|
||||
{
|
||||
id: "011_v01_cancel_lifecycle",
|
||||
checksum: checksumSql(cancelLifecycleMigrationSql),
|
||||
sql: cancelLifecycleMigrationSql,
|
||||
},
|
||||
];
|
||||
|
||||
export function postgresMigrationContract(): JsonRecord {
|
||||
return {
|
||||
migrationIds: postgresMigrations.map((migration) => migration.id),
|
||||
latestMigrationId: latestMigrationId(),
|
||||
requiredTables: ["agentrun_schema_migrations", "agentrun_runs", "agentrun_commands", "agentrun_events", "agentrun_runners", "agentrun_backends", "agentrun_leases", "agentrun_sessions", "agentrun_session_read_cursors", "agentrun_runner_jobs", "agentrun_queue_tasks", "agentrun_queue_read_cursors"],
|
||||
requiredTables: ["agentrun_schema_migrations", "agentrun_runs", "agentrun_commands", "agentrun_events", "agentrun_runners", "agentrun_backends", "agentrun_leases", "agentrun_sessions", "agentrun_session_read_cursors", "agentrun_runner_jobs", "agentrun_queue_tasks", "agentrun_queue_read_cursors", "agentrun_cancel_requests"],
|
||||
checksums: Object.fromEntries(postgresMigrations.map((migration) => [migration.id, migration.checksum])),
|
||||
};
|
||||
}
|
||||
@@ -452,7 +490,7 @@ CREATE TABLE IF NOT EXISTS agentrun_schema_migrations (
|
||||
const at = nowIso();
|
||||
return this.withTransaction(async (client) => {
|
||||
const sessionRef = await this.resolveSessionForRun(client, input, at);
|
||||
const run: RunRecord = { ...input, sessionRef, resourceBundleRef: input.resourceBundleRef ?? null, id: newId("run"), status: "pending", terminalStatus: null, failureKind: null, failureMessage: null, createdAt: at, updatedAt: at, claimedBy: null, leaseExpiresAt: null };
|
||||
const run: RunRecord = { ...input, sessionRef, resourceBundleRef: input.resourceBundleRef ?? null, id: newId("run"), status: "pending", terminalStatus: null, failureKind: null, failureMessage: null, cancelEpoch: 0, cancelRequestId: null, cancelRequestedAt: null, cancelReason: null, createdAt: at, updatedAt: at, claimedBy: null, leaseExpiresAt: null };
|
||||
await client.query(
|
||||
`INSERT INTO agentrun_runs (id, tenant_id, project_id, workspace_ref, session_ref, resource_bundle_ref, provider_id, backend_profile, execution_policy, trace_sink, status, terminal_status, failure_kind, failure_message, created_at, updated_at, claimed_by, lease_expires_at)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5::jsonb, $6::jsonb, $7, $8, $9::jsonb, $10::jsonb, $11, $12, $13, $14, $15, $16, $17, $18)`,
|
||||
@@ -509,7 +547,7 @@ CREATE TABLE IF NOT EXISTS agentrun_schema_migrations (
|
||||
}
|
||||
const seq = await this.nextSeq(client, "agentrun_commands", runId);
|
||||
const at = nowIso();
|
||||
const command: CommandRecord = { ...input, id: newId("cmd"), runId, seq, state: "pending", payloadHash, createdAt: at, updatedAt: at, acknowledgedAt: null };
|
||||
const command: CommandRecord = { ...input, id: newId("cmd"), runId, seq, state: "pending", payloadHash, cancelEpoch: 0, cancelRequestId: null, cancelRequestedAt: null, cancelReason: null, createdAt: at, updatedAt: at, acknowledgedAt: null };
|
||||
await client.query(
|
||||
`INSERT INTO agentrun_commands (id, run_id, seq, type, payload, payload_hash, idempotency_key, state, created_at, updated_at, acknowledged_at)
|
||||
VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9, $10, $11)`,
|
||||
@@ -684,7 +722,13 @@ CREATE TABLE IF NOT EXISTS agentrun_schema_migrations (
|
||||
const row = existing.rows[0];
|
||||
if (!row) throw new AgentRunError("schema-invalid", `command ${commandId} was not found`, { httpStatus: 404 });
|
||||
const command = commandFromRow(row);
|
||||
if (isTerminalCommandState(command.state)) return command;
|
||||
if (isTerminalCommandState(command.state)) {
|
||||
if (command.state === "cancelled" && result.terminalStatus !== "cancelled") {
|
||||
const run = await this.requireRunForUpdate(client, command.runId);
|
||||
await this.appendEventWithLockedRun(client, command.runId, "backend_status", lateWriteRejectedPayload(run, command, { source: "command-status", terminalStatus: result.terminalStatus, failureKind: result.failureKind, message: result.failureMessage ?? null, threadId: result.threadId ?? null, turnId: result.turnId ?? null }));
|
||||
}
|
||||
return command;
|
||||
}
|
||||
const state = commandStateFromTerminal(result.terminalStatus);
|
||||
const updated = await client.query("UPDATE agentrun_commands SET state = $2, updated_at = $3 WHERE id = $1 RETURNING *", [commandId, state, nowIso()]);
|
||||
const run = await this.requireRunForUpdate(client, command.runId);
|
||||
@@ -705,7 +749,10 @@ CREATE TABLE IF NOT EXISTS agentrun_schema_migrations (
|
||||
async finishRun(runId: string, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">): Promise<RunRecord> {
|
||||
return this.withTransaction(async (client) => {
|
||||
const existing = await this.requireRunForUpdate(client, runId);
|
||||
if (isTerminalRunStatus(existing.status)) return existing;
|
||||
if (isTerminalRunStatus(existing.status)) {
|
||||
if (existing.status === "cancelled" && result.terminalStatus !== "cancelled") await this.appendEventWithLockedRun(client, runId, "backend_status", lateWriteRejectedPayload(existing, null, { source: "run-status", terminalStatus: result.terminalStatus, failureKind: result.failureKind, message: result.failureMessage ?? null, threadId: result.threadId ?? null, turnId: result.turnId ?? null }));
|
||||
return existing;
|
||||
}
|
||||
const status = statusFromTerminal(result.terminalStatus);
|
||||
const updated = await client.query(
|
||||
`UPDATE agentrun_runs SET status = $2, terminal_status = $3, failure_kind = $4, failure_message = $5, updated_at = $6 WHERE id = $1 RETURNING *`,
|
||||
@@ -724,18 +771,32 @@ CREATE TABLE IF NOT EXISTS agentrun_schema_migrations (
|
||||
const run = await this.requireRunForUpdate(client, runId);
|
||||
if (isTerminalRunStatus(run.status)) return run;
|
||||
const at = nowIso();
|
||||
const cancel = await this.createCancelRequest(client, { targetKind: "run", targetId: runId, run, command: null, reason, at, stage: "accepted" });
|
||||
await this.appendCancelStage(client, runId, "accepted", cancel);
|
||||
const persistedRunResult = await client.query(
|
||||
`UPDATE agentrun_runs SET cancel_epoch = $2, cancel_request_id = $3, cancel_requested_at = $4, cancel_reason = $5, updated_at = $4 WHERE id = $1 RETURNING *`,
|
||||
[runId, cancel.epoch, cancel.id, at, reason],
|
||||
);
|
||||
const persistedRun = runFromRow(persistedRunResult.rows[0]);
|
||||
await this.appendCancelStage(client, runId, "persisted", cancel);
|
||||
const leaseExpired = Boolean(persistedRun.claimedBy && isLeaseExpired(persistedRun.leaseExpiresAt));
|
||||
if (leaseExpired) await this.appendCancelStage(client, runId, "fenced", cancel, { claimedBy: persistedRun.claimedBy, leaseExpiresAt: persistedRun.leaseExpiresAt });
|
||||
else if (persistedRun.claimedBy) {
|
||||
await this.appendCancelStage(client, runId, "delivered", cancel, { claimedBy: persistedRun.claimedBy });
|
||||
await this.appendCancelStage(client, runId, "aborting", cancel, { claimedBy: persistedRun.claimedBy });
|
||||
}
|
||||
const commands = await client.query("SELECT * FROM agentrun_commands WHERE run_id = $1 AND state NOT IN ('completed', 'failed', 'cancelled') FOR UPDATE", [runId]);
|
||||
for (const row of commands.rows) {
|
||||
const command = commandFromRow(row);
|
||||
await client.query("UPDATE agentrun_commands SET state = 'cancelled', updated_at = $2 WHERE id = $1", [command.id, at]);
|
||||
await this.appendEventWithLockedRun(client, runId, "backend_status", { phase: "command-terminal", commandId: command.id, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled" });
|
||||
await client.query("UPDATE agentrun_commands SET state = 'cancelled', cancel_epoch = $2, cancel_request_id = $3, cancel_requested_at = $4, cancel_reason = $5, updated_at = $4 WHERE id = $1", [command.id, cancel.epoch, cancel.id, at, reason]);
|
||||
await this.appendEventWithLockedRun(client, runId, "backend_status", { phase: "command-terminal", commandId: command.id, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
||||
}
|
||||
await this.appendEventWithLockedRun(client, runId, "backend_status", { phase: "cancel-requested", reason });
|
||||
const updated = await client.query(
|
||||
`UPDATE agentrun_runs SET status = 'cancelled', terminal_status = 'cancelled', failure_kind = 'cancelled', failure_message = $2, updated_at = $3 WHERE id = $1 RETURNING *`,
|
||||
[runId, reason, at],
|
||||
);
|
||||
await this.appendEventWithLockedRun(client, runId, "terminal_status", { terminalStatus: "cancelled", failureKind: "cancelled", message: reason });
|
||||
await this.appendEventWithLockedRun(client, runId, "terminal_status", { terminalStatus: "cancelled", failureKind: "cancelled", message: reason, cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
||||
await this.appendCancelStage(client, runId, "terminalized", cancel);
|
||||
const next = runFromRow(updated.rows[0]);
|
||||
await this.touchSessionForRun(client, next, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: runId, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: at }, { bumpVersion: true, at });
|
||||
return next;
|
||||
@@ -749,11 +810,22 @@ CREATE TABLE IF NOT EXISTS agentrun_schema_migrations (
|
||||
if (!row) throw new AgentRunError("schema-invalid", `command ${commandId} was not found`, { httpStatus: 404 });
|
||||
const command = commandFromRow(row);
|
||||
if (isTerminalCommandState(command.state)) return command;
|
||||
const updated = await client.query("UPDATE agentrun_commands SET state = 'cancelled', updated_at = $2 WHERE id = $1 RETURNING *", [commandId, nowIso()]);
|
||||
await this.appendEventWithLockedRun(client, command.runId, "backend_status", { phase: "command-terminal", commandId, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", message: reason });
|
||||
const run = await this.requireRunForUpdate(client, command.runId);
|
||||
const at = nowIso();
|
||||
const cancel = await this.createCancelRequest(client, { targetKind: "command", targetId: commandId, run, command, reason, at, stage: "accepted" });
|
||||
await this.appendCancelStage(client, command.runId, "accepted", cancel);
|
||||
const updated = await client.query("UPDATE agentrun_commands SET state = 'cancelled', cancel_epoch = $2, cancel_request_id = $3, cancel_requested_at = $4, cancel_reason = $5, updated_at = $4 WHERE id = $1 RETURNING *", [commandId, cancel.epoch, cancel.id, at, reason]);
|
||||
await this.appendCancelStage(client, command.runId, "persisted", cancel);
|
||||
const leaseExpired = Boolean(run.claimedBy && isLeaseExpired(run.leaseExpiresAt));
|
||||
if (leaseExpired) await this.appendCancelStage(client, command.runId, "fenced", cancel, { claimedBy: run.claimedBy, leaseExpiresAt: run.leaseExpiresAt });
|
||||
else if (run.claimedBy || command.state === "acknowledged") {
|
||||
await this.appendCancelStage(client, command.runId, "delivered", cancel, { claimedBy: run.claimedBy, commandState: command.state });
|
||||
await this.appendCancelStage(client, command.runId, "aborting", cancel, { claimedBy: run.claimedBy, commandState: command.state });
|
||||
}
|
||||
await this.appendEventWithLockedRun(client, command.runId, "backend_status", { phase: "command-terminal", commandId, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", message: reason, cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
||||
await this.appendCancelStage(client, command.runId, "terminalized", cancel);
|
||||
if (command.type === "turn") {
|
||||
const run = await this.requireRunForUpdate(client, command.runId);
|
||||
await this.touchSessionForRun(client, run, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: command.runId, lastCommandId: command.id, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: nowIso() }, { bumpVersion: true });
|
||||
await this.touchSessionForRun(client, run, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: command.runId, lastCommandId: command.id, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: at }, { bumpVersion: true, at });
|
||||
}
|
||||
return commandFromRow(updated.rows[0]);
|
||||
});
|
||||
@@ -1094,11 +1166,47 @@ CREATE TABLE IF NOT EXISTS agentrun_schema_migrations (
|
||||
await this.pool.end();
|
||||
}
|
||||
|
||||
private async createCancelRequest(client: PoolClient, input: { targetKind: CancelTargetKind; targetId: string; run: RunRecord; command: CommandRecord | null; reason: string; at: string; stage: CancelStage }): Promise<CancelRequestRecord> {
|
||||
const epoch = input.command ? input.command.cancelEpoch + 1 : input.run.cancelEpoch + 1;
|
||||
const record: CancelRequestRecord = {
|
||||
id: newId("cancel"),
|
||||
targetKind: input.targetKind,
|
||||
targetId: input.targetId,
|
||||
runId: input.run.id,
|
||||
commandId: input.command?.id ?? null,
|
||||
sessionId: input.run.sessionRef?.sessionId ?? null,
|
||||
taskId: null,
|
||||
reason: input.reason,
|
||||
requestedBy: null,
|
||||
epoch,
|
||||
stage: input.stage,
|
||||
metadata: {},
|
||||
createdAt: input.at,
|
||||
updatedAt: input.at,
|
||||
};
|
||||
await client.query(
|
||||
`INSERT INTO agentrun_cancel_requests (id, target_kind, target_id, run_id, command_id, session_id, task_id, reason, requested_by, epoch, stage, metadata, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13, $14)`,
|
||||
[record.id, record.targetKind, record.targetId, record.runId, record.commandId, record.sessionId, record.taskId, record.reason, record.requestedBy, record.epoch, record.stage, JSON.stringify(record.metadata), record.createdAt, record.updatedAt],
|
||||
);
|
||||
return record;
|
||||
}
|
||||
|
||||
private async appendCancelStage(client: PoolClient, runId: string, stage: CancelStage, cancel: CancelRequestRecord, extra: JsonRecord = {}): Promise<CancelRequestRecord> {
|
||||
const at = nowIso();
|
||||
const next: CancelRequestRecord = { ...cancel, stage, updatedAt: at };
|
||||
await client.query("UPDATE agentrun_cancel_requests SET stage = $2, updated_at = $3 WHERE id = $1", [cancel.id, stage, at]);
|
||||
await this.appendEventWithLockedRun(client, runId, "backend_status", cancelStagePayload(next, stage, extra));
|
||||
return next;
|
||||
}
|
||||
|
||||
private async appendEventWithLockedRun(client: PoolClient, runId: string, type: EventType, payload: JsonRecord): Promise<RunEvent> {
|
||||
const run = await this.requireRunForUpdate(client, runId);
|
||||
const eventType = requireEventType(type);
|
||||
const eventPayload = normalizeRunEventPayload(eventType, payload);
|
||||
const fenced = fenceLateEventForCancelledRun(run, eventType, payload);
|
||||
const eventPayload = normalizeRunEventPayload(fenced.type, fenced.payload);
|
||||
const seq = await this.nextSeq(client, "agentrun_events", runId);
|
||||
const event: RunEvent = { id: newId("evt"), runId, seq, type: eventType, payload: redactJson(eventPayload), createdAt: nowIso() };
|
||||
const event: RunEvent = { id: newId("evt"), runId, seq, type: fenced.type, payload: redactJson(eventPayload), createdAt: nowIso() };
|
||||
await client.query("INSERT INTO agentrun_events (id, run_id, seq, type, payload, created_at) VALUES ($1, $2, $3, $4, $5::jsonb, $6)", [event.id, event.runId, event.seq, event.type, JSON.stringify(event.payload), event.createdAt]);
|
||||
await client.query(
|
||||
`UPDATE agentrun_sessions
|
||||
@@ -1362,6 +1470,10 @@ function runFromRow(row: QueryResultRow): RunRecord {
|
||||
terminalStatus: nullableString(row.terminal_status) as TerminalStatus | null,
|
||||
failureKind: nullableString(row.failure_kind) as FailureKind | null,
|
||||
failureMessage: nullableString(row.failure_message),
|
||||
cancelEpoch: Number(row.cancel_epoch ?? 0),
|
||||
cancelRequestId: nullableString(row.cancel_request_id),
|
||||
cancelRequestedAt: nullableIso(row.cancel_requested_at),
|
||||
cancelReason: nullableString(row.cancel_reason),
|
||||
createdAt: iso(row.created_at),
|
||||
updatedAt: iso(row.updated_at),
|
||||
claimedBy: nullableString(row.claimed_by),
|
||||
@@ -1379,6 +1491,10 @@ function commandFromRow(row: QueryResultRow): CommandRecord {
|
||||
payloadHash: stringValue(row.payload_hash),
|
||||
...(nullableString(row.idempotency_key) ? { idempotencyKey: stringValue(row.idempotency_key) } : {}),
|
||||
state: stringValue(row.state) as CommandState,
|
||||
cancelEpoch: Number(row.cancel_epoch ?? 0),
|
||||
cancelRequestId: nullableString(row.cancel_request_id),
|
||||
cancelRequestedAt: nullableIso(row.cancel_requested_at),
|
||||
cancelReason: nullableString(row.cancel_reason),
|
||||
createdAt: iso(row.created_at),
|
||||
updatedAt: iso(row.updated_at),
|
||||
acknowledgedAt: nullableIso(row.acknowledged_at),
|
||||
|
||||
@@ -67,6 +67,7 @@ export async function buildRunResult(store: AgentRunStore, runId: string, comman
|
||||
const terminalClassification = terminalClassificationSummary({ terminal, terminalSource, failureKind, failureMessage, liveness });
|
||||
const diagnosis = runDiagnosis({ run, command, latestJob, events, terminalClassification, liveness, terminalStatus: terminal, failureKind, failureMessage });
|
||||
const steerDelivery = command?.type === "steer" ? steerDeliverySummary(events, command.id) : null;
|
||||
const cancelLifecycle = cancelLifecycleSummary(run, command, scopedEvents);
|
||||
const executionOk = terminal === null ? null : terminal === "completed" && !providerTerminalFailure;
|
||||
return {
|
||||
ok: executionOk === true,
|
||||
@@ -113,6 +114,7 @@ export async function buildRunResult(store: AgentRunStore, runId: string, comman
|
||||
diagnosis,
|
||||
blocker,
|
||||
liveness,
|
||||
cancelLifecycle,
|
||||
...(steerDelivery ? { steerDelivery } : {}),
|
||||
lastSeq: events.at(-1)?.seq ?? 0,
|
||||
eventCount: events.length,
|
||||
@@ -128,6 +130,38 @@ export async function buildRunResult(store: AgentRunStore, runId: string, comman
|
||||
};
|
||||
}
|
||||
|
||||
function cancelLifecycleSummary(run: RunRecord, command: CommandRecord | null, events: RunEvent[]): JsonRecord | null {
|
||||
const cancelEvents = events.filter((event) => {
|
||||
const stage = stringJsonValue(event.payload.cancelStage);
|
||||
const phase = stringJsonValue(event.payload.phase);
|
||||
return Boolean(stage || phase?.startsWith("cancel-") || phase === "late-write-rejected");
|
||||
});
|
||||
const requestId = command?.cancelRequestId ?? run.cancelRequestId ?? stringJsonValue(cancelEvents.at(-1)?.payload.cancelRequestId);
|
||||
if (!requestId && cancelEvents.length === 0) return null;
|
||||
const stages = cancelEvents.map((event) => ({
|
||||
seq: event.seq,
|
||||
at: event.createdAt,
|
||||
phase: stringJsonValue(event.payload.phase),
|
||||
stage: stringJsonValue(event.payload.cancelStage),
|
||||
commandId: stringJsonValue(event.payload.commandId),
|
||||
cancelRequestId: stringJsonValue(event.payload.cancelRequestId),
|
||||
}));
|
||||
const stageNames = Array.from(new Set(stages.map((stage) => stage.stage ?? stage.phase).filter((stage): stage is string => Boolean(stage))));
|
||||
const lateWriteRejected = stages.filter((stage) => stage.stage === "late-write-rejected" || stage.phase === "late-write-rejected").length;
|
||||
return {
|
||||
requestId,
|
||||
cancelEpoch: command?.cancelEpoch || run.cancelEpoch || numberJsonValue(cancelEvents.at(-1)?.payload.cancelEpoch),
|
||||
reason: command?.cancelReason ?? run.cancelReason ?? stringJsonValue(cancelEvents.at(-1)?.payload.reason),
|
||||
requestedAt: command?.cancelRequestedAt ?? run.cancelRequestedAt,
|
||||
stages,
|
||||
stageNames,
|
||||
stageCount: stages.length,
|
||||
terminalized: stageNames.includes("terminalized") || stageNames.includes("cancel-terminalized"),
|
||||
lateWriteRejected,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function livenessSnapshot(run: RunRecord, command: CommandRecord | null, events: RunEvent[], scopedEvents: RunEvent[], terminal: TerminalStatus | null, failureKind: FailureKind | null, failureMessage: string | null, completion: { responseAuthority: string; needsContinuation: boolean }): JsonRecord {
|
||||
const nowMs = Date.now();
|
||||
const active = terminal === null && !runIsTerminal(run) && !commandIsTerminal(command);
|
||||
|
||||
+1
-1
@@ -590,7 +590,7 @@ async function route({ method, url, body, store, sourceCommit, authSummary, runn
|
||||
const reason = typeof record.reason === "string" && record.reason.trim().length > 0 ? record.reason.trim() : undefined;
|
||||
if (session.activeCommandId) return await store.cancelCommand(session.activeCommandId, reason) as unknown as JsonValue;
|
||||
if (session.activeRunId) return await store.cancelRun(session.activeRunId, reason) as unknown as JsonValue;
|
||||
throw new AgentRunError("schema-invalid", `session ${session.sessionId} has no active run or command`, { httpStatus: 409 });
|
||||
return { action: "session-cancel", sessionId: session.sessionId, cancelled: false, reason: reason ?? null, session, valuesPrinted: false } as unknown as JsonValue;
|
||||
}
|
||||
throw new AgentRunError("schema-invalid", `session control action ${action} is not supported`, { httpStatus: 400 });
|
||||
}
|
||||
|
||||
+129
-17
@@ -1,4 +1,4 @@
|
||||
import type { BackendProfile, BackendTurnResult, CommandRecord, CreateCommandInput, CreateQueueTaskInput, CreateRunInput, FailureKind, JsonRecord, ListGcExpiredSessionsInput, QueueAttemptRef, QueueCommanderSnapshot, QueueReadCursorRecord, QueueStats, QueueTaskListResult, QueueTaskRecord, QueueTaskState, QueueTaskSummary, RunEvent, RunnerJobRecord, RunnerRecord, RunRecord, SessionEventPage, SessionListResult, SessionListState, SessionReadCursorRecord, SessionRecord, SessionRef, SessionStoragePatch, SessionSummary, TerminalStatus, UpsertSessionInput } from "../common/types.js";
|
||||
import type { BackendProfile, BackendTurnResult, CancelRequestRecord, CancelStage, CancelTargetKind, CommandRecord, CreateCommandInput, CreateQueueTaskInput, CreateRunInput, FailureKind, JsonRecord, ListGcExpiredSessionsInput, QueueAttemptRef, QueueCommanderSnapshot, QueueReadCursorRecord, QueueStats, QueueTaskListResult, QueueTaskRecord, QueueTaskState, QueueTaskSummary, RunEvent, RunnerJobRecord, RunnerRecord, RunRecord, SessionEventPage, SessionListResult, SessionListState, SessionReadCursorRecord, SessionRecord, SessionRef, SessionStoragePatch, SessionSummary, TerminalStatus, UpsertSessionInput } from "../common/types.js";
|
||||
import { AgentRunError } from "../common/errors.js";
|
||||
import { newId, nowIso, stableHash } from "../common/validation.js";
|
||||
import { redactJson } from "../common/redaction.js";
|
||||
@@ -136,6 +136,7 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
private readonly eventsByRun = new Map<string, RunEvent[]>();
|
||||
private readonly runners = new Map<string, RunnerRecord>();
|
||||
private readonly sessions = new Map<string, SessionRecord>();
|
||||
private readonly cancelRequests = new Map<string, CancelRequestRecord>();
|
||||
private readonly sessionReadCursors = new Map<string, SessionReadCursorRecord>();
|
||||
private readonly runnerJobs = new Map<string, RunnerJobRecord>();
|
||||
private readonly queueTasks = new Map<string, QueueTaskRecord>();
|
||||
@@ -150,7 +151,7 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
createRun(input: CreateRunInput): RunRecord {
|
||||
const at = nowIso();
|
||||
const sessionRef = this.resolveSessionForRun(input, at);
|
||||
const run: RunRecord = { ...input, sessionRef, resourceBundleRef: input.resourceBundleRef ?? null, id: newId("run"), status: "pending", terminalStatus: null, failureKind: null, failureMessage: null, createdAt: at, updatedAt: at, claimedBy: null, leaseExpiresAt: null };
|
||||
const run: RunRecord = { ...input, sessionRef, resourceBundleRef: input.resourceBundleRef ?? null, id: newId("run"), status: "pending", terminalStatus: null, failureKind: null, failureMessage: null, cancelEpoch: 0, cancelRequestId: null, cancelRequestedAt: null, cancelReason: null, createdAt: at, updatedAt: at, claimedBy: null, leaseExpiresAt: null };
|
||||
this.runs.set(run.id, run);
|
||||
this.eventsByRun.set(run.id, []);
|
||||
this.touchSessionForRun(run, { lastRunId: run.id, lastActivityAt: at }, { bumpVersion: false, at });
|
||||
@@ -190,7 +191,7 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
}
|
||||
const at = nowIso();
|
||||
const seq = Array.from(this.commands.values()).filter((command) => command.runId === runId).length + 1;
|
||||
const command: CommandRecord = { ...input, id: newId("cmd"), runId, seq, state: "pending", payloadHash, createdAt: at, updatedAt: at, acknowledgedAt: null };
|
||||
const command: CommandRecord = { ...input, id: newId("cmd"), runId, seq, state: "pending", payloadHash, cancelEpoch: 0, cancelRequestId: null, cancelRequestedAt: null, cancelReason: null, createdAt: at, updatedAt: at, acknowledgedAt: null };
|
||||
this.commands.set(command.id, command);
|
||||
if (command.type === "turn") this.touchSessionForRun(run, { executionState: "running", lastRunId: run.id, lastCommandId: command.id, activeRunId: run.id, activeCommandId: command.id, terminalStatus: null, failureKind: null, title: sessionTitleFromCommand(command), lastActivityAt: at }, { bumpVersion: true, at });
|
||||
else if (command.type === "steer") this.touchSessionForRun(run, { executionState: "running", lastRunId: run.id, lastCommandId: command.id, activeRunId: run.id, lastActivityAt: at }, { bumpVersion: true, at });
|
||||
@@ -288,7 +289,10 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
|
||||
finishCommand(commandId: string, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">): CommandRecord {
|
||||
const command = this.getCommand(commandId);
|
||||
if (isTerminalCommandState(command.state)) return command;
|
||||
if (isTerminalCommandState(command.state)) {
|
||||
if (command.state === "cancelled" && result.terminalStatus !== "cancelled") this.appendLateWriteRejected(command.runId, command, result, "command-status");
|
||||
return command;
|
||||
}
|
||||
const next = { ...command, state: commandStateFromTerminal(result.terminalStatus), updatedAt: nowIso() };
|
||||
this.commands.set(commandId, next);
|
||||
const run = this.getRun(command.runId);
|
||||
@@ -299,11 +303,12 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
}
|
||||
|
||||
appendEvent(runId: string, type: RunEvent["type"], payload: JsonRecord): RunEvent {
|
||||
this.getRun(runId);
|
||||
const run = this.getRun(runId);
|
||||
const eventType = requireEventType(type);
|
||||
const eventPayload = normalizeRunEventPayload(eventType, payload);
|
||||
const fenced = fenceLateEventForCancelledRun(run, eventType, payload);
|
||||
const eventPayload = normalizeRunEventPayload(fenced.type, fenced.payload);
|
||||
const events = this.eventsByRun.get(runId) ?? [];
|
||||
const event: RunEvent = { id: newId("evt"), runId, seq: events.length + 1, type: eventType, payload: redactJson(eventPayload), createdAt: nowIso() };
|
||||
const event: RunEvent = { id: newId("evt"), runId, seq: events.length + 1, type: fenced.type, payload: redactJson(eventPayload), createdAt: nowIso() };
|
||||
events.push(event);
|
||||
this.eventsByRun.set(runId, events);
|
||||
this.touchSessionForRun(this.getRun(runId), { lastEventSeq: event.seq, lastActivityAt: event.createdAt }, { bumpVersion: false, at: event.createdAt });
|
||||
@@ -312,7 +317,10 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
|
||||
finishRun(runId: string, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">): RunRecord {
|
||||
const existing = this.getRun(runId);
|
||||
if (isTerminalRunStatus(existing.status)) return existing;
|
||||
if (isTerminalRunStatus(existing.status)) {
|
||||
if (existing.status === "cancelled" && result.terminalStatus !== "cancelled") this.appendLateWriteRejected(runId, null, result, "run-status");
|
||||
return existing;
|
||||
}
|
||||
const status = statusFromTerminal(result.terminalStatus);
|
||||
const next = this.updateRun(runId, { status, terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: result.failureMessage });
|
||||
if (result.threadId && next.sessionRef?.sessionId) this.upsertSessionThread(next, result.threadId, result.turnId ?? null);
|
||||
@@ -325,14 +333,24 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
const run = this.getRun(runId);
|
||||
if (isTerminalRunStatus(run.status)) return run;
|
||||
const at = nowIso();
|
||||
for (const command of Array.from(this.commands.values()).filter((item) => item.runId === runId && !isTerminalCommandState(item.state))) {
|
||||
const cancelled = { ...command, state: "cancelled" as const, updatedAt: at };
|
||||
this.commands.set(command.id, cancelled);
|
||||
this.appendEvent(runId, "backend_status", { phase: "command-terminal", commandId: command.id, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled" });
|
||||
const cancel = this.createCancelRequest({ targetKind: "run", targetId: runId, run, command: null, reason, at, stage: "accepted" });
|
||||
this.appendCancelStage(runId, "accepted", cancel);
|
||||
const persistedRun = this.updateRun(runId, { cancelEpoch: cancel.epoch, cancelRequestId: cancel.id, cancelRequestedAt: at, cancelReason: reason });
|
||||
this.appendCancelStage(runId, "persisted", cancel);
|
||||
const leaseExpired = Boolean(persistedRun.claimedBy && isLeaseExpired(persistedRun.leaseExpiresAt));
|
||||
if (leaseExpired) this.appendCancelStage(runId, "fenced", cancel, { claimedBy: persistedRun.claimedBy, leaseExpiresAt: persistedRun.leaseExpiresAt });
|
||||
else if (persistedRun.claimedBy) {
|
||||
this.appendCancelStage(runId, "delivered", cancel, { claimedBy: persistedRun.claimedBy });
|
||||
this.appendCancelStage(runId, "aborting", cancel, { claimedBy: persistedRun.claimedBy });
|
||||
}
|
||||
for (const command of Array.from(this.commands.values()).filter((item) => item.runId === runId && !isTerminalCommandState(item.state))) {
|
||||
const cancelled = { ...command, state: "cancelled" as const, cancelEpoch: cancel.epoch, cancelRequestId: cancel.id, cancelRequestedAt: at, cancelReason: reason, updatedAt: at };
|
||||
this.commands.set(command.id, cancelled);
|
||||
this.appendEvent(runId, "backend_status", { phase: "command-terminal", commandId: command.id, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
||||
}
|
||||
this.appendEvent(runId, "backend_status", { phase: "cancel-requested", reason });
|
||||
const next = this.updateRun(runId, { status: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", failureMessage: reason });
|
||||
this.appendEvent(runId, "terminal_status", { terminalStatus: "cancelled", failureKind: "cancelled", message: reason });
|
||||
this.appendEvent(runId, "terminal_status", { terminalStatus: "cancelled", failureKind: "cancelled", message: reason, cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
||||
this.appendCancelStage(runId, "terminalized", cancel);
|
||||
this.touchSessionForRun(next, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: runId, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: next.updatedAt }, { bumpVersion: true, at: next.updatedAt });
|
||||
return next;
|
||||
}
|
||||
@@ -340,10 +358,22 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
cancelCommand(commandId: string, reason = "cancel requested"): CommandRecord {
|
||||
const command = this.getCommand(commandId);
|
||||
if (isTerminalCommandState(command.state)) return command;
|
||||
const next = { ...command, state: "cancelled" as const, updatedAt: nowIso() };
|
||||
const run = this.getRun(command.runId);
|
||||
const at = nowIso();
|
||||
const cancel = this.createCancelRequest({ targetKind: "command", targetId: commandId, run, command, reason, at, stage: "accepted" });
|
||||
this.appendCancelStage(command.runId, "accepted", cancel);
|
||||
const next = { ...command, state: "cancelled" as const, cancelEpoch: cancel.epoch, cancelRequestId: cancel.id, cancelRequestedAt: at, cancelReason: reason, updatedAt: at };
|
||||
this.commands.set(commandId, next);
|
||||
this.appendEvent(command.runId, "backend_status", { phase: "command-terminal", commandId, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", message: reason });
|
||||
if (command.type === "turn") this.touchSessionForRun(this.getRun(command.runId), { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: command.runId, lastCommandId: command.id, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: next.updatedAt }, { bumpVersion: true, at: next.updatedAt });
|
||||
this.appendCancelStage(command.runId, "persisted", cancel);
|
||||
const leaseExpired = Boolean(run.claimedBy && isLeaseExpired(run.leaseExpiresAt));
|
||||
if (leaseExpired) this.appendCancelStage(command.runId, "fenced", cancel, { claimedBy: run.claimedBy, leaseExpiresAt: run.leaseExpiresAt });
|
||||
else if (run.claimedBy || command.state === "acknowledged") {
|
||||
this.appendCancelStage(command.runId, "delivered", cancel, { claimedBy: run.claimedBy, commandState: command.state });
|
||||
this.appendCancelStage(command.runId, "aborting", cancel, { claimedBy: run.claimedBy, commandState: command.state });
|
||||
}
|
||||
this.appendEvent(command.runId, "backend_status", { phase: "command-terminal", commandId, state: "cancelled", terminalStatus: "cancelled", failureKind: "cancelled", message: reason, cancelRequestId: cancel.id, cancelEpoch: cancel.epoch });
|
||||
this.appendCancelStage(command.runId, "terminalized", cancel);
|
||||
if (command.type === "turn") this.touchSessionForRun(run, { executionState: "terminal", activeRunId: null, activeCommandId: null, lastRunId: command.runId, lastCommandId: command.id, terminalStatus: "cancelled", failureKind: "cancelled", lastActivityAt: next.updatedAt }, { bumpVersion: true, at: next.updatedAt });
|
||||
return next;
|
||||
}
|
||||
|
||||
@@ -580,6 +610,39 @@ export class MemoryAgentRunStore implements AgentRunStore {
|
||||
return next;
|
||||
}
|
||||
|
||||
private createCancelRequest(input: { targetKind: CancelTargetKind; targetId: string; run: RunRecord; command: CommandRecord | null; reason: string; at: string; stage: CancelStage }): CancelRequestRecord {
|
||||
const sessionId = input.run.sessionRef?.sessionId ?? null;
|
||||
const epoch = input.command ? input.command.cancelEpoch + 1 : input.run.cancelEpoch + 1;
|
||||
const record: CancelRequestRecord = {
|
||||
id: newId("cancel"),
|
||||
targetKind: input.targetKind,
|
||||
targetId: input.targetId,
|
||||
runId: input.run.id,
|
||||
commandId: input.command?.id ?? null,
|
||||
sessionId,
|
||||
taskId: null,
|
||||
reason: input.reason,
|
||||
requestedBy: null,
|
||||
epoch,
|
||||
stage: input.stage,
|
||||
metadata: {},
|
||||
createdAt: input.at,
|
||||
updatedAt: input.at,
|
||||
};
|
||||
this.cancelRequests.set(record.id, record);
|
||||
return record;
|
||||
}
|
||||
|
||||
private appendCancelStage(runId: string, stage: CancelStage, cancel: CancelRequestRecord, extra: JsonRecord = {}): void {
|
||||
const next: CancelRequestRecord = { ...cancel, stage, updatedAt: nowIso() };
|
||||
this.cancelRequests.set(cancel.id, next);
|
||||
this.appendEvent(runId, "backend_status", cancelStagePayload(next, stage, extra));
|
||||
}
|
||||
|
||||
private appendLateWriteRejected(runId: string, command: CommandRecord | null, result: Pick<BackendTurnResult, "terminalStatus" | "failureKind" | "failureMessage" | "threadId" | "turnId">, source: string): void {
|
||||
this.appendEvent(runId, "backend_status", lateWriteRejectedPayload(this.getRun(runId), command, { source, terminalStatus: result.terminalStatus, failureKind: result.failureKind, message: result.failureMessage ?? null, threadId: result.threadId ?? null, turnId: result.turnId ?? null }));
|
||||
}
|
||||
|
||||
private nextQueueVersion(): number {
|
||||
this.queueVersion += 1;
|
||||
return this.queueVersion;
|
||||
@@ -710,6 +773,55 @@ export function isTerminalCommandState(state: CommandRecord["state"]): boolean {
|
||||
return state === "completed" || state === "failed" || state === "cancelled";
|
||||
}
|
||||
|
||||
export function cancelStagePayload(cancel: CancelRequestRecord, stage: CancelStage, extra: JsonRecord = {}): JsonRecord {
|
||||
return {
|
||||
phase: cancelPhase(stage),
|
||||
cancelStage: stage,
|
||||
cancelRequestId: cancel.id,
|
||||
targetKind: cancel.targetKind,
|
||||
targetId: cancel.targetId,
|
||||
runId: cancel.runId,
|
||||
commandId: cancel.commandId,
|
||||
sessionId: cancel.sessionId,
|
||||
taskId: cancel.taskId,
|
||||
reason: cancel.reason,
|
||||
requestedBy: cancel.requestedBy,
|
||||
cancelEpoch: cancel.epoch,
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
export function fenceLateEventForCancelledRun(run: RunRecord, type: RunEvent["type"], payload: JsonRecord): { type: RunEvent["type"]; payload: JsonRecord } {
|
||||
if (run.status !== "cancelled" || isAllowedAfterCancel(type, payload)) return { type, payload };
|
||||
return { type: "backend_status", payload: lateWriteRejectedPayload(run, null, { eventType: type, payload }) };
|
||||
}
|
||||
|
||||
export function lateWriteRejectedPayload(run: RunRecord, command: CommandRecord | null, details: JsonRecord): JsonRecord {
|
||||
return {
|
||||
phase: "late-write-rejected",
|
||||
cancelStage: "late-write-rejected",
|
||||
cancelRequestId: command?.cancelRequestId ?? run.cancelRequestId,
|
||||
cancelEpoch: command?.cancelEpoch ?? run.cancelEpoch,
|
||||
runId: run.id,
|
||||
commandId: command?.id ?? null,
|
||||
reason: command?.cancelReason ?? run.cancelReason,
|
||||
rejected: redactJson(details),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function cancelPhase(stage: CancelStage): string {
|
||||
return stage === "late-write-rejected" ? "late-write-rejected" : `cancel-${stage}`;
|
||||
}
|
||||
|
||||
function isAllowedAfterCancel(type: RunEvent["type"], payload: JsonRecord): boolean {
|
||||
if (type === "terminal_status" && payload.terminalStatus === "cancelled") return true;
|
||||
const phase = typeof payload.phase === "string" ? payload.phase : "";
|
||||
if (phase.startsWith("cancel-") || phase === "late-write-rejected" || phase === "turn-cancelled") return true;
|
||||
if (phase === "command-terminal" && payload.failureKind === "cancelled") return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function isTerminalQueueTaskState(state: QueueTaskState): boolean {
|
||||
return state === "completed" || state === "failed" || state === "blocked" || state === "cancelled";
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user