feat: 补齐 HWLAB 手动调度能力

2026-06-01 11:40:08 +08:00
parent eb5e0f57a6
commit 62846f6369
25 changed files with 1159 additions and 70 deletions
@@ -141,6 +141,9 @@ function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string
    { name: "AGENTRUN_RUNNER_ID", value: context.runnerId },
    { name: "AGENTRUN_BACKEND_PROFILE", value: options.run.backendProfile },
    { name: "AGENTRUN_EXECUTION_POLICY_JSON", value: JSON.stringify(options.run.executionPolicy) },
+    { name: "AGENTRUN_SESSION_REF_JSON", value: JSON.stringify(options.run.sessionRef ?? null) },
+    { name: "AGENTRUN_RESOURCE_BUNDLE_JSON", value: JSON.stringify(options.run.resourceBundleRef ?? null) },
+    { name: "AGENTRUN_WORKSPACE_ROOT", value: "/home/agentrun/workspaces" },
    { name: "AGENTRUN_SOURCE_COMMIT", value: context.sourceCommit },
    { name: "AGENTRUN_RUNTIME_NAMESPACE", value: context.namespace },
    { name: "AGENTRUN_K8S_JOB_NAME", value: context.jobName },
@@ -55,6 +55,14 @@ export class RunnerManagerApi {
    return await this.client.patch(`/api/v1/runs/${encodeURIComponent(runId)}/lease`, { runnerId, leaseMs }) as RunRecord;
  }

+  async getRun(runId: string): Promise<RunRecord> {
+    return await this.client.get(`/api/v1/runs/${encodeURIComponent(runId)}`) as RunRecord;
+  }
+
+  async getCommand(runId: string, commandId: string): Promise<CommandRecord> {
+    return await this.client.get(`/api/v1/runs/${encodeURIComponent(runId)}/commands/${encodeURIComponent(commandId)}`) as CommandRecord;
+  }
+
  async pollCommands(runId: string, options: { afterSeq?: number; limit?: number; commandId?: string }): Promise<PollCommandsResult> {
    const afterSeq = options.afterSeq ?? 0;
    const limit = options.limit ?? 20;
@@ -72,7 +80,7 @@ export class RunnerManagerApi {
    return await this.client.post(`/api/v1/runs/${encodeURIComponent(runId)}/events`, event as unknown as JsonRecord) as JsonRecord;
  }

-  async reportStatus(runId: string, report: { terminalStatus: TerminalStatus; failureKind: FailureKind | null; failureMessage: string | null }): Promise<RunRecord> {
+  async reportStatus(runId: string, report: { terminalStatus: TerminalStatus; failureKind: FailureKind | null; failureMessage: string | null; threadId?: string; turnId?: string }): Promise<RunRecord> {
    return await this.client.patch(`/api/v1/runs/${encodeURIComponent(runId)}/status`, report as unknown as JsonRecord) as RunRecord;
  }

@@ -0,0 +1,81 @@
+import { spawn } from "node:child_process";
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { AgentRunError } from "../common/errors.js";
+import { redactText } from "../common/redaction.js";
+import type { JsonRecord, ResourceBundleRef } from "../common/types.js";
+import { stableHash } from "../common/validation.js";
+
+export interface MaterializedResourceBundle {
+  workspacePath: string;
+  event: JsonRecord;
+}
+
+export async function materializeResourceBundle(resourceBundleRef: ResourceBundleRef | null | undefined, env: NodeJS.ProcessEnv = process.env): Promise<MaterializedResourceBundle | null> {
+  if (!resourceBundleRef) return null;
+  const workspaceRoot = path.resolve(env.AGENTRUN_WORKSPACE_ROOT ?? "/home/agentrun/workspaces");
+  const checkoutPath = path.join(workspaceRoot, stableHash({ repoUrl: resourceBundleRef.repoUrl, commitId: resourceBundleRef.commitId }).slice(0, 16));
+  await mkdir(checkoutPath, { recursive: true });
+  await git(["init"], checkoutPath);
+  await git(["remote", "remove", "origin"], checkoutPath, { allowFailure: true });
+  await git(["remote", "add", "origin", resourceBundleRef.repoUrl], checkoutPath);
+  if (resourceBundleRef.sparsePaths && resourceBundleRef.sparsePaths.length > 0) {
+    await git(["config", "core.sparseCheckout", "true"], checkoutPath);
+    await mkdir(path.join(checkoutPath, ".git", "info"), { recursive: true });
+    await writeFile(path.join(checkoutPath, ".git", "info", "sparse-checkout"), `${resourceBundleRef.sparsePaths.join("\n")}\n`, "utf8");
+  }
+  await git(["fetch", "--depth", "1", "origin", resourceBundleRef.commitId], checkoutPath);
+  await git(["checkout", "--detach", resourceBundleRef.commitId], checkoutPath);
+  const actualCommit = (await git(["rev-parse", "HEAD"], checkoutPath)).stdout.trim();
+  if (actualCommit !== resourceBundleRef.commitId) throw new AgentRunError("infra-failed", "resource bundle checkout did not land on requested commit", { httpStatus: 500, details: { expectedCommit: resourceBundleRef.commitId, actualCommit } });
+  const treeId = (await git(["rev-parse", "HEAD^{tree}"], checkoutPath)).stdout.trim();
+  const workspacePath = resolveWorkspacePath(checkoutPath, resourceBundleRef.subdir);
+  return {
+    workspacePath,
+    event: {
+      phase: "resource-bundle-materialized",
+      kind: "git",
+      repoUrl: resourceBundleRef.repoUrl,
+      commitId: resourceBundleRef.commitId,
+      treeId,
+      checkoutPath: pathSummary(checkoutPath),
+      workspacePath: pathSummary(workspacePath),
+      subdir: resourceBundleRef.subdir ?? null,
+      sparsePathCount: resourceBundleRef.sparsePaths?.length ?? 0,
+      valuesPrinted: false,
+    },
+  };
+}
+
+async function git(args: string[], cwd: string, options: { allowFailure?: boolean } = {}): Promise<{ stdout: string; stderr: string }> {
+  const child = spawn("git", args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
+  let stdout = "";
+  let stderr = "";
+  child.stdout.setEncoding("utf8");
+  child.stderr.setEncoding("utf8");
+  child.stdout.on("data", (chunk) => { stdout += String(chunk); });
+  child.stderr.on("data", (chunk) => { stderr += String(chunk); });
+  const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => {
+    child.on("error", reject);
+    child.on("close", (code, signal) => resolve({ code, signal }));
+  }).catch((error: unknown) => {
+    throw new AgentRunError("infra-failed", `failed to start git: ${error instanceof Error ? error.message : String(error)}`, { httpStatus: 503 });
+  });
+  if (result.code !== 0 && !options.allowFailure) {
+    throw new AgentRunError("infra-failed", `git ${args[0] ?? "command"} failed with code ${result.code}`, { httpStatus: 502, details: { stderr: redactText(stderr.slice(-4000)), stdout: redactText(stdout.slice(-1000)), signal: result.signal } });
+  }
+  return { stdout, stderr };
+}
+
+function resolveWorkspacePath(checkoutPath: string, subdir: string | undefined): string {
+  if (!subdir) return checkoutPath;
+  const resolved = path.resolve(checkoutPath, subdir);
+  const root = path.resolve(checkoutPath);
+  if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) throw new AgentRunError("schema-invalid", "resource bundle subdir escaped checkout", { httpStatus: 400 });
+  return resolved;
+}
+
+function pathSummary(value: string): JsonRecord {
+  const parts = value.split(/[\\/]+/u).filter(Boolean);
+  return { absolute: path.isAbsolute(value), basename: parts.at(-1) ?? null, depth: parts.length, fingerprint: stableHash(value).slice(0, 16), valuePrinted: false };
+}
@@ -1,5 +1,6 @@
 import { RunnerManagerApi, failureKindFromError, terminalStatusForFailure, errorMessage } from "./manager-api.js";
 import { runBackendTurn, type BackendAdapterOptions } from "../backend/adapter.js";
+import { materializeResourceBundle } from "./resource-bundle.js";
 import type { BackendProfile, JsonRecord, RunRecord, RunnerRecord } from "../common/types.js";
 import { AgentRunError } from "../common/errors.js";

@@ -21,6 +22,7 @@ export interface RunnerOnceOptions extends BackendAdapterOptions {
 export async function runOnce(options: RunnerOnceOptions): Promise<JsonRecord> {
  const api = new RunnerManagerApi(options.managerUrl);
  const targetRun = await api.client.get(`/api/v1/runs/${encodeURIComponent(options.runId)}`) as RunRecord;
+  if (isTerminalRun(targetRun)) return { terminalStatus: targetRun.terminalStatus, failureKind: targetRun.failureKind, run: targetRun, skipped: "run-terminal" } as JsonRecord;
  if (options.backendProfile && options.backendProfile !== targetRun.backendProfile) {
    throw new AgentRunError("schema-invalid", `runner backendProfile ${options.backendProfile} does not match run backendProfile ${targetRun.backendProfile}`, { httpStatus: 400 });
  }
@@ -55,9 +57,60 @@ export async function runOnce(options: RunnerOnceOptions): Promise<JsonRecord> {
    return { runner, claimed, terminalStatus: "blocked", failureKind: "schema-invalid", polledCommands: commandsResponse.items.length };
  }
  await api.ackCommand(command.id);
-  const result = await runBackendTurn(claimed, command, options);
-  for (const event of result.events) await api.appendEvent(options.runId, event);
-  await api.reportCommandStatus(command.id, { terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: result.failureMessage });
-  const finalRun = await api.reportStatus(options.runId, { terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: result.failureMessage }) as RunRecord;
-  return { runner, commandId: command.id, terminalStatus: result.terminalStatus, failureKind: result.failureKind, run: finalRun } as JsonRecord;
+  const acked = await api.getCommand(options.runId, command.id);
+  if (acked.state === "cancelled") return await reportCancelled(api, options.runId, command.id, runner, claimed, "command cancelled before backend start");
+  await assertNotCancelled(api, options.runId, command.id);
+  const abortController = new AbortController();
+  const stopCancelWatch = watchCancellation(api, options.runId, command.id, abortController);
+  let workspacePath: string | undefined;
+  try {
+    const materialized = await materializeResourceBundle(claimed.resourceBundleRef ?? null, options.env ?? process.env);
+    if (materialized) {
+      workspacePath = materialized.workspacePath;
+      await api.appendEvent(options.runId, { type: "backend_status", payload: materialized.event });
+    }
+    await assertNotCancelled(api, options.runId, command.id);
+    const result = await runBackendTurn(claimed, command, { ...options, ...(workspacePath ? { workspacePath } : {}), abortSignal: abortController.signal });
+    for (const event of result.events) await api.appendEvent(options.runId, event);
+    await api.reportCommandStatus(command.id, { terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: result.failureMessage });
+    const finalRun = await api.reportStatus(options.runId, { terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: result.failureMessage, ...(result.threadId ? { threadId: result.threadId } : {}), ...(result.turnId ? { turnId: result.turnId } : {}) }) as RunRecord;
+    return { runner, commandId: command.id, terminalStatus: result.terminalStatus, failureKind: result.failureKind, run: finalRun } as JsonRecord;
+  } finally {
+    stopCancelWatch();
+  }
+}
+
+function isTerminalRun(run: RunRecord): boolean {
+  return run.status === "completed" || run.status === "failed" || run.status === "blocked" || run.status === "cancelled";
+}
+
+async function assertNotCancelled(api: RunnerManagerApi, runId: string, commandId: string): Promise<void> {
+  const [run, command] = await Promise.all([api.getRun(runId), api.getCommand(runId, commandId)]);
+  if (run.status === "cancelled" || command.state === "cancelled") throw new AgentRunError("cancelled", "run or command was cancelled", { httpStatus: 409 });
+}
+
+function watchCancellation(api: RunnerManagerApi, runId: string, commandId: string, controller: AbortController): () => void {
+  let stopped = false;
+  const check = async (): Promise<void> => {
+    if (stopped || controller.signal.aborted) return;
+    try {
+      const [run, command] = await Promise.all([api.getRun(runId), api.getCommand(runId, commandId)]);
+      if (run.status === "cancelled" || command.state === "cancelled") controller.abort();
+    } catch {
+      // Cancellation polling must not hide the backend's own terminal result.
+    }
+  };
+  const timer = setInterval(() => { void check(); }, 2_000);
+  void check();
+  return () => {
+    stopped = true;
+    clearInterval(timer);
+  };
+}
+
+async function reportCancelled(api: RunnerManagerApi, runId: string, commandId: string, runner: RunnerRecord, claimed: RunRecord, message: string): Promise<JsonRecord> {
+  await api.appendEvent(runId, { type: "terminal_status", payload: { terminalStatus: "cancelled", failureKind: "cancelled", message } });
+  await api.reportCommandStatus(commandId, { terminalStatus: "cancelled", failureKind: "cancelled", failureMessage: message });
+  const finalRun = await api.reportStatus(runId, { terminalStatus: "cancelled", failureKind: "cancelled", failureMessage: message });
+  return { runner, commandId, claimed, terminalStatus: "cancelled", failureKind: "cancelled", run: finalRun };
 }