支持 gitbundle 资源装配

This commit is contained in:
Codex
2026-06-08 11:21:34 +08:00
parent 47b02b5101
commit 52376fb71e
13 changed files with 287 additions and 364 deletions
+135 -125
View File
@@ -1,6 +1,6 @@
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import { chmod, cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { chmod, cp, mkdir, readdir, readFile, rm, stat } from "node:fs/promises";
import path from "node:path";
import { AgentRunError } from "../common/errors.js";
import { redactText } from "../common/redaction.js";
@@ -40,70 +40,135 @@ interface MaterializedSkillRef {
summary: string;
}
interface GitCheckout {
repoUrl: string;
commitId: string;
checkoutPath: string;
treeId: string;
}
interface MaterializedGitBundle {
name: string | null;
repoUrl: string;
commitId: string;
subpath: string;
targetPath: string;
sourceKind: "file" | "directory";
sourceBytes: number | null;
}
export async function materializeResourceBundle(resourceBundleRef: ResourceBundleRef | null | undefined, env: NodeJS.ProcessEnv = process.env): Promise<MaterializedResourceBundle | null> {
if (!resourceBundleRef) return null;
const workspaceRoot = path.resolve(env.AGENTRUN_WORKSPACE_ROOT ?? "/home/agentrun/workspaces");
const checkoutPath = path.join(workspaceRoot, stableHash({ repoUrl: resourceBundleRef.repoUrl, commitId: resourceBundleRef.commitId }).slice(0, 16));
await mkdir(checkoutPath, { recursive: true });
await git(["init"], checkoutPath);
await git(["remote", "remove", "origin"], checkoutPath, { allowFailure: true });
await git(["remote", "add", "origin", resourceBundleRef.repoUrl], checkoutPath);
if (resourceBundleRef.sparsePaths && resourceBundleRef.sparsePaths.length > 0) {
await git(["config", "core.sparseCheckout", "true"], checkoutPath);
await mkdir(path.join(checkoutPath, ".git", "info"), { recursive: true });
await writeFile(path.join(checkoutPath, ".git", "info", "sparse-checkout"), `${resourceBundleRef.sparsePaths.join("\n")}\n`, "utf8");
}
await git(["fetch", "--depth", "1", "origin", resourceBundleRef.commitId], checkoutPath);
await git(["checkout", "--detach", resourceBundleRef.commitId], checkoutPath);
const actualCommit = (await git(["rev-parse", "HEAD"], checkoutPath)).stdout.trim();
if (actualCommit !== resourceBundleRef.commitId) throw new AgentRunError("infra-failed", "resource bundle checkout did not land on requested commit", { httpStatus: 500, details: { expectedCommit: resourceBundleRef.commitId, actualCommit } });
const treeId = (await git(["rev-parse", "HEAD^{tree}"], checkoutPath)).stdout.trim();
const workspacePath = resolveWorkspacePath(checkoutPath, resourceBundleRef.subdir);
const toolAliases = await materializeToolAliases(checkoutPath, resourceBundleRef.toolAliases ?? [], env);
const skills = await materializeSkillRefs(checkoutPath, workspacePath, resourceBundleRef.skillRefs ?? []);
const prompts = await materializePromptRefs(checkoutPath, resourceBundleRef.promptRefs ?? []);
const workspaceFiles = await materializeWorkspaceFiles(workspacePath, resourceBundleRef.workspaceFiles ?? []);
const runScope = env.AGENTRUN_RUN_ID ?? env.AGENTRUN_ATTEMPT_ID ?? "standalone";
const assemblyRoot = path.join(workspaceRoot, `gitbundle-${stableHash({ runScope, resourceBundleRef }).slice(0, 16)}`);
const checkoutRoot = path.join(assemblyRoot, "checkouts");
const workspacePath = path.join(assemblyRoot, "workspace");
await rm(assemblyRoot, { recursive: true, force: true });
await mkdir(checkoutRoot, { recursive: true });
await mkdir(workspacePath, { recursive: true });
const checkoutCache = new Map<string, Promise<GitCheckout>>();
const checkoutFor = (repoUrl: string, commitId: string) => {
const key = stableHash({ repoUrl, commitId });
let checkout = checkoutCache.get(key);
if (!checkout) {
checkout = checkoutGitCommit(checkoutRoot, repoUrl, commitId);
checkoutCache.set(key, checkout);
}
return checkout;
};
const materializedBundles = await materializeGitBundles(workspacePath, resourceBundleRef, checkoutFor);
const defaultCheckout = await checkoutFor(resourceBundleRef.repoUrl, resourceBundleRef.commitId);
const tools = await prepareGitBundleTools(workspacePath);
const skills = await discoverGitBundleSkills(workspacePath);
const prompts = await materializePromptRefs(defaultCheckout.checkoutPath, resourceBundleRef.promptRefs ?? []);
const initialPrompt = assembleInitialPrompt(prompts.items, skills.items);
return {
workspacePath,
...(toolAliases.binPath ? { binPath: toolAliases.binPath } : {}),
...(tools.binPath ? { binPath: tools.binPath } : {}),
...(skills.skillsDir ? { skillsDir: skills.skillsDir } : {}),
...(initialPrompt ? { initialPrompt } : {}),
event: {
phase: "resource-bundle-materialized",
kind: "git",
kind: "gitbundle",
repoUrl: resourceBundleRef.repoUrl,
commitId: resourceBundleRef.commitId,
treeId,
checkoutPath: pathSummary(checkoutPath),
treeId: defaultCheckout.treeId,
checkoutPath: pathSummary(defaultCheckout.checkoutPath),
workspacePath: pathSummary(workspacePath),
subdir: resourceBundleRef.subdir ?? null,
sparsePathCount: resourceBundleRef.sparsePaths?.length ?? 0,
toolAliases: toolAliases.event,
skillRefs: skills.event,
bundles: {
count: materializedBundles.length,
items: materializedBundles.map((item) => ({ ...item, valuesPrinted: false })),
valuesPrinted: false,
},
tools: tools.event,
skillDirs: skills.event,
promptRefs: prompts.event,
workspaceFiles: workspaceFiles.event,
initialPrompt: initialPrompt?.summary ?? { available: false, bytes: 0, sha256: null, promptRefCount: prompts.items.length, skillRefCount: skills.items.length, valuesPrinted: false },
initialPrompt: initialPrompt?.summary ?? { available: false, bytes: 0, sha256: null, promptRefCount: prompts.items.length, skillCount: skills.items.length, valuesPrinted: false },
valuesPrinted: false,
},
};
}
async function materializeToolAliases(checkoutPath: string, aliases: NonNullable<ResourceBundleRef["toolAliases"]>, env: NodeJS.ProcessEnv): Promise<{ binPath?: string; event: JsonRecord }> {
if (aliases.length === 0) return { event: { count: 0, names: [], binPath: null, valuesPrinted: false } };
const binPath = path.resolve(env.AGENTRUN_RESOURCE_BIN_PATH ?? path.join(path.dirname(checkoutPath), ".bin"));
await mkdir(binPath, { recursive: true });
const names: string[] = [];
for (const alias of aliases) {
const target = resolveBundlePath(checkoutPath, alias.path, `toolAliases.${alias.name}.path`);
const wrapper = path.join(binPath, alias.name);
const content = aliasWrapper(alias.kind, target);
await assertAliasWrapperWritable(wrapper, alias.name);
await writeFile(wrapper, content, "utf8");
await chmod(wrapper, 0o755);
names.push(alias.name);
async function checkoutGitCommit(checkoutRoot: string, repoUrl: string, commitId: string): Promise<GitCheckout> {
const checkoutPath = path.join(checkoutRoot, stableHash({ repoUrl, commitId }).slice(0, 16));
await mkdir(checkoutPath, { recursive: true });
await git(["init"], checkoutPath);
await git(["remote", "remove", "origin"], checkoutPath, { allowFailure: true });
await git(["remote", "add", "origin", repoUrl], checkoutPath);
await git(["fetch", "--depth", "1", "origin", commitId], checkoutPath);
await git(["checkout", "--detach", commitId], checkoutPath);
const actualCommit = (await git(["rev-parse", "HEAD"], checkoutPath)).stdout.trim();
if (actualCommit !== commitId) throw new AgentRunError("infra-failed", "gitbundle checkout did not land on requested commit", { httpStatus: 500, details: { expectedCommit: commitId, actualCommit } });
const treeId = (await git(["rev-parse", "HEAD^{tree}"], checkoutPath)).stdout.trim();
return { repoUrl, commitId, checkoutPath, treeId };
}
async function materializeGitBundles(workspacePath: string, resourceBundleRef: ResourceBundleRef, checkoutFor: (repoUrl: string, commitId: string) => Promise<GitCheckout>): Promise<MaterializedGitBundle[]> {
const items: MaterializedGitBundle[] = [];
for (const [index, bundle] of resourceBundleRef.bundles.entries()) {
const repoUrl = bundle.repoUrl ?? resourceBundleRef.repoUrl;
const commitId = bundle.commitId ?? resourceBundleRef.commitId;
const checkout = await checkoutFor(repoUrl, commitId);
const source = resolveBundlePath(checkout.checkoutPath, bundle.subpath, `bundles[${index}].subpath`);
const target = resolveWorkspaceTargetPath(workspacePath, bundle.targetPath, `bundles[${index}].target_path`);
let sourceStat;
try {
sourceStat = await stat(source);
} catch (error) {
throw new AgentRunError("schema-invalid", `gitbundle subpath ${bundle.subpath} is not readable`, { httpStatus: 400, details: { index, subpath: bundle.subpath, error: fileErrorSummary(error), valuesPrinted: false } });
}
await mkdir(path.dirname(target), { recursive: true });
await rm(target, { recursive: true, force: true });
await cp(source, target, { recursive: true, force: true, dereference: false });
items.push({ name: bundle.name ?? null, repoUrl, commitId, subpath: bundle.subpath, targetPath: bundle.targetPath, sourceKind: sourceStat.isDirectory() ? "directory" : "file", sourceBytes: sourceStat.isFile() ? sourceStat.size : null });
}
return { binPath, event: { count: names.length, names, binPath: pathSummary(binPath), valuesPrinted: false } };
return items;
}
async function prepareGitBundleTools(workspacePath: string): Promise<{ binPath?: string; event: JsonRecord }> {
const binPath = path.join(workspacePath, "tools");
let entries;
try {
entries = await readdir(binPath, { withFileTypes: true });
} catch (error) {
if (error && typeof error === "object" && "code" in error && (error as { code?: unknown }).code === "ENOENT") return { event: { count: 0, names: [], binPath: null, valuesPrinted: false } };
throw error;
}
const names: string[] = [];
const items: JsonRecord[] = [];
for (const entry of entries) {
if (!entry.isFile()) continue;
const filePath = path.join(binPath, entry.name);
const text = await readFile(filePath, "utf8");
const firstLine = text.split(/\r?\n/u, 1)[0] ?? "";
if (entry.name.endsWith(".ts") && !firstLine.startsWith("#!")) throw new AgentRunError("schema-invalid", `gitbundle tool ${entry.name} must start with a shebang`, { httpStatus: 400 });
if (!firstLine.startsWith("#!")) continue;
await chmod(filePath, 0o755);
names.push(entry.name);
items.push({ name: entry.name, sha256: sha256Text(text), bytes: Buffer.byteLength(text, "utf8"), shebang: firstLine.slice(0, 80), valuesPrinted: false });
}
return { binPath, event: { count: names.length, names, items, binPath: pathSummary(binPath), valuesPrinted: false } };
}
async function materializePromptRefs(checkoutPath: string, refs: NonNullable<ResourceBundleRef["promptRefs"]>): Promise<{ items: MaterializedPromptRef[]; event: JsonRecord }> {
@@ -142,41 +207,39 @@ async function materializePromptRefs(checkoutPath: string, refs: NonNullable<Res
};
}
async function materializeSkillRefs(checkoutPath: string, workspacePath: string, refs: NonNullable<ResourceBundleRef["skillRefs"]>): Promise<{ items: MaterializedSkillRef[]; skillsDir?: string; event: JsonRecord }> {
if (refs.length === 0) return { items: [], event: { count: 0, materializedCount: 0, names: [], skillsDir: null, items: [], valuesPrinted: false } };
async function discoverGitBundleSkills(workspacePath: string): Promise<{ items: MaterializedSkillRef[]; skillsDir?: string; event: JsonRecord }> {
const skillsDir = path.join(workspacePath, ".agents", "skills");
await mkdir(skillsDir, { recursive: true });
let entries;
try {
entries = await readdir(skillsDir, { withFileTypes: true });
} catch (error) {
if (error && typeof error === "object" && "code" in error && (error as { code?: unknown }).code === "ENOENT") return { items: [], event: { count: 0, materializedCount: 0, names: [], skillsDir: null, items: [], valuesPrinted: false } };
throw error;
}
const items: MaterializedSkillRef[] = [];
const eventItems: JsonRecord[] = [];
for (const ref of refs) {
const manifestPath = resolveBundlePath(checkoutPath, ref.path, `skillRefs.${ref.name}.path`);
const required = ref.required === true;
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const aggregateAs = entry.name;
const manifestPath = path.join(skillsDir, aggregateAs, "SKILL.md");
let manifestText: string;
try {
manifestText = await readFile(manifestPath, "utf8");
} catch (error) {
if (required) throw new AgentRunError("skill-unavailable", `required resource skill ${ref.name} is not readable`, { httpStatus: 400, details: { name: ref.name, path: ref.path, error: fileErrorSummary(error), valuesPrinted: false } });
eventItems.push({ name: ref.name, path: ref.path, required, aggregateAs: ref.aggregateAs ?? ref.name, status: "missing", valuesPrinted: false });
eventItems.push({ name: aggregateAs, path: `.agents/skills/${aggregateAs}/SKILL.md`, required: true, aggregateAs, status: "missing", error: fileErrorSummary(error), valuesPrinted: false });
continue;
}
const aggregateAs = ref.aggregateAs ?? ref.name;
const sourceRoot = path.dirname(manifestPath);
const targetRoot = path.join(skillsDir, aggregateAs);
if (path.resolve(sourceRoot) !== path.resolve(targetRoot)) {
await rm(targetRoot, { recursive: true, force: true });
await cp(sourceRoot, targetRoot, { recursive: true, force: true, dereference: false });
}
const bytes = Buffer.byteLength(manifestText, "utf8");
const sha = sha256Text(manifestText);
const summary = skillSummary(manifestText);
items.push({ name: ref.name, path: ref.path, aggregateAs, required, registryPath: path.join(targetRoot, "SKILL.md"), manifestBytes: bytes, manifestSha256: sha, summary });
eventItems.push({ name: ref.name, path: ref.path, aggregateAs, required, status: "materialized", manifestSha256: sha, manifestBytes: bytes, registryPath: pathSummary(path.join(targetRoot, "SKILL.md")), summary, valuesPrinted: false });
items.push({ name: aggregateAs, path: `.agents/skills/${aggregateAs}/SKILL.md`, aggregateAs, required: true, registryPath: manifestPath, manifestBytes: bytes, manifestSha256: sha, summary });
eventItems.push({ name: aggregateAs, path: `.agents/skills/${aggregateAs}/SKILL.md`, aggregateAs, required: true, status: "materialized", manifestSha256: sha, manifestBytes: bytes, registryPath: pathSummary(manifestPath), summary, valuesPrinted: false });
}
return {
items,
skillsDir,
event: {
count: refs.length,
count: entries.filter((entry) => entry.isDirectory()).length,
materializedCount: items.length,
names: items.map((item) => item.name),
skillsDir: pathSummary(skillsDir),
@@ -186,50 +249,25 @@ async function materializeSkillRefs(checkoutPath: string, workspacePath: string,
};
}
async function materializeWorkspaceFiles(workspacePath: string, files: NonNullable<ResourceBundleRef["workspaceFiles"]>): Promise<{ event: JsonRecord }> {
if (files.length === 0) return { event: { count: 0, materializedCount: 0, items: [], totalBytes: 0, valuesPrinted: false } };
const eventItems: JsonRecord[] = [];
let totalBytes = 0;
for (const file of files) {
const target = resolveWorkspaceFilePath(workspacePath, file.path, `workspaceFiles.${file.path}.path`);
const content = file.content;
const bytes = Buffer.byteLength(content, "utf8");
await mkdir(path.dirname(target), { recursive: true });
await writeFile(target, content, "utf8");
totalBytes += bytes;
eventItems.push({ path: file.path, encoding: file.encoding ?? "utf8", status: "materialized", bytes, sha256: sha256Text(content), valuesPrinted: false });
}
return {
event: {
count: files.length,
materializedCount: eventItems.length,
paths: eventItems.map((item) => item.path),
items: eventItems,
totalBytes,
valuesPrinted: false,
},
};
}
function assembleInitialPrompt(promptRefs: MaterializedPromptRef[], skillRefs: MaterializedSkillRef[]): InitialPromptAssembly | undefined {
if (promptRefs.length === 0 && skillRefs.length === 0) return undefined;
function assembleInitialPrompt(promptRefs: MaterializedPromptRef[], skills: MaterializedSkillRef[]): InitialPromptAssembly | undefined {
if (promptRefs.length === 0 && skills.length === 0) return undefined;
const sections: string[] = [
"AgentRun initial runtime instructions. These instructions are assembled from ResourceBundleRef promptRefs and skillRefs for the first thread-start turn only.",
"AgentRun initial runtime instructions. These instructions are assembled from ResourceBundleRef promptRefs and gitbundle skill directories for the first thread-start turn only.",
];
for (const prompt of promptRefs) {
sections.push([`## Resource Prompt: ${prompt.name}`, `path: ${prompt.path}`, prompt.text].join("\n"));
}
if (skillRefs.length > 0) {
if (skills.length > 0) {
const lines = [
"## Resource Skills",
"The following required runtime skills are mounted in the current workspace. Use these bundle skills instead of default model skill guesses.",
...skillRefs.map((skill) => `- ${skill.name}: ${skill.summary || "No summary provided."} manifest=.agents/skills/${skill.aggregateAs}/SKILL.md source=${skill.path} required=${skill.required}`),
...skills.map((skill) => `- ${skill.name}: ${skill.summary || "No summary provided."} manifest=.agents/skills/${skill.aggregateAs}/SKILL.md source=${skill.path} required=${skill.required}`),
];
sections.push(lines.join("\n"));
}
const text = sections.join("\n\n");
const bytes = Buffer.byteLength(text, "utf8");
if (bytes > maxInitialPromptBytes) throw new AgentRunError("prompt-too-large", "assembled initial prompt exceeds the total size limit", { httpStatus: 400, details: { bytes, maxInitialPromptBytes, promptRefCount: promptRefs.length, skillRefCount: skillRefs.length, valuesPrinted: false } });
if (bytes > maxInitialPromptBytes) throw new AgentRunError("prompt-too-large", "assembled initial prompt exceeds the total size limit", { httpStatus: 400, details: { bytes, maxInitialPromptBytes, promptRefCount: promptRefs.length, skillCount: skills.length, valuesPrinted: false } });
return {
text,
summary: {
@@ -238,8 +276,8 @@ function assembleInitialPrompt(promptRefs: MaterializedPromptRef[], skillRefs: M
sha256: sha256Text(text),
promptRefCount: promptRefs.length,
promptRefNames: promptRefs.map((item) => item.name),
skillRefCount: skillRefs.length,
skillRefNames: skillRefs.map((item) => item.name),
skillCount: skills.length,
skillNames: skills.map((item) => item.name),
valuesPrinted: false,
},
};
@@ -269,25 +307,6 @@ function fileErrorSummary(error: unknown): JsonRecord {
return { code: typeof record.code === "string" ? record.code : null, message: typeof record.message === "string" ? redactText(record.message).slice(0, 300) : null };
}
function aliasWrapper(kind: string, target: string): string {
if (kind === "node-script") return `#!/usr/bin/env sh\n# agentrun-resource-alias-wrapper\nexec node ${shellArg(target)} "$@"\n`;
if (kind === "bun-script") return `#!/usr/bin/env sh\n# agentrun-resource-alias-wrapper\nexec bun ${shellArg(target)} "$@"\n`;
if (kind === "sh-script") return `#!/usr/bin/env sh\n# agentrun-resource-alias-wrapper\nexec sh ${shellArg(target)} "$@"\n`;
return `#!/usr/bin/env sh\n# agentrun-resource-alias-wrapper\nexec ${shellArg(target)} "$@"\n`;
}
async function assertAliasWrapperWritable(wrapper: string, name: string): Promise<void> {
try {
const existing = await readFile(wrapper, "utf8");
if (existing.includes("agentrun-resource-alias-wrapper")) return;
throw new AgentRunError("schema-invalid", `resource bundle tool alias ${name} would overwrite an existing command`, { httpStatus: 409, details: { wrapper: pathSummary(wrapper) } });
} catch (error) {
if (error instanceof AgentRunError) throw error;
if (error && typeof error === "object" && "code" in error && (error as { code?: unknown }).code === "ENOENT") return;
throw error;
}
}
async function git(args: string[], cwd: string, options: { allowFailure?: boolean } = {}): Promise<{ stdout: string; stderr: string }> {
const child = spawn("git", args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
let stdout = "";
@@ -308,11 +327,6 @@ async function git(args: string[], cwd: string, options: { allowFailure?: boolea
return { stdout, stderr };
}
function resolveWorkspacePath(checkoutPath: string, subdir: string | undefined): string {
if (!subdir) return checkoutPath;
return resolveBundlePath(checkoutPath, subdir, "resourceBundleRef.subdir");
}
function resolveBundlePath(checkoutPath: string, relativePath: string, fieldName: string): string {
const resolved = path.resolve(checkoutPath, relativePath);
const root = path.resolve(checkoutPath);
@@ -320,17 +334,13 @@ function resolveBundlePath(checkoutPath: string, relativePath: string, fieldName
return resolved;
}
function resolveWorkspaceFilePath(workspacePath: string, relativePath: string, fieldName: string): string {
function resolveWorkspaceTargetPath(workspacePath: string, relativePath: string, fieldName: string): string {
const resolved = path.resolve(workspacePath, relativePath);
const root = path.resolve(workspacePath);
if (resolved === root || !resolved.startsWith(`${root}${path.sep}`)) throw new AgentRunError("schema-invalid", `${fieldName} escaped workspace`, { httpStatus: 400 });
return resolved;
}
function shellArg(value: string): string {
return `'${value.replace(/'/gu, `'"'"'`)}'`;
}
function pathSummary(value: string): JsonRecord {
const parts = value.split(/[\\/]+/u).filter(Boolean);
return { absolute: path.isAbsolute(value), basename: parts.at(-1) ?? null, depth: parts.length, fingerprint: stableHash(value).slice(0, 16), valuePrinted: false };