Merge pull request #120 from pikasTech/fix/issue-118-runner-egress-secret
修复 runner 代理注入与 transient env SecretRef
This commit is contained in:
@@ -69,6 +69,10 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
|
||||
const serviceAccountName = optionalString(options.input.serviceAccountName) ?? options.defaults.serviceAccountName;
|
||||
const idempotencyKey = optionalString(options.input.idempotencyKey);
|
||||
const transientEnv = assembleToolContextTransientEnv(run.executionPolicy, transientEnvField(options.input.transientEnv), options.defaults);
|
||||
const attemptId = optionalString(options.input.attemptId) ?? `attempt_${Date.now().toString(36)}`;
|
||||
const runnerId = optionalString(options.input.runnerId);
|
||||
const transientEnvSecretName = transientEnv.length > 0 ? transientEnvSecretNameForRun(run.id, commandId, attemptId) : null;
|
||||
const renderTransientEnv = transientEnvSecretName ? transientEnvWithSecretRefs(transientEnv, transientEnvSecretName) : transientEnv;
|
||||
const normalizedPayload = {
|
||||
commandId,
|
||||
image,
|
||||
@@ -105,14 +109,32 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
|
||||
image,
|
||||
namespace,
|
||||
sourceCommit,
|
||||
transientEnv,
|
||||
transientEnv: renderTransientEnv,
|
||||
...(serviceAccountName ? { serviceAccountName } : {}),
|
||||
...(sessionPvc ? { sessionPvc } : {}),
|
||||
};
|
||||
const attemptId = optionalString(options.input.attemptId);
|
||||
const runnerId = optionalString(options.input.runnerId);
|
||||
const render = renderRunnerJobManifest({ ...renderOptions, ...(attemptId ? { attemptId } : {}), ...(runnerId ? { runnerId } : {}) });
|
||||
const created = await kubectlCreate(render.manifest, options.defaults.kubectlCommand ?? "kubectl");
|
||||
const render = renderRunnerJobManifest({ ...renderOptions, attemptId, ...(runnerId ? { runnerId } : {}) });
|
||||
const kubectlCommand = options.defaults.kubectlCommand ?? "kubectl";
|
||||
let transientEnvSecretCreated = false;
|
||||
let transientEnvSecretOwnerAttached = false;
|
||||
let created: JsonRecord | null = null;
|
||||
try {
|
||||
if (transientEnvSecretName) {
|
||||
await kubectlCreate(transientEnvSecretManifest({ namespace: render.namespace, name: transientEnvSecretName, runId: run.id, commandId, attemptId: render.attemptId, runnerId: render.runnerId, jobName: render.jobName, items: transientEnv }), kubectlCommand, "runner transient env secret");
|
||||
transientEnvSecretCreated = true;
|
||||
}
|
||||
created = await kubectlCreate(render.manifest, kubectlCommand, "runner job");
|
||||
} catch (error) {
|
||||
if (transientEnvSecretName && transientEnvSecretCreated) await kubectlDeleteSecret(transientEnvSecretName, render.namespace, kubectlCommand);
|
||||
throw error;
|
||||
}
|
||||
if (!created) throw new AgentRunError("infra-failed", "kubectl did not return created runner job metadata", { httpStatus: 502 });
|
||||
if (transientEnvSecretName) {
|
||||
const owner = await kubectlPatchSecretOwnerReference(transientEnvSecretName, render.namespace, { name: render.jobName, uid: objectPath(created, ["metadata", "uid"]) }, kubectlCommand);
|
||||
transientEnvSecretOwnerAttached = owner.ok;
|
||||
if (!owner.ok) render.warnings.push("transientEnv Secret ownerReference patch failed; Kubernetes TTL may not garbage-collect this per-job Secret automatically");
|
||||
}
|
||||
const transientEnvSecretResponse = transientEnvSecretName ? summarizeTransientEnvSecret(transientEnvSecretName, render.namespace, transientEnv, render.jobName, transientEnvSecretOwnerAttached) : null;
|
||||
const response = {
|
||||
action: "create-kubernetes-job",
|
||||
mutation: true,
|
||||
@@ -143,6 +165,7 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
|
||||
secretRefs: render.secretRefs.map((item) => ({ profile: item.profile, name: item.secretRef.name, namespace: item.secretRef.namespace ?? render.namespace, keys: item.secretRef.keys ?? [], mountPath: item.runtimeMountPath, projectionPath: item.projectionMountPath, writableCopy: true, valuesPrinted: false })),
|
||||
toolCredentials: summarizeToolCredentials(render.toolCredentials, render.namespace),
|
||||
transientEnv: summarizeTransientEnv(transientEnv),
|
||||
transientEnvSecret: transientEnvSecretResponse,
|
||||
retention: {
|
||||
ttlSecondsAfterFinished: render.ttlSecondsAfterFinished,
|
||||
},
|
||||
@@ -184,6 +207,7 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
|
||||
jobName: saved.jobName,
|
||||
idempotencyKey: idempotencyKey ? "present" : null,
|
||||
transientEnv: summarizeTransientEnv(transientEnv),
|
||||
transientEnvSecret: transientEnvSecretResponse,
|
||||
toolCredentials: summarizeToolCredentials(render.toolCredentials, render.namespace),
|
||||
sessionRef: summarizeSessionRef(run.sessionRef ?? null),
|
||||
resourceBundleRef: summarizeResourceBundleRef(run.resourceBundleRef ?? null),
|
||||
@@ -273,7 +297,56 @@ function summarizeTransientEnv(items: RunnerTransientEnv[]): JsonRecord {
|
||||
};
|
||||
}
|
||||
|
||||
async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string): Promise<JsonRecord> {
|
||||
function transientEnvWithSecretRefs(items: RunnerTransientEnv[], secretName: string): RunnerTransientEnv[] {
|
||||
return items.map((item) => ({ ...item, secretRef: { name: secretName, key: item.name } }));
|
||||
}
|
||||
|
||||
function transientEnvSecretNameForRun(runId: string, commandId: string, attemptId: string): string {
|
||||
return `agentrun-v01-runner-env-${stableHash({ runId, commandId, attemptId }).slice(0, 20)}`;
|
||||
}
|
||||
|
||||
function transientEnvSecretManifest(options: { namespace: string; name: string; runId: string; commandId: string; attemptId: string; runnerId: string; jobName: string; items: RunnerTransientEnv[] }): JsonRecord {
|
||||
const stringData: JsonRecord = {};
|
||||
for (const item of options.items) stringData[item.name] = item.value;
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "Secret",
|
||||
metadata: {
|
||||
name: options.name,
|
||||
namespace: options.namespace,
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "agentrun-runner",
|
||||
"app.kubernetes.io/component": "runner-transient-env",
|
||||
"app.kubernetes.io/part-of": "agentrun",
|
||||
"agentrun.pikastech.local/lane": "v0.1",
|
||||
"agentrun.pikastech.local/run-hash": stableHash(options.runId).slice(0, 12),
|
||||
"agentrun.pikastech.local/runner-job": options.jobName,
|
||||
},
|
||||
annotations: {
|
||||
"agentrun.pikastech.local/run-id": options.runId,
|
||||
"agentrun.pikastech.local/command-id": options.commandId,
|
||||
"agentrun.pikastech.local/attempt-id": options.attemptId,
|
||||
"agentrun.pikastech.local/runner-id": options.runnerId,
|
||||
"agentrun.pikastech.local/runner-job": options.jobName,
|
||||
"agentrun.pikastech.local/values-printed": "false",
|
||||
},
|
||||
},
|
||||
type: "Opaque",
|
||||
stringData,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeTransientEnvSecret(name: string, namespace: string, items: RunnerTransientEnv[], jobName: string, ownerReferenceAttached: boolean): JsonRecord {
|
||||
return {
|
||||
name,
|
||||
namespace,
|
||||
keys: items.map((item) => item.name),
|
||||
valuesPrinted: false,
|
||||
ownerReference: { kind: "Job", name: jobName, attached: ownerReferenceAttached },
|
||||
};
|
||||
}
|
||||
|
||||
async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string, label = "runner job"): Promise<JsonRecord> {
|
||||
const child = spawn(kubectlCommand, ["create", "-f", "-", "-o", "json"], { stdio: ["pipe", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
@@ -289,7 +362,7 @@ async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string): Prom
|
||||
throw new AgentRunError("infra-failed", `failed to start kubectl: ${error instanceof Error ? error.message : String(error)}`, { httpStatus: 503 });
|
||||
});
|
||||
if (result.code !== 0) {
|
||||
throw new AgentRunError("infra-failed", `kubectl create runner job failed with code ${result.code}`, { httpStatus: 502, details: redactJson({ stderr: redactText(stderr.slice(-4000)), stdout: redactText(stdout.slice(-2000)), signal: result.signal }) });
|
||||
throw new AgentRunError("infra-failed", `kubectl create ${label} failed with code ${result.code}`, { httpStatus: 502, details: redactJson({ stderr: redactText(stderr.slice(-4000)), stdout: redactText(stdout.slice(-2000)), signal: result.signal }) });
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(stdout) as unknown;
|
||||
@@ -300,6 +373,38 @@ async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string): Prom
|
||||
throw new AgentRunError("infra-failed", "kubectl returned non-object JSON", { httpStatus: 502 });
|
||||
}
|
||||
|
||||
async function kubectlDeleteSecret(name: string, namespace: string, kubectlCommand: string): Promise<void> {
|
||||
await kubectlRun(kubectlCommand, ["delete", "secret", name, "-n", namespace, "--ignore-not-found=true"]);
|
||||
}
|
||||
|
||||
async function kubectlPatchSecretOwnerReference(name: string, namespace: string, owner: { name: string; uid: string | null }, kubectlCommand: string): Promise<{ ok: boolean }> {
|
||||
if (!owner.uid) return { ok: false };
|
||||
const patch = {
|
||||
metadata: {
|
||||
ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: owner.name, uid: owner.uid, controller: false, blockOwnerDeletion: false }],
|
||||
},
|
||||
};
|
||||
const result = await kubectlRun(kubectlCommand, ["patch", "secret", name, "-n", namespace, "--type", "merge", "-p", JSON.stringify(patch), "-o", "json"]);
|
||||
return { ok: result.code === 0 };
|
||||
}
|
||||
|
||||
async function kubectlRun(kubectlCommand: string, args: string[]): Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }> {
|
||||
const child = spawn(kubectlCommand, args, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => { stdout += String(chunk); });
|
||||
child.stderr.on("data", (chunk) => { stderr += String(chunk); });
|
||||
const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("close", (code, signal) => resolve({ code, signal }));
|
||||
}).catch((error: unknown) => {
|
||||
throw new AgentRunError("infra-failed", `failed to start kubectl: ${error instanceof Error ? error.message : String(error)}`, { httpStatus: 503 });
|
||||
});
|
||||
return { ...result, stdout: redactText(stdout), stderr: redactText(stderr) };
|
||||
}
|
||||
|
||||
function stringField(record: JsonRecord, key: string): string {
|
||||
const value = record[key];
|
||||
if (typeof value !== "string" || value.trim().length === 0) throw new AgentRunError("schema-invalid", `${key} is required`, { httpStatus: 400 });
|
||||
|
||||
+77
-4
@@ -5,6 +5,34 @@ import { backendProfileSpec } from "../common/backend-profiles.js";
|
||||
const defaultBootRepoUrl = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git";
|
||||
const defaultResourceBinPath = "/usr/local/bin";
|
||||
const defaultCodexShellSandbox = "danger-full-access";
|
||||
const defaultRunnerEgressProxyUrl = "http://g14-provider-egress-proxy.unidesk.svc.cluster.local:18789";
|
||||
const defaultRunnerNoProxy = [
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"::1",
|
||||
"agentrun-mgr",
|
||||
"agentrun-mgr.agentrun-v01",
|
||||
"agentrun-mgr.agentrun-v01.svc",
|
||||
"agentrun-mgr.agentrun-v01.svc.cluster.local",
|
||||
"agentrun-v01-postgres",
|
||||
"agentrun-v01-postgres.agentrun-v01",
|
||||
"agentrun-v01-postgres.agentrun-v01.svc",
|
||||
"agentrun-v01-postgres.agentrun-v01.svc.cluster.local",
|
||||
"g14-provider-egress-proxy",
|
||||
"g14-provider-egress-proxy.unidesk",
|
||||
"g14-provider-egress-proxy.unidesk.svc",
|
||||
"g14-provider-egress-proxy.unidesk.svc.cluster.local",
|
||||
"g14-tcp-egress-gateway",
|
||||
"g14-tcp-egress-gateway.unidesk",
|
||||
"g14-tcp-egress-gateway.unidesk.svc",
|
||||
"g14-tcp-egress-gateway.unidesk.svc.cluster.local",
|
||||
"hyueapi.com",
|
||||
".hyueapi.com",
|
||||
"10.42.0.0/16",
|
||||
"10.43.0.0/16",
|
||||
".svc",
|
||||
".cluster.local",
|
||||
].join(",");
|
||||
|
||||
export interface RunnerJobRenderOptions {
|
||||
run: RunRecord;
|
||||
@@ -35,6 +63,12 @@ export interface RunnerTransientEnv {
|
||||
name: string;
|
||||
value: string;
|
||||
sensitive?: boolean;
|
||||
secretRef?: RunnerTransientEnvSecretRef;
|
||||
}
|
||||
|
||||
export interface RunnerTransientEnvSecretRef {
|
||||
name: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
interface CredentialProjection {
|
||||
@@ -170,7 +204,7 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
|
||||
function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined }): JsonRecord[] {
|
||||
const selectedSecret = context.secretRefs.find((item) => item.profile === options.run.backendProfile);
|
||||
const codexHome = selectedSecret?.runtimeMountPath ?? defaultRuntimeHome(options.run.backendProfile);
|
||||
return [
|
||||
return dedupeEnvVars([
|
||||
{ name: "AGENTRUN_MGR_URL", value: options.managerUrl },
|
||||
{ name: "AGENTRUN_RUN_ID", value: options.run.id },
|
||||
{ name: "AGENTRUN_COMMAND_ID", value: options.commandId },
|
||||
@@ -202,9 +236,10 @@ function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string
|
||||
{ name: "AGENTRUN_SESSION_PVC_MOUNT_PATH", value: context.sessionPvc.mountPath },
|
||||
{ name: "AGENTRUN_CODEX_ROLLOUT_SUBDIR", value: context.sessionPvc.codexRolloutSubdir },
|
||||
] : []),
|
||||
...runnerEgressProxyEnvVars(),
|
||||
...toolCredentialEnvVars(context.toolCredentials),
|
||||
...transientEnvVars(options.transientEnv ?? []),
|
||||
];
|
||||
]);
|
||||
}
|
||||
|
||||
function codexShellSandbox(policy: ExecutionPolicy): string {
|
||||
@@ -225,7 +260,45 @@ function toolCredentialEnvVars(items: ToolCredentialProjection[]): JsonRecord[]
|
||||
}
|
||||
|
||||
function transientEnvVars(items: RunnerTransientEnv[]): JsonRecord[] {
|
||||
return items.map((item) => ({ name: item.name, value: item.value }));
|
||||
return items.map((item) => {
|
||||
if (item.secretRef) {
|
||||
return {
|
||||
name: item.name,
|
||||
valueFrom: {
|
||||
secretKeyRef: {
|
||||
name: item.secretRef.name,
|
||||
key: item.secretRef.key,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return { name: item.name, value: item.value };
|
||||
});
|
||||
}
|
||||
|
||||
function runnerEgressProxyEnvVars(): JsonRecord[] {
|
||||
return [
|
||||
{ name: "HTTP_PROXY", value: defaultRunnerEgressProxyUrl },
|
||||
{ name: "HTTPS_PROXY", value: defaultRunnerEgressProxyUrl },
|
||||
{ name: "ALL_PROXY", value: defaultRunnerEgressProxyUrl },
|
||||
{ name: "NO_PROXY", value: defaultRunnerNoProxy },
|
||||
{ name: "http_proxy", value: defaultRunnerEgressProxyUrl },
|
||||
{ name: "https_proxy", value: defaultRunnerEgressProxyUrl },
|
||||
{ name: "all_proxy", value: defaultRunnerEgressProxyUrl },
|
||||
{ name: "no_proxy", value: defaultRunnerNoProxy },
|
||||
];
|
||||
}
|
||||
|
||||
function dedupeEnvVars(items: JsonRecord[]): JsonRecord[] {
|
||||
const order: string[] = [];
|
||||
const byName = new Map<string, JsonRecord>();
|
||||
for (const item of items) {
|
||||
const name = typeof item.name === "string" ? item.name : "";
|
||||
if (!name) continue;
|
||||
if (!byName.has(name)) order.push(name);
|
||||
byName.set(name, item);
|
||||
}
|
||||
return order.map((name) => byName.get(name)).filter((item): item is JsonRecord => item !== undefined);
|
||||
}
|
||||
|
||||
function summarizeTransientEnv(items: RunnerTransientEnv[]): JsonRecord {
|
||||
@@ -263,7 +336,7 @@ function redactTransientEnvInManifest(manifest: JsonRecord, items: RunnerTransie
|
||||
for (const container of containers) {
|
||||
const env = Array.isArray(container.env) ? container.env as JsonRecord[] : [];
|
||||
for (const entry of env) {
|
||||
if (typeof entry.name === "string" && names.has(entry.name)) entry.value = "REDACTED";
|
||||
if (typeof entry.name === "string" && names.has(entry.name) && entry.value !== undefined) entry.value = "REDACTED";
|
||||
}
|
||||
}
|
||||
return copy;
|
||||
|
||||
@@ -42,6 +42,7 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
assertRunnerJobUsesWritableCodexHome(rendered.manifest as JsonRecord, context.codexHome, "codex-0", "/var/run/agentrun/secrets/codex-0");
|
||||
assertRunnerJobUsesToolCredential(rendered, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN");
|
||||
assertRunnerJobUsesToolCredential(rendered, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
|
||||
assertRunnerJobUsesG14EgressProxy(rendered.manifest as JsonRecord);
|
||||
assert.equal(runnerEnvValue(rendered.manifest as JsonRecord, "AGENTRUN_CODEX_SHELL_SANDBOX"), "danger-full-access");
|
||||
assert.equal(runnerEnvValue(rendered.manifest as JsonRecord, "HWLAB_API_KEY"), "REDACTED");
|
||||
assert.deepEqual((((rendered.transientEnv as JsonRecord).names) as string[]), ["HWLAB_API_KEY"]);
|
||||
@@ -123,13 +124,35 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
|
||||
const fakeKubectl = path.join(context.tmp, "fake-kubectl.js");
|
||||
const createdManifest = path.join(context.tmp, "created-runner-job.json");
|
||||
const createdTransientEnvSecret = path.join(context.tmp, "created-transient-env-secret.json");
|
||||
const patchedTransientEnvSecret = path.join(context.tmp, "patched-transient-env-secret.json");
|
||||
await writeFile(fakeKubectl, `#!/usr/bin/env bun
|
||||
const chunks = [];
|
||||
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
||||
const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
||||
await Bun.write(${JSON.stringify(createdManifest)}, text);
|
||||
const manifest = JSON.parse(text);
|
||||
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-selftest", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
||||
const args = Bun.argv.slice(2);
|
||||
async function readStdin() {
|
||||
const chunks = [];
|
||||
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
||||
return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
||||
}
|
||||
if (args[0] === "create") {
|
||||
const text = await readStdin();
|
||||
const manifest = JSON.parse(text);
|
||||
if (manifest.kind === "Secret") await Bun.write(${JSON.stringify(createdTransientEnvSecret)}, text);
|
||||
if (manifest.kind === "Job") await Bun.write(${JSON.stringify(createdManifest)}, text);
|
||||
const uid = manifest.kind === "Job" ? "job-uid-selftest" : "secret-uid-selftest";
|
||||
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid, resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (args[0] === "patch" && args[1] === "secret") {
|
||||
await Bun.write(${JSON.stringify(patchedTransientEnvSecret)}, JSON.stringify({ args }, null, 2));
|
||||
console.log(JSON.stringify({ apiVersion: "v1", kind: "Secret", metadata: { uid: "secret-uid-selftest", resourceVersion: "2", name: args[2], namespace: args[4] } }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (args[0] === "delete" && args[1] === "secret") {
|
||||
console.log(JSON.stringify({ kind: "Status", status: "Success" }));
|
||||
process.exit(0);
|
||||
}
|
||||
console.error("unsupported fake kubectl args: " + args.join(" "));
|
||||
process.exit(1);
|
||||
`);
|
||||
await chmod(fakeKubectl, 0o755);
|
||||
await mkdir(path.dirname(fakeKubectl), { recursive: true });
|
||||
@@ -167,12 +190,25 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin
|
||||
assert.equal((created as { mutation?: unknown }).mutation, true);
|
||||
assert.equal(((created as JsonRecord).retention as JsonRecord).ttlSecondsAfterFinished, 86_400);
|
||||
assert.deepEqual((((created as JsonRecord).transientEnv as JsonRecord).names) as string[], ["HWLAB_API_KEY", "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_SOURCE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP"]);
|
||||
const transientEnvSecret = (created as JsonRecord).transientEnvSecret as JsonRecord;
|
||||
assert.match(String(transientEnvSecret.name), /^agentrun-v01-runner-env-[a-f0-9]{20}$/u);
|
||||
assert.equal(transientEnvSecret.namespace, "agentrun-v01");
|
||||
assert.equal(transientEnvSecret.valuesPrinted, false);
|
||||
assert.deepEqual(transientEnvSecret.keys as string[], ["HWLAB_API_KEY", "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_SOURCE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP"]);
|
||||
assert.equal(((transientEnvSecret.ownerReference as JsonRecord).attached), true);
|
||||
const secretManifest = JSON.parse(await readFile(createdTransientEnvSecret, "utf8")) as JsonRecord;
|
||||
assert.equal(secretManifest.kind, "Secret");
|
||||
assert.equal(((secretManifest.metadata as JsonRecord).name), transientEnvSecret.name);
|
||||
assert.equal(Object.keys((secretManifest.stringData as JsonRecord)).length, 9);
|
||||
const ownerPatch = JSON.parse(await readFile(patchedTransientEnvSecret, "utf8")) as JsonRecord;
|
||||
assert.deepEqual((ownerPatch.args as string[]).slice(0, 3), ["patch", "secret", String(transientEnvSecret.name)]);
|
||||
const manifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
|
||||
assert.equal((manifest.spec as JsonRecord).ttlSecondsAfterFinished, 86_400);
|
||||
assert.equal(runnerEnvValue(manifest, "HWLAB_API_KEY"), "hwl_live_selftest");
|
||||
assert.equal(runnerEnvValue(manifest, "HWLAB_RUNTIME_API_URL"), "http://runtime-api.test");
|
||||
assert.equal(runnerEnvValue(manifest, "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME"), "1");
|
||||
assert.equal(runnerEnvValue(manifest, "UNIDESK_MAIN_SERVER_IP"), "https://unidesk.example.test");
|
||||
assertRunnerJobUsesG14EgressProxy(manifest);
|
||||
assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_API_KEY", String(transientEnvSecret.name));
|
||||
assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_RUNTIME_API_URL", String(transientEnvSecret.name));
|
||||
assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", String(transientEnvSecret.name));
|
||||
assertRunnerJobUsesTransientEnvSecret(manifest, "UNIDESK_MAIN_SERVER_IP", String(transientEnvSecret.name));
|
||||
assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN");
|
||||
assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
|
||||
assertNoSecretLeak(created);
|
||||
@@ -183,7 +219,9 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin
|
||||
}) as JsonRecord;
|
||||
assert.deepEqual(((defaultEndpointCreated.transientEnv as JsonRecord).names) as string[], ["UNIDESK_MAIN_SERVER_IP"]);
|
||||
const defaultEndpointManifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
|
||||
assert.equal(runnerEnvValue(defaultEndpointManifest, "UNIDESK_MAIN_SERVER_IP"), "https://unidesk.default.example.test");
|
||||
const defaultEndpointSecret = defaultEndpointCreated.transientEnvSecret as JsonRecord;
|
||||
assertRunnerJobUsesTransientEnvSecret(defaultEndpointManifest, "UNIDESK_MAIN_SERVER_IP", String(defaultEndpointSecret.name));
|
||||
assertRunnerJobUsesG14EgressProxy(defaultEndpointManifest);
|
||||
assertRunnerJobUsesToolCredential({ manifest: defaultEndpointManifest, toolCredentials: defaultEndpointCreated.toolCredentials } as JsonRecord, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
|
||||
assertNoSecretLeak(defaultEndpointCreated);
|
||||
await assert.rejects(
|
||||
@@ -239,7 +277,7 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin
|
||||
assert.equal(envMap.get("AGENTRUN_SESSION_PVC_NAMESPACE"), "agentrun-v01");
|
||||
assert.equal(envMap.get("AGENTRUN_SESSION_PVC_MOUNT_PATH"), "/home/agentrun/.codex-codex/sessions");
|
||||
assert.equal(envMap.get("AGENTRUN_CODEX_ROLLOUT_SUBDIR"), "sessions");
|
||||
return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-codex-shell-sandbox-env", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-dsflash-go-profile-dry-run", "runner-k8s-job-dsflash-go-legacy-secretref-normalized", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-unidesk-ssh-endpoint-auto-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-k8s-job-session-pvc-volume-and-env"] };
|
||||
return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-codex-shell-sandbox-env", "runner-k8s-job-g14-egress-proxy-env", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-dsflash-go-profile-dry-run", "runner-k8s-job-dsflash-go-legacy-secretref-normalized", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-transient-env-secretref", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-unidesk-ssh-endpoint-auto-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-k8s-job-session-pvc-volume-and-env"] };
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
||||
}
|
||||
@@ -247,14 +285,44 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin
|
||||
|
||||
export default selfTest;
|
||||
|
||||
function runnerEnvValue(manifest: JsonRecord, name: string): unknown {
|
||||
function runnerEnvEntry(manifest: JsonRecord, name: string): JsonRecord | undefined {
|
||||
const spec = manifest.spec as JsonRecord;
|
||||
const template = spec.template as JsonRecord;
|
||||
const podSpec = template.spec as JsonRecord;
|
||||
const containers = podSpec.containers as JsonRecord[];
|
||||
const runner = containers[0] as JsonRecord;
|
||||
const env = runner.env as JsonRecord[];
|
||||
return env.find((item) => item.name === name)?.value;
|
||||
return env.find((item) => item.name === name) as JsonRecord | undefined;
|
||||
}
|
||||
|
||||
function runnerEnvValue(manifest: JsonRecord, name: string): unknown {
|
||||
return runnerEnvEntry(manifest, name)?.value;
|
||||
}
|
||||
|
||||
function assertRunnerJobUsesTransientEnvSecret(manifest: JsonRecord, envName: string, secretName: string): void {
|
||||
const entry = runnerEnvEntry(manifest, envName);
|
||||
assert.ok(entry, `${envName} env should be present`);
|
||||
assert.equal(entry.value, undefined);
|
||||
const valueFrom = entry.valueFrom as JsonRecord;
|
||||
const secretKeyRef = valueFrom.secretKeyRef as JsonRecord;
|
||||
assert.equal(secretKeyRef.name, secretName);
|
||||
assert.equal(secretKeyRef.key, envName);
|
||||
}
|
||||
|
||||
function assertRunnerJobUsesG14EgressProxy(manifest: JsonRecord): void {
|
||||
const proxy = "http://g14-provider-egress-proxy.unidesk.svc.cluster.local:18789";
|
||||
assert.equal(runnerEnvValue(manifest, "HTTP_PROXY"), proxy);
|
||||
assert.equal(runnerEnvValue(manifest, "HTTPS_PROXY"), proxy);
|
||||
assert.equal(runnerEnvValue(manifest, "ALL_PROXY"), proxy);
|
||||
assert.equal(runnerEnvValue(manifest, "http_proxy"), proxy);
|
||||
assert.equal(runnerEnvValue(manifest, "https_proxy"), proxy);
|
||||
assert.equal(runnerEnvValue(manifest, "all_proxy"), proxy);
|
||||
const noProxy = String(runnerEnvValue(manifest, "NO_PROXY"));
|
||||
assert.equal(runnerEnvValue(manifest, "no_proxy"), noProxy);
|
||||
assert.ok(noProxy.includes("hyueapi.com"), "NO_PROXY must keep hyueapi.com direct");
|
||||
assert.ok(noProxy.includes(".hyueapi.com"), "NO_PROXY must keep .hyueapi.com direct");
|
||||
assert.ok(noProxy.includes("g14-provider-egress-proxy.unidesk.svc.cluster.local"), "NO_PROXY must include the proxy Service itself");
|
||||
assert.ok(noProxy.includes(".svc"), "NO_PROXY must include Kubernetes Service domains");
|
||||
}
|
||||
|
||||
function assertRunnerJobUsesToolCredential(rendered: JsonRecord, envName: string, secretName: string, secretKey: string): void {
|
||||
|
||||
@@ -11,12 +11,30 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
const fakeKubectl = path.join(context.tmp, "fake-kubectl-queue-q2.js");
|
||||
const createdManifest = path.join(context.tmp, "created-queue-q2-runner-job.json");
|
||||
await writeFile(fakeKubectl, `#!/usr/bin/env bun
|
||||
const chunks = [];
|
||||
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
||||
const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
||||
await Bun.write(${JSON.stringify(createdManifest)}, text);
|
||||
const manifest = JSON.parse(text);
|
||||
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-queue-q2", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
||||
const args = Bun.argv.slice(2);
|
||||
async function readStdin() {
|
||||
const chunks = [];
|
||||
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
||||
return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
||||
}
|
||||
if (args[0] === "create") {
|
||||
const text = await readStdin();
|
||||
const manifest = JSON.parse(text);
|
||||
if (manifest.kind === "Job") await Bun.write(${JSON.stringify(createdManifest)}, text);
|
||||
const uid = manifest.kind === "Job" ? "job-uid-queue-q2" : "secret-uid-queue-q2";
|
||||
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid, resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (args[0] === "patch" && args[1] === "secret") {
|
||||
console.log(JSON.stringify({ apiVersion: "v1", kind: "Secret", metadata: { uid: "secret-uid-queue-q2", resourceVersion: "2", name: args[2], namespace: args[4] } }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (args[0] === "delete" && args[1] === "secret") {
|
||||
console.log(JSON.stringify({ kind: "Status", status: "Success" }));
|
||||
process.exit(0);
|
||||
}
|
||||
console.error("unsupported fake kubectl args: " + args.join(" "));
|
||||
process.exit(1);
|
||||
`);
|
||||
await chmod(fakeKubectl, 0o755);
|
||||
const store = new MemoryAgentRunStore();
|
||||
@@ -132,8 +150,9 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin
|
||||
}) as QueueTaskRecord;
|
||||
const unideskDispatched = await client.post(`/api/v1/queue/tasks/${unideskCreated.id}/dispatch`, { attemptId: "attempt_queue_q2_unidesk_ssh_selftest" }) as QueueDispatchResult;
|
||||
assert.deepEqual((((unideskDispatched.runnerJob as JsonRecord).transientEnv as JsonRecord).names) as string[], ["UNIDESK_MAIN_SERVER_IP"]);
|
||||
assert.equal((((unideskDispatched.runnerJob as JsonRecord).transientEnvSecret as JsonRecord).valuesPrinted), false);
|
||||
const unideskManifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
|
||||
assert.equal(runnerEnvValue(unideskManifest, "UNIDESK_MAIN_SERVER_IP"), "https://unidesk.default.example.test");
|
||||
assert.equal(runnerEnvValue(unideskManifest, "UNIDESK_MAIN_SERVER_IP"), "secretRef");
|
||||
assert.equal(runnerEnvValue(unideskManifest, "UNIDESK_SSH_CLIENT_TOKEN"), "secretRef");
|
||||
assertNoSecretLeak(unideskDispatched);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user