diff --git a/src/mgr/kubernetes-runner-job.ts b/src/mgr/kubernetes-runner-job.ts index 8670176..283d261 100644 --- a/src/mgr/kubernetes-runner-job.ts +++ b/src/mgr/kubernetes-runner-job.ts @@ -69,6 +69,10 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore; const serviceAccountName = optionalString(options.input.serviceAccountName) ?? options.defaults.serviceAccountName; const idempotencyKey = optionalString(options.input.idempotencyKey); const transientEnv = assembleToolContextTransientEnv(run.executionPolicy, transientEnvField(options.input.transientEnv), options.defaults); + const attemptId = optionalString(options.input.attemptId) ?? `attempt_${Date.now().toString(36)}`; + const runnerId = optionalString(options.input.runnerId); + const transientEnvSecretName = transientEnv.length > 0 ? transientEnvSecretNameForRun(run.id, commandId, attemptId) : null; + const renderTransientEnv = transientEnvSecretName ? transientEnvWithSecretRefs(transientEnv, transientEnvSecretName) : transientEnv; const normalizedPayload = { commandId, image, @@ -105,14 +109,32 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore; image, namespace, sourceCommit, - transientEnv, + transientEnv: renderTransientEnv, ...(serviceAccountName ? { serviceAccountName } : {}), ...(sessionPvc ? { sessionPvc } : {}), }; - const attemptId = optionalString(options.input.attemptId); - const runnerId = optionalString(options.input.runnerId); - const render = renderRunnerJobManifest({ ...renderOptions, ...(attemptId ? { attemptId } : {}), ...(runnerId ? { runnerId } : {}) }); - const created = await kubectlCreate(render.manifest, options.defaults.kubectlCommand ?? "kubectl"); + const render = renderRunnerJobManifest({ ...renderOptions, attemptId, ...(runnerId ? { runnerId } : {}) }); + const kubectlCommand = options.defaults.kubectlCommand ?? "kubectl"; + let transientEnvSecretCreated = false; + let transientEnvSecretOwnerAttached = false; + let created: JsonRecord | null = null; + try { + if (transientEnvSecretName) { + await kubectlCreate(transientEnvSecretManifest({ namespace: render.namespace, name: transientEnvSecretName, runId: run.id, commandId, attemptId: render.attemptId, runnerId: render.runnerId, jobName: render.jobName, items: transientEnv }), kubectlCommand, "runner transient env secret"); + transientEnvSecretCreated = true; + } + created = await kubectlCreate(render.manifest, kubectlCommand, "runner job"); + } catch (error) { + if (transientEnvSecretName && transientEnvSecretCreated) await kubectlDeleteSecret(transientEnvSecretName, render.namespace, kubectlCommand); + throw error; + } + if (!created) throw new AgentRunError("infra-failed", "kubectl did not return created runner job metadata", { httpStatus: 502 }); + if (transientEnvSecretName) { + const owner = await kubectlPatchSecretOwnerReference(transientEnvSecretName, render.namespace, { name: render.jobName, uid: objectPath(created, ["metadata", "uid"]) }, kubectlCommand); + transientEnvSecretOwnerAttached = owner.ok; + if (!owner.ok) render.warnings.push("transientEnv Secret ownerReference patch failed; Kubernetes TTL may not garbage-collect this per-job Secret automatically"); + } + const transientEnvSecretResponse = transientEnvSecretName ? summarizeTransientEnvSecret(transientEnvSecretName, render.namespace, transientEnv, render.jobName, transientEnvSecretOwnerAttached) : null; const response = { action: "create-kubernetes-job", mutation: true, @@ -143,6 +165,7 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore; secretRefs: render.secretRefs.map((item) => ({ profile: item.profile, name: item.secretRef.name, namespace: item.secretRef.namespace ?? render.namespace, keys: item.secretRef.keys ?? [], mountPath: item.runtimeMountPath, projectionPath: item.projectionMountPath, writableCopy: true, valuesPrinted: false })), toolCredentials: summarizeToolCredentials(render.toolCredentials, render.namespace), transientEnv: summarizeTransientEnv(transientEnv), + transientEnvSecret: transientEnvSecretResponse, retention: { ttlSecondsAfterFinished: render.ttlSecondsAfterFinished, }, @@ -184,6 +207,7 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore; jobName: saved.jobName, idempotencyKey: idempotencyKey ? "present" : null, transientEnv: summarizeTransientEnv(transientEnv), + transientEnvSecret: transientEnvSecretResponse, toolCredentials: summarizeToolCredentials(render.toolCredentials, render.namespace), sessionRef: summarizeSessionRef(run.sessionRef ?? null), resourceBundleRef: summarizeResourceBundleRef(run.resourceBundleRef ?? null), @@ -273,7 +297,56 @@ function summarizeTransientEnv(items: RunnerTransientEnv[]): JsonRecord { }; } -async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string): Promise { +function transientEnvWithSecretRefs(items: RunnerTransientEnv[], secretName: string): RunnerTransientEnv[] { + return items.map((item) => ({ ...item, secretRef: { name: secretName, key: item.name } })); +} + +function transientEnvSecretNameForRun(runId: string, commandId: string, attemptId: string): string { + return `agentrun-v01-runner-env-${stableHash({ runId, commandId, attemptId }).slice(0, 20)}`; +} + +function transientEnvSecretManifest(options: { namespace: string; name: string; runId: string; commandId: string; attemptId: string; runnerId: string; jobName: string; items: RunnerTransientEnv[] }): JsonRecord { + const stringData: JsonRecord = {}; + for (const item of options.items) stringData[item.name] = item.value; + return { + apiVersion: "v1", + kind: "Secret", + metadata: { + name: options.name, + namespace: options.namespace, + labels: { + "app.kubernetes.io/name": "agentrun-runner", + "app.kubernetes.io/component": "runner-transient-env", + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": "v0.1", + "agentrun.pikastech.local/run-hash": stableHash(options.runId).slice(0, 12), + "agentrun.pikastech.local/runner-job": options.jobName, + }, + annotations: { + "agentrun.pikastech.local/run-id": options.runId, + "agentrun.pikastech.local/command-id": options.commandId, + "agentrun.pikastech.local/attempt-id": options.attemptId, + "agentrun.pikastech.local/runner-id": options.runnerId, + "agentrun.pikastech.local/runner-job": options.jobName, + "agentrun.pikastech.local/values-printed": "false", + }, + }, + type: "Opaque", + stringData, + }; +} + +function summarizeTransientEnvSecret(name: string, namespace: string, items: RunnerTransientEnv[], jobName: string, ownerReferenceAttached: boolean): JsonRecord { + return { + name, + namespace, + keys: items.map((item) => item.name), + valuesPrinted: false, + ownerReference: { kind: "Job", name: jobName, attached: ownerReferenceAttached }, + }; +} + +async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string, label = "runner job"): Promise { const child = spawn(kubectlCommand, ["create", "-f", "-", "-o", "json"], { stdio: ["pipe", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; @@ -289,7 +362,7 @@ async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string): Prom throw new AgentRunError("infra-failed", `failed to start kubectl: ${error instanceof Error ? error.message : String(error)}`, { httpStatus: 503 }); }); if (result.code !== 0) { - throw new AgentRunError("infra-failed", `kubectl create runner job failed with code ${result.code}`, { httpStatus: 502, details: redactJson({ stderr: redactText(stderr.slice(-4000)), stdout: redactText(stdout.slice(-2000)), signal: result.signal }) }); + throw new AgentRunError("infra-failed", `kubectl create ${label} failed with code ${result.code}`, { httpStatus: 502, details: redactJson({ stderr: redactText(stderr.slice(-4000)), stdout: redactText(stdout.slice(-2000)), signal: result.signal }) }); } try { const parsed = JSON.parse(stdout) as unknown; @@ -300,6 +373,38 @@ async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string): Prom throw new AgentRunError("infra-failed", "kubectl returned non-object JSON", { httpStatus: 502 }); } +async function kubectlDeleteSecret(name: string, namespace: string, kubectlCommand: string): Promise { + await kubectlRun(kubectlCommand, ["delete", "secret", name, "-n", namespace, "--ignore-not-found=true"]); +} + +async function kubectlPatchSecretOwnerReference(name: string, namespace: string, owner: { name: string; uid: string | null }, kubectlCommand: string): Promise<{ ok: boolean }> { + if (!owner.uid) return { ok: false }; + const patch = { + metadata: { + ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: owner.name, uid: owner.uid, controller: false, blockOwnerDeletion: false }], + }, + }; + const result = await kubectlRun(kubectlCommand, ["patch", "secret", name, "-n", namespace, "--type", "merge", "-p", JSON.stringify(patch), "-o", "json"]); + return { ok: result.code === 0 }; +} + +async function kubectlRun(kubectlCommand: string, args: string[]): Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }> { + const child = spawn(kubectlCommand, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { stdout += String(chunk); }); + child.stderr.on("data", (chunk) => { stderr += String(chunk); }); + const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { + child.on("error", reject); + child.on("close", (code, signal) => resolve({ code, signal })); + }).catch((error: unknown) => { + throw new AgentRunError("infra-failed", `failed to start kubectl: ${error instanceof Error ? error.message : String(error)}`, { httpStatus: 503 }); + }); + return { ...result, stdout: redactText(stdout), stderr: redactText(stderr) }; +} + function stringField(record: JsonRecord, key: string): string { const value = record[key]; if (typeof value !== "string" || value.trim().length === 0) throw new AgentRunError("schema-invalid", `${key} is required`, { httpStatus: 400 }); diff --git a/src/runner/k8s-job.ts b/src/runner/k8s-job.ts index 979d2ce..157a183 100644 --- a/src/runner/k8s-job.ts +++ b/src/runner/k8s-job.ts @@ -5,6 +5,34 @@ import { backendProfileSpec } from "../common/backend-profiles.js"; const defaultBootRepoUrl = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git"; const defaultResourceBinPath = "/usr/local/bin"; const defaultCodexShellSandbox = "danger-full-access"; +const defaultRunnerEgressProxyUrl = "http://g14-provider-egress-proxy.unidesk.svc.cluster.local:18789"; +const defaultRunnerNoProxy = [ + "localhost", + "127.0.0.1", + "::1", + "agentrun-mgr", + "agentrun-mgr.agentrun-v01", + "agentrun-mgr.agentrun-v01.svc", + "agentrun-mgr.agentrun-v01.svc.cluster.local", + "agentrun-v01-postgres", + "agentrun-v01-postgres.agentrun-v01", + "agentrun-v01-postgres.agentrun-v01.svc", + "agentrun-v01-postgres.agentrun-v01.svc.cluster.local", + "g14-provider-egress-proxy", + "g14-provider-egress-proxy.unidesk", + "g14-provider-egress-proxy.unidesk.svc", + "g14-provider-egress-proxy.unidesk.svc.cluster.local", + "g14-tcp-egress-gateway", + "g14-tcp-egress-gateway.unidesk", + "g14-tcp-egress-gateway.unidesk.svc", + "g14-tcp-egress-gateway.unidesk.svc.cluster.local", + "hyueapi.com", + ".hyueapi.com", + "10.42.0.0/16", + "10.43.0.0/16", + ".svc", + ".cluster.local", +].join(","); export interface RunnerJobRenderOptions { run: RunRecord; @@ -35,6 +63,12 @@ export interface RunnerTransientEnv { name: string; value: string; sensitive?: boolean; + secretRef?: RunnerTransientEnvSecretRef; +} + +export interface RunnerTransientEnvSecretRef { + name: string; + key: string; } interface CredentialProjection { @@ -170,7 +204,7 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined }): JsonRecord[] { const selectedSecret = context.secretRefs.find((item) => item.profile === options.run.backendProfile); const codexHome = selectedSecret?.runtimeMountPath ?? defaultRuntimeHome(options.run.backendProfile); - return [ + return dedupeEnvVars([ { name: "AGENTRUN_MGR_URL", value: options.managerUrl }, { name: "AGENTRUN_RUN_ID", value: options.run.id }, { name: "AGENTRUN_COMMAND_ID", value: options.commandId }, @@ -202,9 +236,10 @@ function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string { name: "AGENTRUN_SESSION_PVC_MOUNT_PATH", value: context.sessionPvc.mountPath }, { name: "AGENTRUN_CODEX_ROLLOUT_SUBDIR", value: context.sessionPvc.codexRolloutSubdir }, ] : []), + ...runnerEgressProxyEnvVars(), ...toolCredentialEnvVars(context.toolCredentials), ...transientEnvVars(options.transientEnv ?? []), - ]; + ]); } function codexShellSandbox(policy: ExecutionPolicy): string { @@ -225,7 +260,45 @@ function toolCredentialEnvVars(items: ToolCredentialProjection[]): JsonRecord[] } function transientEnvVars(items: RunnerTransientEnv[]): JsonRecord[] { - return items.map((item) => ({ name: item.name, value: item.value })); + return items.map((item) => { + if (item.secretRef) { + return { + name: item.name, + valueFrom: { + secretKeyRef: { + name: item.secretRef.name, + key: item.secretRef.key, + }, + }, + }; + } + return { name: item.name, value: item.value }; + }); +} + +function runnerEgressProxyEnvVars(): JsonRecord[] { + return [ + { name: "HTTP_PROXY", value: defaultRunnerEgressProxyUrl }, + { name: "HTTPS_PROXY", value: defaultRunnerEgressProxyUrl }, + { name: "ALL_PROXY", value: defaultRunnerEgressProxyUrl }, + { name: "NO_PROXY", value: defaultRunnerNoProxy }, + { name: "http_proxy", value: defaultRunnerEgressProxyUrl }, + { name: "https_proxy", value: defaultRunnerEgressProxyUrl }, + { name: "all_proxy", value: defaultRunnerEgressProxyUrl }, + { name: "no_proxy", value: defaultRunnerNoProxy }, + ]; +} + +function dedupeEnvVars(items: JsonRecord[]): JsonRecord[] { + const order: string[] = []; + const byName = new Map(); + for (const item of items) { + const name = typeof item.name === "string" ? item.name : ""; + if (!name) continue; + if (!byName.has(name)) order.push(name); + byName.set(name, item); + } + return order.map((name) => byName.get(name)).filter((item): item is JsonRecord => item !== undefined); } function summarizeTransientEnv(items: RunnerTransientEnv[]): JsonRecord { @@ -263,7 +336,7 @@ function redactTransientEnvInManifest(manifest: JsonRecord, items: RunnerTransie for (const container of containers) { const env = Array.isArray(container.env) ? container.env as JsonRecord[] : []; for (const entry of env) { - if (typeof entry.name === "string" && names.has(entry.name)) entry.value = "REDACTED"; + if (typeof entry.name === "string" && names.has(entry.name) && entry.value !== undefined) entry.value = "REDACTED"; } } return copy; diff --git a/src/selftest/cases/20-runner-k8s-job.ts b/src/selftest/cases/20-runner-k8s-job.ts index 5cffcfa..321f3a1 100644 --- a/src/selftest/cases/20-runner-k8s-job.ts +++ b/src/selftest/cases/20-runner-k8s-job.ts @@ -42,6 +42,7 @@ const selfTest: SelfTestCase = async (context) => { assertRunnerJobUsesWritableCodexHome(rendered.manifest as JsonRecord, context.codexHome, "codex-0", "/var/run/agentrun/secrets/codex-0"); assertRunnerJobUsesToolCredential(rendered, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN"); assertRunnerJobUsesToolCredential(rendered, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN"); + assertRunnerJobUsesG14EgressProxy(rendered.manifest as JsonRecord); assert.equal(runnerEnvValue(rendered.manifest as JsonRecord, "AGENTRUN_CODEX_SHELL_SANDBOX"), "danger-full-access"); assert.equal(runnerEnvValue(rendered.manifest as JsonRecord, "HWLAB_API_KEY"), "REDACTED"); assert.deepEqual((((rendered.transientEnv as JsonRecord).names) as string[]), ["HWLAB_API_KEY"]); @@ -123,13 +124,35 @@ const selfTest: SelfTestCase = async (context) => { const fakeKubectl = path.join(context.tmp, "fake-kubectl.js"); const createdManifest = path.join(context.tmp, "created-runner-job.json"); + const createdTransientEnvSecret = path.join(context.tmp, "created-transient-env-secret.json"); + const patchedTransientEnvSecret = path.join(context.tmp, "patched-transient-env-secret.json"); await writeFile(fakeKubectl, `#!/usr/bin/env bun -const chunks = []; -for await (const chunk of Bun.stdin.stream()) chunks.push(chunk); -const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8"); -await Bun.write(${JSON.stringify(createdManifest)}, text); -const manifest = JSON.parse(text); -console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-selftest", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } })); +const args = Bun.argv.slice(2); +async function readStdin() { + const chunks = []; + for await (const chunk of Bun.stdin.stream()) chunks.push(chunk); + return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8"); +} +if (args[0] === "create") { + const text = await readStdin(); + const manifest = JSON.parse(text); + if (manifest.kind === "Secret") await Bun.write(${JSON.stringify(createdTransientEnvSecret)}, text); + if (manifest.kind === "Job") await Bun.write(${JSON.stringify(createdManifest)}, text); + const uid = manifest.kind === "Job" ? "job-uid-selftest" : "secret-uid-selftest"; + console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid, resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } })); + process.exit(0); +} +if (args[0] === "patch" && args[1] === "secret") { + await Bun.write(${JSON.stringify(patchedTransientEnvSecret)}, JSON.stringify({ args }, null, 2)); + console.log(JSON.stringify({ apiVersion: "v1", kind: "Secret", metadata: { uid: "secret-uid-selftest", resourceVersion: "2", name: args[2], namespace: args[4] } })); + process.exit(0); +} +if (args[0] === "delete" && args[1] === "secret") { + console.log(JSON.stringify({ kind: "Status", status: "Success" })); + process.exit(0); +} +console.error("unsupported fake kubectl args: " + args.join(" ")); +process.exit(1); `); await chmod(fakeKubectl, 0o755); await mkdir(path.dirname(fakeKubectl), { recursive: true }); @@ -167,12 +190,25 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin assert.equal((created as { mutation?: unknown }).mutation, true); assert.equal(((created as JsonRecord).retention as JsonRecord).ttlSecondsAfterFinished, 86_400); assert.deepEqual((((created as JsonRecord).transientEnv as JsonRecord).names) as string[], ["HWLAB_API_KEY", "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_SOURCE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP"]); + const transientEnvSecret = (created as JsonRecord).transientEnvSecret as JsonRecord; + assert.match(String(transientEnvSecret.name), /^agentrun-v01-runner-env-[a-f0-9]{20}$/u); + assert.equal(transientEnvSecret.namespace, "agentrun-v01"); + assert.equal(transientEnvSecret.valuesPrinted, false); + assert.deepEqual(transientEnvSecret.keys as string[], ["HWLAB_API_KEY", "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_SOURCE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP"]); + assert.equal(((transientEnvSecret.ownerReference as JsonRecord).attached), true); + const secretManifest = JSON.parse(await readFile(createdTransientEnvSecret, "utf8")) as JsonRecord; + assert.equal(secretManifest.kind, "Secret"); + assert.equal(((secretManifest.metadata as JsonRecord).name), transientEnvSecret.name); + assert.equal(Object.keys((secretManifest.stringData as JsonRecord)).length, 9); + const ownerPatch = JSON.parse(await readFile(patchedTransientEnvSecret, "utf8")) as JsonRecord; + assert.deepEqual((ownerPatch.args as string[]).slice(0, 3), ["patch", "secret", String(transientEnvSecret.name)]); const manifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord; assert.equal((manifest.spec as JsonRecord).ttlSecondsAfterFinished, 86_400); - assert.equal(runnerEnvValue(manifest, "HWLAB_API_KEY"), "hwl_live_selftest"); - assert.equal(runnerEnvValue(manifest, "HWLAB_RUNTIME_API_URL"), "http://runtime-api.test"); - assert.equal(runnerEnvValue(manifest, "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME"), "1"); - assert.equal(runnerEnvValue(manifest, "UNIDESK_MAIN_SERVER_IP"), "https://unidesk.example.test"); + assertRunnerJobUsesG14EgressProxy(manifest); + assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_API_KEY", String(transientEnvSecret.name)); + assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_RUNTIME_API_URL", String(transientEnvSecret.name)); + assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", String(transientEnvSecret.name)); + assertRunnerJobUsesTransientEnvSecret(manifest, "UNIDESK_MAIN_SERVER_IP", String(transientEnvSecret.name)); assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN"); assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN"); assertNoSecretLeak(created); @@ -183,7 +219,9 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin }) as JsonRecord; assert.deepEqual(((defaultEndpointCreated.transientEnv as JsonRecord).names) as string[], ["UNIDESK_MAIN_SERVER_IP"]); const defaultEndpointManifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord; - assert.equal(runnerEnvValue(defaultEndpointManifest, "UNIDESK_MAIN_SERVER_IP"), "https://unidesk.default.example.test"); + const defaultEndpointSecret = defaultEndpointCreated.transientEnvSecret as JsonRecord; + assertRunnerJobUsesTransientEnvSecret(defaultEndpointManifest, "UNIDESK_MAIN_SERVER_IP", String(defaultEndpointSecret.name)); + assertRunnerJobUsesG14EgressProxy(defaultEndpointManifest); assertRunnerJobUsesToolCredential({ manifest: defaultEndpointManifest, toolCredentials: defaultEndpointCreated.toolCredentials } as JsonRecord, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN"); assertNoSecretLeak(defaultEndpointCreated); await assert.rejects( @@ -239,7 +277,7 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin assert.equal(envMap.get("AGENTRUN_SESSION_PVC_NAMESPACE"), "agentrun-v01"); assert.equal(envMap.get("AGENTRUN_SESSION_PVC_MOUNT_PATH"), "/home/agentrun/.codex-codex/sessions"); assert.equal(envMap.get("AGENTRUN_CODEX_ROLLOUT_SUBDIR"), "sessions"); - return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-codex-shell-sandbox-env", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-dsflash-go-profile-dry-run", "runner-k8s-job-dsflash-go-legacy-secretref-normalized", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-unidesk-ssh-endpoint-auto-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-k8s-job-session-pvc-volume-and-env"] }; + return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-codex-shell-sandbox-env", "runner-k8s-job-g14-egress-proxy-env", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-dsflash-go-profile-dry-run", "runner-k8s-job-dsflash-go-legacy-secretref-normalized", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-transient-env-secretref", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-unidesk-ssh-endpoint-auto-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-k8s-job-session-pvc-volume-and-env"] }; } finally { await new Promise((resolve) => server.server.close(() => resolve())); } @@ -247,14 +285,44 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin export default selfTest; -function runnerEnvValue(manifest: JsonRecord, name: string): unknown { +function runnerEnvEntry(manifest: JsonRecord, name: string): JsonRecord | undefined { const spec = manifest.spec as JsonRecord; const template = spec.template as JsonRecord; const podSpec = template.spec as JsonRecord; const containers = podSpec.containers as JsonRecord[]; const runner = containers[0] as JsonRecord; const env = runner.env as JsonRecord[]; - return env.find((item) => item.name === name)?.value; + return env.find((item) => item.name === name) as JsonRecord | undefined; +} + +function runnerEnvValue(manifest: JsonRecord, name: string): unknown { + return runnerEnvEntry(manifest, name)?.value; +} + +function assertRunnerJobUsesTransientEnvSecret(manifest: JsonRecord, envName: string, secretName: string): void { + const entry = runnerEnvEntry(manifest, envName); + assert.ok(entry, `${envName} env should be present`); + assert.equal(entry.value, undefined); + const valueFrom = entry.valueFrom as JsonRecord; + const secretKeyRef = valueFrom.secretKeyRef as JsonRecord; + assert.equal(secretKeyRef.name, secretName); + assert.equal(secretKeyRef.key, envName); +} + +function assertRunnerJobUsesG14EgressProxy(manifest: JsonRecord): void { + const proxy = "http://g14-provider-egress-proxy.unidesk.svc.cluster.local:18789"; + assert.equal(runnerEnvValue(manifest, "HTTP_PROXY"), proxy); + assert.equal(runnerEnvValue(manifest, "HTTPS_PROXY"), proxy); + assert.equal(runnerEnvValue(manifest, "ALL_PROXY"), proxy); + assert.equal(runnerEnvValue(manifest, "http_proxy"), proxy); + assert.equal(runnerEnvValue(manifest, "https_proxy"), proxy); + assert.equal(runnerEnvValue(manifest, "all_proxy"), proxy); + const noProxy = String(runnerEnvValue(manifest, "NO_PROXY")); + assert.equal(runnerEnvValue(manifest, "no_proxy"), noProxy); + assert.ok(noProxy.includes("hyueapi.com"), "NO_PROXY must keep hyueapi.com direct"); + assert.ok(noProxy.includes(".hyueapi.com"), "NO_PROXY must keep .hyueapi.com direct"); + assert.ok(noProxy.includes("g14-provider-egress-proxy.unidesk.svc.cluster.local"), "NO_PROXY must include the proxy Service itself"); + assert.ok(noProxy.includes(".svc"), "NO_PROXY must include Kubernetes Service domains"); } function assertRunnerJobUsesToolCredential(rendered: JsonRecord, envName: string, secretName: string, secretKey: string): void { diff --git a/src/selftest/cases/75-queue-q2-dispatch.ts b/src/selftest/cases/75-queue-q2-dispatch.ts index b290678..2912348 100644 --- a/src/selftest/cases/75-queue-q2-dispatch.ts +++ b/src/selftest/cases/75-queue-q2-dispatch.ts @@ -11,12 +11,30 @@ const selfTest: SelfTestCase = async (context) => { const fakeKubectl = path.join(context.tmp, "fake-kubectl-queue-q2.js"); const createdManifest = path.join(context.tmp, "created-queue-q2-runner-job.json"); await writeFile(fakeKubectl, `#!/usr/bin/env bun -const chunks = []; -for await (const chunk of Bun.stdin.stream()) chunks.push(chunk); -const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8"); -await Bun.write(${JSON.stringify(createdManifest)}, text); -const manifest = JSON.parse(text); -console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-queue-q2", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } })); +const args = Bun.argv.slice(2); +async function readStdin() { + const chunks = []; + for await (const chunk of Bun.stdin.stream()) chunks.push(chunk); + return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8"); +} +if (args[0] === "create") { + const text = await readStdin(); + const manifest = JSON.parse(text); + if (manifest.kind === "Job") await Bun.write(${JSON.stringify(createdManifest)}, text); + const uid = manifest.kind === "Job" ? "job-uid-queue-q2" : "secret-uid-queue-q2"; + console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid, resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } })); + process.exit(0); +} +if (args[0] === "patch" && args[1] === "secret") { + console.log(JSON.stringify({ apiVersion: "v1", kind: "Secret", metadata: { uid: "secret-uid-queue-q2", resourceVersion: "2", name: args[2], namespace: args[4] } })); + process.exit(0); +} +if (args[0] === "delete" && args[1] === "secret") { + console.log(JSON.stringify({ kind: "Status", status: "Success" })); + process.exit(0); +} +console.error("unsupported fake kubectl args: " + args.join(" ")); +process.exit(1); `); await chmod(fakeKubectl, 0o755); const store = new MemoryAgentRunStore(); @@ -132,8 +150,9 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin }) as QueueTaskRecord; const unideskDispatched = await client.post(`/api/v1/queue/tasks/${unideskCreated.id}/dispatch`, { attemptId: "attempt_queue_q2_unidesk_ssh_selftest" }) as QueueDispatchResult; assert.deepEqual((((unideskDispatched.runnerJob as JsonRecord).transientEnv as JsonRecord).names) as string[], ["UNIDESK_MAIN_SERVER_IP"]); + assert.equal((((unideskDispatched.runnerJob as JsonRecord).transientEnvSecret as JsonRecord).valuesPrinted), false); const unideskManifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord; - assert.equal(runnerEnvValue(unideskManifest, "UNIDESK_MAIN_SERVER_IP"), "https://unidesk.default.example.test"); + assert.equal(runnerEnvValue(unideskManifest, "UNIDESK_MAIN_SERVER_IP"), "secretRef"); assert.equal(runnerEnvValue(unideskManifest, "UNIDESK_SSH_CLIENT_TOKEN"), "secretRef"); assertNoSecretLeak(unideskDispatched);