Files
pikasTech-unidesk/scripts/src/hwlab-fake-model-provider.ts
T

950 lines
45 KiB
TypeScript

// SPEC: pikasTech/unidesk#1190 fake Responses provider for HWLAB v0.3 / AgentRun v0.2.
// Responsibility: YAML-first fake model provider materialization, k3s apply, status, and smoke checks.
import { createHash, randomBytes } from "node:crypto";
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "./config";
import { runCommand, type CommandResult } from "./command";
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
import { resolveAgentRunLaneTarget, type AgentRunLaneSpec } from "./agentrun-lanes";
import { resolveSecretSourceRoot } from "./agentrun/secrets";
import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
import { parseEnvFile, sha256Fingerprint, shQuote } from "./platform-infra-ops-library";
type FakeModelProviderAction = "plan" | "materialize" | "apply" | "status" | "smoke";
interface FakeModelProviderOptions {
action: FakeModelProviderAction;
node: string;
lane: HwlabRuntimeLane;
provider: string;
confirm: boolean;
dryRun: boolean;
timeoutSeconds: number;
full: boolean;
}
interface FakeModelProviderState {
options: FakeModelProviderOptions;
spec: HwlabRuntimeLaneSpec;
rootPath: string;
root: Record<string, unknown>;
runtimeRef: string;
secretsRef: string;
profileRef: string;
runtime: Record<string, unknown>;
secrets: Record<string, unknown>;
profile: Record<string, unknown>;
agentrun: {
configPath: string;
spec: AgentRunLaneSpec;
secretSourceRoot: string;
};
}
interface SourceMaterial {
purpose: string;
sourceRef: string;
sourceKey: string | null;
sourcePath: string;
existsBefore: boolean;
mutation: boolean;
valueBytes: number;
fingerprint: string;
valuesPrinted: false;
}
interface MaterializationResult {
ok: boolean;
mutation: boolean;
secretSourceRoot: string;
sources: SourceMaterial[];
files: Array<{
purpose: string;
sourceRef: string;
sourcePath: string;
existsBefore: boolean;
mutation: boolean;
byteCount: number;
fingerprint: string;
valuesPrinted: false;
}>;
providerApiKey: string;
valuesPrinted: false;
}
export function fakeModelProviderHelp(): Record<string, unknown> {
return {
ok: true,
command: "hwlab nodes fake-model-provider",
description: "YAML-first fake OpenAI-compatible Responses provider for HWLAB/AgentRun sentinel smoke.",
examples: [
"bun scripts/cli.ts hwlab nodes fake-model-provider plan --node D518 --lane v03 --provider fake-echo",
"bun scripts/cli.ts hwlab nodes fake-model-provider materialize --node D518 --lane v03 --provider fake-echo --confirm",
"bun scripts/cli.ts hwlab nodes fake-model-provider apply --node D518 --lane v03 --provider fake-echo --confirm",
"bun scripts/cli.ts hwlab nodes fake-model-provider status --node D518 --lane v03 --provider fake-echo",
"bun scripts/cli.ts hwlab nodes fake-model-provider smoke --node D518 --lane v03 --provider fake-echo",
],
actions: {
plan: "Read YAML configRefs and show local source/materialization and target k3s objects without mutation.",
materialize: "Create/update local .state/secrets source files for provider auth, Codex config, and sentinel prompt set.",
apply: "Materialize local sources, then apply ConfigMap/Secret/Deployment/Service to the selected node k3s namespace.",
status: "Inspect remote Deployment/Service/Secret/ConfigMap/pod readiness and /healthz without printing values.",
smoke: "Exec a deterministic ECHO streaming and non-ECHO error check inside the fake provider pod.",
},
notes: [
"Mutation actions require --confirm; --dry-run is accepted for apply.",
"Secret values are never printed; output is limited to sourceRef, key names, byte counts, and fingerprints.",
],
};
}
export async function runHwlabFakeModelProviderCommand(_config: Config, args: string[]): Promise<Record<string, unknown>> {
if (args.length === 0 || args.includes("--help") || args.includes("-h") || args[0] === "help") return fakeModelProviderHelp();
const options = parseFakeModelProviderOptions(args);
const state = readFakeModelProviderState(options);
if (options.action === "plan") return planFakeModelProvider(state);
if (options.action === "materialize") {
if (!options.confirm) throw new Error("fake-model-provider materialize requires --confirm");
const materialized = materializeFakeModelProviderSources(state);
return {
ok: materialized.ok,
command: "hwlab nodes fake-model-provider materialize",
node: options.node,
lane: options.lane,
provider: options.provider,
mutation: materialized.mutation,
materialized: redactMaterialization(materialized),
next: {
providerApply: `bun scripts/cli.ts hwlab nodes fake-model-provider apply --node ${options.node} --lane ${options.lane} --provider ${options.provider} --confirm`,
agentrunSecretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${options.node} --lane ${state.agentrun.spec.lane} --confirm`,
},
valuesPrinted: false,
};
}
if (options.action === "apply") return applyFakeModelProvider(state);
if (options.action === "status") return remoteFakeModelProviderStatus(state);
if (options.action === "smoke") return remoteFakeModelProviderSmoke(state);
throw new Error(`unsupported fake-model-provider action: ${options.action}`);
}
function parseFakeModelProviderOptions(args: string[]): FakeModelProviderOptions {
const [actionRaw] = args;
if (actionRaw !== "plan" && actionRaw !== "materialize" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "smoke") {
throw new Error(`fake-model-provider action must be plan, materialize, apply, status, or smoke; got ${actionRaw ?? "<missing>"}`);
}
const knownString = new Set(["--node", "--lane", "--provider", "--timeout-seconds"]);
const knownFlags = new Set(["--confirm", "--dry-run", "--full", "--raw"]);
for (let index = 1; index < args.length; index += 1) {
const arg = args[index];
if (knownString.has(arg)) {
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
index += 1;
continue;
}
if (knownFlags.has(arg)) continue;
throw new Error(`unsupported fake-model-provider option: ${arg}`);
}
const node = requiredOption(args, "--node");
if (!/^[A-Z][A-Z0-9_-]*$/u.test(node)) throw new Error("--node must be a simple node id such as D518");
const laneRaw = requiredOption(args, "--lane");
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`);
const provider = optionValue(args, "--provider") ?? "fake-echo";
if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/u.test(provider)) throw new Error("--provider must be a Kubernetes-safe id");
const confirm = args.includes("--confirm");
const dryRun = args.includes("--dry-run");
if (confirm && dryRun) throw new Error("fake-model-provider accepts only one of --confirm or --dry-run");
if (actionRaw === "apply" && !confirm && !dryRun) throw new Error("fake-model-provider apply requires --dry-run or --confirm");
if (actionRaw !== "apply" && dryRun) throw new Error(`fake-model-provider ${actionRaw} does not accept --dry-run`);
if ((actionRaw === "status" || actionRaw === "smoke" || actionRaw === "plan") && confirm) throw new Error(`fake-model-provider ${actionRaw} is read-only and does not accept --confirm`);
return {
action: actionRaw,
node,
lane: laneRaw,
provider,
confirm,
dryRun,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 900),
full: args.includes("--full") || args.includes("--raw"),
};
}
function readFakeModelProviderState(options: FakeModelProviderOptions): FakeModelProviderState {
const spec = hwlabRuntimeLaneSpecForNode(options.lane, options.node);
const configPath = rootPath("config", "hwlab-fake-model-provider", `${options.node.toLowerCase()}-${options.lane}`, `${options.provider}.yaml`);
if (!existsSync(configPath)) throw new Error(`fake provider config is missing: ${repoRelative(configPath)}`);
const root = readYamlRecord(configPath, "HwlabFakeModelProvider");
const provider = record(root.provider, "provider");
if (stringAt(provider, "id") !== options.provider) throw new Error(`${repoRelative(configPath)} provider.id must be ${options.provider}`);
if (provider.enabled !== true) throw new Error(`${repoRelative(configPath)} provider.enabled must be true`);
const target = record(provider.target, "provider.target");
if (stringAt(target, "node") !== options.node) throw new Error(`${repoRelative(configPath)} provider.target.node must be ${options.node}`);
if (stringAt(target, "lane") !== options.lane) throw new Error(`${repoRelative(configPath)} provider.target.lane must be ${options.lane}`);
const configRefs = record(provider.configRefs, "provider.configRefs");
const runtimeRef = stringAt(configRefs, "runtime");
const secretsRef = stringAt(configRefs, "secrets");
const profileRef = stringAt(configRefs, "profile");
const runtime = record(readConfigRefTarget(runtimeRef), runtimeRef);
const secrets = record(readConfigRefTarget(secretsRef), secretsRef);
const profile = record(readConfigRefTarget(profileRef), profileRef);
const runtimeTarget = record(runtime.target, "provider.runtime.target");
const agentrunLane = stringAt(runtimeTarget, "agentrunLane");
const agentrunTarget = resolveAgentRunLaneTarget({ node: options.node, lane: agentrunLane });
return {
options,
spec,
rootPath: configPath,
root,
runtimeRef,
secretsRef,
profileRef,
runtime,
secrets,
profile,
agentrun: {
configPath: agentrunTarget.configPath,
spec: agentrunTarget.spec,
secretSourceRoot: resolveSecretSourceRoot(agentrunTarget.spec),
},
};
}
function planFakeModelProvider(state: FakeModelProviderState): Record<string, unknown> {
const runtime = state.runtime;
const profile = state.profile;
const materialization = plannedMaterialSources(state);
return {
ok: true,
command: "hwlab nodes fake-model-provider plan",
node: state.options.node,
lane: state.options.lane,
provider: state.options.provider,
configRefs: {
root: repoRelative(state.rootPath),
runtime: state.runtimeRef,
secrets: state.secretsRef,
profile: state.profileRef,
},
runtime: {
namespace: stringAt(runtime, "namespace"),
deployment: stringAt(runtime, "deploymentName"),
service: stringAt(runtime, "serviceName"),
configMap: stringAt(runtime, "configMapName"),
secret: stringAt(runtime, "secretName"),
image: record(runtime.image, "provider.runtime.image").imageRef,
model: record(runtime.config, "provider.runtime.config").modelId,
endpoint: `http://${stringAt(runtime, "serviceName")}.${stringAt(runtime, "namespace")}.svc.cluster.local:${numberAt(runtime, "servicePort")}/v1`,
},
agentrun: {
configPath: repoRelative(state.agentrun.configPath.startsWith("/") ? state.agentrun.configPath : rootPath(state.agentrun.configPath)),
node: state.agentrun.spec.nodeId,
lane: state.agentrun.spec.lane,
namespace: state.agentrun.spec.runtime.namespace,
providerCredential: record(profile.providerCredential, "provider.profile.providerCredential"),
secretSourceRoot: displayPath(state.agentrun.secretSourceRoot),
},
localSources: materialization,
next: {
materialize: `bun scripts/cli.ts hwlab nodes fake-model-provider materialize --node ${state.options.node} --lane ${state.options.lane} --provider ${state.options.provider} --confirm`,
apply: `bun scripts/cli.ts hwlab nodes fake-model-provider apply --node ${state.options.node} --lane ${state.options.lane} --provider ${state.options.provider} --confirm`,
agentrunSecretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${state.options.node} --lane ${state.agentrun.spec.lane} --confirm`,
sentinelPlan: `bun scripts/cli.ts web-probe sentinel plan --node ${state.options.node} --lane ${state.options.lane} --sentinel workbench-fake-echo-session-invariance-10x`,
},
valuesPrinted: false,
};
}
function plannedMaterialSources(state: FakeModelProviderState): Record<string, unknown> {
const sources = [];
const providerSource = sourceByPurpose(state.secrets, "provider-api-key");
const promptSource = sourceByPurpose(state.secrets, "sentinel-prompts");
for (const source of [providerSource, promptSource]) {
const sourceRef = stringAt(source, "sourceRef");
const sourceKey = stringAt(source, "sourceKey");
const sourcePath = secretSourcePath(state, sourceRef);
const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
const value = values[sourceKey] ?? "";
sources.push({
purpose: stringAt(source, "purpose"),
sourceRef,
sourceKey,
sourcePath: displayPath(sourcePath),
exists: existsSync(sourcePath),
keyPresent: value.length > 0,
valueBytes: value.length > 0 ? Buffer.byteLength(value) : 0,
fingerprint: value.length > 0 ? sha256Fingerprint(value) : null,
valuesPrinted: false,
});
}
const credential = record(state.profile.providerCredential, "provider.profile.providerCredential");
const files = [stringAt(credential, "authJsonSourceRef"), stringAt(credential, "configTomlSourceRef")].map((sourceRef) => {
const sourcePath = secretSourcePath(state, sourceRef);
const value = existsSync(sourcePath) ? readFileSync(sourcePath, "utf8") : "";
return {
sourceRef,
sourcePath: displayPath(sourcePath),
exists: existsSync(sourcePath),
byteCount: Buffer.byteLength(value),
fingerprint: value.length > 0 ? sha256Fingerprint(value) : null,
valuesPrinted: false,
};
});
return {
secretSourceRoot: displayPath(state.agentrun.secretSourceRoot),
sources,
files,
valuesPrinted: false,
};
}
function materializeFakeModelProviderSources(state: FakeModelProviderState): MaterializationResult {
const providerSource = sourceByPurpose(state.secrets, "provider-api-key");
const promptSource = sourceByPurpose(state.secrets, "sentinel-prompts");
const apiKey = ensureEnvSourceValue(state, providerSource, () => randomBytes(randomHexBytes(providerSource, 24)).toString("hex"));
const promptJson = JSON.stringify(fakeEchoPrompts());
const prompts = ensureEnvSourceExactValue(state, promptSource, promptJson);
const credential = record(state.profile.providerCredential, "provider.profile.providerCredential");
const authJsonRef = stringAt(credential, "authJsonSourceRef");
const configTomlRef = stringAt(credential, "configTomlSourceRef");
const authJson = `${JSON.stringify({ OPENAI_API_KEY: apiKey.value }, null, 2)}\n`;
const configToml = renderFakeEchoCodexConfig(state);
const authFile = writeSecretFileSource(state, authJsonRef, authJson, "provider-auth-json");
const configFile = writeSecretFileSource(state, configTomlRef, configToml, "provider-config-toml");
const mutation = apiKey.mutation || prompts.mutation || authFile.mutation || configFile.mutation;
return {
ok: true,
mutation,
secretSourceRoot: displayPath(state.agentrun.secretSourceRoot),
sources: [apiKey, prompts],
files: [authFile, configFile],
providerApiKey: apiKey.value,
valuesPrinted: false,
};
}
function redactMaterialization(materialized: MaterializationResult): Record<string, unknown> {
return {
ok: materialized.ok,
mutation: materialized.mutation,
secretSourceRoot: materialized.secretSourceRoot,
sources: materialized.sources.map(({ value: _value, ...item }) => item),
files: materialized.files,
valuesPrinted: false,
};
}
function applyFakeModelProvider(state: FakeModelProviderState): Record<string, unknown> {
if (state.options.dryRun) {
const preview = renderFakeModelProviderManifests(state, null);
return {
ok: true,
command: "hwlab nodes fake-model-provider apply --dry-run",
node: state.options.node,
lane: state.options.lane,
provider: state.options.provider,
mutation: false,
manifest: preview.summary,
localSources: plannedMaterialSources(state),
confirm: `bun scripts/cli.ts hwlab nodes fake-model-provider apply --node ${state.options.node} --lane ${state.options.lane} --provider ${state.options.provider} --confirm`,
valuesPrinted: false,
};
}
if (!state.options.confirm) throw new Error("fake-model-provider apply requires --confirm");
const materialized = materializeFakeModelProviderSources(state);
const rendered = renderFakeModelProviderManifests(state, materialized.providerApiKey);
const applyResult = runRemoteApply(state, rendered.yaml);
const applyResolution = resolveFakeModelProviderRemotePayload(applyResult, "fake-model-provider apply JSON");
const applyPayload = applyResolution.parsed ?? {};
const status = remoteFakeModelProviderStatus(state);
return {
ok: applyResult.exitCode === 0 && applyPayload.ok === true && status.ok === true,
command: "hwlab nodes fake-model-provider apply",
node: state.options.node,
lane: state.options.lane,
provider: state.options.provider,
mutation: true,
materialized: redactMaterialization(materialized),
manifest: rendered.summary,
apply: Object.keys(applyPayload).length > 0 ? { ...applyPayload, stdoutRecovery: applyResolution.diagnostics, valuesRedacted: true } : { ...compactCommand(applyResult, state.options.full), stdoutRecovery: applyResolution.diagnostics },
status,
next: {
smoke: `bun scripts/cli.ts hwlab nodes fake-model-provider smoke --node ${state.options.node} --lane ${state.options.lane} --provider ${state.options.provider}`,
agentrunSecretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${state.options.node} --lane ${state.agentrun.spec.lane} --confirm`,
sentinelTrigger: "bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node D518 --lane v03 --sentinel workbench-fake-echo-session-invariance-10x --confirm --wait",
},
valuesPrinted: false,
};
}
function renderFakeModelProviderManifests(state: FakeModelProviderState, apiKey: string | null): { yaml: string; summary: Record<string, unknown> } {
const runtime = state.runtime;
const namespace = stringAt(runtime, "namespace");
const serviceName = stringAt(runtime, "serviceName");
const deploymentName = stringAt(runtime, "deploymentName");
const configMapName = stringAt(runtime, "configMapName");
const secretName = stringAt(runtime, "secretName");
const containerName = stringAt(runtime, "containerName");
const servicePort = numberAt(runtime, "servicePort");
const image = record(runtime.image, "provider.runtime.image");
const config = record(runtime.config, "provider.runtime.config");
const resources = record(runtime.resources, "provider.runtime.resources");
const probes = record(runtime.probes, "provider.runtime.probes");
const source = record(runtime.source, "provider.runtime.source");
const files = arrayAt(source, "files").map((item) => {
if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("provider.runtime.source.files entries must be relative safe paths");
const path = rootPath(item);
if (!existsSync(path)) throw new Error(`fake provider source file is missing: ${item}`);
return { path: item, key: sourceConfigKey(item), content: readFileSync(path, "utf8") };
});
const labels = {
"app.kubernetes.io/name": serviceName,
"app.kubernetes.io/part-of": "hwlab-fake-model-provider",
"app.kubernetes.io/managed-by": "unidesk",
"unidesk.ai/node": state.options.node,
"unidesk.ai/lane": state.options.lane,
"unidesk.ai/provider": state.options.provider,
};
const objects: Record<string, unknown>[] = [{
apiVersion: "v1",
kind: "Namespace",
metadata: { name: namespace },
}, {
apiVersion: "v1",
kind: "ConfigMap",
metadata: { name: configMapName, namespace, labels },
data: Object.fromEntries(files.map((file) => [file.key, file.content])),
}];
if (apiKey !== null) {
objects.push({
apiVersion: "v1",
kind: "Secret",
metadata: { name: secretName, namespace, labels },
type: "Opaque",
stringData: { "api-key": apiKey },
});
}
objects.push({
apiVersion: "apps/v1",
kind: "Deployment",
metadata: { name: deploymentName, namespace, labels },
spec: {
replicas: 1,
selector: { matchLabels: { "app.kubernetes.io/name": serviceName, "unidesk.ai/provider": state.options.provider } },
template: {
metadata: { labels },
spec: {
serviceAccountName: stringAt(runtime, "serviceAccountName"),
volumes: [{ name: "provider-source", configMap: { name: configMapName } }],
containers: [{
name: containerName,
image: stringAt(image, "imageRef"),
imagePullPolicy: stringAt(image, "imagePullPolicy"),
command: ["/bin/sh", "-ec"],
args: [providerContainerCommand(files, stringAt(source, "entrypoint"))],
env: [
{ name: "LISTEN_HOST", value: stringAt(runtime, "listenHost") },
{ name: "PORT", value: String(servicePort) },
{ name: "FAKE_RESPONSES_MODEL_ID", value: stringAt(config, "modelId") },
{ name: "FAKE_RESPONSES_MODE", value: stringAt(config, "mode") },
{ name: "FAKE_RESPONSES_DELAY_MS", value: String(numberAt(config, "responseDelayMs")) },
{ name: "FAKE_RESPONSES_VERSION", value: sourceDigest(files).slice(0, 16) },
{ name: "FAKE_ECHO_API_KEY", valueFrom: { secretKeyRef: { name: secretName, key: "api-key", optional: true } } },
],
ports: [{ name: "http", containerPort: servicePort }],
resources,
readinessProbe: {
httpGet: { path: stringAt(runtime, "healthPath"), port: "http" },
initialDelaySeconds: numberAt(probes, "initialDelaySeconds"),
periodSeconds: numberAt(probes, "periodSeconds"),
timeoutSeconds: numberAt(probes, "timeoutSeconds"),
failureThreshold: numberAt(probes, "failureThreshold"),
},
livenessProbe: {
httpGet: { path: stringAt(runtime, "healthPath"), port: "http" },
initialDelaySeconds: numberAt(probes, "initialDelaySeconds"),
periodSeconds: numberAt(probes, "periodSeconds"),
timeoutSeconds: numberAt(probes, "timeoutSeconds"),
failureThreshold: numberAt(probes, "failureThreshold"),
},
volumeMounts: [{ name: "provider-source", mountPath: "/config/provider-source", readOnly: true }],
}],
},
},
},
}, {
apiVersion: "v1",
kind: "Service",
metadata: { name: serviceName, namespace, labels },
spec: {
type: "ClusterIP",
selector: { "app.kubernetes.io/name": serviceName, "unidesk.ai/provider": state.options.provider },
ports: [{ name: "http", port: servicePort, targetPort: "http" }],
},
});
const yaml = `${objects.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
return {
yaml,
summary: {
namespace,
objects: objects.map((item) => `${item.kind}/${record(item.metadata, "metadata").name}`),
manifestSha256: sha256Fingerprint(yaml),
sourceFiles: files.map((file) => ({ path: file.path, key: file.key, bytes: Buffer.byteLength(file.content), sha256: sha256Fingerprint(file.content) })),
secretIncluded: apiKey !== null,
valuesPrinted: false,
},
};
}
function providerContainerCommand(files: Array<{ path: string; key: string }>, entrypoint: string): string {
if (entrypoint.startsWith("/") || entrypoint.includes("..")) throw new Error("provider.runtime.source.entrypoint must be a relative safe path");
return [
"set -eu",
"rm -rf /work",
"mkdir -p /work",
...files.map((file) => {
const target = `/work/${file.path}`;
return `mkdir -p ${shQuote(dirname(target))}; cp ${shQuote(`/config/provider-source/${file.key}`)} ${shQuote(target)}`;
}),
"cd /work",
`exec bun ${shQuote(entrypoint)}`,
].join("\n");
}
function runRemoteApply(state: FakeModelProviderState, manifestYaml: string): CommandResult {
const runtime = state.runtime;
const namespace = stringAt(runtime, "namespace");
const deployment = stringAt(runtime, "deploymentName");
const timeout = Math.min(state.options.timeoutSeconds, 300);
const script = [
"set +e",
`namespace=${shQuote(namespace)}`,
`deployment=${shQuote(deployment)}`,
"tmp=$(mktemp -d)",
"trap 'rm -rf \"$tmp\"' EXIT",
"manifest=\"$tmp/fake-model-provider.yaml\"",
"cat >\"$manifest\"",
"kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-fake-model-provider -f \"$manifest\" >/tmp/fake-provider-apply.out 2>/tmp/fake-provider-apply.err",
"apply_rc=$?",
"rollout_rc=0",
"if [ \"$apply_rc\" -eq 0 ]; then",
` kubectl -n "$namespace" rollout status deployment/"$deployment" --timeout=${timeout}s >/tmp/fake-provider-rollout.out 2>/tmp/fake-provider-rollout.err`,
" rollout_rc=$?",
"fi",
"python3 - \"$apply_rc\" \"$rollout_rc\" <<'PY'",
"import json, pathlib, sys",
"apply_rc = int(sys.argv[1]); rollout_rc = int(sys.argv[2])",
"def tail(path, n=4000):",
" try: return pathlib.Path(path).read_text(errors='replace')[-n:]",
" except FileNotFoundError: return ''",
"print(json.dumps({'ok': apply_rc == 0 and rollout_rc == 0, 'applyExitCode': apply_rc, 'rolloutExitCode': rollout_rc, 'applyStdoutTail': tail('/tmp/fake-provider-apply.out'), 'applyStderrTail': tail('/tmp/fake-provider-apply.err'), 'rolloutStdoutTail': tail('/tmp/fake-provider-rollout.out'), 'rolloutStderrTail': tail('/tmp/fake-provider-rollout.err'), 'valuesPrinted': False}, ensure_ascii=False))",
"PY",
].join("\n");
return runCommand([transPath(), state.spec.nodeKubeRoute, "sh", "--", script], repoRoot, {
input: manifestYaml,
timeoutMs: (state.options.timeoutSeconds + 30) * 1000,
});
}
function remoteFakeModelProviderStatus(state: FakeModelProviderState): Record<string, unknown> {
const runtime = state.runtime;
const script = remoteStatusScript(runtime);
const result = runCommand([transPath(), state.spec.nodeKubeRoute, "sh", "--", script], repoRoot, { timeoutMs: state.options.timeoutSeconds * 1000 });
const payloadResolution = resolveFakeModelProviderRemotePayload(result, "fake-model-provider status JSON");
const payload = payloadResolution.parsed ?? {};
return {
ok: result.exitCode === 0 && payload.ok === true,
command: "hwlab nodes fake-model-provider status",
node: state.options.node,
lane: state.options.lane,
provider: state.options.provider,
target: {
route: state.spec.nodeKubeRoute,
namespace: stringAt(runtime, "namespace"),
deployment: stringAt(runtime, "deploymentName"),
service: stringAt(runtime, "serviceName"),
},
...(Object.keys(payload).length > 0 ? payload : { result: compactCommand(result, state.options.full) }),
stdoutRecovery: payloadResolution.diagnostics,
valuesPrinted: false,
};
}
function remoteFakeModelProviderSmoke(state: FakeModelProviderState): Record<string, unknown> {
const runtime = state.runtime;
const script = remoteSmokeScript(runtime);
const result = runCommand([transPath(), state.spec.nodeKubeRoute, "sh", "--", script], repoRoot, { timeoutMs: state.options.timeoutSeconds * 1000 });
const payloadResolution = resolveFakeModelProviderRemotePayload(result, "fake-model-provider smoke JSON");
const payload = payloadResolution.parsed ?? {};
return {
ok: result.exitCode === 0 && payload.ok === true,
command: "hwlab nodes fake-model-provider smoke",
node: state.options.node,
lane: state.options.lane,
provider: state.options.provider,
target: {
route: state.spec.nodeKubeRoute,
namespace: stringAt(runtime, "namespace"),
deployment: stringAt(runtime, "deploymentName"),
service: stringAt(runtime, "serviceName"),
},
...(Object.keys(payload).length > 0 ? payload : { result: compactCommand(result, state.options.full) }),
stdoutRecovery: payloadResolution.diagnostics,
valuesPrinted: false,
};
}
function remoteStatusScript(runtime: Record<string, unknown>): string {
return remoteCommonNodeScript(runtime, `
const deploy = kubectlJson(['-n', ns, 'get', 'deployment', deployment, '-o', 'json']);
const svc = kubectlJson(['-n', ns, 'get', 'service', service, '-o', 'json']);
const cm = kubectlJson(['-n', ns, 'get', 'configmap', configMap, '-o', 'json']);
const secret = kubectlJson(['-n', ns, 'get', 'secret', secretName, '-o', 'json']);
const pods = kubectlJson(['-n', ns, 'get', 'pod', '-l', selector, '-o', 'json']);
const pod = selectPod(pods.value);
let health = { ok: false, skipped: pod === null, status: null, body: null, valuesPrinted: false };
if (pod !== null) {
const js = "const r=await fetch('http://127.0.0.1:" + port + "/healthz'); const t=await r.text(); console.log(JSON.stringify({status:r.status, body:t, valuesPrinted:false}));";
const out = run(['kubectl', '-n', ns, 'exec', pod.name, '-c', container, '--', 'bun', '-e', js]);
const parsed = parseJson(out.stdout);
health = { ok: out.status === 0 && parsed.status === 200, status: parsed.status ?? null, body: parseJson(parsed.body || ''), exitCode: out.status, stderrTail: out.stderr.slice(-1000), valuesPrinted: false };
}
const available = Number(deploy.value?.status?.availableReplicas || 0);
const ready = Number(deploy.value?.status?.readyReplicas || 0);
const data = secret.value?.data && typeof secret.value.data === 'object' ? secret.value.data : {};
const configData = cm.value?.data && typeof cm.value.data === 'object' ? cm.value.data : {};
const ok = deploy.ok && svc.ok && cm.ok && secret.ok && available >= 1 && health.ok;
console.log(JSON.stringify({
ok,
deployment: { exists: deploy.ok, availableReplicas: available, readyReplicas: ready, observedGeneration: deploy.value?.status?.observedGeneration ?? null },
service: { exists: svc.ok, clusterIP: svc.value?.spec?.clusterIP ?? null, ports: (svc.value?.spec?.ports || []).map((item) => ({ name: item.name, port: item.port, targetPort: item.targetPort })) },
configMap: { exists: cm.ok, keys: Object.keys(configData).sort(), valuesPrinted: false },
secret: { exists: secret.ok, keys: Object.keys(data).sort(), valuesPrinted: false },
pod,
health,
valuesPrinted: false
}, null, 0));
`);
}
function remoteSmokeScript(runtime: Record<string, unknown>): string {
const model = stringAt(record(runtime.config, "provider.runtime.config"), "modelId");
return remoteCommonNodeScript(runtime, `
const pods = kubectlJson(['-n', ns, 'get', 'pod', '-l', selector, '-o', 'json']);
const pod = selectPod(pods.value);
if (pod === null) {
console.log(JSON.stringify({ ok: false, error: 'fake-provider-pod-missing', valuesPrinted: false }));
process.exit(0);
}
const js = ${JSON.stringify(`
const base = 'http://127.0.0.1:${numberAt(runtime, "servicePort")}';
const body = { model: ${JSON.stringify(model)}, input: [{ role: 'user', content: [{ type: 'input_text', text: 'ECHO AGENTRUN_FAKE_OK' }] }], stream: true };
const response = await fetch(base + '/v1/responses', { method: 'POST', headers: { 'content-type': 'application/json', authorization: 'Bearer test-redacted' }, body: JSON.stringify(body) });
const text = await response.text();
const bad = await fetch(base + '/v1/responses', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ model: ${JSON.stringify(model)}, input: 'non echo', stream: false }) });
const badText = await bad.text();
const ok = response.status === 200 && text.includes('AGENTRUN_FAKE_OK') && text.includes('response.completed') && bad.status === 400;
console.log(JSON.stringify({ ok, streamStatus: response.status, deltaSeen: text.includes('AGENTRUN_FAKE_OK'), completedSeen: text.includes('response.completed'), doneSeen: text.includes('[DONE]'), nonEchoStatus: bad.status, nonEchoStructuredError: badText.includes('non_echo_prompt'), valuesPrinted: false }));
`)};
const out = run(['kubectl', '-n', ns, 'exec', pod.name, '-c', container, '--', 'bun', '-e', js]);
const parsed = parseJson(out.stdout);
console.log(JSON.stringify({
ok: out.status === 0 && parsed.ok === true,
pod,
smoke: Object.keys(parsed).length > 0 ? parsed : null,
exec: { exitCode: out.status, stderrTail: out.stderr.slice(-1200), valuesPrinted: false },
valuesPrinted: false
}));
`);
}
function remoteCommonNodeScript(runtime: Record<string, unknown>, body: string): string {
const namespace = stringAt(runtime, "namespace");
const deployment = stringAt(runtime, "deploymentName");
const service = stringAt(runtime, "serviceName");
const configMap = stringAt(runtime, "configMapName");
const secretName = stringAt(runtime, "secretName");
const container = stringAt(runtime, "containerName");
const port = numberAt(runtime, "servicePort");
const selector = `app.kubernetes.io/name=${service},unidesk.ai/provider=fake-echo`;
return [
"set +e",
`NS=${shQuote(namespace)} DEPLOYMENT=${shQuote(deployment)} SERVICE=${shQuote(service)} CONFIG_MAP=${shQuote(configMap)} SECRET_NAME=${shQuote(secretName)} CONTAINER=${shQuote(container)} PORT=${shQuote(String(port))} SELECTOR=${shQuote(selector)} node <<'NODE'`,
"const cp = require('node:child_process');",
"const ns = process.env.NS;",
"const deployment = process.env.DEPLOYMENT;",
"const service = process.env.SERVICE;",
"const configMap = process.env.CONFIG_MAP;",
"const secretName = process.env.SECRET_NAME;",
"const container = process.env.CONTAINER;",
"const port = process.env.PORT;",
"const selector = process.env.SELECTOR;",
"function run(argv) { return cp.spawnSync(argv[0], argv.slice(1), { encoding: 'utf8', maxBuffer: 8 * 1024 * 1024 }); }",
"function parseJson(text) { try { return JSON.parse(String(text || '').trim()); } catch { const s=String(text||''); const a=s.indexOf('{'); const b=s.lastIndexOf('}'); if (a>=0&&b>a) { try { return JSON.parse(s.slice(a,b+1)); } catch {} } return {}; } }",
"function kubectlJson(args) { const out = run(['kubectl', ...args]); return { ok: out.status === 0, exitCode: out.status, value: out.status === 0 ? parseJson(out.stdout) : {}, stderrTail: out.stderr.slice(-1200) }; }",
"function selectPod(pods) { const items = Array.isArray(pods?.items) ? pods.items : []; const mapped = items.map((item) => { const statuses = Array.isArray(item.status?.containerStatuses) ? item.status.containerStatuses : []; const target = statuses.find((status) => status.name === container); return { name: item.metadata?.name || '', phase: item.status?.phase || null, ready: item.status?.phase === 'Running' && target?.ready === true, containerReady: target?.ready === true, restartCount: target?.restartCount ?? null }; }).filter((item) => item.name); return mapped.find((item) => item.ready) || mapped.find((item) => item.phase === 'Running') || mapped[0] || null; }",
body,
"NODE",
].join("\n");
}
function sourceByPurpose(secrets: Record<string, unknown>, purpose: string): Record<string, unknown> {
const source = arrayAt(secrets, "sources").map((item) => record(item, "provider.secrets.sources[]")).find((item) => item.purpose === purpose);
if (source === undefined) throw new Error(`provider.secrets.sources is missing purpose=${purpose}`);
return source;
}
function ensureEnvSourceValue(state: FakeModelProviderState, source: Record<string, unknown>, createValue: () => string): SourceMaterial & { value: string } {
const purpose = stringAt(source, "purpose");
const sourceRef = stringAt(source, "sourceRef");
const sourceKey = stringAt(source, "sourceKey");
const sourcePath = secretSourcePath(state, sourceRef);
const existsBefore = existsSync(sourcePath);
const existingText = existsBefore ? readFileSync(sourcePath, "utf8") : "";
const values = parseEnvFile(existingText);
const existing = values[sourceKey];
const value = existing && existing.length > 0 ? existing : createValue();
let nextText = existingText;
let mutation = false;
if (!existing || existing.length === 0) {
nextText = upsertEnvLine(existingText, sourceKey, value);
writePrivateFile(sourcePath, nextText);
mutation = true;
}
return {
purpose,
sourceRef,
sourceKey,
sourcePath: displayPath(sourcePath),
existsBefore,
mutation,
value,
valueBytes: Buffer.byteLength(value),
fingerprint: sha256Fingerprint(value),
valuesPrinted: false,
};
}
function ensureEnvSourceExactValue(state: FakeModelProviderState, source: Record<string, unknown>, value: string): SourceMaterial & { value: string } {
const purpose = stringAt(source, "purpose");
const sourceRef = stringAt(source, "sourceRef");
const sourceKey = stringAt(source, "sourceKey");
const sourcePath = secretSourcePath(state, sourceRef);
const existsBefore = existsSync(sourcePath);
const existingText = existsBefore ? readFileSync(sourcePath, "utf8") : "";
const values = parseEnvFile(existingText);
const existing = values[sourceKey] ?? "";
const mutation = existing !== value;
if (mutation) writePrivateFile(sourcePath, upsertEnvLine(existingText, sourceKey, value));
return {
purpose,
sourceRef,
sourceKey,
sourcePath: displayPath(sourcePath),
existsBefore,
mutation,
value,
valueBytes: Buffer.byteLength(value),
fingerprint: sha256Fingerprint(value),
valuesPrinted: false,
};
}
function writeSecretFileSource(state: FakeModelProviderState, sourceRef: string, content: string, purpose: string): MaterializationResult["files"][number] {
const sourcePath = secretSourcePath(state, sourceRef);
const existsBefore = existsSync(sourcePath);
const current = existsBefore ? readFileSync(sourcePath, "utf8") : "";
const mutation = current !== content;
if (mutation) writePrivateFile(sourcePath, content);
return {
purpose,
sourceRef,
sourcePath: displayPath(sourcePath),
existsBefore,
mutation,
byteCount: Buffer.byteLength(content),
fingerprint: sha256Fingerprint(content),
valuesPrinted: false,
};
}
function renderFakeEchoCodexConfig(state: FakeModelProviderState): string {
const config = record(state.profile.codexConfig, "provider.profile.codexConfig");
return [
`model = ${tomlString(stringAt(config, "model"))}`,
`model_provider = ${tomlString(stringAt(config, "modelProvider"))}`,
`review_model = ${tomlString(stringAt(config, "model"))}`,
`model_context_window = ${numberAt(config, "modelContextWindow")}`,
`model_auto_compact_token_limit = ${numberAt(config, "modelAutoCompactTokenLimit")}`,
`model_supports_reasoning_summaries = ${config.modelSupportsReasoningSummaries === true ? "true" : "false"}`,
"",
`[model_providers.${stringAt(config, "modelProvider")}]`,
`name = ${tomlString(stringAt(config, "modelProvider"))}`,
`base_url = ${tomlString(stringAt(config, "baseUrl"))}`,
`wire_api = ${tomlString(stringAt(config, "wireApi"))}`,
`requires_openai_auth = ${config.requiresOpenaiAuth === true ? "true" : "false"}`,
"",
].join("\n");
}
function fakeEchoPrompts(): string[] {
return Array.from({ length: 10 }, (_, index) => {
const marker = `sentinel-${String(index + 1).padStart(2, "0")}`;
return `ECHO ${marker} fake-echo session invariance round ${index + 1}`;
});
}
function randomHexBytes(source: Record<string, unknown>, fallback: number): number {
const policy = record(source.createIfMissing, "createIfMissing");
const value = policy.randomHexBytes;
return typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 256 ? value : fallback;
}
function secretSourcePath(state: FakeModelProviderState, sourceRef: string): string {
if (sourceRef.includes("..")) throw new Error(`secret sourceRef must not contain ..: ${sourceRef}`);
if (sourceRef.startsWith("/")) return sourceRef;
return join(state.agentrun.secretSourceRoot, ...sourceRef.split("/"));
}
function sourceConfigKey(path: string): string {
return path.replace(/[^A-Za-z0-9_.-]+/gu, "__");
}
function sourceDigest(files: Array<{ path: string; content: string }>): string {
const hash = createHash("sha256");
for (const file of files.slice().sort((a, b) => a.path.localeCompare(b.path))) {
hash.update(file.path);
hash.update("\0");
hash.update(file.content);
hash.update("\0");
}
return hash.digest("hex");
}
function writePrivateFile(path: string, content: string): void {
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, content, { mode: 0o600 });
try {
chmodSync(path, 0o600);
} catch {
// chmod is best-effort on non-POSIX filesystems.
}
}
function upsertEnvLine(text: string, key: string, value: string): string {
const quoted = `${key}=${envSingleQuote(value)}`;
const lines = text.length === 0 ? [] : text.replace(/\n?$/u, "").split(/\r?\n/u);
const index = lines.findIndex((line) => line.trim().startsWith(`${key}=`));
if (index >= 0) lines[index] = quoted;
else lines.push(quoted);
return `${lines.join("\n")}\n`;
}
function envSingleQuote(value: string): string {
if (value.includes("'")) throw new Error("fake provider source values containing single quotes are not supported by the current env parser");
return `'${value}'`;
}
function readConfigRefTarget(ref: string): unknown {
const [pathPart, selector] = ref.split("#");
if (!pathPart || !selector) throw new Error(`configRef must use path#selector: ${ref}`);
const parsed = readYamlRecord(rootPath(pathPart));
let current: unknown = parsed;
for (const rawPart of selector.split(".")) {
const part = rawPart.trim();
if (part.length === 0) continue;
current = record(current, ref)[part];
}
return current;
}
function readYamlRecord(path: string, expectedKind?: string): Record<string, unknown> {
const parsed = Bun.YAML.parse(readFileSync(path, "utf8")) as unknown;
const value = record(parsed, repoRelative(path));
if (expectedKind !== undefined && value.kind !== expectedKind) throw new Error(`${repoRelative(path)} kind must be ${expectedKind}`);
return value;
}
function optionValue(args: string[], name: string): string | null {
const index = args.indexOf(name);
return index >= 0 ? args[index + 1] ?? null : null;
}
function requiredOption(args: string[], name: string): string {
const value = optionValue(args, name);
if (value === null || value.length === 0 || value.startsWith("--")) throw new Error(`${name} is required`);
return value;
}
function positiveIntegerOption(args: string[], name: string, defaultValue: number, max: number): number {
const raw = optionValue(args, name);
if (raw === null) return defaultValue;
const value = Number(raw);
if (!Number.isInteger(value) || value <= 0 || value > max) throw new Error(`${name} must be an integer in 1..${max}`);
return value;
}
function arrayAt(obj: Record<string, unknown>, key: string): unknown[] {
const value = obj[key];
if (!Array.isArray(value)) throw new Error(`${key} must be an array`);
return value;
}
function record(value: unknown, label: string): Record<string, unknown> {
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${label} must be an object`);
return value as Record<string, unknown>;
}
function stringAt(obj: Record<string, unknown>, key: string): string {
const value = obj[key];
if (typeof value !== "string" || value.length === 0) throw new Error(`${key} must be a non-empty string`);
return value;
}
function numberAt(obj: Record<string, unknown>, key: string): number {
const value = obj[key];
if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${key} must be a number`);
return value;
}
function tomlString(value: string): string {
return JSON.stringify(value);
}
function resolveFakeModelProviderRemotePayload(result: CommandResult, requestedStdoutType: string): { parsed: Record<string, unknown> | null; diagnostics: Record<string, unknown> } {
const resolved = resolveCliChildJsonCommandResult({
result,
requestedStdoutType,
acceptParsed: (value) => typeof value.ok === "boolean" || typeof value.error === "string" || typeof value.failureKind === "string" || Object.keys(value).length > 1,
});
return { parsed: resolved.parsed, diagnostics: resolved.diagnostics };
}
function compactCommand(result: CommandResult, full = false): Record<string, unknown> {
return {
exitCode: result.exitCode,
timedOut: result.timedOut,
stdoutBytes: Buffer.byteLength(result.stdout),
stderrBytes: Buffer.byteLength(result.stderr),
stdoutTail: full || result.exitCode !== 0 ? result.stdout.trim().slice(-4000) : "",
stderrTail: full || result.exitCode !== 0 ? result.stderr.trim().slice(-4000) : "",
};
}
function transPath(): string {
return join(repoRoot, "scripts", "trans");
}
function repoRelative(path: string): string {
return path.startsWith(`${repoRoot}/`) ? path.slice(repoRoot.length + 1) : path;
}
function displayPath(path: string): string {
if (path.startsWith(`${repoRoot}/`)) return path.slice(repoRoot.length + 1);
const marker = "/.state/secrets/";
const index = path.indexOf(marker);
if (index >= 0) return `.state/secrets/${path.slice(index + marker.length)}`;
if (path.endsWith("/.state/secrets")) return ".state/secrets";
return path;
}