ymal: compose repeated node lane templates

scripts/src/agentrun-lanes.ts

@@ -13,6 +13,7 @@ import {
   stringArrayField,
   stringField,
 } from "./platform-infra-ops-library";
+import { materializeYamlComposition } from "./yaml-composition";
 
 export const AGENTRUN_CONFIG_PATH = "config/agentrun.yaml";
 
@@ -464,7 +465,7 @@ export function agentRunSourceSnapshotRef(spec: AgentRunLaneSpec, sourceCommit:
 
 function readAgentRunControlPlaneConfig(env: NodeJS.ProcessEnv): AgentRunControlPlaneConfig {
   const configPath = env.AGENTRUN_CONTROL_PLANE_CONFIG ?? rootPath(AGENTRUN_CONFIG_PATH);
-  const root = readYamlRecord<Record<string, unknown>>(configPath);
+  const root = materializeYamlComposition(readYamlRecord<Record<string, unknown>>(configPath), { label: configPath }).value;
   validateConfigEnvelope(root, configPath);
   const controlPlane = recordField(root, "controlPlane", configPath);
   const defaultTarget = recordField(controlPlane, "default", `${configPath}.controlPlane`);

scripts/src/hwlab-node-lanes.ts

@@ -5,6 +5,7 @@
 // Responsibility: YAML source-of-truth parsing for HWLAB node/lane Workbench observability.
 import { readFileSync } from "node:fs";
 import { rootPath } from "./config";
+import { materializeYamlComposition } from "./yaml-composition";
 
 export type HwlabRuntimeLane = string;
 
@@ -1685,7 +1686,10 @@ function runtimeImageBuildsConfig(value: unknown, path: string): HwlabRuntimeIma
 function readHwlabNodeLaneConfig(): HwlabNodeLaneConfig {
   const path = rootPath(HWLAB_NODE_LANE_CONFIG_PATH);
   const raw = readFileSync(path, "utf8");
-  const parsed = asRecord(Bun.YAML.parse(raw) as unknown, HWLAB_NODE_LANE_CONFIG_PATH);
+  const parsed = asRecord(
+    materializeYamlComposition(Bun.YAML.parse(raw) as unknown, { label: HWLAB_NODE_LANE_CONFIG_PATH }).value,
+    HWLAB_NODE_LANE_CONFIG_PATH,
+  );
   validateConfigEnvelope(parsed);
   const defaultTarget = parseDefaultTarget(asRecord(parsed.defaults, `${HWLAB_NODE_LANE_CONFIG_PATH}.defaults`));
   const requiredNoProxy = stringArrayField(parsed, "requiredNoProxy", HWLAB_NODE_LANE_CONFIG_PATH);

scripts/src/hwlab-node-web-sentinel-config-ref.ts

@@ -3,6 +3,7 @@
 import { createHash } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { rootPath } from "./config";
+import { materializeYamlComposition } from "./yaml-composition";
 
 export interface WebProbeSentinelTemplateContext {
   readonly nodeId: string;
@@ -34,7 +35,7 @@ export function readWebProbeSentinelConfigRef(context: WebProbeSentinelTemplateC
   const absPath = rootPath(parsed.file);
   if (!existsSync(absPath)) throw new Error(`${parsed.file} does not exist`);
   const text = readFileSync(absPath, "utf8");
-  const doc = Bun.YAML.parse(text) as unknown;
+  const doc = materializeYamlComposition(Bun.YAML.parse(text) as unknown, { label: parsed.file }).value;
   const vars = { ...builtinVars, ...documentVars(doc, builtinVars) };
   const path = renderTemplateString(parsed.path, vars, `${parsed.file}#path`);
   const target = valueAtConfigPath(doc, path);

scripts/src/hwlab-node-web-sentinel-resolver.ts

@@ -4,6 +4,7 @@ import { existsSync, readFileSync } from "node:fs";
 import { rootPath } from "./config";
 import { readWebProbeSentinelConfigRefTarget, type WebProbeSentinelTemplateContext } from "./hwlab-node-web-sentinel-config-ref";
 import { HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS, type HwlabRuntimeLaneSpec, type HwlabRuntimeWebProbeSentinelConfigRefKey, type HwlabRuntimeWebProbeSentinelRegistryItemSpec } from "./hwlab-node-lanes";
+import { materializeYamlComposition } from "./yaml-composition";
 
 export interface ResolvedWebProbeSentinel {
   readonly id: string;
@@ -169,7 +170,7 @@ export function readConfigRefTarget(ref: string, context?: WebProbeSentinelTempl
   }
   const absPath = rootPath(file);
   if (!existsSync(absPath)) throw new Error(`${file} does not exist`);
-  const doc = Bun.YAML.parse(readFileSync(absPath, "utf8")) as unknown;
+  const doc = materializeYamlComposition(Bun.YAML.parse(readFileSync(absPath, "utf8")) as unknown, { label: file }).value;
   return valueAtPath(doc, path);
 }
 

scripts/src/platform-infra-host-proxy.ts

@@ -9,6 +9,7 @@ import type { RenderedCliResult } from "./output";
 import { shQuote } from "./platform-infra-public-service";
 import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets";
 import { transHostProxyEnvSummary } from "./trans-host-proxy";
+import { materializeYamlComposition } from "./yaml-composition";
 
 const configPath = "config/platform-infra/host-proxy.yaml";
 
@@ -161,7 +162,7 @@ function parseOptions(args: string[]): HostProxyOptions {
 }
 
 function readHostProxyConfig(): HostProxyConfig {
-  const root = record(Bun.YAML.parse(readFileSync(rootPath(configPath), "utf8")), configPath);
+  const root = record(materializeYamlComposition(Bun.YAML.parse(readFileSync(rootPath(configPath), "utf8")) as unknown, { label: configPath }).value, configPath);
   if (root.kind !== "platform-infra-host-proxy") throw new Error(`${configPath}.kind must be platform-infra-host-proxy`);
   const defaultsRaw = record(root.defaults, `${configPath}.defaults`);
   const server = parseServer(record(root.server, `${configPath}.server`), `${configPath}.server`);
@@ -172,7 +173,7 @@ function readHostProxyConfig(): HostProxyConfig {
   }));
   const targets: Record<string, HostProxyTarget> = {};
   for (const [id, value] of Object.entries(record(root.targets, `${configPath}.targets`))) {
-    targets[id] = parseTarget(id, record(value, `${configPath}.targets.${id}`), sources);
+    targets[id] = parseTarget(id, record(value, `${configPath}.targets.${id}`), sources, root);
   }
   return {
     defaults: { targetId: stringField(defaultsRaw, "targetId", `${configPath}.defaults`) },
@@ -315,7 +316,7 @@ function podAccessSpec(raw: Record<string, unknown>, label: string): HostProxyPo
   return { enabled, listenHost, listenPort, proxyUrl };
 }
 
-function parseTarget(id: string, raw: Record<string, unknown>, sources: Record<string, HostProxySource>): HostProxyTarget {
+function parseTarget(id: string, raw: Record<string, unknown>, sources: Record<string, HostProxySource>, root: Record<string, unknown>): HostProxyTarget {
   const sourceRef = stringField(raw, "sourceRef", `${configPath}.targets.${id}`);
   if (!sourceRef.startsWith("sources.")) throw new Error(`${configPath}.targets.${id}.sourceRef must use sources.<id>`);
   const sourceId = sourceRef.slice("sources.".length);
@@ -324,7 +325,9 @@ function parseTarget(id: string, raw: Record<string, unknown>, sources: Record<s
   const envRaw = record(raw.env, `${configPath}.targets.${id}.env`);
   const filesRaw = record(raw.files, `${configPath}.targets.${id}.files`);
   const applyRaw = record(raw.apply, `${configPath}.targets.${id}.apply`);
-  const noProxy = stringArrayField(envRaw, "noProxy", `${configPath}.targets.${id}.env`);
+  const noProxy = envRaw.noProxy === undefined
+    ? noProxyFromRefs(envRaw, root, `${configPath}.targets.${id}.env`)
+    : stringArrayField(envRaw, "noProxy", `${configPath}.targets.${id}.env`);
   if (!noProxy.includes("hyueapi.com") || !noProxy.includes(".hyueapi.com")) {
     throw new Error(`${configPath}.targets.${id}.env.noProxy must preserve hyueapi.com and .hyueapi.com`);
   }
@@ -359,6 +362,30 @@ function parseTarget(id: string, raw: Record<string, unknown>, sources: Record<s
   return target;
 }
 
+function noProxyFromRefs(envRaw: Record<string, unknown>, root: Record<string, unknown>, label: string): string[] {
+  const refs = record(envRaw.noProxyRefs, `${label}.noProxyRefs`);
+  const values: string[] = [];
+  for (const key of ["common", "extras"] as const) {
+    const ref = stringField(refs, key, `${label}.noProxyRefs`);
+    values.push(...stringArrayAtPath(root, ref, `${label}.noProxyRefs.${key}`));
+  }
+  return [...new Set(values)];
+}
+
+function stringArrayAtPath(root: unknown, ref: string, label: string): string[] {
+  let current: unknown = root;
+  for (const segment of ref.split(".")) {
+    if (!/^[A-Za-z0-9_-]+$/u.test(segment) || typeof current !== "object" || current === null || Array.isArray(current)) {
+      throw new Error(`${label} references invalid path ${ref}`);
+    }
+    current = (current as Record<string, unknown>)[segment];
+  }
+  if (!Array.isArray(current) || !current.every((item) => typeof item === "string" && item.length > 0)) {
+    throw new Error(`${label} must reference a string array`);
+  }
+  return [...current];
+}
+
 function resolveTarget(config: HostProxyConfig, targetId: string): HostProxyTarget {
   const target = config.targets[targetId];
   if (target === undefined) throw new Error(`${configPath}.targets.${targetId} is not defined`);

scripts/src/platform-infra-pipelines-as-code.ts

@@ -15,6 +15,7 @@ import {
   shQuote,
   sha256Fingerprint,
 } from "./platform-infra-ops-library";
+import { materializeYamlComposition } from "./yaml-composition";
 
 const configFile = rootPath("config", "platform-infra", "pipelines-as-code.yaml");
 const configLabel = "config/platform-infra/pipelines-as-code.yaml";
@@ -213,7 +214,7 @@ function help(): Record<string, unknown> {
 }
 
 function readPacConfig(): PacConfig {
-  const root = readYamlRecord<Record<string, unknown>>(configFile, "platform-infra-pipelines-as-code");
+  const root = materializeYamlComposition(readYamlRecord<Record<string, unknown>>(configFile, "platform-infra-pipelines-as-code"), { label: configLabel }).value;
   const release = y.objectField(root, "release", "");
   const closeout = y.objectField(root, "closeout", "");
   const gitOpsMirrorFlush = y.objectField(closeout, "gitOpsMirrorFlush", "closeout");

scripts/src/trans-host-proxy.ts

@@ -1,6 +1,7 @@
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { repoRoot } from "./config";
+import { materializeYamlComposition } from "./yaml-composition";
 
 export const transHostProxyConfigPath = "config/platform-infra/host-proxy.yaml";
 
@@ -19,7 +20,7 @@ export interface TransHostProxyEnvRule {
 export function readTransHostProxyEnvRule(targetId: string): TransHostProxyEnvRule | null {
   const absolutePath = join(repoRoot, transHostProxyConfigPath);
   if (!existsSync(absolutePath)) return null;
-  const root = record(Bun.YAML.parse(readFileSync(absolutePath, "utf8")), transHostProxyConfigPath);
+  const root = record(materializeYamlComposition(Bun.YAML.parse(readFileSync(absolutePath, "utf8")) as unknown, { label: transHostProxyConfigPath }).value, transHostProxyConfigPath);
   if (root.kind !== "platform-infra-host-proxy") throw new Error(`${transHostProxyConfigPath}.kind must be platform-infra-host-proxy`);
   const targets = record(root.targets, `${transHostProxyConfigPath}.targets`);
   const targetRaw = targets[targetId];

scripts/src/yaml-composition.ts

@@ -0,0 +1,166 @@
+// SPEC: pikasTech/unidesk#1579 YAML-first composition.
+// Responsibility: materialize lightweight YAML templates before domain parsers validate shapes.
+
+export interface YamlCompositionResult<T> {
+  readonly value: T;
+  readonly traces: readonly YamlCompositionTrace[];
+}
+
+export interface YamlCompositionTrace {
+  readonly path: string;
+  readonly extends: readonly string[];
+  readonly overrideKeys: readonly string[];
+  readonly variables: Record<string, string>;
+}
+
+interface CompositionOptions {
+  readonly label: string;
+  readonly stripTemplateKeys?: boolean;
+}
+
+const COMPOSITION_KEYS = new Set(["extends", "overrides", "variables"]);
+const TEMPLATE_ROOT_KEYS = new Set(["templates", "baselines"]);
+
+export function materializeYamlComposition<T>(input: T, options: CompositionOptions): YamlCompositionResult<T> {
+  const traces: YamlCompositionTrace[] = [];
+  const root = input as unknown;
+  const value = materializeValue(root, root, options.label, {}, traces, []);
+  if (options.stripTemplateKeys === true && isRecord(value)) {
+    const stripped = { ...value };
+    for (const key of TEMPLATE_ROOT_KEYS) delete stripped[key];
+    return { value: stripped as T, traces };
+  }
+  return { value: value as T, traces };
+}
+
+export function renderYamlTemplateStrings<T>(input: T, variables: Record<string, string>): T {
+  return renderValue(input as unknown, deriveVariables(variables)) as T;
+}
+
+function materializeValue(
+  value: unknown,
+  root: unknown,
+  path: string,
+  inheritedVariables: Record<string, string>,
+  traces: YamlCompositionTrace[],
+  stack: readonly string[],
+): unknown {
+  if (Array.isArray(value)) {
+    return value.map((item, index) => materializeValue(item, root, `${path}[${index}]`, inheritedVariables, traces, stack));
+  }
+  if (!isRecord(value)) return renderValue(value, inheritedVariables);
+  const localVariables = deriveVariables({ ...inheritedVariables, ...stringRecord(value.variables) });
+  const refs = extendsRefs(value.extends);
+  let merged: Record<string, unknown> = {};
+  if (refs.length > 0) {
+    for (const ref of refs) {
+      if (stack.includes(ref)) throw new Error(`${path}.extends has cyclic template reference ${ref}`);
+      const template = valueAtPath(root, ref);
+      if (!isRecord(template)) throw new Error(`${path}.extends references missing object ${ref}`);
+      const renderedTemplate = materializeValue(template, root, ref, localVariables, traces, [...stack, ref]);
+      merged = deepMerge(merged, asRecord(renderedTemplate, ref));
+    }
+  }
+  const own = stripCompositionFields(value);
+  const overrides = value.overrides === undefined ? {} : asRecord(value.overrides, `${path}.overrides`);
+  merged = deepMerge(merged, own);
+  merged = deepMerge(merged, overrides);
+  const rendered = renderObjectScalars(merged, deriveVariables({ ...variablesFromRecord(merged), ...localVariables }));
+  const result = Object.fromEntries(
+    Object.entries(asRecord(rendered, path)).map(([key, item]) => [
+      key,
+      materializeValue(item, root, `${path}.${key}`, deriveVariables({ ...localVariables, ...variablesFromRecord(rendered) }), traces, stack),
+    ]),
+  );
+  if (refs.length > 0) {
+    traces.push({
+      path,
+      extends: refs,
+      overrideKeys: Object.keys(overrides).sort(),
+      variables: deriveVariables({ ...localVariables, ...variablesFromRecord(result) }),
+    });
+  }
+  return result;
+}
+
+function stripCompositionFields(value: Record<string, unknown>): Record<string, unknown> {
+  return Object.fromEntries(Object.entries(value).filter(([key]) => !COMPOSITION_KEYS.has(key)));
+}
+
+function extendsRefs(value: unknown): string[] {
+  if (value === undefined) return [];
+  if (typeof value === "string" && value.trim().length > 0) return [value.trim()];
+  if (Array.isArray(value) && value.every((item) => typeof item === "string" && item.trim().length > 0)) {
+    return value.map((item) => item.trim());
+  }
+  throw new Error("extends must be a non-empty string or string array");
+}
+
+function renderValue(value: unknown, variables: Record<string, string>): unknown {
+  if (typeof value === "string") {
+    return value.replace(/\$\{([A-Za-z][A-Za-z0-9_]*)\}/gu, (match, name: string) => variables[name] ?? match);
+  }
+  if (Array.isArray(value)) return value.map((item) => renderValue(item, variables));
+  if (!isRecord(value)) return value;
+  return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, renderValue(item, variables)]));
+}
+
+function renderObjectScalars(value: Record<string, unknown>, variables: Record<string, string>): Record<string, unknown> {
+  return Object.fromEntries(Object.entries(value).map(([key, item]) => [
+    key,
+    typeof item === "string" ? renderValue(item, variables) : item,
+  ]));
+}
+
+function variablesFromRecord(value: unknown): Record<string, string> {
+  if (!isRecord(value)) return {};
+  const result: Record<string, string> = {};
+  if (typeof value.node === "string" && value.node.length > 0) result.NODE = value.node;
+  if (typeof value.lane === "string" && value.lane.length > 0) result.LANE = value.lane;
+  if (typeof value.id === "string" && value.id.length > 0) result.targetId = value.id;
+  return deriveVariables(result);
+}
+
+function deriveVariables(input: Record<string, string>): Record<string, string> {
+  const result = { ...input };
+  if (result.NODE !== undefined) result.nodeLower = result.NODE.toLowerCase();
+  if (result.LANE !== undefined) result.lane = result.LANE;
+  if (result.lane !== undefined && result.LANE === undefined) result.LANE = result.lane;
+  return result;
+}
+
+function stringRecord(value: unknown): Record<string, string> {
+  if (value === undefined) return {};
+  const raw = asRecord(value, "variables");
+  return Object.fromEntries(Object.entries(raw).map(([key, item]) => {
+    if (typeof item !== "string" || item.length === 0) throw new Error(`variables.${key} must be a non-empty string`);
+    return [key, item];
+  }));
+}
+
+function deepMerge(base: Record<string, unknown>, override: Record<string, unknown>): Record<string, unknown> {
+  const result: Record<string, unknown> = { ...base };
+  for (const [key, value] of Object.entries(override)) {
+    const existing = result[key];
+    result[key] = isRecord(existing) && isRecord(value) ? deepMerge(existing, value) : value;
+  }
+  return result;
+}
+
+function valueAtPath(root: unknown, ref: string): unknown {
+  let current: unknown = root;
+  for (const segment of ref.split(".")) {
+    if (!/^[A-Za-z0-9_-]+$/u.test(segment) || !isRecord(current)) return undefined;
+    current = current[segment];
+  }
+  return current;
+}
+
+function asRecord(value: unknown, path: string): Record<string, unknown> {
+  if (!isRecord(value)) throw new Error(`${path} must be an object`);
+  return value;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}